Merge branch 'master' into ids

Author: martinfleis

.github/workflows/unittests.yml

@@ -19,12 +19,18 @@
      strategy:
        matrix:
          os: ['ubuntu-latest']
-         environment-file: [ci/37-minimal.yaml, ci/37.yaml, ci/38.yaml, ci/39.yaml, ci/310.yaml]
+         environment-file: [
+            ci/38.yaml,
+            ci/39.yaml,
+            ci/310.yaml,
+            ci/310_shapely_dev.yaml
+         ]
          include:
            - environment-file: ci/310.yaml
              os: macos-latest
            - environment-file: ci/310.yaml
              os: windows-latest
+       fail-fast: false
 
      steps:
        - name: checkout repo
@@ -35,6 +41,7 @@
          with:
            environment-file: ${{ matrix.environment-file }}
            micromamba-version: 'latest'
+           channel-priority: 'flexible'
 
        - name: run tests - bash
          shell: bash -l {0}

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos: 
--   repo: https://github.com/psf/black
-    rev: 21.12b0
-    hooks:
-    - id: black
-      language_version: python3.9
+- repo: https://github.com/psf/black
+  rev: 22.10.0
+  hooks:
+  - id: black
+    language_version: python3

ci/37.yaml ci/310_shapely_dev.yamlci/37.yaml renamed to ci/310_shapely_dev.yaml

@@ -1,8 +1,9 @@
 name: test
 channels:
   - conda-forge
+  - conda-forge/label/shapely_dev
 dependencies:
-  - python=3.7
+  - python=3.10
   - platformdirs
   - beautifulsoup4
   - jinja2
@@ -15,11 +16,18 @@ dependencies:
   - pytest
   - pytest-cov
   - pytest-xdist
-  - networkx=2.7
   # optional
-  - geopandas>=0.7.0
+  - geopandas>=0.12.0
   - joblib
+  - networkx
   - numba
   - packaging
+  - shapely>=2.0b1
   - xarray
   - zstd
+  # for docs build action (this env only)
+  - nbsphinx
+  - numpydoc
+  - sphinx
+  - sphinxcontrib-bibtex
+  - sphinx_bootstrap_theme

ci/37-minimal.yaml

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 46,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -11,7 +11,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 47,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -25,7 +25,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 48,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -83,7 +83,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 71,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -126,28 +126,41 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 72,
+   "execution_count": 5,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "FileNotFoundError",
+     "evalue": "[Errno 2] No such file or directory: 'libpysal/examples/examples.json'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
+      "Input \u001b[0;32mIn [5]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mlibpysal/examples/examples.json\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m efile:\n\u001b[1;32m      2\u001b[0m     examples  \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mload(efile)\n",
+      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'libpysal/examples/examples.json'"
+     ]
+    }
+   ],
    "source": [
     "with open('libpysal/examples/examples.json', 'r') as efile:\n",
     "    examples  = json.load(efile)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 73,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/plain": [
-       "dict_keys(['nat', 'south', 'rio', 'mexico', 'baltimore'])"
-      ]
-     },
-     "execution_count": 73,
-     "metadata": {},
-     "output_type": "execute_result"
+     "ename": "NameError",
+     "evalue": "name 'examples' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "Input \u001b[0;32mIn [6]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mexamples\u001b[49m\u001b[38;5;241m.\u001b[39mkeys()\n",
+      "\u001b[0;31mNameError\u001b[0m: name 'examples' is not defined"
+     ]
     }
    ],
    "source": [
@@ -555,7 +568,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -569,9 +582,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.3"
+   "version": "3.10.6"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

examples.ipynb

@@ -1,23 +1,30 @@
-""" The :mod:`libpysal.examples` module includes a number of small built-in
-    example datasets as well as functions to fetch larger datasets.
+"""The :mod:`libpysal.examples` module provides example datasets.
+
+The datasets consist of two sets, built-ins which are installed with
+this module and remotes that can be downloaded.
+
+This module provides functionality for working with these example datasets.
 """
 
 
 import pandas as pd
 from .base import example_manager
 from .remotes import datasets as remote_datasets
 from .builtin import datasets as builtin_datasets
+from typing import Union
 
 
-from typing import Union
+available_datasets = builtin_datasets.copy()
+available_datasets.update(remote_datasets.datasets)
 
-__all__ = ["get_path", "available", "explain", "fetch_all"]
+__all__ = ["get_path", "available", "explain", "fetch_all",
+           "get_url", "load_example", "summary"]
+
+example_manager.add_examples(available_datasets)
 
-example_manager.add_examples(builtin_datasets)
 
 def fetch_all():
-    """Fetch and install all remote datasets
-    """
+    """Fetch and install all remote datasets."""
     datasets = remote_datasets.datasets
     names = list(datasets.keys())
     names.sort()
@@ -32,35 +39,36 @@ def fetch_all():
 
 def available() -> pd.DataFrame:
     """Return a dataframe with available datasets."""
-    fetch_all()
-
     return example_manager.available()
 
 
 def explain(name: str) -> str:
     """Explain a dataset by name."""
-
     return example_manager.explain(name)
 
 
+def get_url(name: str) -> str:
+    """Get url for remote dataset."""
+    return example_manager.get_remote_url(name)
+
+
 def load_example(example_name: str) -> Union[base.Example, builtin.LocalExample]:
     """Load example dataset instance."""
     example = example_manager.load(example_name)
-
-    if example is None:
-        fetch_all()  # refresh remotes
-        example = example_manager.load(example_name)
-
     return example
 
 
 def get_path(file_name: str) -> str:
     """Get the path for a file by searching installed datasets."""
-
     installed = example_manager.get_installed_names()
     for name in installed:
         example = example_manager.datasets[name]
         pth = example.get_path(file_name, verbose=False)
         if pth:
             return pth
     print("{} is not a file in any installed dataset.".format(file_name))
+
+
+def summary():
+    """Summary of datasets."""
+    example_manager.summary()

libpysal/examples/__init__.py

@@ -8,8 +8,6 @@
 import io
 import os
 import webbrowser
-from os import environ, makedirs
-from os.path import exists, expanduser, join
 from platformdirs import user_data_dir
 import zipfile
 import requests
@@ -35,9 +33,9 @@ def get_data_home():
 
     appname = "pysal"
     appauthor = "pysal"
-    data_home =  user_data_dir(appname, appauthor) 
-    if not exists(data_home):
-        makedirs(data_home, exist_ok=True)
+    data_home = user_data_dir(appname, appauthor)
+    if not os.path.exists(data_home):
+        os.makedirs(data_home, exist_ok=True)
     return data_home
 
 
@@ -122,7 +120,9 @@ class Example:
 
     """
 
-    def __init__(self, name, description, n, k, download_url, explain_url):
+    def __init__(self, name, description, n, k, download_url,
+                 explain_url):
+        """Initialze Example."""
         self.name = name
         self.description = description
         self.n = n
@@ -134,12 +134,10 @@ def __init__(self, name, description, n, k, download_url, explain_url):
 
     def get_local_path(self, path=get_data_home()) -> str:
         """Get the local path for example."""
-
-        return join(path, self.root)
+        return os.path.join(path, self.root)
 
     def get_path(self, file_name, verbose=True) -> Union[str, None]:
         """Get the path for local file."""
-
         file_list = self.get_file_list()
         for file_path in file_list:
             base_name = os.path.basename(file_path)
@@ -151,7 +149,6 @@ def get_path(self, file_name, verbose=True) -> Union[str, None]:
 
     def downloaded(self) -> bool:
         """Check if the example has already been installed."""
-
         path = self.get_local_path()
         if os.path.isdir(path):
             self.installed = True
@@ -180,7 +177,7 @@ def download(self, path=get_data_home()):
         if not self.downloaded():
             request = requests.get(self.download_url)
             archive = zipfile.ZipFile(io.BytesIO(request.content))
-            target = join(path, self.root)
+            target = os.path.join(path, self.root)
             print("Downloading {} to {}".format(self.name, target))
             archive.extractall(path=target)
             self.zipfile = archive
@@ -213,8 +210,8 @@ def load(self, file_name) -> io.FileIO:
 class Examples:
     """Manager for pysal example datasets."""
 
-    def __init__(self):
-        self.datasets = {}
+    def __init__(self, datasets={}):
+        self.datasets = datasets
 
     def add_examples(self, examples):
         """Add examples to the set of datasets available."""
@@ -273,5 +270,23 @@ def get_installed_names(self) -> list:
         ds = self.datasets
         return [name for name in ds if ds[name].installed]
 
+    def get_remote_url(self, name):
+        if name in self.datasets: 
+            try:
+                return self.datasets[name].download_url
+            except:
+                print(f'{name} is a built-in dataset, no url.')
+        else:
+            print(f'{name} is not an available dataset.')
+                
+
+    def summary(self):
+        """Report on datasets."""
+        available = self.available()
+        n = available.shape[0]
+        n_installed = available.Installed.sum()
+        n_remote = n - n_installed
+        print(f'{n} datasets available, {n_installed} installed, {n_remote} remote.')
+
 
 example_manager = Examples()