get local file, web endpoints working

Author: fvankrieken

dcpy/connectors/registry.py

@@ -167,5 +167,8 @@ def get_subregistry(self, cls: Type[_C2]) -> ConnectorRegistry[_C2]:
         connectors = {
             t: conn for (t, conn) in self._connectors.items() if isinstance(conn, cls)
         }
-        print(connectors)
-        return ConnectorRegistry(connectors=connectors)  # type: ignore
+        return ConnectorRegistry(connectors=connectors)
+
+    @property
+    def nonversioned(self) -> ConnectorRegistry[NonVersionedConnector]:
+        return self.get_subregistry(NonVersionedConnector)

dcpy/connectors/web.py

@@ -2,6 +2,7 @@
 import requests
 
 from dcpy.utils.logging import logger
+from dcpy.connectors import NonVersionedConnector
 
 
 def download_file(url: str, path: Path) -> None:
@@ -14,3 +15,22 @@ def download_file(url: str, path: Path) -> None:
     response.raise_for_status()
     with open(path, "wb") as f:
         f.write(response.content)
+
+
+class Connector(NonVersionedConnector):
+    conn_type = "file_download"
+
+    def push(self, key: str, version, push_conf: dict | None = {}) -> dict:
+        raise NotImplementedError("Sorry :)")
+
+    def pull(
+        self,
+        key: str,
+        destination_path: Path,
+        pull_conf: dict | None = None,
+    ) -> dict:
+        download_file(key, destination_path)
+        return {"path": destination_path}
+
+    def get_current_version(self, key: str) -> str | None:
+        return None

dcpy/lifecycle/connector_registry.py

@@ -1,11 +1,11 @@
 from dcpy.connectors.edm import recipes, publishing
 from dcpy.connectors.socrata import connector as soc_connector
 from dcpy.connectors.esri import arcgis_feature_service
+from dcpy.connectors import web
 from dcpy.connectors.registry import (
     ConnectorRegistry,
-    Connector,
-    VersionedConnector,
     NonVersionedConnector,
+    Connector,
 )
 from dcpy.utils.logging import logger
 
@@ -20,11 +20,9 @@ def _set_default_connectors():
     connectors.register(connector=publishing.GisDatasetsConnector())
     connectors.register(connector=soc_connector.Connector())
     connectors.register(connector=arcgis_feature_service.Connector())
+    connectors.register(connector=web.Connector())
+    connectors.register(connector=web.Connector(), conn_type="api")
     logger.info(f"Registered Connectors: {connectors.list_registered()}")
 
 
 _set_default_connectors()
-
-
-versioned_connectors = connectors.get_subregistry(VersionedConnector)
-nonversioned_connectors = connectors.get_subregistry(NonVersionedConnector)

dcpy/lifecycle/ingest/__init__.py

@@ -1,11 +1,16 @@
 from pathlib import Path
+import shutil
 import typer
 
 from .run import ingest
+from .connector_registry import LocalFileConnector
 from dcpy.configuration import TEMPLATE_DIR
+from dcpy.lifecycle.connector_registry import connectors
 
 app = typer.Typer(add_completion=False)
 
+connectors.register(LocalFileConnector())
+
 
 @app.command("ingest")
 def _cli_wrapper_run(

dcpy/lifecycle/ingest/configure.py

@@ -5,13 +5,13 @@
 from pathlib import Path
 from urllib.parse import urlparse
 import yaml
-from dcpy.lifecycle.connector_registry import nonversioned_connectors
+from dcpy.lifecycle.connector_registry import connectors
 
 from dcpy.models.lifecycle.ingest import (
     ArchivalMetadata,
     Ingestion,
     LocalFileSource,
-    ConnectorSource,
+    _Source,
     S3Source,
     GisDataset,
     FileDownloadSource,
@@ -59,8 +59,8 @@ def read_template(
 
 def get_version(source: Source, timestamp: datetime) -> str:
     match source:
-        case ConnectorSource():
-            connector = nonversioned_connectors[source.type]
+        case _Source():
+            connector = connectors.nonversioned[source.type]
             version = connector.get_version(source.key, source.model_dump())
         case _:
             version = None

dcpy/lifecycle/ingest/connector_registry.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+import shutil
+
+from dcpy.lifecycle.connector_registry import NonVersionedConnector
+
+
+class LocalFileConnector(NonVersionedConnector):
+    conn_type = "local_file"
+
+    def pull(
+        self, key: str, destination_path: Path, pull_conf: dict | None = None
+    ) -> dict:
+        if Path(key) != destination_path:
+            shutil.copy(key, destination_path)
+        return {"path": destination_path}

dcpy/lifecycle/ingest/extract.py

@@ -1,21 +1,13 @@
 from pathlib import Path
-import shutil
 
 from dcpy.models.lifecycle.ingest import Source
-from dcpy.lifecycle.connector_registry import nonversioned_connectors
+from dcpy.lifecycle.connector_registry import connectors
 
 
 def download_file_from_source(source: Source, filename: str, dir: Path) -> None:
     """
     From parsed config template and version, download raw data from source to provided path
     """
     path = dir / filename
-    if source.type == "local_file":
-        if source.path != path:
-            shutil.copy(source.path, path)
-    else:
-        connector = nonversioned_connectors[source.type]
-        print(source.model_dump())
-        connector.pull(
-            key=source.key, destination_path=path, pull_conf=source.model_dump()
-        )
+    connector = connectors.nonversioned[source.type]
+    connector.pull(key=source.key, destination_path=path, pull_conf=source.model_dump())

dcpy/models/lifecycle/ingest.py

@@ -16,7 +16,7 @@
 from dcpy.connectors.esri import arcgis_feature_service
 
 
-class ConnectorSource(ABC):
+class _Source(ABC):
     type: str
 
     @property
@@ -25,29 +25,41 @@ def key(self) -> str:
         return ""
 
 
-class LocalFileSource(BaseModel, extra="forbid"):
+class LocalFileSource(_Source, BaseModel, extra="forbid"):
     type: Literal["local_file"]
     path: Path
 
+    @property
+    def key(self) -> str:
+        return str(self.path)
+
 
 class S3Source(BaseModel, extra="forbid"):
     type: Literal["s3"]
     bucket: str
     key: str
 
 
-class FileDownloadSource(BaseModel, extra="forbid"):
+class FileDownloadSource(_Source, BaseModel, extra="forbid"):
     type: Literal["file_download"]
     url: str
 
+    @property
+    def key(self) -> str:
+        return self.url
 
-class GenericApiSource(BaseModel, extra="forbid"):
+
+class GenericApiSource(_Source, BaseModel, extra="forbid"):
     type: Literal["api"]
     endpoint: str
     format: Literal["json", "csv"]
 
+    @property
+    def key(self) -> str:
+        return self.endpoint
+
 
-class DEPublished(BaseModel, ConnectorSource, extra="forbid"):
+class DEPublished(_Source, BaseModel, extra="forbid"):
     type: Literal["edm.publishing.published"]
     product: str
     filename: str
@@ -57,7 +69,7 @@ def key(self) -> str:
         return self.product
 
 
-class GisDataset(BaseModel, ConnectorSource, extra="forbid"):
+class GisDataset(_Source, BaseModel, extra="forbid"):
     """Dataset published by GIS in edm-publishing/datasets"""
 
     # Some datasets here will phased out if we eventually get data
@@ -70,7 +82,7 @@ def key(self) -> str:
         return self.name
 
 
-class SocrataSource(BaseModel, ConnectorSource, extra="forbid"):
+class SocrataSource(_Source, BaseModel, extra="forbid"):
     type: Literal["socrata"]
     org: socrata.Org
     uid: str
@@ -88,7 +100,7 @@ def key(self) -> str:
         return self.uid
 
 
-class ESRIFeatureServer(BaseModel, ConnectorSource, extra="forbid"):
+class ESRIFeatureServer(_Source, BaseModel, extra="forbid"):
     type: Literal["arcgis_feature_server"]
     server: esri.Server
     dataset: str