API Import + QA Maturity Levels

MANIFEST.in

@@ -6,4 +6,7 @@ recursive-include ckanext/udc/migration *.ini *.py *.mako
 recursive-include ckanext/udc_import *.html *.json *.js *.less *.css *.mo *.yml
 recursive-include ckanext/udc_import/migration *.ini *.py *.mako
 recursive-include ckanext/udc_theme *.html *.json *.js *.less *.css *.mo *.yml
-recursive-include ckanext/udc_theme/migration *.ini *.py *.mako
+recursive-include ckanext/udc_theme/migration *.ini *.py *.mako
+recursive-include ckanext/udc_import_other_portals *.html *.json *.js *.less *.css *.mo *.yml
+recursive-include ckanext/udc_import_other_portals/migration *.ini *.py *.mako
+recursive-include ckanext/udc_react *.html *.json *.js *.less *.css *.mo *.yml

ckanext/udc/cli/udc.py

@@ -1,6 +1,7 @@
 import click
 import ckan.model as model
 import logging
+from importlib import import_module
 
 
 @click.group(short_help=u"UDC commands.")
@@ -42,4 +43,12 @@ def initdb():
 
     from ..licenses.model import init_tables
     init_tables()
+
+    # Try import "import_other_portals" models
+    try:
+        lib = import_module("ckanext.udc_import_other_portals.model")
+        lib.init_tables()
+    except:
+        log.warning("Cannot init DB in import_other_portals plugin")
+
     log.info("DB tables are setup")

ckanext/udc_import_other_portals/jobs.py

@@ -0,0 +1,99 @@
+import logging
+import json
+from typing import Any, List, Dict, cast
+from datetime import datetime
+
+from ckanext.udc_import_other_portals.logger import ImportLogger
+
+from ckan.types import Context
+import ckan.logic as logic
+import ckan.authz as authz
+import ckan.lib.jobs as jobs
+import ckan.model as model
+
+from ckanext.udc_import_other_portals.model import CUDCImportConfig, CUDCImportLog
+from .logic.base import BaseImport
+
+
+def job_run_import(import_config_id: str, run_by: str):
+    """
+    Run imports.
+    Raises:
+        logic.NotAuthorized
+        logic.ValidationError
+        logic.ValidationError
+    """
+    logger = ImportLogger()
+    import_log = CUDCImportLog(import_config_id=import_config_id, 
+                               run_at=datetime.utcnow(),
+                               run_by=run_by)
+    userobj = model.User.get(run_by)
+
+    import_instance = None
+    try:
+        if not import_config_id:
+            raise logger.exception(
+                logic.ValidationError("import_config_id should be provided.")
+            )
+
+        import_config = CUDCImportConfig.get(import_config_id)
+
+        if not import_config:
+            raise logger.exception(
+                logic.ValidationError(f"import_config not found: {import_config_id}")
+            )
+
+        # Check existing import instance
+        if import_config.is_running:
+            raise logger.exception(logic.ValidationError("Already running."))
+
+        code = import_config.code
+        if not code:
+            raise logger.exception(logic.ValidationError("Code does not existed."))
+
+        # Run the provided code
+        locals = {}
+        try:
+            exec(code, globals(), locals)
+
+        except Exception as e:
+            raise logger.exception(e)
+
+        DefaultImportClass = locals.get("DefaultImportClass")
+
+
+        if not DefaultImportClass:
+            raise logic.ValidationError("DefaultImportClass is not defined.")
+
+        try:
+            import_instance = DefaultImportClass()
+            context = cast(
+                Context,
+                {
+                    "model": model,
+                    "session": model.Session,
+                    "user": userobj.name,
+                    "auth_user_obj": userobj,
+                },
+            )
+            import_instance.run_imports(context)
+        except Exception as e:
+            raise logger.exception(e)
+
+    except Exception as e:
+        raise logger.exception(e)
+    finally:
+        # Finished
+        if import_instance:
+            import_log.has_error = logger.has_error or import_instance.logger.has_error
+            import_log.has_warning = logger.has_warning or import_instance.logger.has_warning
+            import_log.logs = "\n".join([*logger.logs, *import_instance.logger.logs])
+        else:
+            import_log.has_error = logger.has_error
+            import_log.has_warning = logger.has_warning
+            import_log.logs = "\n".join([*logger.logs].join("\n"))
+        import_log.finished_at = datetime.utcnow()
+        print(import_log.logs)
+
+        model.Session.add(import_log)
+        model.Session.commit()

ckanext/udc_import_other_portals/logger.py

@@ -0,0 +1,36 @@
+import logging
+import traceback
+
+
+log = logging.getLogger(__name__)
+
+
+class ImportLogger:
+    """Log to file and memory."""
+
+    def __init__(self, base_logger=log):
+        self.base_logger = base_logger
+        self.logs = []
+        self.has_error = False
+        self.has_warning = False
+
+    def exception(self, e):
+        trace = "".join(traceback.TracebackException.from_exception(e).format())
+        self.logs.append(f"Exception: {trace}")
+        self.base_logger.exception(e)
+        self.has_error = True
+        return e
+
+    def warning(self, s):
+        self.logs.append(f"Warning: {s}")
+        self.base_logger.warning(s)
+        self.has_error = True
+
+    def error(self, s):
+        self.logs.append(f"Error: {s}")
+        self.base_logger.error(s)
+        self.has_warning = True
+
+    def info(self, s):
+        self.logs.append(f"Info: {s}")
+        self.base_logger.info(s)

ckanext/udc_import_other_portals/logic/__init__.py

@@ -0,0 +1 @@
+from .ckan_based.base import CKANBasedImport