Include definitions from GitHub repo

nomenclature/codelist.py

@@ -9,7 +9,7 @@
 from pydantic import BaseModel, validator
 
 from nomenclature.code import Code, MetaCode, RegionCode, VariableCode
-from nomenclature.config import DataStructureConfig
+from nomenclature.config import CodeListConfig, RegionCodeListConfig
 from nomenclature.countries import countries
 from nomenclature.error.codelist import DuplicateCodeError
 from nomenclature.error.variable import (
@@ -170,7 +170,7 @@ def from_directory(
         cls,
         name: str,
         path: Path,
-        config: DataStructureConfig = None,
+        config: CodeListConfig = None,
         file_glob_pattern: str = "**/*",
     ):
         """Initialize a CodeList from a directory with codelist files
@@ -181,7 +181,7 @@ def from_directory(
             Name of the CodeList
         path : :class:`pathlib.Path` or path-like
             Directory with the codelist files
-        config: :class:`DataStructureConfig`, optional
+        config: :class:`CodeListConfig`, optional
             Attributes for configuring the CodeList
         file_glob_pattern : str, optional
             Pattern to downselect codelist files by name
@@ -525,7 +525,7 @@ def from_directory(
         cls,
         name: str,
         path: Path,
-        config: DataStructureConfig = None,
+        config: RegionCodeListConfig = None,
         file_glob_pattern: str = "**/*",
     ):
         """Initialize a RegionCodeList from a directory with codelist files
@@ -536,7 +536,7 @@ def from_directory(
             Name of the CodeList
         path : :class:`pathlib.Path` or path-like
             Directory with the codelist files
-        config : :class:`DataStructureConfig`, optional
+        config : :class:`RegionCodeListConfig`, optional
             Attributes for configuring the CodeList
         file_glob_pattern : str, optional
             Pattern to downselect codelist files by name, default: "**/*" (i.e. all
@@ -551,9 +551,9 @@ def from_directory(
         code_list: List[RegionCode] = []
 
         # initializing from general configuration
-        if config is not None and config.region is not None:
+        if config is not None:
             # adding all countries
-            if config.region.country is True:
+            if config.country is True:
                 for c in countries:
                     try:
                         code_list.append(
@@ -566,15 +566,18 @@ def from_directory(
                         code_list.append(RegionCode(name=c.name, hierarchy="Country"))
 
             # importing from an external repository
-            if repo := config.region.repository:
-                repo_path = path.parents[1] / repo
-                if not repo_path.exists():
-                    raise FileNotFoundError(f"Repository not found: {repo}")
+            if config.repository:
+                repo_path = path.parent / config.repository_name
+                config.fetch_repo(repo_path)
+
                 code_list = cls._parse_region_code_dir(
-                    code_list, repo_path, file_glob_pattern, repository=repo
+                    code_list,
+                    repo_path / "definitions" / "region",
+                    file_glob_pattern,
+                    repository=config.repository,
                 )
                 code_list = cls._parse_and_replace_tags(
-                    code_list, repo_path, file_glob_pattern
+                    code_list, repo_path / "definitions" / "region", file_glob_pattern
                 )
 
         # parse from current repository

nomenclature/config.py

@@ -1,12 +1,44 @@
 from pathlib import Path
-from typing import Dict, Optional
-from pydantic import BaseModel
+from typing import Optional
+from pydantic import BaseModel, root_validator
 
 import yaml
+from git import Repo
 
 
 class CodeListConfig(BaseModel):
-    repository: Optional[Path]
+    repository: Optional[str]
+    hash: Optional[str]
+    release: Optional[str]
+
+    @root_validator()
+    def check_hash_and_release(cls, v):
+        if v.get("hash") and v.get("release"):
+            raise ValueError("Either 'hash' or 'release' can be provided, not both.")
+        return v
+
+    @property
+    def revision(self):
+        return self.hash or self.release or "main"
+
+    @property
+    def repository_name(self):
+        return self.repository.split("/")[-1].split(".")[0]
+
+    def fetch_repo(self, to_path="./common-definitions"):
+        to_path = to_path if isinstance(to_path, Path) else Path(to_path)
+
+        if not to_path.is_dir():
+            repo = Repo.clone_from(self.repository, to_path)
+        else:
+            repo = Repo(to_path)
+            repo.remotes.origin.fetch()
+        repo.git.reset("--hard")
+        repo.git.checkout(self.revision)
+        repo.git.reset("--hard")
+        repo.git.clean("-xdf")
+        if self.revision == "main":
+            repo.remotes.origin.pull()
 
 
 class RegionCodeListConfig(CodeListConfig):

nomenclature/definition.py

@@ -56,7 +56,10 @@ def __init__(self, path, dimensions=None):
         for dim in self.dimensions:
             codelist_cls = SPECIAL_CODELIST.get(dim, CodeList)
             self.__setattr__(
-                dim, codelist_cls.from_directory(dim, path / dim, self.config)
+                dim,
+                codelist_cls.from_directory(
+                    dim, path / dim, getattr(self.config, dim, None)
+                ),
             )
 
         empty = [d for d in self.dimensions if not self.__getattribute__(d)]

setup.cfg

@@ -23,6 +23,7 @@ install_requires =
     pandas >= 1.5.2
     numpy
     pycountry
+    gitpython
 setup_requires =
     setuptools >= 41
     setuptools_scm

tests/data/general-config-definitions/config.yaml

@@ -1,3 +1,3 @@
 region:
-  repository: validation_nc/region
+  repository: git@github.com:IAMconsortium/common-definitions.git
   country: true

tests/test_definition.py

@@ -1,3 +1,4 @@
+import shutil
 import pytest
 import pandas as pd
 from nomenclature import DataStructureDefinition, create_yaml_from_xlsx
@@ -45,17 +46,26 @@ def test_definition_from_general_config():
         TEST_DATA_DIR / "general-config-definitions",
         dimensions=["region"],
     )
-
-    # explicitly defined in `general-config-definitions/region/regions.yaml`
-    assert "Region A" in obs.region
-    # imported from `validation_nc` repo
-    assert "World" in obs.region
-    # added via general-config definitions
-    assert "Austria" in obs.region
-    # added via general-config definitions renamed from pycountry name
-    assert "Bolivia" in obs.region
-    # added via general-config definitions in addition to pycountry.countries
-    assert "Kosovo" in obs.region
+    try:
+        # explicitly defined in `general-config-definitions/region/regions.yaml`
+        assert "Region A" in obs.region
+        # imported from https://github.com/IAMconsortium/common-definitions repo
+        assert "World" in obs.region
+        # added via general-config definitions
+        assert "Austria" in obs.region
+        # added via general-config definitions renamed from pycountry name
+        assert "Bolivia" in obs.region
+        # added via general-config definitions in addition to pycountry.countries
+        assert "Kosovo" in obs.region
+    finally:
+        # clean up the external repo
+        repo_dir = (
+            TEST_DATA_DIR
+            / "general-config-definitions"
+            / obs.config.region.repository_name
+        )
+        if repo_dir.exists():
+            shutil.rmtree(repo_dir)
 
 
 def test_to_excel(simple_definition, tmpdir):