Merge 73a3377 into 6f2ec8d

.github/workflows/pythonpackage.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.5, 3.6, 3.7]
+        python-version: [3.6, 3.7, 3.8]
 
     steps:
     - uses: actions/checkout@v1
@@ -20,15 +20,15 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        pip install .
+    - name: Test with unit test
+      run: |
+        python -m pyrfume.unit_test 
     - name: Lint with flake8
       run: |
         pip install flake8
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test with pytest
-      run: |
-        pip install pytest
-        pytest
+    

.travis.yml

@@ -1,7 +1,9 @@
 language: python
+dist: bionic
 python:
   - "3.7"
   - "3.8"
+  - "3.9"
 install:
   - sudo apt update
   - python -m pip install --upgrade pip

mysql-docker/docker-compose.yml

@@ -0,0 +1,12 @@
+version: '2.4'
+services:
+  db:
+    # image: datajoint/mysql:5.6
+    # image: datajoint/mysql:5.7
+    image: datajoint/mysql:8.0
+    ports:
+      - "3306:3306"
+    environment:
+      - MYSQL_ROOT_PASSWORD=simple
+    volumes:
+      - ./data:/var/lib/mysql

pyrfume/__init__.py

@@ -10,7 +10,7 @@
 from tqdm.auto import tqdm, trange
 
 from .base import CONFIG_PATH, DEFAULT_DATA_PATH
-from typing import Any, List
+from typing import Any
 
 logger = logging.getLogger("pyrfume")
 
@@ -19,6 +19,7 @@ def init_config(overwrite=False):
     if overwrite or not CONFIG_PATH.exists():
         config = configparser.ConfigParser()
         config["PATHS"] = {"pyrfume-data": str(DEFAULT_DATA_PATH)}
+        config["DATABASE"] = {"schema_name": "UNDEFINED"}
         with open(CONFIG_PATH, "w") as f:
             config.write(f)
 

pyrfume/datajoint_tools.py

@@ -3,9 +3,12 @@
 import re
 from importlib import import_module
 from inspect import isclass
-
+from typing import Any, ForwardRef, _GenericAlias
 import datajoint as dj
+from .dbtables import QuantityAdapter
+dj.errors._switch_adapted_types(True)
 
+QUANTITY_ADAPTER = None
 
 def schematize(cls, schema: dj.schema):
     """Take a Python class and build a Datajoint table from it.
@@ -18,10 +21,77 @@ def schematize(cls, schema: dj.schema):
         cls: The schematized class (now for use with DataJoint)
     """
     cls = type(cls.__name__, (dj.Manual, cls, object), {})
-    set_dj_definition(cls)
+    cls = set_dj_definition(cls)
+    global QUANTITY_ADAPTER
+    if QUANTITY_ADAPTER:
+        schema.context.update({'QUANTITY_ADAPTER': QUANTITY_ADAPTER})
     cls = schema(cls)
     return cls
 
+def create_quantity_adapter() -> None:
+    """ Create an datajoint adapter class, `QuantityAdapter`, that puts and gets 
+        Python Quantity objects to and from the datajoint database server.
+        The adapter will be assigned to the global variable `QUANTITY_ADAPTER`
+        in this module.
+    """
+
+    global QUANTITY_ADAPTER
+    QUANTITY_ADAPTER = QuantityAdapter()
+
+
+def handle_dict(cls, _type_map: dict, attr: Any, type_hint: _GenericAlias):
+    """Using master-part relation to store a dict. It is assumed that 
+        the type of keys have corresponding tables in the database.
+
+        It is assumed that values of the dict are:
+        primitive type which is in `_type_map`
+        OR
+        `quantities.Quantity` instance.
+
+    Args:
+        _type_map (dict): A map that maps type hint to data type that accepted by datajoint.
+        attr (Any): Variable name of the dict.
+        type_hint (typing._GenericAlias): Required to be a type hint like `Dict[TypeA, int]`.
+                                            A type hint of `dict` will cause an exception.
+
+    Returns:
+        type: `cls` that contains a part class for the keys of the dict.
+    """
+
+    # For example, components: Dict[ClassA, int] = {a: 1, b: 2}
+    # key_cls_name will be "ClassA"
+    # part_cls_name will be "Component", 
+    # note that the "s" at the end of the dict name will be removed.
+
+    part_cls_name = attr[0].upper() + attr[1:]
+    part_cls_name = part_cls_name[:-1] if part_cls_name[-1] == 's' else part_cls_name
+
+    key_type = type_hint.__args__[0]
+    value_type = type_hint.__args__[1]
+
+    key_cls_name = key_type.__forward_arg__ if isinstance(key_type, ForwardRef) else key_type.__name__
+    value_type = value_type.__forward_arg__ if isinstance(value_type, ForwardRef) else value_type.__name__
+
+    if value_type == 'Quantity':
+        if not QUANTITY_ADAPTER:
+            create_quantity_adapter()
+        value_type = '<QUANTITY_ADAPTER>'
+    else:
+        assert value_type in _type_map
+        value_type = _type_map[value_type]
+
+    part_cls = type(
+        part_cls_name,
+        (dj.Part, object),
+        {
+            "definition": "\n-> %s\n-> %s\n---\nvalue = NULL : %s"
+            % (cls.__name__, key_cls_name, value_type)
+        }
+    )
+    cls_dict = dict(vars(cls))
+    cls_dict[part_cls_name] = part_cls
+    cls = type(cls.__name__, tuple(cls.__bases__), {part_cls_name: part_cls})
+    return cls
 
 def set_dj_definition(cls, type_map: dict = None) -> None:
     """Set the definition property of a class by inspecting its attributes.
@@ -31,29 +101,45 @@ def set_dj_definition(cls, type_map: dict = None) -> None:
         type_map: Optional additional type mappings
     """
     # A mapping between python types and DataJoint types
-    _type_map = {"int": "int", "str": "varchar(256)", "float": "float", "datetime": "datetime", "bool": "tinyint"}
+    _type_map = {
+        "int": "int", 
+        "str": "varchar(256)", 
+        "float": "float",
+        "Quantity": "float",
+        "datetime": "datetime", 
+        "datetime.datetime": "datetime", 
+        "bool": "tinyint",
+        "list": "longblob",
+    }
     # A list of python types which have no DataJoint
     # equivalent and so are unsupported
-    unsupported = [list, dict]
+    unsupported = [dict]
     if type_map:
         _type_map.update(type_map)
-    dj_def = "id: int auto_increment\n---\n"
-    for attr, type_hint in cls.__annotations__.items():
+    dj_def = "%s_id: int auto_increment\n---\n" % cls.__name__.lower()
+    cls_items = cls.__annotations__.items()
+    for attr, type_hint in cls_items:
         if type_hint in unsupported:
             continue
         name = getattr(type_hint, "__name__", type_hint)
-        default = getattr(cls, attr)
+        default = getattr(cls, attr)            
+
         if isinstance(default, str):
             default = '"%s"' % default
         elif isinstance(default, bool):
             default = int(default)
-        elif default is None:
+        else:
             default = "NULL"
-        if name in _type_map:
+
+        if getattr(type_hint, '_name', "") == 'Dict':
+            cls = handle_dict(cls, _type_map, attr, type_hint)
+            continue
+        elif name in _type_map:
             dj_def += "%s = %s : %s\n" % (attr, default, _type_map[name])
         else:
             dj_def += "-> %s\n" % name
     cls.definition = dj_def
+    return cls
 
 
 def import_classes(module_name: str, match: str = None) -> dict: