Merge pull request #31 from octue/v0.0.10

V0.0.10

.pre-commit-config.yaml

@@ -29,6 +29,11 @@ repos:
     rev: 3.8.1
     hooks:
       - id: flake8
-#        args: ['--config=setup.cfg']
         additional_dependencies: [flake8-isort]
         language_version: python3
+
+  - repo: https://github.com/thclark/pre-commit-sphinx
+    rev: 0.0.1
+    hooks:
+      - id: build-docs
+        language_version: python3

docs/.gitignore

@@ -0,0 +1,2 @@
+doctrees
+html

docs/source/quick_start.rst

@@ -77,7 +77,7 @@ Load the twine
 ==============
 
 **twined** provides a `Twine()` class to load a twine (from a file or a json string).
-The loading process checks the twine is valid. It's as simple as:
+The loading process checks the twine itself is valid. It's as simple as:
 
 .. code-block:: py
 
@@ -91,5 +91,27 @@ The loading process checks the twine is valid. It's as simple as:
 Validate some inputs
 ====================
 
+Say we have some json that we want to parse and validate, to make sure it matches what's required for input values.
+
+.. code-block:: py
+
+    my_input_values = my_twine.validate_input_values(json='{"foo": 30, "baz": 500}')
+
+You can read the values from a file too. Paste the following into a file named ``input_values.json``:
+
+.. code-block:: javascript
+
+    {
+      "foo": 30,
+      "baz": 500
+    }
+
+Then parse and validate directly from the file:
+
+.. code-block:: py
+
+    my_input_values = my_twine.validate_input_values(file="input_values.json")
+
+
 .. ATTENTION::
-    LIBRARY IS UNDER CONSTRUCTION! WATCH THIS SPACE!
+    LIBRARY IS UNDER CONSTRUCTION! WATCH THIS SPACE FOR MORE!

docs/source/schema.rst

@@ -278,8 +278,15 @@ Here, we describe how each of these data classes is described by **twined**.
 
          SERVICE_API_KEY=someLongTokenTHatYouProbablyHaveToPayTheThirdPartyProviderLoadsOfMoneyFor
 
-      **twined** helps by providing a small shim to check for their presence and bring these environment variables
-      into your configuration.
+      Credentials may also reside in a ``.env`` file in the current directory, either in the format above
+      (with a new line for each variable) or, for convenience, as bash exports like:
+
+      .. code-block:: javascript
+
+         export SERVICE_API_KEY=someLongTokenTHatYouProbablyHaveToPayTheThirdPartyProviderLoadsOfMoneyFor
+
+      The ``validate_credentials()`` method of the ``Twine class checks for their presence and, where contained in a
+      ``.env`` file, ensures they are loaded into the environment.
 
       .. ATTENTION::
 

requirements-dev.txt

@@ -10,3 +10,20 @@ flake8==3.8.3  # https://github.com/PyCQA/flake8
 flake8-isort==3.0.0  # https://github.com/gforcada/flake8-isort
 black==19.10.b0  # https://github.com/ambv/black
 pre-commit  # https://github.com/pre-commit/pre-commit
+
+
+# Pre-deploy checks
+# ------------------------------------------------------------------------------
+setuptools
+wheel
+twine  # <---- nothing to do with the twined library!
+
+
+# Building documentation
+# ------------------------------------------------------------------------------
+Sphinx==1.8.3
+sphinx-rtd-theme==0.4.2
+sphinx-tabs==1.1.10
+breathe==4.11.1
+exhale==0.2.1
+pre-commit try-repo ../hook-repo foo --verbose --all-files

setup.cfg

@@ -1,3 +1,3 @@
 [metadata]
 long_description = file: README.md
-long_description_content_type = text/markdown; charset=UTF-8
+long_description_content_type = text/markdown; charset=UTF-8

setup.py

@@ -15,11 +15,11 @@
 
 setup(
     name="twined",
-    version="0.0.9",
+    version="0.0.10",
     py_modules=[],
-    install_requires=["jsonschema ~= 3.2.0"],
+    install_requires=["jsonschema ~= 3.2.0", "python-dotenv"],
     url="https://www.github.com/octue/twined",
-    license='MIT',
+    license=license_text,
     author="Octue (github: octue)",
     description="A library to help digital twins and data services talk to one another",
     long_description=readme_text,

tests/data/children/extra_key.json

@@ -0,0 +1,13 @@
+[
+  {
+    "key": "gis",
+    "id": "some-id",
+    "uri_env_name": "SOME_ENV_VAR_NAME",
+    "some_extra_key": "should not be a problem if present"
+  },
+  {
+    "key": "some_weird_other_child",
+    "id": "some-other-id",
+    "uri_env_name": "SOME_ENV_VAR_NAME"
+  }
+]

tests/data/children/extra_property.json

@@ -0,0 +1,8 @@
+[
+  {
+    "key": "gis",
+    "id": "some-id",
+    "uri_env_name": "SOME_ENV_VAR_NAME",
+    "some_extra_property": "should not be a problem if present"
+  }
+]

tests/data/children/invalid_env_name.json

@@ -0,0 +1,7 @@
+[
+  {
+    "key": "gis",
+    "id": "some-id",
+    "uri_env_name": "an environment variable which isnt in CAPS_CASE is invalid per the credentials spec"
+  }
+]

tests/data/children/valid.json

@@ -0,0 +1,7 @@
+[
+  {
+    "key": "gis",
+    "id": "some-id",
+    "uri_env_name": "NAME_OF_SOME_ENV_VAR_THAT_CONTAINS_A_URI"
+  }
+]

tests/data/manifests/inputs/input_valid.json

@@ -0,0 +1,38 @@
+{
+  "id": "8ead7669-8162-4f64-8cd5-4abe92509e17",
+  "datasets": [
+    {
+      "id": "7ead7669-8162-4f64-8cd5-4abe92509e17",
+      "name": "my meteorological dataset",
+      "tags": "met, mast, wind",
+      "files": [
+        {
+          "path": "input/datasets/7ead7669/file_1.csv",
+          "cluster": 0,
+          "sequence": 0,
+          "extension": "csv",
+          "tags": "",
+          "posix_timestamp": 0,
+          "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86",
+          "last_modified": "2019-02-28T22:40:30.533005Z",
+          "name": "file_1.csv",
+          "size_bytes": 59684813,
+          "sha-512/256": "somesha"
+        },
+        {
+          "path": "input/datasets/7ead7669/file_2.csv",
+          "cluster": 0,
+          "sequence": 1,
+          "extension": "csv",
+          "tags": "",
+          "posix_timestamp": 0,
+          "id": "bbff07bc-7c19-4ed5-be6d-a6546eae8e45",
+          "last_modified": "2019-02-28T22:40:40.633001Z",
+          "name": "file_2.csv",
+          "size_bytes": 59684813,
+          "sha-512/256": "someothersha"
+        }
+      ]
+    }
+  ]
+}

tests/data/manifests/outputs/output_valid.json

@@ -0,0 +1,38 @@
+{
+  "id": "8ead7669-8162-4f64-8cd5-4abe92509e17",
+  "datasets": [
+    {
+      "id": "7ead7669-8162-4f64-8cd5-4abe92509e17",
+      "name": "my meteorological dataset",
+      "tags": "met, mast, wind",
+      "files": [
+        {
+          "path": "input/datasets/7ead7669/file_1.csv",
+          "cluster": 0,
+          "sequence": 0,
+          "extension": "csv",
+          "tags": "",
+          "posix_timestamp": 0,
+          "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86",
+          "last_modified": "2019-02-28T22:40:30.533005Z",
+          "name": "file_1.csv",
+          "size_bytes": 59684813,
+          "sha-512/256": "somesha"
+        },
+        {
+          "path": "input/datasets/7ead7669/file_2.csv",
+          "cluster": 0,
+          "sequence": 1,
+          "extension": "csv",
+          "tags": "",
+          "posix_timestamp": 0,
+          "id": "bbff07bc-7c19-4ed5-be6d-a6546eae8e45",
+          "last_modified": "2019-02-28T22:40:40.633001Z",
+          "name": "file_2.csv",
+          "size_bytes": 59684813,
+          "sha-512/256": "someothersha"
+        }
+      ]
+    }
+  ]
+}

tests/data/twines/valid_credentials_twine.json

@@ -0,0 +1,17 @@
+{
+	"credentials": [
+		{
+			"name": "SECRET_THE_FIRST",
+			"purpose": "Token for accessing a 3rd party API service"
+		},
+		{
+			"name": "SECRET_THE_SECOND",
+			"purpose": "Token for accessing a 3rd party API service"
+		},
+		{
+			"name": "SECRET_THE_THIRD",
+			"purpose": "Usually a big secret but sometimes has a convenient non-secret default, like a sandbox or local database",
+			"default": "postgres://pguser:pgpassword@localhost:5432/pgdb"
+		}
+	]
+}

tests/data/twines/valid_empty_children_twine.json

@@ -0,0 +1,4 @@
+{
+	"children": [
+	]
+}

tests/data/twines/valid_manifest_twine.json

@@ -0,0 +1,14 @@
+{
+	"input_manifest": [
+		{
+			"key": "met_mast_data",
+			"purpose": "A dataset containing meteorological mast data",
+			"filters": "tags:(met* AND mast AND location) files:(extension:csv AND sequence:>=0) location:10"
+		},
+		{
+			"key": "scada_data",
+			"purpose": "A dataset containing scada data",
+			"filters": "tags:(met* AND mast) files:(extension:csv AND sequence:>=0) location:10"
+		}
+	]
+}

tests/test_children.py

@@ -33,30 +33,73 @@ def test_valid_children(self):
         twine = Twine(file=twine_file)
         self.assertEqual(len(twine._raw["children"]), 1)
 
+    def test_empty_children(self):
+        """ Ensures that a twine file will validate with an empty list object as children
+        """
+        twine_file = self.path + "twines/valid_empty_children_twine.json"
+        twine = Twine(file=twine_file)
+        self.assertEqual(len(twine._raw["children"]), 0)
+
+
+class TestChildrenValidation(BaseTestCase):
+    """ Tests related to whether validation of children occurs successfully (given a valid twine)
+    """
+
+    def test_no_children(self):
+        """ Test that a twine with no children will validate on an empty children input
+        """
+        twine = Twine()  # Creates empty twine
+        twine.validate_children(json="[]")
+
+    def test_missing_children(self):
+        """ Test that a twine with children will not validate on an empty children input
+        """
+        twine = Twine(file=self.path + "twines/valid_children_twine.json")
+        with self.assertRaises(exceptions.InvalidValuesContents):
+            twine.validate_children(json="[]")
+
+    def test_extra_children(self):
+        """ Test that a twine with no children will not validate a non-empty children input
+        """
+        twine = Twine()  # Creates empty twine
+        with self.assertRaises(exceptions.InvalidValuesContents):
+            twine.validate_children(file=self.path + "children/valid.json")
 
-# class TestChildrenValidation(unittest.TestCase):
-#     """ Tests related to whether validation of children occurs successfully (given a valid twine)
-#     """
-#
-#     def test_no_children(self):
-#         """ Test that a twine with no children will validate on an empty children input
-#         """
-#         raise exceptions.NotImplementedYet()
-#
-#     def test_missing_children(self):
-#         """ Test that a twine with children will not validate on an empty children input
-#         """
-#         raise exceptions.NotImplementedYet()
-#
-#     def test_extra_children(self):
-#         """ Test that a twine with no children will not validate a non-empty children input
-#         """
-#         raise exceptions.NotImplementedYet()
-#
-#     def test_matched_children(self):
-#         """ Test that a twine with children required will validate when the children input matches
-#         """
-#         raise exceptions.NotImplementedYet()
+    def test_extra_key(self):
+        """ Test that children with extra data will not raise validation error
+        """
+        twine = Twine()  # Creates empty twine
+        with self.assertRaises(exceptions.InvalidValuesContents):
+            twine.validate_children(file=self.path + "children/extra_key.json")
+
+    def test_extra_property(self):
+        """ Test that children with extra data will not raise validation error
+        # TODO review this behaviour - possibly should raise an error but allow for a user specified extra_data property
+        """
+        twine = Twine(file=self.path + "twines/valid_children_twine.json")
+        twine.validate_children(file=self.path + "children/extra_property.json")
+
+    def test_invalid_env_name(self):
+        """ Test that a child uri env name not in ALL_CAPS_SNAKE_CASE doesn't validate
+        """
+        twine = Twine()  # Creates empty twine
+        with self.assertRaises(exceptions.InvalidValuesContents):
+            twine.validate_children(file=self.path + "children/invalid_env_name.json")
+
+    def test_invalid_json(self):
+        """ Tests that a children entry with invalid json will raise an error
+        """
+        twine = Twine(file=self.path + "twines/valid_children_twine.json")
+        with self.assertRaises(exceptions.InvalidValuesJson):
+            twine.validate_children(json="[")
+
+    def test_valid(self):
+        """ Test that a valid twine will validate valid children
+        Valiantly and Validly validating validity since 1983.
+        To those reading this, know that YOU'RE valid.
+        """
+        twine = Twine(file=self.path + "twines/valid_children_twine.json")
+        twine.validate_children(file=self.path + "children/valid.json")
 
 
 if __name__ == "__main__":