adding update task endpoint.

Note that spack currently is not representing build hashes / transitive specs,
so if one of those fails we dont have a record in the database. So the functoin
to do this update will be changed when we add this

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.gitignore

@@ -1,5 +1,6 @@
 backup/
 secret_key.py
+creation_date.py
 app.yaml
 app-dev.yaml
 *.pyc

docs/getting_started/api.rst

@@ -87,46 +87,20 @@ must provide the following endpoints:
 Response Details
 ================
 
-Successful Responses
---------------------
-
-Generally, a successful response will return a json object that shows a message
-with "success" along with a data object with metadata specific to the endpoint.
+Generally, a response will return a json object that shows a message. A successful
+response will have a message of "success" to go along with a 200 or 201 response code,
+while an unsuccessful response will have a message indicating the error, and an error
+code (e.g., 400, 500, etc.). Each reponse will have metadata specific to the endpoint.
 
 .. code-block:: python
 
     {"message": "success", "data" {...}}
 
 
-
-Errors
-------
-
-For all error responses, the server can (OPTIONAL) return in the body a nested structure of errors,
-each including a message and error code. For example:
-
-
-.. code-block:: python
-
-    {
-        "errors": [
-            {
-                "code": "<error code>",
-                "message": "<error message>",
-                "detail": ...
-            },
-            ...
-        ]
-    }
-
-
-Currently we don't have a namespace for errors, but this can be developed if/when needed.
-For now, the code can be a standard server error code.
-
 Timestamps
 ----------
 
-For all fields that return a timestamp, we are tentatively going to use the stringified
+For all fields that will return a timestamp, we are tentatively going to use the stringified
 version of a ``datetime.now()``, which looks like this:
 
 .. code-block:: console
@@ -237,12 +211,12 @@ For each of the above, if the server does not return a Location header, the clie
 should issue an error.
 
 
-New Config
-----------
+New Spec
+--------
 
-``POST /ms1/config/new/``
+``POST /ms1/specs/new/``
 
-If you have a configuration file, you can load it into Python and issue a request
+If you have a spec configuration file, you can load it into Python and issue a request
 to this endpoint. The response can be any of the following:
 
 - `404 <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404>`_: not implemented
@@ -256,7 +230,7 @@ to this endpoint. The response can be any of the following:
 New Config Created 201
 ''''''''''''''''''''''
 
-If the configuration is created, you'll get a 201 response with data that
+If the set of specs are created from the configuration file, you'll get a 201 response with data that
 includes the configuration id (the full_hash) along with full hashes
 for each package included:
 
@@ -265,35 +239,21 @@ for each package included:
     {
         "message": "success",
         "data": {
-            "full_hash": "hxkll3hd7eb7qp7oos4utjjyjq7v3kek",
+            "full_hash": "xttimnxa2kc4rc33axvrcpzejiil6wbn",
             "packages": {
-                "autoconf": "qobbaw2iiotqd6zllbmckbxacw6h2ivk",
-                "m4": "nhqxxeekukxwmss7juovorh74liclysw",
-                "libsigsegv": "r2wa677ntwzamepabphbhwfyikiyg37j",
-                "perl": "bcivb4krzgesrlcdhsz6k5ul3vlzdd7w",
-                "berkeley-db": "la233dfen54tshpywjqhjq446j7o4hqr",
-                "gdbm": "fjy3imkxjkvqemteo2pysxmuwfb32ely",
-                "readline": "gml7funyikp2tu4ngg4jiexp6otysay4",
-                "ncurses": "zuimpjfdx7gd6o3xhhzqekxazfgdgivh",
-                "pkgconf": "faqkgt22jrp7wnmqal5zsl7olisc22qk",
-                "automake": "y2mlu7ou6wx54i5qwagrviy2foeefaho",
-                "libiconv": "qhvlpedcedmc7akojaftw46i5daybsfr",
-                "libxml2": "o5bqxfuieaf37mc2gdvnfdqqthzpbmis",
-                "xz": "w7qivbfd35zjgjavs6kl36yhl6oev75y",
-                "libtool": "witmr33yv7xogeaxyynobdblkuzwenwr",
-                "openssl": "u3aemyff3aw4nnh3igoyktpwukal3zjv",
-                "zlib": "nfa3orbnlq6az76gfdesjmej62pvjh7x",
-                "hdf5": "hxkll3hd7eb7qp7oos4utjjyjq7v3kek",
-                "util-macros": "yeype6hteniz6bj3ec4dt2evcvslxi63",
-                "libevent": "xcvquzui2mjux3hg2qarpbwv42u6cnfv",
-                "openmpi": "pekjjw3a5qbgwfwktvv3jqie4veeq7m6",
-                "numactl": "bvjlcnxyyumevwp2wvc3ht2uudje7owh",
-                "hwloc": "vifktdoq6zle3rfjplmzxoltht5iral5",
-                "libpciaccess": "rr2nr5f4lxs53onoycnq47j7yhygibg2"
+                "cryptsetup": "4riqvvabzho7qyzxumc7csmtcatnfbqd",
+                "go": "2dhsyo2cvpyft5u2ptza7j7kvk5r6626",
+                "libgpg-error": "5fmyz5bhnsaw5vvtbgt3m6cujrw2ajbc",
+                "libseccomp": "3mmhto5wulorfps33lzkzr5ynyanmefn",
+                "shadow": "aozeq6ybtsnrs5phtonutwes7fe6yhcy",
+                "squashfs": "mxfspfx44aforrx6shx6r6nu3th6mca3",
+                "util-linux-uuid": "46cwzqnbfi3xdxlrm76z5gazhvog3n3t"
             }
         }
     }
 
+All of the above are full hashes, which we can use as unique identifiers for the builds.
+
 
 New Config Already Exists 200
 '''''''''''''''''''''''''''''
@@ -302,3 +262,29 @@ If the configuration in question already exists, you'll get the same data respon
 but a status code of 200 to indicate success (but not create).
 
 
+
+Update Build Task Status
+------------------------
+
+``POST /ms1/tasks/update/``
+
+When Spack is running builds, each spec will either succeed or fail. In each case,
+we need to update Spack Monitor with the status for the spec. The default status for
+a build task is ``NOTRUN``. Once the builds start, given a failure,
+this means that the spec that failed is marked as ``FAILURE``, and the main spec 
+along with the other specs that were not installed are marked as ``CANCELLED``.
+In the case of success for any package, we mark with ``SUCCESS``. If Spack has a setting
+to "rollback" we will need to account for that (not currently implemented).
+
+- `404 <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404>`_: not implemented or spec not found
+- `200 <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/200>`_: success
+- `503 <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503>`_: service not available
+- `400 <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/400>`_: bad request
+- `403 <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403>`_: permission denied
+
+
+Build Task Updated 200
+''''''''''''''''''''''
+
+When you want to update the status of a spec build, a successful update will
+return a 200 response.

spackmon/apps/api/urls.py

@@ -25,6 +25,11 @@
         api_views.NewSpec.as_view(),
         name="new_spec",
     ),
+    path(
+        "%s/tasks/update/" % cfg.URL_API_PREFIX,
+        api_views.UpdateTaskStatus.as_view(),
+        name="update_task_status",
+    ),
 ]
 
 

spackmon/apps/api/views/__init__.py

@@ -1,3 +1,4 @@
 from .auth import GetAuthToken
 from .base import ServiceInfo
 from .specs import NewSpec
+from .tasks import UpdateTaskStatus

spackmon/apps/api/views/base.py

@@ -31,18 +31,16 @@ def get(self, request):
 
         data = {
             "id": "spackmon",
-            "status": "running",  # Extra field looked for by Snakemake
+            "status": "running",
             "name": "Spack Monitor (Spackmon)",
             "description": "This service provides a database to monitor spack builds.",
             "organization": {"name": "spack", "url": "https://github.com/spack"},
             "contactUrl": cfg.HELP_CONTACT_URL,
             "documentationUrl": "https://spack-monitor.readthedocs.io",
-            # This is when the function was written, should be when server created
-            "createdAt": "2021-02-10T10:40:19Z",
+            "createdAt": settings.SERVER_CREATION_DATE,
             "updatedAt": cfg.UPDATED_AT,
             "environment": cfg.ENVIRONMENT,
             "version": __version__,
-            # TODO: We will provide this for the user to authenticate
             "auth_instructions_url": cfg.AUTH_INSTRUCTIONS,
         }
 

spackmon/apps/api/views/specs.py

@@ -45,7 +45,7 @@ class NewSpec(APIView):
         )
     )
     def post(self, request, *args, **kwargs):
-        """POST /v2/specs/new/ to upload a configuration file"""
+        """POST /ms1/specs/new/ to upload a specs file"""
 
         # If allow_continue False, return response
         allow_continue, response, _ = is_authenticated(request)
@@ -66,7 +66,7 @@ def post(self, request, *args, **kwargs):
             if result["created"]:
                 return Response(status=201, data=data)
 
-            # 409 conflict means that it already exists
+            # 200 is success, but already exists
             return Response(status=200, data=data)
 
         # 400 Bad request, there was an error parsing the data

spackmon/apps/main/models.py

@@ -12,6 +12,7 @@
 from django.db.models.signals import m2m_changed
 from django.dispatch import receiver
 
+from .utils import BUILD_STATUS
 
 import json
 
@@ -201,14 +202,6 @@ def __repr__(self):
         return str(self)
 
 
-BUILD_STATUS = [
-    ("CANCELLED", "CANCELLED"),
-    ("SUCCESS", "SUCCESS"),
-    ("NOTRUN", "NOTRUN"),
-    ("FAILED", "FAILED"),
-]
-
-
 class Spec(BaseModel):
     """A spec is a descriptor for a package, or a particular build configuration
     for it. It doesn't just include package information, but also information
@@ -306,6 +299,15 @@ def __str__(self):
     def __repr__(self):
         return str(self)
 
+    def cancel_dependencies(self):
+        """Given that a spec is failed or cancelled, we also cancel any
+        dependencies that aren't already successful.
+        """
+        for dep in self.dependencies.all():
+            if dep.package.build_status != "SUCCESS":
+                dep.package.build_status = "CANCELLED"
+                dep.package.save()
+
     def to_dict_ids(self):
         """This function is intended to return a simple json response that
         includes the configuration and spec ids, but not additional

spackmon/apps/main/tasks.py

@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+from django.shortcuts import get_object_or_404
+
 from spackmon.apps.main.models import (
     Spec,
     Architecture,
@@ -13,7 +15,6 @@
 )
 from spackmon.apps.main.utils import read_json
 
-from django.db.models import Count
 import os
 import re
 
@@ -22,6 +23,21 @@
 logger = logging.getLogger(__name__)
 
 
+def update_task_status(full_hash, status):
+    """Given a full hash to identify a spec and a status, update the status
+    for the spec. Given that we are cancelling a spec, this means that all
+    dependencies are cancelled too.
+    """
+    spec = get_object_or_404(Spec, full_hash=full_hash)
+    if spec:
+        spec.build_status = status
+        spec.save()
+
+        # If we cancel or fail, cancel all dependencies that were not successful
+        if status in ["CANCELLED", "FAILED"]:
+            spec.cancel_dependencies()
+
+
 def get_target(meta):
     """Given a section of metadata for a target (expected to have name, vendor,
     features, and parents) create the Target objects, which includes also
@@ -135,7 +151,8 @@ def import_configuration(config):
         # Create the spec (full hash and name are unique together)
         spec, created = get_spec(name, meta, arch, compiler)
 
-        if not created:
+        # Add dependencies if not added yet
+        if spec.dependencies.count() == 0:
             spec = add_dependencies(spec, meta.get("dependencies", {}))
             spec.save()
 

spackmon/apps/main/utils.py

@@ -9,6 +9,15 @@
 
 logger = logging.getLogger(__name__)
 
+# Valid build statuses
+
+BUILD_STATUS = [
+    ("CANCELLED", "CANCELLED"),
+    ("SUCCESS", "SUCCESS"),
+    ("NOTRUN", "NOTRUN"),
+    ("FAILED", "FAILED"),
+]
+
 
 def read_json(filename):
     with open(filename, "r") as fd:

spackmon/settings.py

@@ -15,6 +15,7 @@
 # Build paths inside the project with the base directory
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 
+
 # The spackmon global conflict contains all settings.
 SETTINGS_FILE = os.path.join(BASE_DIR, "settings.yml")
 if not os.path.exists(SETTINGS_FILE):
@@ -50,7 +51,7 @@ def __iter__(self):
     if envar:
         setattr(cfg, key, envar)
 
-# Secret Key
+# Secret Key and Dates
 
 
 def generate_secret_keys(filename):
@@ -61,6 +62,13 @@ def generate_secret_keys(filename):
             fd.writelines("%s = '%s'\n" % (keyname, key))
 
 
+def generate_creation_date(filename):
+    """Keep track of when the server was generated for metadata"""
+    created_at = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
+    with open(filename, "w") as fd:
+        fd.writelines("SERVER_CREATION_DATE = '%s'\n" % created_at)
+
+
 # Generate secret keys if do not exist, and not defined in environment
 SECRET_KEY = os.environ.get("SECRET_KEY")
 JWT_SERVER_SECRET = os.environ.get("JWT_SERVER_SECRET")
@@ -69,10 +77,16 @@ def generate_secret_keys(filename):
     try:
         from .secret_key import SECRET_KEY, JWT_SERVER_SECRET
     except ImportError:
-        SETTINGS_DIR = os.path.abspath(os.path.dirname(__file__))
-        generate_secret_keys(os.path.join(SETTINGS_DIR, "secret_key.py"))
+        generate_secret_keys(os.path.join(BASE_DIR, "secret_key.py"))
         from .secret_key import SECRET_KEY, JWT_SERVER_SECRET
 
+# A record of the server creation date
+try:
+    from .creation_date import SERVER_CREATION_DATE
+except ImportError:
+    generate_creation_date(os.path.join(BASE_DIR, "creation_date.py"))
+    from .creation_date import SERVER_CREATION_DATE
+
 # Set the domain name
 DOMAIN_NAME = cfg.DOMAIN_NAME
 if cfg.DOMAIN_PORT: