start of work to add Build table

We need to include BuildEnvironment and objects alongside a Spec build,
and we need it to be possible to build the same spec in different environments
and have different ids/rows in the table. This is start of work to do that,
and also clean up some of the API namespace. I next need to go into spack
and refactor for this structure, and figure out how to get a list of object
files after a build to (eventually) parse and send to the spack monitor

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

Dockerfile

@@ -3,7 +3,7 @@ ENV PYTHONUNBUFFERED 1
 ENV PATH /opt/conda/bin:${PATH}
 ENV LANG C.UTF-8
 ENV SHELL /bin/bash
-RUN install_packages wget curl bzip2 ca-certificates openssl gnupg2 git vim pkg-config
+RUN install_packages wget curl bzip2 ca-certificates openssl gnupg2 git vim python3-pygraphviz gcc pkg-config
 COPY requirements.txt /requirements.txt
 RUN /bin/bash -c "curl -L https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh && \
     bash miniconda.sh -b -p /opt/conda && \

README.md

@@ -13,15 +13,16 @@ to get started, but keep in mind this is under development. 👇️
 
 ⚠️ *This respository is under development! All is subject to change. Use at your own risk!* ⚠️
 
-## Development Plan
+## Next TODO
 
-1. Develop the base models for a spec
-2. Get a working container orchestration (database and application)
-3. Create endpoint for uploading simple spec (akin to cray's)
-4. Identify locations in spack where we want to ping the server with an update
-5. Create API endpoints for doing so
+1. Update spack to include build environment information
+2. Create endpoint to accept a new spec, and build environment, should return a build ID
+3. Update spack monitor to be able to reference the build id for endpoints instead of spec full_hash
+4. Create simple endpoint to retrieve a build_id based on a spec, spack_version, and environment.
+
+5. Develop separate tool to parse libabigail xml
+6. Somehow get list of files that are objects generated, send them to Spack Monitor (even if we don't have ABI yet)
 
 ## License
 
  * Free software: Apache License 2.0 or MIT. See the LICENSE files and COPYRIGHT notice files in the root folder, and [AUTHORS](AUTHORS) for a list of contributions and maintainers.
-

docs/development/index.rst

@@ -13,5 +13,6 @@ If you have a question please `let us know <https://github.com/spack/spack-monit
 
    background
    setup
+   tables
    documentation
 

docs/development/tables.rst

@@ -0,0 +1,24 @@
+.. _development-tables:
+
+======
+Tables
+======
+
+The table design is represented in the models.py file of each app.
+Once you have the application running, you can generate a graph as follows:
+
+.. code-block:: console
+
+    $ python manage.py graph_models main -o tables.png
+    
+    
+Or from the outside of the container:
+
+.. code-block:: console
+
+    $ docker exec -it spack-monitor_uwsgi_1 python manage.py graph_models main -o tables.png
+    $ mv tables.png docs/development/img/
+    
+The output looks like this:
+
+.. image:: img/tables.png

docs/getting_started/api.rst

@@ -266,7 +266,7 @@ but a status code of 200 to indicate success (but not create).
 Update Build Task Status
 ------------------------
 
-``POST /ms1/tasks/update/``
+``POST /ms1/builds/update/``
 
 When Spack is running builds, each spec will either succeed or fail. In each case,
 we need to update Spack Monitor with the status for the spec. The default status for
@@ -290,10 +290,10 @@ When you want to update the status of a spec build, a successful update will
 return a 200 response.
 
 
-Package Metadata
-----------------
+Specs Metadata
+--------------
 
-``POST /ms1/packages/metadata/``
+``POST /ms1/specs/metadata/``
 
 When a package is finished installing, we have a metadata folder, usually within
 the spack root located at ``opt/<system>/<compiler>/<package>/.spack`` 

requirements.txt

@@ -1,6 +1,8 @@
 Django==3.0.8
 django-ratelimit==3.0.0
 django-extensions==3.0.2
+graphviz
+pydotplus
 sendgrid==6.4.3
 # just for postgres
 psycopg2-binary==2.8.5

spackmon/apps/api/urls.py

@@ -26,21 +26,20 @@
         name="new_spec",
     ),
     path(
-        "%s/tasks/update/" % cfg.URL_API_PREFIX,
-        api_views.UpdateTaskStatus.as_view(),
-        name="update_task_status",
+        "%s/specs/metadata/" % cfg.URL_API_PREFIX,
+        api_views.UpdateBuildMetadata.as_view(),
+        name="update_build_metadata",
     ),
     path(
-        "%s/phases/metadata/" % cfg.URL_API_PREFIX,
-        api_views.UpdatePhaseStatus.as_view(),
-        name="update_phase_status",
+        "%s/builds/update/" % cfg.URL_API_PREFIX,
+        api_views.UpdateBuildStatus.as_view(),
+        name="update_build_status",
     ),
     path(
-        "%s/packages/metadata/" % cfg.URL_API_PREFIX,
-        api_views.UpdatePackageMetadata.as_view(),
-        name="update_package_metadata",
+        "%s/builds/phases/update/" % cfg.URL_API_PREFIX,
+        api_views.UpdatePhaseStatus.as_view(),
+        name="update_phase_status",
     ),
 ]
 
-
 app_name = "api"

spackmon/apps/api/views/__init__.py

@@ -1,5 +1,4 @@
 from .auth import GetAuthToken
 from .base import ServiceInfo
-from .specs import NewSpec
-from .tasks import UpdateTaskStatus, UpdatePhaseStatus
-from .packages import UpdatePackageMetadata
+from .specs import NewSpec, UpdateSpecMetadata
+from .builds import UpdateBuildStatus, UpdatePhaseStatus

spackmon/apps/api/views/packages.py

@@ -19,56 +19,4 @@
 
 import json
 
-
-class UpdatePackageMetadata(APIView):
-    """Given a finished package install, receive content from the metadata
-    folder. Any metadata that is missing will expect a None response. Fields
-    that we expect to parse include:
-
-     - errors: goes into the spec.errors field
-     - manifest: includes a json object with install files. Install files are
-                 linked to a spec and go into the ManyToMany install_files
-                 field, which points to the InstallFile table.
-     - environ: should be a list of parsed environment variables, which go
-                into the spec.envars field, pointing to EnvironmentVariable.
-     - config: text of config arguments.
-    """
-
-    permission_classes = []
-    allowed_methods = ("POST",)
-
-    @never_cache
-    @method_decorator(
-        ratelimit(
-            key="ip",
-            rate=settings.VIEW_RATE_LIMIT,
-            method="POST",
-            block=settings.VIEW_RATE_LIMIT_BLOCK,
-        )
-    )
-    def post(self, request, *args, **kwargs):
-        """POST /ms1/packages/metadata/ to add or update a package metadata"""
-
-        # If allow_continue False, return response
-        allow_continue, response, _ = is_authenticated(request)
-        if not allow_continue:
-            return response
-
-        # Get the data, including output, error, environ, manifest, config
-        data = json.loads(request.body)
-        spack_version = data.get("spack_version")
-
-        # The spack version is required
-        if not spack_version:
-            return Response(
-                status=400, data={"message": "A spack_version string is required"}
-            )
-
-        # Look up the spec based on the full hash and spack version
-        spec = get_object_or_404(
-            Spec, full_hash=data.get("full_hash"), spack_version=spack_version
-        )
-
-        # Update the output, error, install files, config, and environment
-        spec = update_package_metadata(spec, data)
-        return Response(status=200, data={"full_hash": spec.full_hash})
+from spackmon.apps.main.tasks import update_package_metadata

spackmon/apps/api/views/specs.py

@@ -9,7 +9,7 @@
 from django.views.decorators.cache import never_cache
 from django.utils.decorators import method_decorator
 
-from spackmon.apps.main.tasks import import_configuration
+from spackmon.apps.main.tasks import import_configuration, update_spec_metadata
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
@@ -72,3 +72,57 @@ def post(self, request, *args, **kwargs):
 
         # 400 Bad request, there was an error parsing the data
         return Response(status=400, data=data)
+
+
+class UpdateSpecMetadata(APIView):
+    """Given a finished spec install, receive content from the metadata
+    folder. Any metadata that is missing will expect a None response. Fields
+    that we expect to parse include:
+
+     - errors: goes into the spec.errors field
+     - manifest: includes a json object with install files. Install files are
+                 linked to a spec and go into the ManyToMany install_files
+                 field, which points to the InstallFile table.
+     - environ: should be a list of parsed environment variables, which go
+                into the spec.envars field, pointing to EnvironmentVariable.
+     - config: text of config arguments.
+    """
+
+    permission_classes = []
+    allowed_methods = ("POST",)
+
+    @never_cache
+    @method_decorator(
+        ratelimit(
+            key="ip",
+            rate=settings.VIEW_RATE_LIMIT,
+            method="POST",
+            block=settings.VIEW_RATE_LIMIT_BLOCK,
+        )
+    )
+    def post(self, request, *args, **kwargs):
+        """POST /ms1/packages/metadata/ to add or update a package metadata"""
+
+        # If allow_continue False, return response
+        allow_continue, response, _ = is_authenticated(request)
+        if not allow_continue:
+            return response
+
+        # Get the data, including output, error, environ, manifest, config
+        data = json.loads(request.body)
+        spack_version = data.get("spack_version")
+
+        # The spack version is required
+        if not spack_version:
+            return Response(
+                status=400, data={"message": "A spack_version string is required"}
+            )
+
+        # Look up the spec based on the full hash and spack version
+        spec = get_object_or_404(
+            Spec, full_hash=data.get("full_hash"), spack_version=spack_version
+        )
+
+        # Update the output, error, install files, config, and environment
+        spec = update_spec_metadata(spec, data)
+        return Response(status=200, data={"full_hash": spec.full_hash})

spackmon/apps/base/apps.py

@@ -0,0 +1,7 @@
+from __future__ import unicode_literals
+
+from django.apps import AppConfig
+
+
+class BaseConfig(AppConfig):
+    name = "base"

spackmon/apps/base/templates/base/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /