Release with Multiparty Examples

Jenkinsfile.groovy

@@ -538,11 +538,6 @@ String createK8sTestJob(String deploymentName, String imageNameWithTag, String s
                                                           "name": "results"
                                                   ]]
                                           ]
-                                  ],
-                                  "imagePullSecrets": [
-                                          [
-                                                  "name": "n1-quay-pull-secret"
-                                          ]
                                   ]
                           ]
 

backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.8
+FROM alpine:3.9
 
 EXPOSE 8000
 ADD requirements.txt /var/www/requirements.txt

backend/entityservice/__init__.py

@@ -61,10 +61,15 @@ def before_request():
     g.flask_tracer = flask_tracer
 
 
-@app.teardown_request
-def teardown_request(exception):
-    if hasattr(g, 'db'):
-        g.db.close()
+@app.teardown_appcontext
+def close_db(error):
+    db = g.pop('db', None)
+
+    if db is not None:
+        g.log.debug("Closing database connection")
+        for notice in db.notices:
+            g.log.debug(notice)
+        db.close()
 
 
 if __name__ == '__main__':

backend/entityservice/database/util.py

@@ -49,7 +49,11 @@ def connect_db():
 
 
 def init_db(delay=0.5):
-    """Initializes the database."""
+    """
+    Initializes the database by creating the required tables.
+
+    @param float delay: Number of seconds to wait before executing the SQL.
+    """
     time.sleep(delay)
     db = connect_db()
     with current_app.open_resource('init-db-schema.sql', mode='r') as f:
@@ -60,16 +64,17 @@ def init_db(delay=0.5):
 
 class DBConn:
 
+    def __init__(self, conn=None):
+        self.conn = conn if conn is not None else connect_db()
+
     def __enter__(self):
-        self.conn = connect_db()
         return self.conn
 
     def __exit__(self, exc_type, exc_val, traceback):
         if not exc_type:
             self.conn.commit()
             for notice in self.conn.notices:
                 logger.debug(notice)
-            self.conn.close()
         else:
             # There was an exception in the DBConn body
             if isinstance(exc_type, psycopg2.Error):
@@ -96,7 +101,8 @@ def execute_returning_id(cur, query, args):
 
 def get_db():
     """Opens a new database connection if there is none yet for the
-    current flask application context.
+    current flask application context. The connection will be closed
+    at the end of the request.
     """
     conn = getattr(g, 'db', None)
     if conn is None:

backend/entityservice/error_checking.py

@@ -28,8 +28,6 @@ def check_dataproviders_encoding(project_id, encoding_size):
         """))
 
 
-
-
 def handle_invalid_encoding_data(project_id, dp_id):
     with DBConn() as conn:
         filename = get_filter_metadata(conn, dp_id)

backend/entityservice/logger_setup.py

@@ -22,7 +22,7 @@ def setup_logging(
     except yaml.YAMLError as e:
         raise InvalidConfiguration("Parsing YAML logging config failed") from e
     except FileNotFoundError as e:
-        raise InvalidConfiguration("Logging config YAML file doesn't exist. Falling back to defaults") from e
+        raise InvalidConfiguration(f"Logging config YAML file '{path}' doesn't exist.") from e
 
     # Configure Structlog wrapper for client use
     setup_structlog()

backend/entityservice/settings.py

@@ -20,7 +20,6 @@ class Config(object):
     LOGFILE = os.getenv("LOGFILE")
     LOG_HTTP_HEADER_FIELDS = os.getenv("LOG_HTTP_HEADER_FIELDS")
 
-
     REDIS_SERVER = os.getenv('REDIS_SERVER', 'redis')
     REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')
     REDIS_USE_SENTINEL = os.getenv('REDIS_USE_SENTINEL', 'false').lower() == "true"

backend/entityservice/tests/test_project.py

@@ -216,9 +216,9 @@ def test_invalid_result_type_number_parties_does_not_side_effect(
         invalid_result_type_number_parties):
     result_type, number_parties = invalid_result_type_number_parties
 
-    original_project_list_respose = requests.get(url + '/projects').json()
+    original_project_list_response = requests.get(url + '/projects').json()
     original_project_ids = {
-        p['project_id'] for p in original_project_list_respose}
+        p['project_id'] for p in original_project_list_response}
 
     requests.post(url + '/projects', json={
         'schema': {},

backend/entityservice/verbose_logging.yaml

@@ -52,14 +52,14 @@ loggers:
 
   entityservice.tasks:
     level: DEBUG
-    propagate: no
+    propagate: yes
 
-  entityservice.database.util:
-    level: WARNING
-    propagate: no
+  entityservice.database:
+    level: INFO
+    propagate: yes
 
   celery:
-    level: INFO
+    level: WARNING
     propogate: yes
 
   jaeger_tracing:

backend/entityservice/views/project.py

@@ -35,21 +35,21 @@ def projects_post(project):
     There are multiple result types, see documentation for how these effect information leakage
     and the resulting data.
     """
-    logger = get_logger()
     logger.debug("Processing request to add a new project", project=project)
     try:
         project_model = models.Project.from_json(project)
-        logger = logger.bind(pid=project_model.project_id)
     except models.InvalidProjectParametersException as e:
+        logger.info(f"Denied request to add a new project - {e.msg}", project=project)
         safe_fail_request(400, message=e.msg)
 
     # Persist the new project
-    logger.info("Adding new project to database")
+    log = logger.bind(pid=project_model.project_id)
+    log.info("Adding new project to database")
     try:
-        with DBConn() as conn:
+        with DBConn(get_db()) as conn:
             project_model.save(conn)
     except Exception as e:
-        logger.warn(e)
+        log.warn(e)
         safe_fail_request(500, 'Problem creating new project')
 
     return NewProjectResponse().dump(project_model), 201
@@ -65,7 +65,7 @@ def project_delete(project_id):
     abort_if_invalid_results_token(project_id, request.headers.get('Authorization'))
     log.info("Marking project for deletion")
 
-    with DBConn() as db_conn:
+    with DBConn(get_db()) as db_conn:
         db.mark_project_deleted(db_conn, project_id)
 
     log.info("Queuing authorized request to delete project resources")
@@ -82,7 +82,7 @@ def project_get(project_id):
     log.info("Getting detail for a project")
     abort_if_project_doesnt_exist(project_id)
     authorise_get_request(project_id)
-    with DBConn() as db_conn:
+    with DBConn(get_db()) as db_conn:
         project_object = db.get_project(db_conn, project_id)
         # Expose the number of data providers who have uploaded clks
         parties_contributed = db.get_number_parties_uploaded(db_conn, project_id)
@@ -112,19 +112,20 @@ def project_clks_post(project_id):
             safe_fail_request(401, message="Authentication token required")
 
         token = headers['Authorization']
+        #span.set_tag("headers", headers)
 
         # Check the caller has valid token -> otherwise 403
         abort_if_invalid_dataprovider_token(token)
 
-    with DBConn() as conn:
+    with DBConn(db.get_db()) as conn:
         dp_id = db.get_dataprovider_id(conn, token)
-        project_encoding_size = db.get_project_schema_encoding_size(get_db(), project_id)
+        project_encoding_size = db.get_project_schema_encoding_size(conn, project_id)
 
     log = log.bind(dp_id=dp_id)
     log.info("Receiving CLK data.")
     receipt_token = None
 
-    with opentracing.tracer.start_span('upload-data', child_of=parent_span) as span:
+    with opentracing.tracer.start_span('upload-clk-data', child_of=parent_span) as span:
         span.set_tag("project_id", project_id)
         if headers['Content-Type'] == "application/json":
             span.set_tag("content-type", 'json')
@@ -202,7 +203,7 @@ def authorise_get_request(project_id):
     dp_id = None
     # Check the resource exists
     abort_if_project_doesnt_exist(project_id)
-    with DBConn() as dbinstance:
+    with DBConn(get_db()) as dbinstance:
         project_object = db.get_project(dbinstance, project_id)
     logger.info("Checking credentials")
     result_type = project_object['result_type']
@@ -224,7 +225,7 @@ def upload_clk_data_binary(project_id, dp_id, raw_stream, count, size=128):
     receipt_token = generate_code()
     filename = Config.BIN_FILENAME_FMT.format(receipt_token)
     # Set the state to 'pending' in the bloomingdata table
-    with DBConn() as conn:
+    with DBConn(get_db()) as conn:
         db.insert_encoding_metadata(conn, filename, dp_id, receipt_token, count)
         db.update_encoding_metadata_set_encoding_size(conn, dp_id, size)
     logger.info(f"Storing supplied binary clks of individual size {size} in file: {filename}")
@@ -237,16 +238,16 @@ def upload_clk_data_binary(project_id, dp_id, raw_stream, count, size=128):
     logger.info(f"Uploading {count} binary encodings to object store. Total size: {fmt_bytes(num_bytes)}")
     parent_span = g.flask_tracer.get_span()
 
-    with opentracing.tracer.start_span('save-to-minio', child_of=parent_span) as span:
+    with opentracing.tracer.start_span('save-to-minio', child_of=parent_span):
         mc = connect_to_object_store()
         try:
             mc.put_object(Config.MINIO_BUCKET, filename, data=raw_stream, length=num_bytes)
         except (minio.error.InvalidSizeError, minio.error.InvalidArgumentError, minio.error.ResponseError):
             logger.info("Mismatch between expected stream length and header info")
             raise ValueError("Mismatch between expected stream length and header info")
 
-    with opentracing.tracer.start_span('update-database', child_of=parent_span) as span:
-        with DBConn() as conn:
+    with opentracing.tracer.start_span('update-database-with-metadata', child_of=parent_span):
+        with DBConn(get_db()) as conn:
             db.update_encoding_metadata(conn, filename, dp_id, 'ready')
             db.set_dataprovider_upload_state(conn, dp_id, True)
 
@@ -273,7 +274,7 @@ def upload_json_clk_data(dp_id, clk_json, parent_span):
     filename = Config.RAW_FILENAME_FMT.format(receipt_token)
     logger.info("Storing user {} supplied clks from json".format(dp_id))
 
-    with opentracing.tracer.start_span('clk-splitting', child_of=parent_span) as span:
+    with opentracing.tracer.start_span('splitting-json-clks', child_of=parent_span) as span:
         count = len(clk_json['clks'])
         span.set_tag("clks", count)
         data = b''.join(''.join(clk.split('\n')).encode() + b'\n' for clk in clk_json['clks'])
@@ -283,7 +284,7 @@ def upload_json_clk_data(dp_id, clk_json, parent_span):
         buffer = BytesIO(data)
 
     logger.info(f"Received {count} encodings. Uploading {fmt_bytes(num_bytes)} to object store")
-    with opentracing.tracer.start_span('save-to-quarantine', child_of=parent_span) as span:
+    with opentracing.tracer.start_span('save-clk-file-to-quarantine', child_of=parent_span) as span:
         span.set_tag('filename', filename)
         mc = connect_to_object_store()
         mc.put_object(
@@ -293,8 +294,8 @@ def upload_json_clk_data(dp_id, clk_json, parent_span):
             length=num_bytes
         )
 
-    with opentracing.tracer.start_span('update-db', child_of=parent_span) as span:
-        with DBConn() as conn:
+    with opentracing.tracer.start_span('update-encoding-metadata', child_of=parent_span):
+        with DBConn(get_db()) as conn:
             db.insert_encoding_metadata(conn, filename, dp_id, receipt_token, count)
 
     return receipt_token, filename

backend/entityservice/views/run/list.py

@@ -40,7 +40,7 @@ def post(project_id, run):
 
     log.debug("Saving run")
 
-    with db.DBConn() as db_conn:
+    with db.DBConn(get_db()) as db_conn:
         run_model.save(db_conn)
         project_object = db.get_project(db_conn, project_id)
         parties_contributed = db.get_number_parties_uploaded(db_conn, project_id)

deployment/entity-service/requirements.yaml

@@ -8,6 +8,6 @@ dependencies:
     repository: https://kubernetes-charts.storage.googleapis.com
     condition: provision.minio
   - name: postgresql
-    version: 3.15.0
+    version: 5.2.1
     repository: https://kubernetes-charts.storage.googleapis.com
     condition: provision.postgresql

deployment/entity-service/templates/api-deployment.yaml

@@ -94,5 +94,7 @@ spec:
             initialDelaySeconds: 60
             periodSeconds: 60
             timeoutSeconds: 5
+      {{- if .Values.api.pullSecret }}
       imagePullSecrets:
         - name: {{ .Values.api.pullSecret }}
+      {{- end }}

deployment/entity-service/templates/highmemory-worker-deployment.yaml

@@ -73,5 +73,7 @@ spec:
           {{- range $key, $value := .Values.workers.extraArgs }}
             - --{{ $key }}={{ $value }}
           {{- end }}
+      {{- if .Values.api.pullSecret }}
       imagePullSecrets:
       - name: {{ .Values.api.pullSecret }}
+      {{- end}}

deployment/entity-service/templates/init-db-job.yaml

@@ -43,6 +43,8 @@ spec:
           - "flask"
           - "initdb"
       restartPolicy: Never
+      {{- if .Values.api.pullSecret }}
       imagePullSecrets:
       - name: {{ .Values.api.pullSecret }}
+      {{- end }}
 {{- end }}

deployment/entity-service/templates/monitor-deployment.yaml

@@ -36,6 +36,8 @@ spec:
           - "--port=8888"
           - "--broker=redis://:{{ .Values.redis.redisPassword }}@{{ .Release.Name }}-{{ .Values.redis.nameOverride }}:6379/0"
           - "-Q celery,compute"
+      {{- if .Values.api.pullSecret }}
       imagePullSecrets:
       - name: {{ .Values.api.pullSecret }}
+      {{- end }}
 {{- end }}

deployment/entity-service/templates/worker-deployment.yaml

@@ -73,5 +73,7 @@ spec:
           {{- range $key, $value := .Values.workers.extraArgs }}
             - --{{ $key }}={{ $value }}
           {{- end }}
+      {{- if .Values.api.pullSecret }}
       imagePullSecrets:
       - name: {{ .Values.api.pullSecret }}
+      {{- end}}

deployment/entity-service/values.yaml

@@ -27,7 +27,7 @@ api:
 
     image:
       repository: quay.io/n1analytics/entity-nginx
-      tag: "v1.4.1"
+      tag: "v1.4.2"
       ## "IfNotPresent" or "Always"
       pullPolicy: IfNotPresent
 
@@ -82,7 +82,7 @@ api:
 
   ## A shared pull secret for obtaining docker images
   ## Needs to be pushed into the namespace prior to installation
-  pullSecret: n1-quay-pull-secret
+  #pullSecret: name-of-k8s-secret-in-namespace
 
   ingress:
     enabled: true
@@ -134,7 +134,7 @@ api:
     ## to a given IP range. Provide a list of IPs that are allowed via a
     ## security group.
     loadBalancerSourceRanges: []
-    ##  - 130.155.157.0/24
+
 
 workers:
   name: "matcher"
@@ -143,7 +143,7 @@ workers:
     repository: "quay.io/n1analytics/entity-app"
     tag: "v1.11.0"
     pullPolicy: Always
-    pullSecret: "n1-quay-pull-secret"
+    #pullSecret: "name-of-k8s-secret-in-namespace"
 
   ## Usually this is set to 1 per compute node
   replicaCount: 2

deployment/jobs/benchmark/timing-test-job.yaml

@@ -40,8 +40,6 @@ spec:
             mountPath: /config
           - name: data-volume
             mountPath: /cache
-      imagePullSecrets:
-        - name: n1-quay-pull-secret
       volumes:
         - name: experiments-volume
           configMap:

deployment/jobs/clear-in-progress-job.yaml

@@ -29,5 +29,4 @@ spec:
           - "async_worker"
           - "purge"
           - "-f"
-      imagePullSecrets:
-        - name: n1-quay-pull-secret
+

deployment/jobs/integration-test-job.yaml

@@ -28,5 +28,3 @@ spec:
           - "pytest"
           - "entityservice/tests"
           - "-x"
-      imagePullSecrets:
-      - name: n1-quay-pull-secret