Merge 5253a1d into cadc107

.ebextensions/05_set_wsgi.config

@@ -107,6 +107,10 @@ files:
       WSGIDaemonProcess encoded-indexer user=wsgi group=wsgi threads=1 python-path=/opt/python/current/app:/opt/python/run/venv/lib64/python3.6/site-packages:/opt/python/run/venv/lib/python3.6/site-packages display-name=encoded-indexer
       WSGIScriptAlias /_indexer /opt/python/current/app/parts/production-indexer/wsgi process-group=encoded-indexer application-group=%{GLOBAL}
 
+      # Ingestion listener (very similar config to the indexer, but not multi-processed the same way)
+      WSGIDaemonProcess encoded-ingestion-listener user=wsgi group=wsgi threads=1 python-path=/opt/python/current/app:/opt/python/run/venv/lib64/python3.6/site-packages:/opt/python/run/venv/lib/python3.6/site-packages display-name=encoded-ingestion-listener
+      WSGIScriptAlias /_ingestion_listener /opt/python/current/app/parts/production-ingestion-listener/wsgi process-group=encoded-ingestion-listener application-group=%{GLOBAL}
+
       # https://github.com/GrahamDumpleton/mod_wsgi/issues/2
       # Try Graham Dumpleton's fix since we have upgraded WSGI
       # Original fix (next 3 lines)

Makefile

@@ -62,10 +62,10 @@ deploy1:  # starts postgres/ES locally and loads inserts
 	@SNOVAULT_DB_TEST_PORT=`grep 'sqlalchemy[.]url =' development.ini | sed -E 's|.*:([0-9]+)/.*|\1|'` dev-servers development.ini --app-name app --clear --init --load
 
 deploy2:  # spins up waittress to serve the application
-	pserve development.ini
+	@SNOVAULT_DB_TEST_PORT=`grep 'sqlalchemy[.]url =' development.ini | sed -E 's|.*:([0-9]+)/.*|\1|'` pserve development.ini	
 
 deploy3:  # uploads: GeneAnnotationFields, then Genes, then AnnotationFields, then Variant + VariantSamples
-	python src/encoded/commands/ingestion.py src/encoded/annotations/variant_table_v0.4.6.csv src/encoded/schemas/annotation_field.json src/encoded/schemas/variant.json src/encoded/schemas/variant_sample.json src/encoded/annotations/vcf_v0.4.6.vcf hms-dbmi hms-dbmi src/encoded/annotations/gene_table_v0.4.5.csv src/encoded/schemas/gene_annotation_field.json src/encoded/schemas/gene.json src/encoded/annotations/gene_inserts_v0.4.5.json hms-dbmi hms-dbmi development.ini --post-variant-consequences --post-variants --post-gene-annotation-field-inserts --post-gene-inserts --app-name app
+	python src/encoded/commands/ingestion.py src/encoded/annotations/variant_table_v0.4.6.csv src/encoded/schemas/annotation_field.json src/encoded/schemas/variant.json src/encoded/schemas/variant_sample.json src/encoded/annotations/GAPFI3JX5D2J.vcf hms-dbmi hms-dbmi src/encoded/annotations/gene_table_v0.4.5.csv src/encoded/schemas/gene_annotation_field.json src/encoded/schemas/gene.json src/encoded/annotations/gene_inserts_v0.4.5.json hms-dbmi hms-dbmi development.ini --post-variant-consequences --post-variants --post-gene-annotation-field-inserts --post-gene-inserts --post-vcf-file --app-name app
 
 psql-dev:  # starts psql with the url after 'sqlalchemy.url =' in development.ini
 	@psql `grep 'sqlalchemy[.]url =' development.ini | sed -E 's/^.* = (.*)/\1/'`

base.ini

@@ -53,6 +53,12 @@ timeout = 60
 set embed_cache.capacity = 5000
 set indexer = true
 
+[composite:ingester]
+use = egg:encoded#ingester
+app = app
+path = /ingest
+timeout = 60
+
 [filter:memlimit]
 use = egg:encoded#memlimit
 rss_limit = 500MB

deploy/ini_files/any.ini

@@ -27,6 +27,9 @@ sqlalchemy.url = postgresql://${RDS_USERNAME}:${RDS_PASSWORD}@${RDS_HOSTNAME}:${
 [composite:indexer]
 use = config:base.ini#indexer
 
+[composite:ingester]
+use = config:base.ini#ingester
+
 [pipeline:main]
 pipeline =
     config:base.ini#memlimit

deploy/ini_files/cgap.ini

@@ -26,6 +26,9 @@ sqlalchemy.url = postgresql://${RDS_USERNAME}:${RDS_PASSWORD}@${RDS_HOSTNAME}:${
 [composite:indexer]
 use = config:base.ini#indexer
 
+[composite:ingester]
+use = config:base.ini#ingester
+
 [pipeline:main]
 pipeline =
     config:base.ini#memlimit

deploy/ini_files/cgapdev.ini

@@ -26,6 +26,9 @@ sqlalchemy.url = postgresql://${RDS_USERNAME}:${RDS_PASSWORD}@${RDS_HOSTNAME}:${
 [composite:indexer]
 use = config:base.ini#indexer
 
+[composite:ingester]
+use = config:base.ini#ingester
+
 [pipeline:main]
 pipeline =
     config:base.ini#memlimit

deploy/ini_files/cgaptest.ini

@@ -26,6 +26,9 @@ sqlalchemy.url = postgresql://${RDS_USERNAME}:${RDS_PASSWORD}@${RDS_HOSTNAME}:${
 [composite:indexer]
 use = config:base.ini#indexer
 
+[composite:ingester]
+use = config:base.ini#ingester
+
 [pipeline:main]
 pipeline =
     config:base.ini#memlimit

deploy/ini_files/cgapwolf.ini

@@ -26,6 +26,9 @@ sqlalchemy.url = postgresql://${RDS_USERNAME}:${RDS_PASSWORD}@${RDS_HOSTNAME}:${
 [composite:indexer]
 use = config:base.ini#indexer
 
+[composite:ingester]
+use = config:base.ini#ingester
+
 [pipeline:main]
 pipeline =
     config:base.ini#memlimit

parts/production-ingestion-listener/wsgi

@@ -0,0 +1,24 @@
+import os
+
+from logging.config import fileConfig
+from paste.deploy import loadapp
+
+
+# This is sort of like as if we did:
+#  CONFIG_FILE = "/opt/python/current/app/production.ini"
+# except we want to bind to the bundle name so that when links change
+# between /opt/python/ondeck/app and /opt/python/current/app
+# we continue to point to the right directory.
+# NOTE: If this doesn't work, we might try using "../../production.ini"
+
+FILE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+PARTS_DIR = os.path.dirname(FILE_DIR)
+
+HOME_DIR = os.path.dirname(PARTS_DIR)
+
+CONFIG_FILE = os.path.join(HOME_DIR, "production.ini")
+
+fileConfig(CONFIG_FILE)
+
+application = loadapp("config:" + CONFIG_FILE, name="ingester")

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 # Note: Various modules refer to this system as "encoded", not "cgap-portal".
 name = "encoded"
-version = "2.1.11"
+version = "2.2.0"
 description = "Clinical Genomics Analysis Platform"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"
@@ -186,12 +186,14 @@ ingest-genes = "encoded.commands.ingest_genes:main"
 gene-ingestion = "encoded.commands.gene_ingestion:main"
 ingestion = "encoded.commands.ingestion:main"
 clear-variants-and-genes = "encoded.commands.clear_variants_and_genes:main"
+ingestion-listener = "encoded.ingestion_listener:main"
 
 [paste.app_factory]
 main = "encoded:main"
 
 [paste.composite_factory]
 indexer = "snovault.elasticsearch.es_index_listener:composite"
+ingester = "encoded.ingestion_listener:composite"
 
 [paste.filter_app_factory]
 memlimit = "encoded.memlimit:filter_app"

src/encoded/__init__.py

@@ -12,6 +12,7 @@
 from pyramid.config import Configurator
 from pyramid.settings import asbool
 from snovault.app import STATIC_MAX_AGE, session, json_from_path, configure_dbsession, changelogs, json_asset
+from .ingestion_listener import INGESTION_QUEUE
 
 
 if sys.version_info.major < 3:
@@ -76,22 +77,6 @@ def load_workbook(app, workbook_filename, docsdir):
     load_all(testapp, workbook_filename, docsdir)
 
 
-def source_beanstalk_env_vars(config_file=BEANSTALK_ENV_PATH):
-    """
-    set environment variables if we are on Elastic Beanstalk
-    AWS_ACCESS_KEY_ID is indicative of whether or not env vars are sourced
-    Args:
-        config_file (str): filepath to load env vars from
-    """
-    if os.path.exists(config_file) and not os.environ.get("AWS_ACCESS_KEY_ID"):
-        command = ['bash', '-c', 'source ' + config_file + ' && env']
-        proc = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True)
-        for line in proc.stdout:
-            key, _, value = line.partition("=")
-            os.environ[key] = value[:-1]
-        proc.communicate()
-
-
 # This key is best interpreted not as the 'snovault version' but rather the 'version of the app built on snovault'.
 # As such, it should be left this way, even though it may appear redundant with the 'eb_app_version' registry key
 # that we also have, which tries to be the value eb uses. -kmp 28-Apr-2020
@@ -120,59 +105,6 @@ def app_version(config):
     # Fourfront does GA stuff here that makes no sense in CGAP (yet).
 
 
-# This function no longer exists in Fourfront either. Remove it here? -kmp 8-Apr-2008
-'''
-def add_schemas_to_html_responses(config):
-
-    from pyramid.events import BeforeRender
-    from snovault.schema_views import schemas
-    from .renderers import should_transform
-
-    # Exclude some keys, to make response smaller.
-    exclude_schema_keys = [
-        'AccessKey', 'Image', 'ImagingPath', 'PublicationTracking', 'Modification',
-        'QualityMetricBamqc', 'QualityMetricFastqc', 'QualityMetricFlag', 'QualityMetricPairsqc',
-        'TestingDependencies', 'TestingDownload', 'TestingKey', 'TestingLinkSource', 'TestingPostPutPatch',
-        'TestingServerDefault'
-    ]
-
-    def add_schemas(event):
-        request = event.get('request')
-        if request is not None:
-
-            if event.get('renderer_name') != 'null_renderer' and ('application/html' in request.accept or 'text/html' in request.accept):
-                #print('\n\n\n\n')
-                #print(event.keys())
-                #print(event.get('renderer_name'))
-                #print(should_transform(request, request.response))
-                #print(request.response.content_type)
-
-                if event.rendering_val.get('@type') is not None and event.rendering_val.get('@id') is not None and event.rendering_val.get('schemas') is None:
-                    schemasDict = {
-                        k:v for k,v in schemas(None, request).items() if k not in exclude_schema_keys
-                    }
-                    for schema in schemasDict.values():
-                        if schema.get('@type') is not None:
-                            del schema['@type']
-                        if schema.get('mixinProperties') is not None:
-                            del schema['mixinProperties']
-                        if schema.get('properties') is not None:
-                            if schema['properties'].get('@id') is not None:
-                                del schema['properties']['@id']
-                            if schema['properties'].get('@type') is not None:
-                                del schema['properties']['@type']
-                            if schema['properties'].get('display_title') is not None:
-                                del schema['properties']['display_title']
-                            if schema['properties'].get('schema_version') is not None:
-                                del schema['properties']['schema_version']
-                            if schema['properties'].get('uuid') is not None:
-                                del schema['properties']['uuid']
-                    event.rendering_val['schemas'] = schemasDict
-
-    config.add_subscriber(add_schemas, BeforeRender)
-'''
-
-
 def main(global_config, **local_config):
     """
     This function returns a Pyramid WSGI application.
@@ -241,6 +173,7 @@ def main(global_config, **local_config):
     # config.include('.batch_download')
     config.include('.loadxl')
     config.include('.visualization')
+    config.include('.ingestion_listener')
 
     if 'elasticsearch.server' in config.registry.settings:
         config.include('snovault.elasticsearch')