Merge pull request #174 from CSCfi/dev

Release v1.4.3

.github/workflows/codeql-analysis.yml

@@ -39,7 +39,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/int.yml

@@ -27,7 +27,7 @@ jobs:
         name: Get sources
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
         with:
           driver-opts: network=host
 
@@ -65,7 +65,7 @@ jobs:
           sleep 30
           
       - name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: true

aggregator/aggregator.py

@@ -1,5 +1,6 @@
 """Beacon Aggregator API."""
 
+
 import sys
 
 import aiohttp_cors
@@ -47,7 +48,6 @@ async def query(request):
         # Use asynchronous websocket connection
         # Send request for processing
         websocket = await send_beacon_query_websocket(request)
-
         # Return websocket connection
         return websocket
     else:

aggregator/endpoints/query.py

@@ -16,17 +16,20 @@ async def send_beacon_query(request):
     """Send Beacon queries and respond synchronously."""
     LOG.debug("Normal response (sync).")
 
-    # Task variables
-    params = request.query_string  # query parameters (variant search)
     tasks = []  # requests to be done
     services = await get_services(request.host)  # service urls (beacons, aggregators) to be queried
     access_token = await get_access_token(request)  # Get access token if one exists
 
     for service in services:
         # Generate task queue
-        task = asyncio.ensure_future(query_service(service, params, access_token))
-        tasks.append(task)
-
+        if "&filters=filter" in request.query_string:
+            task = asyncio.ensure_future(query_service(service, request.query_string.replace("&filters=filter", ""), access_token))
+            tasks.append(task)
+            task = asyncio.ensure_future(query_service(service, "filter", access_token))
+            tasks.append(task)
+        else:
+            task = asyncio.ensure_future(query_service(service, request.query_string, access_token))
+            tasks.append(task)
     # Prepare and initiate co-routines
     results = await asyncio.gather(*tasks)
 
@@ -46,17 +49,21 @@ async def send_beacon_query_websocket(request):
     await ws.prepare(request)
 
     # Task variables
-    params = request.query_string  # query parameters (variant search)
     tasks = []  # requests to be done
     services = await get_services(request.host)  # service urls (beacons, aggregators) to be queried
     access_token = await get_access_token(request)  # Get access token if one exists
 
     for service in services:
         # Generate task queue
         LOG.debug(f"Query service: {service}")
-        task = asyncio.ensure_future(query_service(service, params, access_token, ws=ws))
-        tasks.append(task)
-
+        if "&filters=filter" in request.query_string:
+            task = asyncio.ensure_future(query_service(service, request.query_string.replace("&filters=filter", ""), access_token, ws=ws))
+            tasks.append(task)
+            task = asyncio.ensure_future(query_service(service, "filter", access_token, ws=ws))
+            tasks.append(task)
+        else:
+            task = asyncio.ensure_future(query_service(service, request.query_string, access_token, ws=ws))
+            tasks.append(task)
     # Prepare and initiate co-routines
     await asyncio.gather(*tasks)
     # Close websocket after all results have been sent

aggregator/utils/utils.py

@@ -102,7 +102,7 @@ async def process_url(url):
     # Check which endpoint to use, Beacon 1.0 or 2.0
     query_endpoints = ["query"]
     if url[1] == 2:
-        query_endpoints = ["individuals", "g_variants", "biosamples", "runs", "analyses", "interactors", "cohorts"]
+        query_endpoints = ["individuals", "g_variants", "biosamples", "runs", "analyses", "interactors", "cohorts", "filtering_terms"]
 
     LOG.debug(f"Using endpoint {query_endpoints}")
     urls = []
@@ -189,45 +189,53 @@ async def pre_process_payload(version, params):
     if version == 2:
         # checks if a query is a listing search
         if (raw_data.get("referenceName")) is not None:
-            # default data which is always present
-            data = {
-                "assemblyId": raw_data.get("assemblyId"),
-                "includeDatasetResponses": raw_data.get("includeDatasetResponses"),
-            }
-
-            # optionals
-            if (rn := raw_data.get("referenceName")) is not None:
-                data["referenceName"] = rn
-            if (vt := raw_data.get("variantType")) is not None:
-                data["variantType"] = vt
-            if (rb := raw_data.get("referenceBases")) is not None:
-                data["referenceBases"] = rb
-            if (ab := raw_data.get("alternateBases")) is not None:
-                data["alternateBases"] = ab
-
-            # exact coordinates
-            if (s := raw_data.get("start")) is not None:
-                data["start"] = s
-            if (e := raw_data.get("end")) is not None:
-                data["end"] = e
-
-            # range coordinates
-            if (smin := raw_data.get("startMin")) is not None and (smax := raw_data.get("startMax")) is not None:
-                data["start"] = ",".join([smin, smax])
-            if (emin := raw_data.get("endMin")) is not None and (emax := raw_data.get("endMax")) is not None:
-                data["end"] = ",".join([emin, emax])
+            data = pre_process_beacon2(raw_data)
         else:
-            # beaconV2 expects some data but in listing search these are not needed thus they are empty
+            # beaconV2 expects some data but in listing search these are not needed and therefore they are empty
             data = {"assemblyId": "", "includeDatasetResponses": ""}
-
+            if (filter := raw_data.get("filters")) != "None" and (raw_data.get("filters")) != "null":
+                data["filters"] = filter
+        return data
     else:
         # convert string digits into integers
         # Beacon 1.0 uses integer coordinates, while Beacon 2.0 uses string coordinates (ignore referenceName, it should stay as a string)
         raw_data = {k: int(v) if v.isdigit() and k != "referenceName" else v for k, v in raw_data.items()}
         # Beacon 1.0
         # Unmodified structure for version 1, straight parsing from GET query string to POST payload
         data = raw_data
+        # datasetIds must be a list instead of a string
+        if "datasetIds" in data:
+            data["datasetIds"] = data["datasetIds"].split(",")
+    return data
 
+
+def pre_process_beacon2(raw_data):
+    """Pre-process GET query string into POST payload for beacon2."""
+    # default data which is always present
+    data = {"assemblyId": raw_data.get("assemblyId"), "includeDatasetResponses": raw_data.get("includeDatasetResponses")}
+    # optionals
+    if (rn := raw_data.get("referenceName")) is not None:
+        data["referenceName"] = rn
+    if (vt := raw_data.get("variantType")) is not None:
+        data["variantType"] = vt
+    if (rb := raw_data.get("referenceBases")) is not None:
+        data["referenceBases"] = rb
+    if (ab := raw_data.get("alternateBases")) is not None:
+        data["alternateBases"] = ab
+    if (di := raw_data.get("datasetIds")) is not None:
+        data["datasetIds"] = di.split(",")
+    # exact coordinates
+    if (s := raw_data.get("start")) is not None:
+        data["start"] = s
+    if (e := raw_data.get("end")) is not None:
+        data["end"] = e
+    # range coordinates
+    if (smin := raw_data.get("startMin")) is not None and (smax := raw_data.get("startMax")) is not None:
+        data["start"] = ",".join([smin, smax])
+    if (emin := raw_data.get("endMin")) is not None and (emax := raw_data.get("endMax")) is not None:
+        data["end"] = ",".join([emin, emax])
+    if (filter := raw_data.get("filters")) is not None:
+        data["filters"] = filter
     return data
 
 
@@ -241,6 +249,8 @@ async def find_query_endpoint(service, params):
         return service[0]
     else:
         for endpoint in endpoints:
+            if params == "filter" and "filtering_terms" in endpoint[0]:
+                return endpoint
             if raw_data.get("searchInInput") is not None:
                 if raw_data.get("searchInInput") in endpoint[0]:
                     if raw_data.get("id") != "0" and raw_data.get("id") is not None:
@@ -309,7 +319,6 @@ async def query_service(service, params, access_token, ws=None):
     # Pre-process query string into payload format
     if endpoint is not None:
         data = await pre_process_payload(endpoint[1], params)
-
         # Query service in a session
         async with aiohttp.ClientSession() as session:
             try:

requirements.txt

@@ -3,9 +3,9 @@ aiohttp==3.8.1
 aiohttp-cors==0.7.0
 aiocache==0.11.1
 aiomcache==0.7.0
-ujson==5.1.0
+ujson==5.2.0
 uvloop==0.14.0; python_version < '3.7'
 uvloop==0.16.0; python_version >= '3.7'
 asyncpg==0.25.0
-jsonschema==4.4.0
+jsonschema==4.5.1
 gunicorn==20.1.0

setup.py

@@ -34,11 +34,11 @@
         "aiohttp-cors==0.7.0",
         "aiocache==0.11.1",
         "aiomcache==0.7.0",
-        "ujson==5.1.0",
+        "ujson==5.2.0",
         "uvloop==0.14.0; python_version < '3.7'",
         "uvloop==0.16.0; python_version >= '3.7'",
         "asyncpg==0.25.0",
-        "jsonschema==4.4.0",
+        "jsonschema==4.5.1",
         "gunicorn==20.1.0",
     ],
     extras_require={
@@ -47,12 +47,12 @@
             "pytest<7.2",
             "pytest-cov==3.0.0",
             "testfixtures==6.18.5",
-            "tox==3.24.5",
+            "tox==3.25.0",
             "flake8==4.0.1",
             "flake8-docstrings==1.6.0",
             "asynctest==0.13.0",
             "aioresponses==0.7.3",
-            "black==22.1.0",
+            "black==22.3.0",
         ],
         "docs": ["sphinx >= 1.4", "sphinx_rtd_theme==1.0.0"],
     },

tests/aggregator/test_aggregator_utils.py

@@ -119,7 +119,6 @@ async def test_process_url_4(self):
         params = "searchInInput=g_variants&id=0&searchByInput="
         processed = await process_url(("https://beacon.fi", 2))
         findQuery = await find_query_endpoint(processed, params)
-        print("\x1b[6;30;42m" + str(findQuery) + "\x1b[0m")
         self.assertEqual("https://beacon.fi/g_variants", findQuery[0])
 
     async def test_remove_self(self):
@@ -283,6 +282,8 @@ async def test_pre_process_payload(self):
             "assemblyId=GRCh38&referenceName=MT&start=9&end=10&referenceBases=T&alternateBases=C&includeDatasetResponses=HIT",
             "assemblyId=GRCh38&referenceName=MT&startMin=5&startMax=10&endMin=5&endMax=15&referenceBases=T&alternateBases=C&variantType=SNP\
 &includeDatasetResponses=HIT",
+            "filters=filter",
+            "assemblyId=GRCh38&referenceName=MT&start=9&end=10&referenceBases=T&alternateBases=C&includeDatasetResponses=HIT&filters=filter",
         ]
         expected_v1 = [
             {"assemblyId": "GRCh38", "referenceName": "MT", "start": 9, "referenceBases": "T", "alternateBases": "C", "includeDatasetResponses": "HIT"},
@@ -329,13 +330,26 @@ async def test_pre_process_payload(self):
                 "variantType": "SNP",
                 "includeDatasetResponses": "HIT",
             },
+            {"assemblyId": "", "filters": "filter", "includeDatasetResponses": ""},
+            {
+                "assemblyId": "GRCh38",
+                "referenceName": "MT",
+                "start": "9",
+                "end": "10",
+                "referenceBases": "T",
+                "alternateBases": "C",
+                "includeDatasetResponses": "HIT",
+                "filters": "filter",
+            },
         ]
         self.assertEqual(await pre_process_payload(1, query_strings[0]), expected_v1[0])
         self.assertEqual(await pre_process_payload(1, query_strings[1]), expected_v1[1])
         self.assertEqual(await pre_process_payload(1, query_strings[2]), expected_v1[2])
         self.assertEqual(await pre_process_payload(2, query_strings[0]), expected_v2[0])
         self.assertEqual(await pre_process_payload(2, query_strings[1]), expected_v2[1])
         self.assertEqual(await pre_process_payload(2, query_strings[2]), expected_v2[2])
+        self.assertEqual(await pre_process_payload(2, query_strings[3]), expected_v2[3])
+        self.assertEqual(await pre_process_payload(2, query_strings[4]), expected_v2[4])
 
 
 if __name__ == "__main__":