Removing deprecated endpoints (#1212)

* Removing deprecated endpoints

* Cleaning up bdd tests

* Cleaning up infrastructure

* Adding README for bdd-tests

.happy/terraform/modules/ecs-stack/main.tf

@@ -98,9 +98,6 @@ module frontend_service {
   wait_for_steady_state = local.wait_for_steady_state
 }
 
-resource "random_uuid" "api_key" {
-}
-
 module install_dynamodb_table {
   source              = "../dynamo"
   table_name          = "${local.custom_stack_name}-install-activity"
@@ -286,7 +283,6 @@ module backend_lambda {
     "DD_SERVICE" = local.custom_stack_name
     "API_URL" = var.env == "dev" ? module.api_gateway_proxy_stage.invoke_url : ""
     "PLUGINS_LAMBDA_NAME" = local.plugins_function_name
-    "API_KEY" = random_uuid.api_key.result
     "STACK_NAME" = local.custom_stack_name
   }
 
@@ -417,18 +413,6 @@ resource "aws_cloudwatch_event_rule" "update_rule" {
   tags                = var.tags
 }
 
-resource aws_cloudwatch_event_target update_target {
-    rule = aws_cloudwatch_event_rule.update_rule.name
-    arn = module.backend_lambda.function_arn
-    input_transformer {
-        input_template = jsonencode({
-          path = "/update",
-          httpMethod = "POST",
-          headers = {"X-API-Key": random_uuid.api_key.result}
-        })
-    }
-}
-
 resource aws_cloudwatch_event_target plugin_target_sqs {
     rule = aws_cloudwatch_event_rule.update_rule.name
     arn = aws_sqs_queue.data_workflows_queue.arn
@@ -458,11 +442,7 @@ locals {
     LambdaPermission = {
       service    = "apigateway"
       source_arn = "${local.execution_arn}*"
-    },
-    AllowExecutionFromCloudWatchForPlugin = {
-      service    = "events"
-      source_arn = aws_cloudwatch_event_rule.update_rule.arn
-    },
+    }
   }
 }
 
@@ -504,7 +484,6 @@ data aws_iam_policy_document backend_policy {
       module.category_dynamodb_table.table_arn,
       module.plugin_metadata_dynamodb_table.table_arn,
       module.plugin_dynamodb_table.table_arn,
-      module.plugin_blocked_dynamodb_table.table_arn,
       "${module.plugin_dynamodb_table.table_arn}/index/*",
     ]
   }

backend/api/app.py

@@ -10,12 +10,7 @@
 from api.home import get_plugin_sections
 from api.plugin_collections import get_collections, get_collection
 from api.custom_wsgi import script_path_middleware
-from api.model import (
-    get_index,
-    get_excluded_plugins,
-    move_artifact_to_s3,
-    get_manifest,
-)
+from api.model import get_index, move_artifact_to_s3, get_manifest
 from api.metrics import get_metrics_for_plugin
 from api.models import (category as categories, plugin as plugin_model)
 from api.shield import get_shield
@@ -76,11 +71,6 @@ def plugin_index_all() -> Response:
     return jsonify(get_index(None, False))
 
 
-@app.route("/plugins")
-def plugins() -> Response:
-    return jsonify(plugin_model.get_latest_by_visibility())
-
-
 @app.route("/plugins/<plugin>", defaults={"version": None})
 @app.route("/plugins/<plugin>/versions/<version>")
 def versioned_plugin(plugin: str, version: str = None) -> Response:
@@ -130,11 +120,6 @@ def shield(plugin: str) -> Response:
     return jsonify(get_shield(plugin))
 
 
-@app.route("/plugins/excluded")
-def get_exclusion_list() -> Response:
-    return jsonify(get_excluded_plugins())
-
-
 @app.route(
     "/categories",
     defaults={"version": os.getenv("category_version", "EDAM-BIOIMAGING:alpha06")}

backend/api/model.py

@@ -5,7 +5,6 @@
 from api.models import (
     install_activity,
     plugin as plugin_model,
-    plugin_blocked,
     plugin_metadata as plugin_metadata_model,
 )
 from utils.github import get_artifact
@@ -58,17 +57,6 @@ def get_index(
     return plugins
 
 
-def get_excluded_plugins() -> Dict[str, str]:
-    """
-    Get the excluded plugins.
-    :return: dict for excluded plugins and their exclusion status
-    """
-    return {
-        **plugin_model.get_excluded_plugins(),
-        **plugin_blocked.get_blocked_plugins()
-    }
-
-
 def move_artifact_to_s3(payload, client):
     """
     move preview page build artifact zip to public s3.

backend/api/models/_tests/test_plugin.py

@@ -162,38 +162,6 @@ def _put_items(cls, table, data):
                 item["release_date"] = item["data"]["release_date"]
             table.put_item(Item=item)
 
-    def test_get_latest_by_visibility_default_visibility(
-            self, plugin_table, data
-    ):
-        self._put_items(plugin_table, data)
-
-        actual = plugin.get_latest_by_visibility()
-
-        assert actual == {"plugin-1": "2.3", "plugin-4": "5.0"}
-
-    @pytest.mark.parametrize("visibility", [
-        "PUBLIC", "HIDDEN", "DISABLED",
-    ])
-    def test_get_latest_by_visibility_by_visibility_without_data(
-            self, plugin_table, visibility
-    ):
-        actual = plugin.get_latest_by_visibility(visibility)
-
-        assert actual == {}
-
-    @pytest.mark.parametrize("visibility, expected", [
-        ("PUBLIC", {"plugin-1": "2.3", "plugin-4": "5.0"}),
-        ("HIDDEN", {"plugin-2": "1.0.0"}),
-        ("DISABLED", {"plugin-3": "1.6"}),
-    ])
-    def test_get_latest_by_visibility_by_visibility_with_data(
-            self, plugin_table, data, visibility, expected
-    ):
-        self._put_items(plugin_table, data)
-
-        actual = plugin.get_latest_by_visibility(visibility)
-        assert actual == expected
-
     @pytest.mark.parametrize("visibility, expected_fixtures", [
         (
                 {"PUBLIC"},
@@ -269,13 +237,6 @@ def test_get_plugin_without_version(
         expected = get_fixture(fixture_name)
         assert actual == expected
 
-    def test_get_excluded_plugins(self, plugin_table, data):
-        self._put_items(plugin_table, data)
-
-        actual = plugin.get_excluded_plugins()
-
-        assert actual == {"plugin-2": "hidden", "plugin-3": "disabled"}
-
     @pytest.mark.parametrize("name, expected", [
         ("plugin-1", "2.3"),
         ("plugin-2", "1.0.0"),

backend/api/models/plugin.py

@@ -8,7 +8,7 @@
 )
 from pynamodb.expressions.condition import Condition
 from pynamodb.indexes import (
-    GlobalSecondaryIndex, AllProjection, IncludeProjection
+    GlobalSecondaryIndex, AllProjection
 )
 from pynamodb.models import Model
 
@@ -36,15 +36,6 @@ class Meta:
     visibility = UnicodeAttribute(null=True)
 
 
-class _ExcludedPluginIndex(GlobalSecondaryIndex):
-    class Meta:
-        index_name = f"{os.getenv('STACK_NAME')}-excluded-plugins"
-        projection = IncludeProjection(["last_updated_timestamp", "is_latest"])
-
-    name = UnicodeAttribute(hash_key=True)
-    excluded = UnicodeAttribute(range_key=True)
-
-
 @set_ddb_metadata("plugin")
 class _Plugin(Model):
     class Meta:
@@ -66,16 +57,6 @@ class Meta:
     visibility = UnicodeAttribute(null=True)
 
     latest_plugin_index = _LatestPluginIndex()
-    excluded_plugin_index = _ExcludedPluginIndex()
-
-
-def get_latest_by_visibility(visibility: str = "PUBLIC") -> Dict[str, str]:
-    return _scan_index(
-        index=_Plugin.latest_plugin_index,
-        attributes_to_get=["name", "version"],
-        filter_conditions=_to_visibility_condition({visibility}),
-        mapper=_to_plugin_version_dict
-    )
 
 
 def get_index(visibility: Set[str]) -> List[Dict[str, Any]]:
@@ -107,14 +88,6 @@ def get_plugin(name: str, version: str = None) -> Dict[str, Any]:
         return {}
 
 
-def get_excluded_plugins() -> Dict[str, str]:
-    return _scan_index(
-        index=_Plugin.excluded_plugin_index,
-        attributes_to_get=["name", "excluded"],
-        mapper=lambda iterator: {p.name: p.excluded.lower() for p in iterator}
-    )
-
-
 def get_latest_version(name: str) -> Optional[str]:
     result = _query_index({
         "hash_key": name,
@@ -184,10 +157,6 @@ def _get_latest_version_plugin(plugins: List[_Plugin]) -> Optional[_Plugin]:
     return sorted(plugins, key=lambda p: p.release_date, reverse=True)[0]
 
 
-def _to_plugin_version_dict(iterator: Iterator[_Plugin]) -> Dict[str, str]:
-    return {plugin.name: plugin.version for plugin in iterator}
-
-
 def _index_list_mapper(
         fields: Set[str]
 ) -> Callable[[Iterator[_Plugin]], List[Dict[str, Any]]]:

backend/bdd_tests/README.md

@@ -0,0 +1,35 @@
+## Behavioral Driven Development for backend
+
+### Setting up to run BDD tests
+
+To ensure you have all the dependencies needed for the BDD tests, run `pip install` on 'backend\bdd-test-requirements.txt`.
+```
+pip install -r bdd-test-requirements.txt
+```
+
+### Running BDD tests locally
+
+To run the tests against your local environment, ensure you have the backend application running locally in another terminal, and run the following command from the backend folder:
+```
+python -m pytest bdd_tests
+```
+
+If you are running your local Flask application on a non-default port (The default port is `12345`), you can leverage the `PORT` variable. For example, if you are running the backend application on port 8080, you can specify it as follows:
+```
+PORT=8080 python -m pytest bdd_tests
+```
+
+### Running BDD tests against staging and prod
+
+The prefix variable is helpful in cases where you are running against non-local environments. For staging and production, it can be run by setting the `PREFIX` value to `staging` and `prod`, respectively. For example, to run the BDD tests against the prod-environment, use the following command:
+```
+PREFIX=prod python -m pytest bdd_tests
+```
+
+### Running BDD tests against dev environments
+
+The `PREFIX` variable also comes to the rescue in this case. The dev environment can have any prefix, so you would have to specify the dev-branch name that acts as the prefix for the environment. For example, to run the BDD tests against the `dev-shared` dev-environment, use the following command:
+
+```
+PREFIX=dev-shared python -m pytest bdd_tests
+```