Adding plugins api for homepage (#1132)

* Adding plugins api for homepage

* Adding guardrails

* Adding bdd tests

* Cleaning up

backend/api/app.py

@@ -7,6 +7,7 @@
 from flask import Flask, Response, jsonify, render_template, request
 from flask_githubapp.core import GitHubApp
 
+from api.home import get_plugin_sections
 from api.plugin_collections import get_collections, get_collection
 from api.custom_wsgi import script_path_middleware
 from api.model import (
@@ -87,6 +88,17 @@ def versioned_plugin(plugin: str, version: str = None) -> Response:
     return jsonify(get_plugin(plugin, version, use_dynamo_plugins))
 
 
+@app.route("/plugin/home/sections/<sections>")
+def plugin_home_page_sections(sections: str = "") -> Response:
+    use_dynamo_plugins = _is_query_param_true("use_dynamo_plugin")
+    unique_sections = set(sections.split(","))
+    limit = int(request.args.get("limit", "3"))
+    plugins_by_sections = get_plugin_sections(
+        unique_sections, use_dynamo_plugins, limit
+    )
+    return jsonify(plugins_by_sections)
+
+
 @app.route("/manifest/<plugin>", defaults={"version": None})
 @app.route("/manifest/<plugin>/versions/<version>")
 def plugin_manifest(plugin: str, version: str = None) -> Response:

backend/api/home.py

@@ -0,0 +1,110 @@
+import logging
+from typing import List, Dict, Set, Callable, Union
+
+from api.model import get_index
+from random import sample
+from datetime import datetime
+
+
+DEFAULT_FIELDS = {
+    "authors", "display_name", "first_released", "name", "release_date",
+    "summary", "total_installs",
+}
+PLUGIN_TYPES = ["reader", "sample_data", "widget", "writer"]
+
+
+logger = logging.getLogger(__name__)
+
+
+def _filtered(data: Dict) -> Dict:
+    return {field: data.get(field) for field in DEFAULT_FIELDS}
+
+
+def _get_plugin_type() -> str:
+    index = int((datetime.now().minute / 5) % 4)
+    return PLUGIN_TYPES[index]
+
+
+def _add_to_exclusions(exclude: Set[str], plugins: List[Dict]) -> None:
+    for plugin in plugins:
+        exclude.add(plugin.get("name"))
+
+
+def _get_plugins_by_type(index: List[Dict], limit: int, exclude: Set) -> Dict:
+    plugin_type = _get_plugin_type()
+    logger.info(f"plugin_type section of type={plugin_type}")
+    plugins_of_type = list(
+        filter(lambda item: plugin_type in item.get("plugin_types", []), index)
+    )
+    rand_sample = sample(plugins_of_type, min(limit, len(index)))
+    sampled_plugins = [_filtered(plugin) for plugin in rand_sample]
+    _add_to_exclusions(exclude, sampled_plugins)
+    return {"type": plugin_type, "plugins": sampled_plugins}
+
+
+def _get_plugins_by_sort(
+        index: List[Dict],
+        limit: int,
+        key: str,
+        default_val: Union[str, int],
+        exclude: Set[str]
+) -> Dict[str, List]:
+    index.sort(key=lambda item: item.get(key, default_val))
+    upper_limit = min(limit, len(index))
+    plugins = []
+    while len(plugins) < upper_limit and len(index) > 0:
+        plugin = index.pop()
+        name = plugin.get("name")
+        if name in exclude:
+            continue
+        exclude.add(name)
+        plugins.append(_filtered(plugin))
+
+    return {"plugins": plugins}
+
+
+def _get_newest_plugins(index: List[Dict], limit: int, exclude: Set) -> Dict:
+    return _get_plugins_by_sort(index, limit, "first_released", "", exclude)
+
+
+def _get_recently_updated_plugins(
+        index: List[Dict], limit: int, exclude: Set
+) -> Dict:
+    return _get_plugins_by_sort(index, limit, "release_date", "", exclude)
+
+
+def _get_top_installed_plugins(
+        index: List[Dict], limit: int, exclude: Set
+) -> Dict:
+    return _get_plugins_by_sort(index, limit, "total_installs", 0, exclude)
+
+
+def get_handler_by_section_name() -> Dict[str, Callable]:
+    return {
+        "plugin_types": _get_plugins_by_type,
+        "newest": _get_newest_plugins,
+        "recently_updated": _get_recently_updated_plugins,
+        "top_installed": _get_top_installed_plugins,
+    }
+
+
+def _has_no_valid_sections(sections: Set):
+    return set(get_handler_by_section_name().keys()).isdisjoint(sections)
+
+
+def get_plugin_sections(
+        sections: Set[str], use_dynamo: bool, limit: int = 3
+) -> Dict[str, Dict]:
+    if _has_no_valid_sections(sections):
+        logger.warning("No processing as there are no valid sections")
+        return {}
+    response = {}
+
+    plugins_encountered = set()
+    index = get_index(use_dynamo)
+    for name, handler in get_handler_by_section_name().items():
+        if name in sections:
+            response[name] = handler(index, limit, plugins_encountered)
+            logger.info(f"fetched data for {name} section")
+
+    return response

backend/bdd_tests/scenarios/homepage_plugins.feature

@@ -0,0 +1,60 @@
+Feature: HomepagePlugins
+  Homepage helps the discoverability of plugins
+
+  Scenario: get plugin_types sections of homepage_plugins without limit
+    Given we call plugins home api for plugin_types with query_param=use_dynamo_plugin=False
+    Then response status is 200
+    And it will have only the plugin_types sections
+    And it will have valid type for plugin_types
+    And each sections will have 3 valid plugins
+
+  Scenario: get newest sections of homepage_plugins without limit
+    Given we call plugins home api for newest with query_param=use_dynamo_plugin=False
+    Then response status is 200
+    And it will have only the newest sections
+    And each sections will have 3 valid plugins
+    And the newest section is sorted by first_released field
+
+  Scenario: get recently_updated sections of homepage_plugins without limit
+    Given we call plugins home api for recently_updated with query_param=use_dynamo_plugin=False
+    Then response status is 200
+    And it will have only the recently_updated sections
+    And each sections will have 3 valid plugins
+    And the recently_updated section is sorted by release_date field
+
+  Scenario: get top_installed sections of homepage_plugins without limit
+    Given we call plugins home api for top_installed with query_param=use_dynamo_plugin=False
+    Then response status is 200
+    And it will have only the top_installed sections
+    And each sections will have 3 valid plugins
+    And the top_installed section is sorted by total_installs field
+
+  Scenario: get all sections of homepage_plugins without limit
+    Given we call plugins home api for newest,recently_updated,top_installed,plugin_types with query_param=use_dynamo_plugin=False
+    Then response status is 200
+    And it will have only the newest,recently_updated,top_installed,plugin_types sections
+    And it will have valid type for plugin_types
+    And each sections will have 3 valid plugins
+    And the newest section is sorted by first_released field
+    And the recently_updated section is sorted by release_date field
+    And the top_installed section is sorted by total_installs field
+
+  Scenario: get all sections of homepage_plugins with limit
+    Given we call plugins home api for newest,recently_updated,top_installed,plugin_types having a limit of 5 and query_param=use_dynamo_plugin=False
+    Then response status is 200
+    And it will have only the newest,recently_updated,top_installed,plugin_types sections
+    And it will have valid type for plugin_types
+    And each sections will have 5 valid plugins
+    And the newest section is sorted by first_released field
+    And the recently_updated section is sorted by release_date field
+    And the top_installed section is sorted by total_installs field
+
+  Scenario: get all sections of homepage_plugins with limit and use_dynamo_plugin=True
+    Given we call plugins home api for newest,recently_updated,top_installed,plugin_types having a limit of 5 and query_param=use_dynamo_plugin=True
+    Then response status is 200
+    And it will have only the newest,recently_updated,top_installed,plugin_types sections
+    And it will have valid type for plugin_types
+    And each sections will have 5 valid plugins
+    And the newest section is sorted by first_released field
+    And the recently_updated section is sorted by release_date field
+    And the top_installed section is sorted by total_installs field

backend/bdd_tests/test_homepage_plugin.py

@@ -0,0 +1,57 @@
+from pytest_bdd import given, scenarios, then, parsers
+from test_utils import call_api
+
+default_plugin_keys = {
+    "authors", "display_name", "first_released", "name", "release_date",
+    "summary", "total_installs"
+}
+valid_plugin_types = {"reader", "sample_data", "widget", "writer"}
+
+scenarios("homepage_plugins.feature")
+
+
+@given(parsers.parse("we call plugins home api for {sections} having a limit of"
+                     " {limit} and query_param={query_param}"))
+def call_plugin_home_with_limit(sections, limit, query_param, context):
+    url = f"/plugin/home/sections/{sections}?limit={limit}&{query_param}"
+    call_api(context, url)
+
+
+@given(parsers.parse("we call plugins home api for {sections} with query_param="
+                     "{query_param}"))
+def call_plugin_home_without_limit(sections, query_param, context):
+    call_api(context, f"/plugin/home/sections/{sections}?{query_param}")
+
+
+@then(parsers.parse("it will have only the {sections} sections"))
+def verify_sections_are_valid(sections, context):
+    response = context["response"].json()
+    sections_list = set(sections.split(","))
+    assert response.keys() == sections_list
+
+
+@then("it will have valid type for plugin_types")
+def verify_plugin_types_valid(context):
+    response = context["response"].json()
+    assert response.get("plugin_types").get("type") in valid_plugin_types
+
+
+@then(parsers.cfparse(
+    "each sections will have {limit:Number} valid plugins",
+    extra_types={"Number": int}
+))
+def verify_section_response_valid(limit, context):
+    response = context["response"].json()
+    for key, items in response.items():
+        plugins = items["plugins"]
+        assert len(plugins) == limit, f"{key} has fewer than expected plugins"
+        for plugin_data in plugins:
+            assert set(plugin_data.keys()).issubset(default_plugin_keys)
+
+
+@then(parsers.parse("the {section_name} section is sorted by {sort_key} field"))
+def verify_section_sort_is_valid(section_name, sort_key, context):
+    response = context["response"].json()
+    section = response[section_name]
+    sort_fields = [item.get(sort_key) for item in section["plugins"]]
+    assert sort_fields == sorted(sort_fields, reverse=True)

backend/bdd_tests/test_utils.py

@@ -20,13 +20,8 @@ def _get_base_url():
 headers = {'User-Agent': 'bdd-test'}
 
 
-def verify_response_status_code(context, status_code):
-    actual = context["response"].status_code
-    assert int(status_code) == actual, f"status code of {actual} was unexpected"
-
-
 def call_api(context, endpoint):
-    context['response'] = requests.get(f'{base_url}{endpoint}', headers=headers)
+    context["response"] = requests.get(f"{base_url}{endpoint}", headers=headers)
 
 
 def valid_str(value):