Context lookup API endpoint part 1 (#2440)

* ContextLinks part #1:
+ Configuration schema & file
+ API endpoint resource and routes
+ Some minor adjustments to make them work

* Bug fixes

* Added missing fields to the VT example.

* Making the black-formatter happy.

* More fixes for the black-formatter.

* Another linter fix.

* Refactored the API endpoint.
It now includes the changes suggested by the reviewer.
+ Better readable input verification
+ More granular error message on bad entries
+ Less log entries

* linter

* black-formatter

* Adding a unittest for the API endpoint.

* black-formatter

* Refactored the API endpoint based on review comments.
+ Removed the validation from the API endpoint.
+ Moved validation to tsctl
+ Changed validation to use jsonschema
+ Updated tests

* black-formatter

* bug fix

* Adding jsonschema to requirements.txt

data/context_links.yaml

@@ -0,0 +1,46 @@
+# ------------------------------------------------------------------------
+#                       -- CONTEXT LINKS --
+# ------------------------------------------------------------------------
+#
+# This is a config file to define context links for event attributes.
+#
+# Each context link consists of the following fields:
+#
+#   context_link_name:
+#
+#       short_name:         Type: str | The name for the context link.
+#                           Will be displayed in the context link submenu.
+#
+#       match_fields:       Type: list[str] | List of field keys where
+#                           this context link should be available. Will
+#                           be checked as case insensitive!
+#
+#       validation_regex:   Type: str | OPTIONAL
+#                           A regex pattern that needs to be
+#                           matched by the field value to to make the
+#                           context link available. This can be used to
+#                           validate the format of a value (e.g. a hash).
+#
+#       context_link:       Type: str | The link that will be opened in a
+#                           new tab when the context link is clicked.
+#                           IMPORTANT: Add the placeholder "<ATTR_VALUE>"
+#                           where the attribute value should be inserted
+#                           into the link.
+#
+#        redirect_warning:   [TRUE]: If the context link is clicked it will
+#                                   open a pop-up dialog first that asks the
+#                                   user if they would like to proceed to
+#                                   the linked page. (Recommended for
+#                                   external pages.)
+#                           [FALSE]: The linked page will be opened without
+#                                    any pop-up. (Recommended for internal
+#                                    pages.)
+#
+# ------------------------------------------------------------------------
+## Virustotal Example:
+# virustotal_hash_lookup:
+#   short_name: 'VirusTotal'
+#   match_fields: ['hash', 'sha256_hash', 'sha256', 'sha1_hash', 'sha1', 'md5_hash', 'md5']
+#   validation_regex: '/^[0-9a-f]{64}$|^[0-9a-f]{40}$|^[0-9a-f]{32}$/i'
+#   context_link: 'https://www.virustotal.com/gui/search/<ATTR_VALUE>'
+#   redirect_warning: TRUE

data/timesketch.conf

@@ -320,3 +320,6 @@ QUESTIONS_PATH = '/etc/timesketch/scenarios/questions.yaml'
 
 # Intelligence tag metadata configuration
 INTELLIGENCE_TAG_METADATA = '/etc/timesketch/intelligence_tag_metadata.yaml'
+
+# Context links configuration
+CONTEXT_LINKS_CONFIG_PATH = '/etc/timesketch/context_links.yaml'

docker/dev/build/docker-entrypoint.sh

@@ -20,6 +20,8 @@ if [ "$1" = 'timesketch' ]; then
   ln -s /usr/local/src/timesketch/data/sigma_config.yaml /etc/timesketch/sigma_config.yaml
   ln -s /usr/local/src/timesketch/data/sigma_rule_status.csv /etc/timesketch/sigma_rule_status.csv
   ln -s /usr/local/src/timesketch/data/sigma /etc/timesketch/
+  ln -s /usr/local/src/timesketch/data/scenarios /etc/timesketch/
+  ln -s /usr/local/src/timesketch/data/context_links.yaml /etc/timesketch/context_links.yaml
 
 
   # Set SECRET_KEY in /etc/timesketch/timesketch.conf if it isn't already set

requirements.txt

@@ -35,3 +35,4 @@ prometheus-client==0.9.0
 prometheus-flask-exporter==0.18.1
 decorator==5.0.5
 geoip2==4.2.0
+jsonschema~=4.0

test_tools/test_events/mock_context_links.yaml

@@ -0,0 +1,20 @@
+## Mock configuration file for testing the contrext links API endpoint!
+lookupone:
+  short_name: 'LookupOne'
+  match_fields: ['hash']
+  validation_regex: '/^[0-9a-f]{40}$|^[0-9a-f]{32}$/i'
+  context_link: 'https://lookupone.local/q=<ATTR_VALUE>'
+  redirect_warning: TRUE
+
+lookuptwo:
+  short_name: 'LookupTwo'
+  match_fields: ['sha256_hash', 'hash']
+  validation_regex: '/^[0-9a-f]{64}$/i'
+  context_link: 'https://lookuptwo.local/q=<ATTR_VALUE>'
+  redirect_warning: FALSE
+
+lookupthree:
+  short_name: 'LookupThree'
+  match_fields: ['url']
+  context_link: 'https://lookupthree.local/q=<ATTR_VALUE>'
+  redirect_warning: TRUE

timesketch/api/v1/resources/contextlinks.py

@@ -0,0 +1,57 @@
+# Copyright 2022 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Context link API for version 1 of the Timesketch API."""
+
+from copy import deepcopy
+from flask import jsonify
+from flask_restful import Resource
+from flask_login import login_required
+
+from timesketch.api.v1 import resources
+from timesketch.api.v1.utils import load_yaml_config
+
+
+class ContextLinkConfigResource(resources.ResourceMixin, Resource):
+    """Resource to get context link information."""
+
+    @login_required
+    def get(self):
+        """Handles GET request to the resource.
+
+        HINT:
+            In case of errors with loading the context links, use the
+            tsctl tool to validate your config file!
+            Example: tsctl validate-context-links-conf ./data/context_links.yaml
+
+        Returns:
+            JSON object including version info
+        """
+        # HINT: In case of errors with loading the context links, use the
+        # tsctl tool to validate your config file!
+        # Example: tsctl validate-context-links-conf ./data/context_links.yaml
+
+        context_link_yaml = load_yaml_config("CONTEXT_LINKS_CONFIG_PATH")
+
+        response = {}
+        if not context_link_yaml:
+            return jsonify(response)
+
+        for entry in context_link_yaml:
+            entry_dict = context_link_yaml[entry]
+            context_link_config = deepcopy(entry_dict)
+            del context_link_config["match_fields"]
+            for field in entry_dict.get("match_fields"):
+                response.setdefault(field.lower(), []).append(context_link_config)
+
+        return jsonify(response)

timesketch/api/v1/resources_test.py

@@ -1095,3 +1095,48 @@ def test_get_intelligence_tag_metadata(self):
         self.assertIsNotNone(response)
         self.assertEqual(response.status_code, HTTP_STATUS_CODE_OK)
         self.assertEqual(data, expected_tag_metadata)
+
+
+class ContextLinksResourceTest(BaseTest):
+    """Test Context Links resources."""
+
+    def test_get_context_links_config(self):
+        """Authenticated request to get the context links configuration."""
+
+        expected_configuration = {
+            "hash": [
+                {
+                    "short_name": "LookupOne",
+                    "validation_regex": "/^[0-9a-f]{40}$|^[0-9a-f]{32}$/i",
+                    "context_link": "https://lookupone.local/q=<ATTR_VALUE>",
+                    "redirect_warning": True,
+                },
+                {
+                    "short_name": "LookupTwo",
+                    "validation_regex": "/^[0-9a-f]{64}$/i",
+                    "context_link": "https://lookuptwo.local/q=<ATTR_VALUE>",
+                    "redirect_warning": False,
+                },
+            ],
+            "sha256_hash": [
+                {
+                    "short_name": "LookupTwo",
+                    "validation_regex": "/^[0-9a-f]{64}$/i",
+                    "context_link": "https://lookuptwo.local/q=<ATTR_VALUE>",
+                    "redirect_warning": False,
+                },
+            ],
+            "url": [
+                {
+                    "short_name": "LookupThree",
+                    "context_link": "https://lookupthree.local/q=<ATTR_VALUE>",
+                    "redirect_warning": True,
+                },
+            ],
+        }
+        self.login()
+        response = self.client.get("/api/v1/contextlinks/")
+        data = json.loads(response.get_data(as_text=True))
+        self.assertIsNotNone(response)
+        self.assertEqual(response.status_code, HTTP_STATUS_CODE_OK)
+        self.assertDictEqual(data, expected_configuration)

timesketch/api/v1/routes.py

@@ -73,6 +73,7 @@
 from .resources.graph import GraphPluginListResource
 from .resources.graph import GraphCacheResource
 from .resources.intelligence import TagMetadataResource
+from .resources.contextlinks import ContextLinkConfigResource
 
 from .resources.scenarios import ScenarioTemplateListResource
 from .resources.scenarios import ScenarioListResource
@@ -170,4 +171,5 @@
         "/sketches/<int:sketch_id>/scenarios/<int:scenario_id>/status/",
     ),
     (TagMetadataResource, "/intelligence/tagmetadata/"),
+    (ContextLinkConfigResource, "/contextlinks/"),
 ]

timesketch/lib/testlib.py

@@ -79,6 +79,7 @@ class TestConfig(object):
     SIMILARITY_DATA_TYPES = []
     SIGMA_RULES_FOLDERS = ["./data/sigma/rules/"]
     INTELLIGENCE_TAG_METADATA = "./data/intelligence_tag_metadata.yaml"
+    CONTEXT_LINKS_CONFIG_PATH = "./test_tools/test_events/mock_context_links.yaml"
 
 
 class MockOpenSearchClient(object):

timesketch/tsctl.py

@@ -21,6 +21,7 @@
 import click
 from flask.cli import FlaskGroup
 from sqlalchemy.exc import IntegrityError
+from jsonschema import validate, ValidationError
 
 from timesketch import version
 from timesketch.app import create_app
@@ -442,3 +443,62 @@ def export_sigma_rules(path):
             fw.write(rule.rule_yaml.encode("utf-8"))
         n = n + 1
     print(f"{n} Sigma rules exported")
+
+
+@cli.command(name="validate-context-links-conf")
+@click.argument("path")
+def validate_context_links_conf(path):
+    """Validates the provided context link yaml configuration file."""
+
+    schema = {
+        "type": "object",
+        "properties": {
+            "context_link": {
+                "type": "string",
+                "pattern": "<ATTR_VALUE>",
+            },
+            "match_fields": {
+                "type": "array",
+                "minItems": 1,
+                "items": [
+                    {
+                        "type": "string",
+                    },
+                ],
+            },
+            "redirect_warning": {
+                "type": "boolean",
+            },
+            "short_name": {
+                "type": "string",
+                "minLength": 1,
+            },
+            "validation_regex": {
+                "type": "string",
+            },
+        },
+        "required": [
+            "context_link",
+            "match_fields",
+            "redirect_warning",
+            "short_name",
+        ],
+    }
+
+    if not os.path.isfile(path):
+        print(f"Cannot load the config file: {path} does not exist!")
+        return
+
+    with open(path, "r") as fh:
+        context_link_config = yaml.safe_load(fh)
+
+    if not context_link_config:
+        print("The provided config file is empty.")
+        return
+
+    for entry in context_link_config:
+        try:
+            validate(instance=context_link_config[entry], schema=schema)
+            print(f'=> OK: "{entry}"')
+        except ValidationError as err:
+            print(f'=> ERROR: "{entry}" >> {err}\n')