Fixed a bug in the stories (#2225)

* Fixing a bug in the stories

* Fixing a bug in the search file in the API client

* update

* update

* update

Co-authored-by: Johan Berggren <jberggren@gmail.com>
Co-authored-by: Alexander J <741037+jaegeral@users.noreply.github.com>

api_client/python/timesketch_api_client/search.py

@@ -15,6 +15,7 @@
 import datetime
 import json
 import logging
+import re
 
 import pandas
 
@@ -232,11 +233,14 @@ class DateRangeChip(Chip):
     _DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
     _DATE_FORMAT_MICROSECONDS = "%Y-%m-%dT%H:%M:%S.%f"
 
+    _DATE_RE = r'^[0-9]{4}-[0-9]{1,2}-[0-9]{2}$'
+
     def __init__(self):
         """Initialize the date range."""
         super().__init__()
         self._start_date = None
         self._end_date = None
+        self._date_re = re.compile(self._DATE_RE)
 
     def add_end_time(self, end_time):
         """Add an end time to the range.
@@ -249,6 +253,11 @@ def add_end_time(self, end_time):
         """
         if end_time.endswith("Z"):
             end_time = end_time[:-1]
+
+        # Check for a whole day, YYYY-MM-DD.
+        if self._date_re.match(end_time):
+            end_time = f'{end_time}T23:59:59'
+
         try:
             dt = datetime.datetime.strptime(end_time, self._DATE_FORMAT_MICROSECONDS)
         except ValueError as exc:
@@ -274,6 +283,11 @@ def add_start_time(self, start_time):
         """
         if start_time.endswith("Z"):
             start_time = start_time[:-1]
+
+        # Check for a whole day, YYYY-MM-DD.
+        if self._date_re.match(start_time):
+            start_time = f'{start_time}T00:00:00'
+
         try:
             dt = datetime.datetime.strptime(start_time, self._DATE_FORMAT_MICROSECONDS)
         except ValueError as exc:
@@ -501,7 +515,7 @@ def _execute_query(self, file_name="", count=False):
             "filter": query_filter,
             "dsl": self._query_dsl,
             "count": count,
-            "fields": self._return_fields,
+            "fields": self.return_fields,
             "enable_scroll": scrolling,
             "file_name": file_name,
         }
@@ -726,7 +740,11 @@ def from_saved(self, search_id):  # pylint: disable=arguments-differ
             if "fields" in filter_dict:
                 fields = filter_dict.pop("fields")
                 return_fields = [x.get("field") for x in fields]
-                self.return_fields = ",".join(return_fields)
+                self._return_fields = ",".join(return_fields)
+
+            indices = filter_dict.get("indices", [])
+            if indices:
+                self.indices = indices
 
             self.query_filter = filter_dict
         self._query_string = data.get("query_string", "")
@@ -745,7 +763,6 @@ def indices(self):
     @indices.setter
     def indices(self, indices):
         """Make changes to the current set of indices."""
-
         if indices == "_all":
             self._indices = "_all"
             self.commit()
@@ -937,6 +954,11 @@ def remove_chip(self, chip_index):
     @property
     def return_fields(self):
         """Property that returns the return_fields."""
+        if self._return_fields:
+            items = self._return_fields.split(",")
+            if "datetime" not in items:
+                items.append("datetime")
+            return ",".join(items)
         return self._return_fields
 
     @return_fields.setter
@@ -1082,7 +1104,7 @@ def to_pandas(self):
         timelines = {t.id: t.name for t in self._sketch.list_timelines()}
 
         return_field_list = []
-        return_fields = self._return_fields
+        return_fields = self.return_fields
         if return_fields:
             if return_fields.startswith("'"):
                 return_fields = return_fields[1:]

contrib/deploy_timesketch.sh

@@ -85,6 +85,7 @@ curl -s $GITHUB_BASE_URL/data/plaso.mappings > timesketch/etc/timesketch/plaso.m
 curl -s $GITHUB_BASE_URL/data/generic.mappings > timesketch/etc/timesketch/generic.mappings
 curl -s $GITHUB_BASE_URL/data/features.yaml > timesketch/etc/timesketch/features.yaml
 curl -s $GITHUB_BASE_URL/data/ontology.yaml > timesketch/etc/timesketch/ontology.yaml
+curl -s $GITHUB_BASE_URL/data/sigma_rule_status.csv > timesketch/etc/timesketch/sigma_rule_status.csv
 curl -s $GITHUB_BASE_URL/data/tags.yaml > timesketch/etc/timesketch/tags.yaml
 curl -s $GITHUB_BASE_URL/data/intelligence_tag_metadata.yaml > timesketch/etc/timesketch/intelligence_tag_metadata.yaml
 curl -s $GITHUB_BASE_URL/data/sigma_config.yaml > timesketch/etc/timesketch/sigma_config.yaml

end_to_end_tests/query_test.py

@@ -84,7 +84,8 @@ def extract_strings(row):
         if search_obj is None:
             raise RuntimeError("Unable to find the saved search.")
         self.assertions.assertEqual(
-            saved_search.return_fields, "computer_name,data_type,strings,user_sid"
+            saved_search.return_fields,
+            "computer_name,data_type,strings,user_sid,datetime"
         )
 
         self.assertions.assertEqual(

timesketch/lib/aggregators/interface.py

@@ -330,10 +330,14 @@ def format_field_by_type(self, field_name):
         field_format = field_name
         field_type = None
 
+        indices = self.indices
+        if isinstance(indices, (list, tuple)):
+            indices = ','.join(indices)
+
         # Get the mapping for the field.
         try:
             mapping = self.opensearch.client.indices.get_field_mapping(
-                index=self.indices, fields=field_name
+                index=indices, fields=field_name
             )
         except opensearchpy.NotFoundError:
             mapping = {}

timesketch/lib/stories/api_fetcher.py

@@ -74,8 +74,12 @@ def get_aggregation(self, agg_dict):
 
         parameter_string = aggregation.parameters
         parameters = json.loads(parameter_string)
-        index = parameters.pop("index", None)
-        aggregator = agg_class(sketch_id=self._sketch_id, index=index)
+        parameter_index = parameters.pop("index", None)
+        indices, timeline_ids = self.get_indices_and_timelines(
+            parameter_index)
+        aggregator = agg_class(
+            sketch_id=self._sketch_id, indices=indices,
+            timeline_ids=timeline_ids)
 
         _ = parameters.pop("supported_charts", None)
         chart_color = parameters.pop("chart_color", "N/A")
@@ -129,8 +133,12 @@ def get_aggregation_group(self, agg_dict):
             if not agg_class:
                 continue
 
-            index = aggregator_parameters.pop("index", None)
-            aggregator_obj = agg_class(sketch_id=self._sketch_id, index=index)
+            parameter_index = aggregator_parameters.pop("index", None)
+            indices, timeline_ids = self.get_indices_and_timelines(
+                parameter_index)
+            aggregator_obj = agg_class(
+                sketch_id=self._sketch_id,
+                indices=indices, timeline_ids=timeline_ids)
             chart_type = aggregator_parameters.pop("supported_charts", None)
             color = aggregator_parameters.pop("chart_color", "")
             chart_title = aggregator_parameters.pop("chart_title", None)
@@ -165,6 +173,31 @@ def get_aggregation_group(self, agg_dict):
         }
         return data
 
+    def get_indices_and_timelines(self, index_list):
+        """Returns a tuple with two lists from indices and timeline IDs.
+
+        Args:
+            index_list (list): A list of timeline IDs (int) and
+                indices (str).
+
+        Returns:
+            A tuple with two items, a list of indices and a list of
+            timeline IDs.
+        """
+        indices = []
+        timeline_ids = []
+
+        if isinstance(index_list, str):
+            index_list = index_list.split(',')
+
+        for index in index_list:
+            if isinstance(index, str):
+                indices.append(index)
+            if isinstance(index, int):
+                timeline_ids.append(index)
+
+        return indices, timeline_ids
+
     def get_view(self, view_dict):
         """Returns a data frame from a view dict.