Handle begin date complications

CHANGELOG.md

@@ -8,6 +8,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 The intended audience of this file is for py42 consumers -- as such, changes that don't affect
 how a consumer would use the library (e.g. adding unit tests, updating documentation, etc) are not captured here.
 
+## Unreleased
+
+### Fixed
+
+- Fixed bug where port attached to `securitydata send-to` command was not properly applied.
+
+### Changed
+
+- Begin dates are no longer required for subsequent interactive `securitydata` commands.
+- When provided, begin dates are now ignored on subsequent interactive `securitydata` commands.
+
 ## 0.3.0 - 2020-03-04
 
 ### Added

README.md

@@ -2,7 +2,8 @@
 
 Use the `code42` command to interact with your Code42 environment.
 `code42 securitydata` is a CLI tool for extracting AED events.
-Additionally, `code42 securitydata` can record a checkpoint so that you only get events you have not previously gotten.
+Additionally, you can choose to only get events that Code42 previously did not observe since you last recorded a checkpoint
+(provided you do not change your query).
 
 ## Requirements
 
@@ -46,19 +47,34 @@ Next, you can query for events and send them to three possible destination types
 
 To print events to stdout, do:
 ```bash
-code42 securitydata print
+code42 securitydata print -b 2020-02-02
 ```
 
+Note that `-b` or `--begin` is usually required.
+To specify a time, do:
+
+```bash
+code42 securitydata print -b 2020-02-02 12:51
+```
+Begin date will be ignored if provided on subsequent queries using `-i`.
+
 To write events to a file, do:
 ```bash
-code42 securitydata write-to filename.txt
+code42 securitydata write-to filename.txt -b 2020-02-02
 ```
 
 To send events to a server, do:
 ```bash
-code42 securitydata send-to https://syslog.company.com -p TCP
+code42 securitydata send-to syslog.company.com -p TCP -b 2020-02-02
 ```
 
+To only get events that Code42 previously did not observe since you last recorded a checkpoint, use the `-i` flag.
+```bash
+code42 securitydata send-to syslog.company.com -i
+```
+This is only guaranteed if you did not change your query.
+
+
 Each destination-type subcommand shares query parameters
 * `-t` (exposure types)
 * `-b` (begin date)
@@ -75,8 +91,7 @@ Each destination-type subcommand shares query parameters
 * `--include-non-exposure` (does not work with `-t`)
 * `--advanced-query` (raw JSON query)
 
-Note that you cannot use other query parameters if you use `--advanced-query`.
-
+You cannot use other query parameters if you use `--advanced-query`.
 To learn more about acceptable arguments, add the `-h` flag to `code42` or and of the destination-type subcommands.
 
 

src/code42cli/securitydata/cursor_store.py

@@ -7,7 +7,7 @@
 _INSERTION_TIMESTAMP_FIELD_NAME = u"insertionTimestamp"
 
 
-class SecurityEventCursorStore(object):
+class BaseCursorStore(object):
     _PRIMARY_KEY_COLUMN_NAME = u"cursor_id"
 
     def __init__(self, db_table_name, db_file_path=None):
@@ -52,11 +52,11 @@ def _is_empty(self):
                 return int(query_result[0]) <= 0
 
 
-class AEDCursorStore(SecurityEventCursorStore):
+class FileEventCursorStore(BaseCursorStore):
     _PRIMARY_KEY = 1
 
     def __init__(self, db_file_path=None):
-        super(AEDCursorStore, self).__init__(u"aed_checkpoint", db_file_path)
+        super(FileEventCursorStore, self).__init__(u"aed_checkpoint", db_file_path)
         if self._is_empty():
             self._init_table()
 

src/code42cli/securitydata/date_helper.py

@@ -3,28 +3,45 @@
 from c42eventextractor.common import convert_datetime_to_timestamp
 from py42.sdk.file_event_query.event_query import EventTimestamp
 
-_DEFAULT_LOOK_BACK_DAYS = 60
 _MAX_LOOK_BACK_DAYS = 90
 _FORMAT_VALUE_ERROR_MESSAGE = u"input must be a date in YYYY-MM-DD or YYYY-MM-DD HH:MM:SS format."
 
 
-def create_event_timestamp_range(begin_date, end_date=None):
-    """Creates a `py42.sdk.file_event_query.event_query.EventTimestamp.in_range` filter
-            using the provided dates.  If begin_date is None, it uses a date that is 60 days back.
-            If end_date is None, it uses the current UTC time.
+def create_event_timestamp_filter(begin_date=None, end_date=None):
+    """Creates a `py42.sdk.file_event_query.event_query.EventTimestamp` filter using the given dates.
+        Returns None if not given a begin_date or an end_date.
 
         Args:
-            begin_date: The begin date for the range. If None, defaults to 60 days back from the current UTC time.
-            end_date: The end date for the range. If None, defaults to the current time.
+            begin_date: The begin date for the range.
+            end_date: The end date for the range.
 
     """
     end_date = _get_end_date_with_eod_time_if_needed(end_date)
+    if begin_date and end_date:
+        return _create_in_range_filter(begin_date, end_date)
+    elif begin_date and not end_date:
+        return _create_on_or_after_filter(begin_date)
+    elif end_date and not begin_date:
+        return _create_on_or_before_filter(end_date)
+
+
+def _create_in_range_filter(begin_date, end_date):
     min_timestamp = _parse_min_timestamp(begin_date)
-    max_timestamp = _parse_max_timestamp(end_date)
+    max_timestamp = _parse_timestamp(end_date)
     _verify_timestamp_order(min_timestamp, max_timestamp)
     return EventTimestamp.in_range(min_timestamp, max_timestamp)
 
 
+def _create_on_or_after_filter(begin_date):
+    min_timestamp = _parse_min_timestamp(begin_date)
+    return EventTimestamp.on_or_after(min_timestamp)
+
+
+def _create_on_or_before_filter(end_date):
+    max_timestamp = _parse_timestamp(end_date)
+    return EventTimestamp.on_or_before(max_timestamp)
+
+
 def _get_end_date_with_eod_time_if_needed(end_date):
     if end_date and len(end_date) == 1:
         return end_date[0], "23:59:59"
@@ -40,12 +57,6 @@ def _parse_min_timestamp(begin_date_str):
     return min_timestamp
 
 
-def _parse_max_timestamp(end_date_str):
-    if not end_date_str:
-        return _get_default_max_timestamp()
-    return _parse_timestamp(end_date_str)
-
-
 def _verify_timestamp_order(min_timestamp, max_timestamp):
     if min_timestamp is None or max_timestamp is None:
         return
@@ -74,7 +85,3 @@ def _join_date_tuple(date_tuple):
     else:
         raise ValueError(_FORMAT_VALUE_ERROR_MESSAGE)
     return date_str
-
-
-def _get_default_max_timestamp():
-    return convert_datetime_to_timestamp(datetime.utcnow())

src/code42cli/securitydata/extraction.py

@@ -18,7 +18,7 @@
 from code42cli.securitydata import date_helper as date_helper
 from code42cli.securitydata.arguments.main import IS_INCREMENTAL_KEY
 from code42cli.securitydata.arguments.search import SearchArguments
-from code42cli.securitydata.cursor_store import AEDCursorStore
+from code42cli.securitydata.cursor_store import FileEventCursorStore
 from code42cli.securitydata.logger_factory import get_error_logger
 from code42cli.securitydata.options import ExposureType as ExposureTypeOptions
 from code42cli.util import print_error, print_bold, is_interactive
@@ -37,15 +37,21 @@ def extract(output_logger, args):
             args:
                 Command line args used to build up file event query filters.
     """
-    handlers = _create_event_handlers(output_logger, args.is_incremental)
+    store = _create_cursor_store(args)
+    handlers = _create_event_handlers(output_logger, store)
     profile = get_profile()
     sdk = _get_sdk(profile, args.is_debug_mode)
     extractor = FileEventExtractor(sdk, handlers)
-    _call_extract(extractor, args)
+    _call_extract(extractor, store, args)
     _handle_result()
 
 
-def _create_event_handlers(output_logger, is_incremental):
+def _create_cursor_store(args):
+    if args.is_incremental:
+        return FileEventCursorStore()
+
+
+def _create_event_handlers(output_logger, cursor_store):
     handlers = FileEventHandlers()
     error_logger = get_error_logger()
 
@@ -56,10 +62,9 @@ def handle_error(exception):
 
     handlers.handle_error = handle_error
 
-    if is_incremental:
-        store = AEDCursorStore()
-        handlers.record_cursor_position = store.replace_stored_insertion_timestamp
-        handlers.get_cursor_position = store.get_stored_insertion_timestamp
+    if cursor_store:
+        handlers.record_cursor_position = cursor_store.replace_stored_insertion_timestamp
+        handlers.get_cursor_position = cursor_store.get_stored_insertion_timestamp
 
     def handle_response(response):
         response_dict = json.loads(response.text)
@@ -86,9 +91,9 @@ def _get_sdk(profile, is_debug_mode):
         exit(1)
 
 
-def _call_extract(extractor, args):
+def _call_extract(extractor, cursor_store, args):
     if not _determine_if_advanced_query(args):
-        _verify_begin_date(args.begin_date)
+        _verify_begin_date_requirements(args, cursor_store)
         _verify_exposure_types(args.exposure_types)
         filters = _create_filters(args)
         extractor.extract(*filters)
@@ -116,23 +121,26 @@ def _verify_compatibility_with_advanced_query(key, val):
     return True
 
 
-def _get_event_timestamp_filter(args):
-    try:
-        return date_helper.create_event_timestamp_range(args.begin_date, args.end_date)
-    except ValueError as ex:
-        print_error(str(ex))
-        exit(1)
-
-
-def _verify_begin_date(begin_date):
-    if not begin_date:
+def _verify_begin_date_requirements(args, cursor_store):
+    if _begin_date_is_required(args, cursor_store) and not args.begin_date:
         print_error(u"'begin date' is required.")
         print(u"")
         print_bold(u"Try using  '-b' or '--begin'. Use `-h` for more info.")
         print(u"")
         exit(1)
 
 
+def _begin_date_is_required(args, cursor_store):
+    if not args.is_incremental:
+        return True
+    required = cursor_store is not None and cursor_store.get_stored_insertion_timestamp() is None
+
+    # Ignore begin date when is incremental mode, it is not required, and it was passed an argument.
+    if not required and args.begin_date:
+        args.begin_date = None
+    return required
+
+
 def _verify_exposure_types(exposure_types):
     if exposure_types is None:
         return
@@ -143,13 +151,25 @@ def _verify_exposure_types(exposure_types):
             exit(1)
 
 
+def _get_event_timestamp_filter(args):
+    try:
+        return date_helper.create_event_timestamp_filter(args.begin_date, args.end_date)
+    except ValueError as ex:
+        print_error(str(ex))
+        exit(1)
+
+
 def _handle_result():
     if is_interactive() and _EXCEPTIONS_OCCURRED:
         print_error(u"View exceptions that occurred at [HOME]/.code42cli/log/code42_errors.")
 
 
 def _create_filters(args):
-    filters = [_get_event_timestamp_filter(args)]
+    filters = []
+    event_timestamp_filter = _get_event_timestamp_filter(args)
+    if event_timestamp_filter:
+        filters.append(event_timestamp_filter)
+
     not args.c42username or filters.append(DeviceUsername.eq(args.c42username))
     not args.actor or filters.append(Actor.eq(args.actor))
     not args.md5 or filters.append(MD5.eq(args.md5))

src/code42cli/securitydata/logger_factory.py

@@ -12,7 +12,7 @@
 
 from code42cli.compat import str
 from code42cli.securitydata.options import OutputFormat
-from code42cli.util import get_user_project_path, print_error
+from code42cli.util import get_user_project_path, print_error, get_url_parts
 
 _logger_deps_lock = Lock()
 
@@ -56,7 +56,7 @@ def get_logger_for_server(hostname, protocol, output_format):
     """Gets the logger that sends logs to a server for the given format.
 
         Args:
-            hostname: The hostname of the server.
+            hostname: The hostname of the server. It may include the port.
             protocol: The transfer protocol for sending logs.
             output_format: CEF, JSON, or RAW_JSON. Each type results in a different logger instance.
     """
@@ -66,7 +66,11 @@ def get_logger_for_server(hostname, protocol, output_format):
 
     with _logger_deps_lock:
         if not _logger_has_handlers(logger):
-            handler = NoPrioritySysLogHandlerWrapper(hostname, protocol=protocol).handler
+            url_parts = get_url_parts(hostname)
+            port = url_parts[1] or 514
+            handler = NoPrioritySysLogHandlerWrapper(
+                url_parts[0], port=port, protocol=protocol
+            ).handler
             return _init_logger(logger, handler, output_format)
     return logger
 

src/code42cli/securitydata/subcommands/clear_checkpoint.py

@@ -1,4 +1,4 @@
-from code42cli.securitydata.cursor_store import AEDCursorStore
+from code42cli.securitydata.cursor_store import FileEventCursorStore
 
 
 def init(subcommand_parser):
@@ -15,7 +15,7 @@ def clear_checkpoint(*args):
         To use, run `code42 clear-checkpoint`.
         This affects `incremental` mode by causing it to behave like it has never been run before.
     """
-    AEDCursorStore().reset()
+    FileEventCursorStore().reset()
 
 
 if __name__ == "__main__":

src/code42cli/util.py

@@ -3,6 +3,8 @@
 import sys
 from os import path, makedirs
 
+from code42cli.compat import urlparse
+
 
 def get_input(prompt):
     """Uses correct input function based on Python version."""
@@ -42,3 +44,11 @@ def print_bold(bold_text):
 
 def is_interactive():
     return sys.stdin.isatty()
+
+
+def get_url_parts(url_str):
+    parts = url_str.split(u":")
+    port = None
+    if len(parts) > 1 and parts[1] != u"":
+        port = int(parts[1])
+    return parts[0], port

tests/conftest.py

@@ -1,7 +1,8 @@
-import pytest
 import json as json_module
-from datetime import datetime, timedelta
 from argparse import Namespace
+from datetime import datetime, timedelta
+
+import pytest
 
 from code42cli.profile.config import ConfigurationKeys
 

tests/profile/test_config.py

@@ -1,4 +1,5 @@
 from __future__ import with_statement
+
 import pytest
 
 import code42cli.profile.config as config

tests/profile/test_profile.py

@@ -1,5 +1,7 @@
-import pytest
 from argparse import ArgumentParser
+
+import pytest
+
 from code42cli.profile import profile
 from .conftest import CONFIG_NAMESPACE, PASSWORD_NAMESPACE, PROFILE_NAMESPACE