patterns.py

"""Classes to aid in working with the STIX2 patterning language."""

import base64
import binascii
import datetime
import re

import six

from .utils import parse_into_datetime


def escape_quotes_and_backslashes(s):
    return s.replace(u'\\', u'\\\\').replace(u"'", u"\\'")


def quote_if_needed(x):
    if isinstance(x, six.string_types):
        if x.find("-") != -1:
            if not x.startswith("'"):
                return "'" + x + "'"
    return x


class _Constant(object):
    pass


class StringConstant(_Constant):
    """Pattern string constant

    Args:
        value (str): string value
    """

    def __init__(self, value, from_parse_tree=False):
        self.needs_to_be_quoted = not from_parse_tree
        self.value = value

    def __str__(self):
        return "'%s'" % (escape_quotes_and_backslashes(self.value) if self.needs_to_be_quoted else self.value)


class TimestampConstant(_Constant):
    """Pattern timestamp constant

    Args:
        value (datetime.datetime OR str): if string, must be a timestamp string
    """
    def __init__(self, value):
        try:
            self.value = parse_into_datetime(value)
        except Exception:
            raise ValueError("Must be a datetime object or timestamp string.")

    def __str__(self):
        return "t%s" % repr(self.value)


class IntegerConstant(_Constant):
    """Pattern interger constant

    Args:
        value (int): integer value
    """
    def __init__(self, value):
        try:
            self.value = int(value)
        except Exception:
            raise ValueError("must be an integer.")

    def __str__(self):
        return "%s" % self.value


class FloatConstant(_Constant):
    def __init__(self, value):
        try:
            self.value = float(value)
        except Exception:
            raise ValueError("must be a float.")

    def __str__(self):
        return "%s" % self.value


class BooleanConstant(_Constant):
    """Pattern boolean constant

    Args:
       value (str OR int):
           (str) 'true', 't' for True; 'false', 'f' for False
           (int) 1 for True; 0 for False
    """
    def __init__(self, value):
        if isinstance(value, bool):
            self.value = value
            return

        trues = ['true', 't', '1']
        falses = ['false', 'f', '0']
        try:
            if value.lower() in trues:
                self.value = True
                return
            elif value.lower() in falses:
                self.value = False
                return
        except AttributeError:
            if value == 1:
                self.value = True
                return
            elif value == 0:
                self.value = False
                return

        raise ValueError("must be a boolean value.")

    def __str__(self):
        return str(self.value).lower()


_HASH_REGEX = {
    "MD5": (r"^[a-fA-F0-9]{32}$", "MD5"),
    "MD6": (r"^[a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{56}|[a-fA-F0-9]{64}|[a-fA-F0-9]{96}|[a-fA-F0-9]{128}$", "MD6"),
    "RIPEMD160": (r"^[a-fA-F0-9]{40}$", "RIPEMD-160"),
    "SHA1": (r"^[a-fA-F0-9]{40}$", "SHA-1"),
    "SHA224": (r"^[a-fA-F0-9]{56}$", "SHA-224"),
    "SHA256": (r"^[a-fA-F0-9]{64}$", "SHA-256"),
    "SHA384": (r"^[a-fA-F0-9]{96}$", "SHA-384"),
    "SHA512": (r"^[a-fA-F0-9]{128}$", "SHA-512"),
    "SHA3224": (r"^[a-fA-F0-9]{56}$", "SHA3-224"),
    "SHA3256": (r"^[a-fA-F0-9]{64}$", "SHA3-256"),
    "SHA3384": (r"^[a-fA-F0-9]{96}$", "SHA3-384"),
    "SHA3512": (r"^[a-fA-F0-9]{128}$", "SHA3-512"),
    "SSDEEP": (r"^[a-zA-Z0-9/+:.]{1,128}$", "SSDEEP"),
    "WHIRLPOOL": (r"^[a-fA-F0-9]{128}$", "WHIRLPOOL"),
    "TLSH": (r"^[a-fA-F0-9]{70}$", "TLSH"),
}


class HashConstant(StringConstant):
    """Pattern hash constant

    Args:
        value (str): hash value
        type (str): hash algorithm name. Supported hash algorithms:
            "MD5", "MD6", RIPEMD160", "SHA1", "SHA224", "SHA256",
            "SHA384", "SHA512", "SHA3224", "SHA3256", "SHA3384",
            "SHA3512", "SSDEEP", "WHIRLPOOL"
    """
    def __init__(self, value, type):
        key = type.upper().replace('-', '')
        if key in _HASH_REGEX:
            vocab_key = _HASH_REGEX[key][1]
            if not re.match(_HASH_REGEX[key][0], value):
                raise ValueError("'%s' is not a valid %s hash" % (value, vocab_key))
            super(HashConstant, self).__init__(value)


class BinaryConstant(_Constant):
    """Pattern binary constant

    Args:
        value (str): base64 encoded string value
    """

    def __init__(self, value, from_parse_tree=False):
        # support with or without a 'b'
        if from_parse_tree:
            m = re.match("^b'(.+)'$", value)
            if m:
                value = m.group(1)
        try:
            base64.b64decode(value)
            self.value = value
        except (binascii.Error, TypeError):
            raise ValueError("must contain a base64 encoded string")

    def __str__(self):
        return "b'%s'" % self.value


class HexConstant(_Constant):
    """Pattern hexadecimal constant

    Args:
        value (str): hexadecimal value
    """
    def __init__(self, value, from_parse_tree=False):
        # support with or without an 'h'
        if not from_parse_tree and re.match('^([a-fA-F0-9]{2})+$', value):
            self.value = value
        else:
            m = re.match("^h'(([a-fA-F0-9]{2})+)'$", value)
            if m:
                self.value = m.group(1)
            else:
                raise ValueError("must contain an even number of hexadecimal characters")

    def __str__(self):
        return "h'%s'" % self.value


class ListConstant(_Constant):
    """Pattern list constant

    Args:
        value (list): list of values
    """
    def __init__(self, values):
        # handle _Constants or make a _Constant
        self.value = [x if isinstance(x, _Constant) else make_constant(x) for x in values]

    def __str__(self):
        return "(" + ", ".join(["%s" % x for x in self.value]) + ")"


def make_constant(value):
    """Convert value to Pattern constant, best effort attempt
    at determining root value type and corresponding conversion

    Args:
        value: value to convert to Pattern constant
    """
    if isinstance(value, _Constant):
        return value

    try:
        return parse_into_datetime(value)
    except (ValueError, TypeError):
        pass

    if isinstance(value, str):
        return StringConstant(value)
    elif isinstance(value, bool):
        return BooleanConstant(value)
    elif isinstance(value, int):
        return IntegerConstant(value)
    elif isinstance(value, float):
        return FloatConstant(value)
    elif isinstance(value, list):
        return ListConstant(value)
    else:
        raise ValueError("Unable to create a constant from %s" % value)


class _ObjectPathComponent(object):
    @staticmethod
    def create_ObjectPathComponent(component_name):
        # first case is to handle if component_name was quoted
        if isinstance(component_name, StringConstant):
            return BasicObjectPathComponent(component_name.value, False)
        elif component_name.endswith("_ref"):
            return ReferenceObjectPathComponent(component_name)
        elif component_name.find("[") != -1:
            parse1 = component_name.split("[")
            return ListObjectPathComponent(parse1[0], parse1[1][:-1])
        else:
            return BasicObjectPathComponent(component_name, False)

    def __str__(self):
        return quote_if_needed(self.property_name)


class BasicObjectPathComponent(_ObjectPathComponent):
    """Basic object path component (for an observation or expression)

    By "Basic", implies that the object path component is not a
    list, object reference or futher referenced property, i.e. terminal
    component

    Args:
        property_name (str): object property name
        is_key (bool): is dictionary key, default: False
    """
    def __init__(self, property_name, is_key):
        self.property_name = property_name
        # TODO: set is_key to True if this component is a dictionary key
        # self.is_key = is_key


class ListObjectPathComponent(_ObjectPathComponent):
    """List object path component (for an observation or expression)

    Args:
        property_name (str): list object property name
        index (int): index of the list property's value that is specified
    """
    def __init__(self, property_name, index):
        self.property_name = property_name
        self.index = index

    def __str__(self):
        return "%s[%s]" % (quote_if_needed(self.property_name), self.index)


class ReferenceObjectPathComponent(_ObjectPathComponent):
    """Reference object path component (for an observation or expression)

    Args:
        reference_property_name (str): reference object property name
    """
    def __init__(self, reference_property_name):
        self.property_name = reference_property_name


class ObjectPath(object):
    """Pattern operand object (property) path

    Args:
        object_type_name (str): name of object type for corresponding object path component
        property_path (_ObjectPathComponent OR str): object path
    """
    def __init__(self, object_type_name, property_path):
        self.object_type_name = object_type_name
        self.property_path = [
            x if isinstance(x, _ObjectPathComponent) else
            _ObjectPathComponent.create_ObjectPathComponent(x)
            for x in property_path
        ]

    def __str__(self):
        return "%s:%s" % (self.object_type_name, ".".join(["%s" % quote_if_needed(x) for x in self.property_path]))

    def merge(self, other):
        """Extend the object property with that of the supplied object property path"""
        self.property_path.extend(other.property_path)
        return self

    @staticmethod
    def make_object_path(lhs):
        """Create ObjectPath from string encoded object path

        Args:
            lhs (str): object path of left-hand-side component of expression
        """
        path_as_parts = lhs.split(":")
        return ObjectPath(path_as_parts[0], path_as_parts[1].split("."))


class _PatternExpression(object):
    pass


class _ComparisonExpression(_PatternExpression):
    """Pattern Comparison Expression

    Args:
        operator (str): operator of comparison expression
        lhs (ObjectPath OR str): object path of left-hand-side component of expression
        rhs (ObjectPath OR str): object path of right-hand-side component of expression
        negated (bool): comparison expression negated. Default: False
    """
    def __init__(self, operator, lhs, rhs, negated=False):
        if operator == "=" and isinstance(rhs, (ListConstant, list)):
            self.operator = "IN"
        else:
            self.operator = operator
        if isinstance(lhs, ObjectPath):
            self.lhs = lhs
        else:
            self.lhs = ObjectPath.make_object_path(lhs)
        if isinstance(rhs, _Constant):
            self.rhs = rhs
        else:
            self.rhs = make_constant(rhs)
        self.negated = negated
        self.root_type = self.lhs.object_type_name

    def __str__(self):
        if self.negated:
            return "%s NOT %s %s" % (self.lhs, self.operator, self.rhs)
        else:
            return "%s %s %s" % (self.lhs, self.operator, self.rhs)


class EqualityComparisonExpression(_ComparisonExpression):
    """Pattern Equality Comparison Expression

    Args:
        lhs (ObjectPath OR str): object path of left-hand-side component of expression
        rhs (ObjectPath OR str): object path of right-hand-side component of expression
        negated (bool): comparison expression negated. Default: False
    """
    def __init__(self, lhs, rhs, negated=False):
        super(EqualityComparisonExpression, self).__init__("=", lhs, rhs, negated)


class GreaterThanComparisonExpression(_ComparisonExpression):
    """Pattern Greater-than Comparison Expression

    Args:
        lhs (ObjectPath OR str): object path of left-hand-side component of expression
        rhs (ObjectPath OR str): object path of right-hand-side component of expression
        negated (bool): comparison expression negated. Default: False
    """
    def __init__(self, lhs, rhs, negated=False):
        super(GreaterThanComparisonExpression, self).__init__(">", lhs, rhs, negated)


class LessThanComparisonExpression(_ComparisonExpression):
    """Pattern Less-than Comparison Expression

    Args:
        lhs (ObjectPath OR str): object path of left-hand-side component of expression
        rhs (ObjectPath OR str): object path of right-hand-side component of expression
        negated (bool): comparison expression negated. Default: False
    """
    def __init__(self, lhs, rhs, negated=False):
        super(LessThanComparisonExpression, self).__init__("<", lhs, rhs, negated)


class GreaterThanEqualComparisonExpression(_ComparisonExpression):
    """Pattern Greater-Than-or-Equal-to Comparison Expression

    Args:
        lhs (ObjectPath OR str): object path of left-hand-side component of expression
        rhs (ObjectPath OR str): object path of right-hand-side component of expression
        negated (bool): comparison expression negated. Default: False
    """
    def __init__(self, lhs, rhs, negated=False):
        super(GreaterThanEqualComparisonExpression, self).__init__(">=", lhs, rhs, negated)


class LessThanEqualComparisonExpression(_ComparisonExpression):
    """Pattern Less-Than-or-Equal-to Comparison Expression

    Args:
        lhs (ObjectPath OR str): object path of left-hand-side component of expression
        rhs (ObjectPath OR str): object path of right-hand-side component of expression
        negated (bool): comparison expression negated. Default: False
    """

    def __init__(self, lhs, rhs, negated=False):
        super(LessThanEqualComparisonExpression, self).__init__("<=", lhs, rhs, negated)


class InComparisonExpression(_ComparisonExpression):
    """'in' Comparison Expression

    Args:
        lhs (ObjectPath OR str): object path of left-hand-side component of expression
        rhs (ObjectPath OR str): object path of right-hand-side component of expression
        negated (bool): comparison expression negated. Default: False
    """
    def __init__(self, lhs, rhs, negated=False):
        super(InComparisonExpression, self).__init__("IN", lhs, rhs, negated)


class LikeComparisonExpression(_ComparisonExpression):
    """'like' Comparison Expression

    Args:
        lhs (ObjectPath OR str): object path of left-hand-side component of expression
        rhs (ObjectPath OR str): object path of right-hand-side component of expression
        negated (bool): comparison expression negated. Default: False
    """

    def __init__(self, lhs, rhs, negated=False):
        super(LikeComparisonExpression, self).__init__("LIKE", lhs, rhs, negated)


class MatchesComparisonExpression(_ComparisonExpression):
    """'Matches' Comparison Expression

    Args:
        lhs (ObjectPath OR str): object path of left-hand-side component of expression
        rhs (ObjectPath OR str): object path of right-hand-side component of expression
        negated (bool): comparison expression negated. Default: False
    """
    def __init__(self, lhs, rhs, negated=False):
        super(MatchesComparisonExpression, self).__init__("MATCHES", lhs, rhs, negated)


class IsSubsetComparisonExpression(_ComparisonExpression):
    """ 'is subset' Comparison Expression

    Args:
        lhs (ObjectPath OR str): object path of left-hand-side component of expression
        rhs (ObjectPath OR str): object path of right-hand-side component of expression
        negated (bool): comparison expression negated. Default: False
    """
    def __init__(self, lhs, rhs, negated=False):
        super(IsSubsetComparisonExpression, self).__init__("ISSUBSET", lhs, rhs, negated)


class IsSupersetComparisonExpression(_ComparisonExpression):
    """ 'is super set' Comparison Expression

    Args:
        lhs (ObjectPath OR str): object path of left-hand-side component of expression
        rhs (ObjectPath OR str): object path of right-hand-side component of expression
        negated (bool): comparison expression negated. Default: False
    """
    def __init__(self, lhs, rhs, negated=False):
        super(IsSupersetComparisonExpression, self).__init__("ISSUPERSET", lhs, rhs, negated)


class _BooleanExpression(_PatternExpression):
    """Boolean Pattern Expression

    Args:
        operator (str): boolean operator
        operands (list): boolean operands
    """
    def __init__(self, operator, operands):
        self.operator = operator
        self.operands = []
        for arg in operands:
            if not hasattr(self, "root_type"):
                self.root_type = arg.root_type
            elif self.root_type and (self.root_type != arg.root_type) and operator == "AND":
                raise ValueError("All operands to an 'AND' expression must have the same object type")
            elif self.root_type and (self.root_type != arg.root_type):
                self.root_type = None
            self.operands.append(arg)

    def __str__(self):
        sub_exprs = []
        for o in self.operands:
            sub_exprs.append(str(o))
        return (" " + self.operator + " ").join(sub_exprs)


class AndBooleanExpression(_BooleanExpression):
    """'AND' Boolean Pattern Expression. Only use if both operands are of
    the same root object.

    Args:
        operands (list): AND operands
    """
    def __init__(self, operands):
        super(AndBooleanExpression, self).__init__("AND", operands)


class OrBooleanExpression(_BooleanExpression):
    """'OR' Boolean Pattern Expression. Only use if both operands are of the same root object

    Args:
        operands (list): OR operands
    """
    def __init__(self, operands):
        super(OrBooleanExpression, self).__init__("OR", operands)


class ObservationExpression(_PatternExpression):
    """Observation Expression

    Args:
        operand (str): observation expression operand
    """
    def __init__(self, operand):
        self.operand = operand

    def __str__(self):
        return "%s" % self.operand if isinstance(self.operand, (ObservationExpression, _CompoundObservationExpression)) else "[%s]" % self.operand


class _CompoundObservationExpression(_PatternExpression):
    """Compound Observation Expression

    Args:
        operator (str): compound observation operator
        operands (str): compound observation operands
    """
    def __init__(self, operator, operands):
        self.operator = operator
        self.operands = operands

    def __str__(self):
        sub_exprs = []
        for o in self.operands:
            sub_exprs.append("%s" % o)
        return (" " + self.operator + " ").join(sub_exprs)


class AndObservationExpression(_CompoundObservationExpression):
    """'AND' Compound Observation Pattern Expression

    Args:
        operands (str): compound observation operands
    """
    def __init__(self, operands):
        super(AndObservationExpression, self).__init__("AND", operands)


class OrObservationExpression(_CompoundObservationExpression):
    """Pattern 'OR' Compound Observation Expression

    Args:
        operands (str): compound observation operands
    """
    def __init__(self, operands):
        super(OrObservationExpression, self).__init__("OR", operands)


class FollowedByObservationExpression(_CompoundObservationExpression):
    """Pattern 'Followed by' Compound Observation Expression

    Args:
        operands (str): compound observation operands
    """
    def __init__(self, operands):
        super(FollowedByObservationExpression, self).__init__("FOLLOWEDBY", operands)


class ParentheticalExpression(_PatternExpression):
    """Pattern Parenthetical Observation Expression

    Args:
       exp (str): observation expression
    """
    def __init__(self, exp):
        self.expression = exp
        if hasattr(exp, "root_type"):
            self.root_type = exp.root_type

    def __str__(self):
        return "(%s)" % self.expression


class _ExpressionQualifier(_PatternExpression):
    pass


class RepeatQualifier(_ExpressionQualifier):
    """Pattern Repeat Qualifier

    Args:
        times_to_repeat (int): times the qualifiers is repeated
    """
    def __init__(self, times_to_repeat):
        if isinstance(times_to_repeat, IntegerConstant):
            self.times_to_repeat = times_to_repeat
        elif isinstance(times_to_repeat, int):
            self.times_to_repeat = IntegerConstant(times_to_repeat)
        else:
            raise ValueError("%s is not a valid argument for a Repeat Qualifier" % times_to_repeat)

    def __str__(self):
        return "REPEATS %s TIMES" % self.times_to_repeat


class WithinQualifier(_ExpressionQualifier):
    """Pattern 'Within' Qualifier

    Args:
        number_of_seconds (int): seconds value for 'within' qualifier
    """
    def __init__(self, number_of_seconds):
        if isinstance(number_of_seconds, IntegerConstant):
            self.number_of_seconds = number_of_seconds
        elif isinstance(number_of_seconds, int):
            self.number_of_seconds = IntegerConstant(number_of_seconds)
        else:
            raise ValueError("%s is not a valid argument for a Within Qualifier" % number_of_seconds)

    def __str__(self):
        return "WITHIN %s SECONDS" % self.number_of_seconds


class StartStopQualifier(_ExpressionQualifier):
    """Pattern Start/Stop Qualifier

    Args:
        start_time (TimestampConstant OR datetime.date): start timestamp for qualifier
        stop_time (TimestampConstant OR datetime.date): stop timestamp for qualifier
    """
    def __init__(self, start_time, stop_time):
        if isinstance(start_time, TimestampConstant):
            self.start_time = start_time
        elif isinstance(start_time, datetime.date):
            self.start_time = TimestampConstant(start_time)
        elif isinstance(start_time, StringConstant):
            self.start_time = StringConstant(start_time.value)
        else:
            raise ValueError("%s is not a valid argument for a Start/Stop Qualifier" % start_time)
        if isinstance(stop_time, TimestampConstant):
            self.stop_time = stop_time
        elif isinstance(stop_time, datetime.date):
            self.stop_time = TimestampConstant(stop_time)
        elif isinstance(stop_time, StringConstant):
            self.stop_time = StringConstant(stop_time.value)
        else:
            raise ValueError("%s is not a valid argument for a Start/Stop Qualifier" % stop_time)

    def __str__(self):
        return "START %s STOP %s" % (self.start_time, self.stop_time)


class QualifiedObservationExpression(_PatternExpression):
    """Pattern Qualified Observation Expression

    Args:
        observation_expression (PatternExpression OR _CompoundObservationExpression OR ): pattern expression
        qualifier (_ExpressionQualifier): pattern expression qualifier
    """
    def __init__(self, observation_expression, qualifier):
        self.observation_expression = observation_expression
        self.qualifier = qualifier

    def __str__(self):
        return "%s %s" % (self.observation_expression, self.qualifier)