Skip to content

Commit

Permalink
Salt-less hubble changes, Pulling 3 salt libraries to our codebase. (h…
Browse files Browse the repository at this point in the history
…ubblestack#6) (hubblestack#876)

* Copying grains into hubble folder

Copying grains into hubble folder, and moving our custom grains into this folder only.

* Adding 3 salt libraries

- salt.exceptions moved to hubblestack.utils.exceptions
  - simplified HubbleException type
  - removed unnecessary exception types
  - Rename SaltException to HubbleException
  - Removed code which was for serialization of exceptions
- Added hubblestack/utils/decorators/memoize.py
  - Copied from salt.decorators
- Moved salt.utils.platform to hubblestack.utils.platform
  - Removed methods for smart os
  - completely independent of salt now

* Updating references

Updating references

* Renaming hubble_core to core.py for grains

Renaming hubble_core to core.py for grains

* Commiting back hubble_core.py

Renaming from core.py as in gitignore core.* is mentioned

* Updating reference of core grain in fqdn grains

Updating reference of core grain in fqdn grains
  • Loading branch information
goravsingal authored and jettero committed Nov 10, 2020
1 parent 03cc37a commit a745153
Show file tree
Hide file tree
Showing 61 changed files with 3,871 additions and 91 deletions.
12 changes: 6 additions & 6 deletions hubblestack/daemon.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import salt.fileserver.gitfs
import salt.modules.cmdmod
import salt.utils
import salt.utils.platform
import hubblestack.utils.platform
import salt.utils.jid
import salt.utils.gitfs
import salt.utils.path
Expand All @@ -49,7 +49,7 @@
HSS = hubblestack.status.HubbleStatus(__name__, 'schedule', 'refresh_grains')

# Importing syslog fails on windows
if not salt.utils.platform.is_windows():
if not hubblestack.utils.platform.is_windows():
import syslog

__opts__ = {}
Expand Down Expand Up @@ -106,7 +106,7 @@ def _emit_and_refresh_grains():
refresh_grains()
last_grains_refresh = time.time()
# Emit syslog at grains refresh frequency
if not (salt.utils.platform.is_windows()) and \
if not (hubblestack.utils.platform.is_windows()) and \
__opts__.get('emit_grains_to_syslog', True):
default_grains_to_emit = ['system_uuid', 'hubble_uuid', 'session_uuid',
'machine_id', 'splunkindex', 'cloud_details',
Expand Down Expand Up @@ -519,7 +519,7 @@ def _setup_signaling():
signal.signal(signal.SIGFPE, clean_up_process)
signal.signal(signal.SIGILL, clean_up_process)
signal.signal(signal.SIGSEGV, clean_up_process)
if not salt.utils.platform.is_windows():
if not hubblestack.utils.platform.is_windows():
signal.signal(signal.SIGHUP, clean_up_process)
signal.signal(signal.SIGQUIT, clean_up_process)

Expand Down Expand Up @@ -591,7 +591,7 @@ def _get_uuid_from_system():
def _load_salt_config(parsed_args):
""" load the configs for salt.DEFAULT_MINION_OPTS """
# Load unique data for Windows or Linux
if salt.utils.platform.is_windows():
if hubblestack.utils.platform.is_windows():
if parsed_args.get('configfile') is None:
parsed_args['configfile'] = 'C:\\Program Files (x86)\\Hubble\\etc\\hubble\\hubble.conf'
salt.config.DEFAULT_MINION_OPTS['cachedir'] = 'C:\\Program Files (x86)\\hubble\\var\\cache'
Expand Down Expand Up @@ -697,7 +697,7 @@ def _setup_dirs():
module_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'modules'))
__opts__['module_dirs'] = module_dirs
grains_dirs = __opts__.get('grains_dirs', [])
grains_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'grains'))
grains_dirs.append(os.path.join(os.path.dirname(__file__), 'grains'))
__opts__['grains_dirs'] = grains_dirs
returner_dirs = __opts__.get('returner_dirs', [])
returner_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'returners'))
Expand Down
157 changes: 157 additions & 0 deletions hubblestack/extmods/audit/grep.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
# -*- encoding: utf-8 -*-
"""
Audit module for using grep to verify settings in files.
Sample YAML data:
.. code-block:: yaml
CIS-6.2.4:
grep.grep:
args:
- /etc/group
kwargs:
pattern: '^+:'
fail_on_match: True
description: Ensure no legacy "+" entries exist in /etc/group
Required args/kwargs:
path
The absolute path of the file to match against
pattern
The pattern to use with the ``grep`` command.
Optional kwargs:
grep_args
A list of args to pass to the ``grep`` command
fail_on_match
Defaults to False. If set to True, then if a match is found it will
count as a failure.
success_on_file_missing
Defaults to False. If set to True, then if a file is missing this check
will be marked as a success.
match_output
String to check for in the output of the grep command. If not provided,
any grep output will be considered a match.
match_output_regex
True/False. Whether to use regex when matching output. Defaults to
False.
match_output_multiline
True/False. Whether to use multiline flag for regex matching with
match_output_regex set to True. Defaults to True.
"""


import logging
import os
import re

from hubblestack.utils.exceptions import CommandExecutionError

log = logging.getLogger(__name__)


def grep(path,
pattern,
grep_args=None,
fail_on_match=False,
success_on_file_missing=False,
match_output=None,
match_output_regex=False,
match_output_multiline=True):
"""
Use grep to match against the contents of a file.
:param path:
See module-level documentation
:param pattern:
See module-level documentation
:param grep_args:
See module-level documentation
:param fail_on_match:
See module-level documentation
:param success_on_file_missing:
See module-level documentation
:param match_output:
See module-level documentation
:param match_output_regex:
See module-level documentation
:param match_output_multiline:
See module-level documentation
:return:
Returns a tuple (success, {'grep_output': output}) where ``success``
is True or False based on the success of the check, and ``output`` is
the output of the ``grep`` command, for documentation purposes.
"""
if not os.path.isfile(path):
if success_on_file_missing:
return True, {'reason': 'File missing'}
return False, {'reason': 'File missing'}

if not grep_args:
grep_args = []

output = _grep(path, pattern, *grep_args)

if not output:
# No output found
if fail_on_match:
return True, {'grep_output': output}
return False, {'grep_output': output}

# We default to ``success = True`` because there was grep output. Now we'll
# check against the various match_output settings to see if we need to
# reverse that decision
success = True
if match_output:
if match_output_regex:
if match_output_multiline:
if not re.search(match_output, output, re.MULTILINE):
success = False
else:
if not re.search(match_output, output):
success = False
else:
if match_output not in output:
success = False

# Reverse our success if ``fail_on_match = True``
if fail_on_match:
success = not success

return success, {'grep_output': output}


def _grep(path,
pattern,
*args):
"""
Grep for a string in the specified file
:param path:
Path to the file to be searched
:param pattern:
Pattern to match. For example: ``test``, or ``a[0-5]``
:param args:
Additional command-line flags to pass to the grep command. For example:
``-v``, or ``-i -B2``
:return:
"""
path = os.path.expanduser(path)

if args:
options = ' '.join(args)
else:
options = ''
cmd = r'grep {options} {pattern} {path}'.format(options=options,
pattern=pattern,
path=path)

try:
ret = __salt__['cmd.run'](cmd, python_shell=False, ignore_retcode=True)
except (IOError, OSError) as exc:
raise CommandExecutionError(exc.strerror)

return ret
2 changes: 1 addition & 1 deletion hubblestack/extmods/fdg/grep.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import logging
import os.path

from salt.exceptions import CommandExecutionError
from hubblestack.utils.exceptions import CommandExecutionError

log = logging.getLogger(__name__)

Expand Down
2 changes: 1 addition & 1 deletion hubblestack/extmods/fdg/process.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import logging
import re

from salt.exceptions import ArgumentValueError
from hubblestack.utils.exceptions import ArgumentValueError
from hubblestack.utils.encoding import encode_base64 as utils_encode_base64

log = logging.getLogger(__name__)
Expand Down
6 changes: 3 additions & 3 deletions hubblestack/extmods/fdg/time_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@


import logging
import salt.utils.platform
import hubblestack.utils.platform

if not salt.utils.platform.is_windows():
if not hubblestack.utils.platform.is_windows():
import ntplib
log = logging.getLogger(__name__)

Expand Down Expand Up @@ -82,7 +82,7 @@ def _query_ntp_server(ntp_server):
string containing the NTP server to query
"""
# use w32tm instead of ntplib
if salt.utils.platform.is_windows():
if hubblestack.utils.platform.is_windows():
ret = __salt__['cmd.run']('w32tm /stripchart /computer:{0} /dataonly /samples:1'.format(
ntp_server))
try:
Expand Down
2 changes: 1 addition & 1 deletion hubblestack/extmods/fileserver/gitfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@

from hubblestack.utils.signing import find_wrapf
from hubblestack.extmods.utils.gitfs import GitFS
from salt.exceptions import FileserverConfigError
from hubblestack.utils.exceptions import FileserverConfigError

log = logging.getLogger(__name__)

Expand Down
4 changes: 2 additions & 2 deletions hubblestack/extmods/fileserver/roots.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import salt.utils.gzip_util
import salt.utils.hashutils
import salt.utils.path
import salt.utils.platform
import hubblestack.utils.platform
import salt.utils.stringutils
import salt.utils.versions
from salt.ext import six
Expand Down Expand Up @@ -360,7 +360,7 @@ def _translate_sep(path):
'roots: %s symlink destination is %s',
abs_path, link_dest
)
if salt.utils.platform.is_windows() \
if hubblestack.utils.platform.is_windows() \
and link_dest.startswith('\\\\'):
# Symlink points to a network path. Since you can't
# join UNC and non-UNC paths, just assume the original
Expand Down
1 change: 1 addition & 0 deletions hubblestack/extmods/modules/audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
import yaml

import hubblestack.extmods.module_runner.runner_factory as runner_factory
from hubblestack.utils.exceptions import CommandExecutionError
from hubblestack.status import HubbleStatus

log = logging.getLogger(__name__)
Expand Down
4 changes: 2 additions & 2 deletions hubblestack/extmods/modules/fdg.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@
import os

import yaml
from salt.exceptions import CommandExecutionError
from hubblestack.exceptions import CommandExecutionError

import hubblestack.extmods.module_runner.runner_factory as runner_factory

Expand Down Expand Up @@ -251,4 +251,4 @@ def _get_fdg_file(fdg_file):
return None
if fdg_file.startswith('salt://'):
return fdg_file
return 'salt://' + BASE_DIR_FDG_PROFILES + os.sep + fdg_file.replace('.', os.sep) + '.fdg'
return 'salt://' + BASE_DIR_FDG_PROFILES + os.sep + fdg_file.replace('.', os.sep) + '.fdg'
2 changes: 1 addition & 1 deletion hubblestack/extmods/modules/hubble.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import traceback
import yaml

from salt.exceptions import CommandExecutionError
from hubblestack.utils.exceptions import CommandExecutionError
from hubblestack import __version__
from hubblestack.status import HubbleStatus

Expand Down
20 changes: 10 additions & 10 deletions hubblestack/extmods/modules/nebula_osquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@

import salt.utils
import salt.utils.files
import salt.utils.platform
import hubblestack.utils.platform

from salt.exceptions import CommandExecutionError
from hubblestack.utils.exceptions import CommandExecutionError
from hubblestack import __version__
import hubblestack.log

Expand Down Expand Up @@ -99,7 +99,7 @@ def queries(query_group,
salt '*' nebula.queries hour pillar_key=sec_osqueries
"""
# sanity check of query_file: if not present, add it
if salt.utils.platform.is_windows():
if hubblestack.utils.platform.is_windows():
query_file = query_file or \
'salt://hubblestack_nebula_v2/hubblestack_nebula_win_queries.yaml'
else:
Expand Down Expand Up @@ -129,7 +129,7 @@ def queries(query_group,
# run the osqueryi queries
success, timing, ret = _run_osquery_queries(query_data, verbose)

if success is False and salt.utils.platform.is_windows():
if success is False and hubblestack.utils.platform.is_windows():
log.error('osquery does not run on windows versions earlier than Server 2008 and Windows 7')
if query_group == 'day':
ret = [
Expand Down Expand Up @@ -350,7 +350,7 @@ def osqueryd_monitor(configfile=None,
databasepath = databasepath or __opts__.get('osquery_dbpath')
pidfile = pidfile or os.path.join(base_path, "hubble_osqueryd.pidfile")
hashfile = hashfile or os.path.join(base_path, "hash_of_flagfile.txt")
if salt.utils.platform.is_windows():
if hubblestack.utils.platform.is_windows():
conftopfile = conftopfile or 'salt://hubblestack_nebula_v2/win_top.osqueryconf'
flagstopfile = flagstopfile or 'salt://hubblestack_nebula_v2/win_top.osqueryflags'

Expand Down Expand Up @@ -506,7 +506,7 @@ def check_disk_usage(path=None):
"""
disk_stats = {}
if salt.utils.platform.is_windows():
if hubblestack.utils.platform.is_windows():
log.info("Platform is windows, skipping disk usage stats")
disk_stats = {"Error": "Platform is windows"}
else:
Expand Down Expand Up @@ -585,7 +585,7 @@ def top(query_group,
"""
Run the queries represented by query_group from the configuration files extracted from topfile
"""
if salt.utils.platform.is_windows():
if hubblestack.utils.platform.is_windows():
topfile = 'salt://hubblestack_nebula_v2/win_top.nebula'

configs = _get_top_data(topfile)
Expand Down Expand Up @@ -1284,7 +1284,7 @@ def _start_osqueryd(pidfile,
This function will start osqueryd
"""
log.info("osqueryd is not running, attempting to start osqueryd")
if salt.utils.platform.is_windows():
if hubblestack.utils.platform.is_windows():
log.info("requesting service manager to start osqueryd")
cmd = ['net', 'start', servicename]
else:
Expand Down Expand Up @@ -1330,7 +1330,7 @@ def _stop_osqueryd(servicename, pidfile):
"""
Thid function will stop osqueryd.
"""
if salt.utils.platform.is_windows():
if hubblestack.utils.platform.is_windows():
stop_cmd = ['net', 'stop', servicename]
else:
stop_cmd = ['pkill', 'hubble_osqueryd']
Expand All @@ -1340,7 +1340,7 @@ def _stop_osqueryd(servicename, pidfile):
ret_stop.get('retcode', None), ret_stop.get('stderr', None))
else:
log.info("Successfully stopped osqueryd")
if not salt.utils.platform.is_windows():
if not hubblestack.utils.platform.is_windows():
remove_pidfile_cmd = ['rm', '-rf', '{0}'.format(pidfile)]
__salt__['cmd.run'](remove_pidfile_cmd, timeout=600)

Expand Down
Loading

0 comments on commit a745153

Please sign in to comment.