Salt-less hubble changes, Pulling 3 salt libraries to our codebase. (h…

…ubblestack#6)

* Copying grains into hubble folder

Copying grains into hubble folder, and moving our custom grains into this folder only.

* Adding 3 salt libraries

- salt.exceptions moved to hubblestack.utils.exceptions
  - simplified HubbleException type
  - removed unnecessary exception types
  - Rename SaltException to HubbleException
  - Removed code which was for serialization of exceptions
- Added hubblestack/utils/decorators/memoize.py
  - Copied from salt.decorators
- Moved salt.utils.platform to hubblestack.utils.platform
  - Removed methods for smart os
  - completely independent of salt now

* Updating references

Updating references

* Renaming hubble_core to core.py for grains

Renaming hubble_core to core.py for grains

* Commiting back hubble_core.py

Renaming from core.py as in gitignore core.* is mentioned

* Updating reference of core grain in fqdn grains

Updating reference of core grain in fqdn grains

hubblestack/daemon.py

@@ -25,7 +25,7 @@
 import salt.fileserver.gitfs
 import salt.modules.cmdmod
 import salt.utils
-import salt.utils.platform
+import hubblestack.utils.platform
 import salt.utils.jid
 import salt.utils.gitfs
 import salt.utils.path
@@ -45,7 +45,7 @@
 HSS = hubblestack.status.HubbleStatus(__name__, 'schedule', 'refresh_grains')
 
 # Importing syslog fails on windows
-if not salt.utils.platform.is_windows():
+if not hubblestack.utils.platform.is_windows():
     import syslog
 
 __opts__ = {}
@@ -102,7 +102,7 @@ def _emit_and_refresh_grains():
     refresh_grains()
     last_grains_refresh = time.time()
     # Emit syslog at grains refresh frequency
-    if not (salt.utils.platform.is_windows()) and \
+    if not (hubblestack.utils.platform.is_windows()) and \
             __opts__.get('emit_grains_to_syslog', True):
         default_grains_to_emit = ['system_uuid', 'hubble_uuid', 'session_uuid',
                                   'machine_id', 'splunkindex', 'cloud_details',
@@ -514,7 +514,7 @@ def _setup_signaling():
     signal.signal(signal.SIGFPE, clean_up_process)
     signal.signal(signal.SIGILL, clean_up_process)
     signal.signal(signal.SIGSEGV, clean_up_process)
-    if not salt.utils.platform.is_windows():
+    if not hubblestack.utils.platform.is_windows():
         signal.signal(signal.SIGHUP, clean_up_process)
         signal.signal(signal.SIGQUIT, clean_up_process)
 
@@ -586,7 +586,7 @@ def _get_uuid_from_system():
 def _load_salt_config(parsed_args):
     """ load the configs for salt.DEFAULT_MINION_OPTS """
     # Load unique data for Windows or Linux
-    if salt.utils.platform.is_windows():
+    if hubblestack.utils.platform.is_windows():
         if parsed_args.get('configfile') is None:
             parsed_args['configfile'] = 'C:\\Program Files (x86)\\Hubble\\etc\\hubble\\hubble.conf'
         salt.config.DEFAULT_MINION_OPTS['cachedir'] = 'C:\\Program Files (x86)\\hubble\\var\\cache'
@@ -686,7 +686,7 @@ def _setup_dirs():
     module_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'modules'))
     __opts__['module_dirs'] = module_dirs
     grains_dirs = __opts__.get('grains_dirs', [])
-    grains_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'grains'))
+    grains_dirs.append(os.path.join(os.path.dirname(__file__), 'grains'))
     __opts__['grains_dirs'] = grains_dirs
     returner_dirs = __opts__.get('returner_dirs', [])
     returner_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'returners'))

hubblestack/extmods/audit/grep.py

@@ -48,7 +48,7 @@
 import os
 import re
 
-from salt.exceptions import CommandExecutionError
+from hubblestack.utils.exceptions import CommandExecutionError
 
 log = logging.getLogger(__name__)
 

hubblestack/extmods/fdg/grep.py

@@ -9,7 +9,7 @@
 import logging
 import os.path
 
-from salt.exceptions import CommandExecutionError
+from hubblestack.utils.exceptions import CommandExecutionError
 
 log = logging.getLogger(__name__)
 

hubblestack/extmods/fdg/process.py

@@ -11,7 +11,7 @@
 import logging
 import re
 
-from salt.exceptions import ArgumentValueError
+from hubblestack.utils.exceptions import ArgumentValueError
 from hubblestack.utils.encoding import encode_base64 as utils_encode_base64
 
 log = logging.getLogger(__name__)

hubblestack/extmods/fdg/time_sync.py

@@ -7,9 +7,9 @@
 
 
 import logging
-import salt.utils.platform
+import hubblestack.utils.platform
 
-if not salt.utils.platform.is_windows():
+if not hubblestack.utils.platform.is_windows():
     import ntplib
 log = logging.getLogger(__name__)
 
@@ -82,7 +82,7 @@ def _query_ntp_server(ntp_server):
         string containing the NTP server to query
     """
     # use w32tm instead of ntplib
-    if salt.utils.platform.is_windows():
+    if hubblestack.utils.platform.is_windows():
         ret = __salt__['cmd.run']('w32tm /stripchart /computer:{0} /dataonly /samples:1'.format(
             ntp_server))
         try:

hubblestack/extmods/fileserver/gitfs.py

@@ -67,7 +67,7 @@
 
 from hubblestack.utils.signing import find_wrapf
 from hubblestack.extmods.utils.gitfs import GitFS
-from salt.exceptions import FileserverConfigError
+from hubblestack.utils.exceptions import FileserverConfigError
 
 log = logging.getLogger(__name__)
 

hubblestack/extmods/fileserver/roots.py

@@ -29,7 +29,7 @@
 import salt.utils.gzip_util
 import salt.utils.hashutils
 import salt.utils.path
-import salt.utils.platform
+import hubblestack.utils.platform
 import salt.utils.stringutils
 import salt.utils.versions
 from salt.ext import six
@@ -360,7 +360,7 @@ def _translate_sep(path):
                         'roots: %s symlink destination is %s',
                         abs_path, link_dest
                     )
-                    if salt.utils.platform.is_windows() \
+                    if hubblestack.utils.platform.is_windows() \
                             and link_dest.startswith('\\\\'):
                         # Symlink points to a network path. Since you can't
                         # join UNC and non-UNC paths, just assume the original

hubblestack/extmods/modules/audit.py

@@ -104,7 +104,7 @@
 from distutils.version import StrictVersion
 import salt.loader
 import salt.utils
-from salt.exceptions import CommandExecutionError
+from hubblestack.utils.exceptions import CommandExecutionError
 from hubblestack.status import HubbleStatus
 
 log = logging.getLogger(__name__)

hubblestack/extmods/modules/fdg.py

@@ -121,7 +121,7 @@
 
 import salt.loader
 import salt.utils
-from salt.exceptions import CommandExecutionError
+from hubblestack.utils.exceptions import CommandExecutionError
 
 log = logging.getLogger(__name__)
 __fdg__ = None

hubblestack/extmods/modules/hubble.py

@@ -18,7 +18,7 @@
 import traceback
 import yaml
 
-from salt.exceptions import CommandExecutionError
+from hubblestack.utils.exceptions import CommandExecutionError
 from hubblestack import __version__
 from hubblestack.status import HubbleStatus
 

hubblestack/extmods/modules/nebula_osquery.py

@@ -40,9 +40,9 @@
 
 import salt.utils
 import salt.utils.files
-import salt.utils.platform
+import hubblestack.utils.platform
 
-from salt.exceptions import CommandExecutionError
+from hubblestack.utils.exceptions import CommandExecutionError
 from hubblestack import __version__
 import hubblestack.log
 
@@ -99,7 +99,7 @@ def queries(query_group,
         salt '*' nebula.queries hour pillar_key=sec_osqueries
     """
     # sanity check of query_file: if not present, add it
-    if salt.utils.platform.is_windows():
+    if hubblestack.utils.platform.is_windows():
         query_file = query_file or \
                      'salt://hubblestack_nebula_v2/hubblestack_nebula_win_queries.yaml'
     else:
@@ -129,7 +129,7 @@ def queries(query_group,
     # run the osqueryi queries
     success, timing, ret = _run_osquery_queries(query_data, verbose)
 
-    if success is False and salt.utils.platform.is_windows():
+    if success is False and hubblestack.utils.platform.is_windows():
         log.error('osquery does not run on windows versions earlier than Server 2008 and Windows 7')
         if query_group == 'day':
             ret = [
@@ -350,7 +350,7 @@ def osqueryd_monitor(configfile=None,
     databasepath = databasepath or __opts__.get('osquery_dbpath')
     pidfile = pidfile or os.path.join(base_path, "hubble_osqueryd.pidfile")
     hashfile = hashfile or os.path.join(base_path, "hash_of_flagfile.txt")
-    if salt.utils.platform.is_windows():
+    if hubblestack.utils.platform.is_windows():
         conftopfile = conftopfile or 'salt://hubblestack_nebula_v2/win_top.osqueryconf'
         flagstopfile = flagstopfile or 'salt://hubblestack_nebula_v2/win_top.osqueryflags'
 
@@ -506,7 +506,7 @@ def check_disk_usage(path=None):
 
     """
     disk_stats = {}
-    if salt.utils.platform.is_windows():
+    if hubblestack.utils.platform.is_windows():
         log.info("Platform is windows, skipping disk usage stats")
         disk_stats = {"Error": "Platform is windows"}
     else:
@@ -585,7 +585,7 @@ def top(query_group,
     """
     Run the queries represented by query_group from the configuration files extracted from topfile
     """
-    if salt.utils.platform.is_windows():
+    if hubblestack.utils.platform.is_windows():
         topfile = 'salt://hubblestack_nebula_v2/win_top.nebula'
 
     configs = _get_top_data(topfile)
@@ -1284,7 +1284,7 @@ def _start_osqueryd(pidfile,
     This function will start osqueryd
     """
     log.info("osqueryd is not running, attempting to start osqueryd")
-    if salt.utils.platform.is_windows():
+    if hubblestack.utils.platform.is_windows():
         log.info("requesting service manager to start osqueryd")
         cmd = ['net', 'start', servicename]
     else:
@@ -1330,7 +1330,7 @@ def _stop_osqueryd(servicename, pidfile):
     """
     Thid function will stop osqueryd.
     """
-    if salt.utils.platform.is_windows():
+    if hubblestack.utils.platform.is_windows():
         stop_cmd = ['net', 'stop', servicename]
     else:
         stop_cmd = ['pkill', 'hubble_osqueryd']
@@ -1340,7 +1340,7 @@ def _stop_osqueryd(servicename, pidfile):
                   ret_stop.get('retcode', None), ret_stop.get('stderr', None))
     else:
         log.info("Successfully stopped osqueryd")
-    if not salt.utils.platform.is_windows():
+    if not hubblestack.utils.platform.is_windows():
         remove_pidfile_cmd = ['rm', '-rf', '{0}'.format(pidfile)]
         __salt__['cmd.run'](remove_pidfile_cmd, timeout=600)
 

hubblestack/extmods/modules/nova_loader.py

@@ -22,16 +22,16 @@
 from zipimport import zipimporter
 
 # Import salt libs
-from salt.exceptions import LoaderError
+from hubblestack.utils.exceptions import LoaderError
 from salt.template import check_render_pipe_str
 from salt.utils.decorators import Depends
 from salt.utils import is_proxy
 import salt.utils.context
-import salt.utils.platform
+import hubblestack.utils.platform
 import salt.utils.lazy
 import salt.utils.event
 import salt.utils.odict
-import salt.exceptions
+import hubblestack.utils.exceptions
 
 # Solve the Chicken and egg problem where grains need to run before any
 # of the modules are loaded and are generally available for any usage.
@@ -95,13 +95,13 @@ def verify_fun(lazy_obj, fun):
     Check that the function passed really exists
     """
     if not fun:
-        raise salt.exceptions.SaltInvocationError(
+        raise hubblestack.utils.exceptions.SaltInvocationError(
             'Must specify a function to run!\n'
             'ex: manage.up'
         )
     if fun not in lazy_obj:
         # If the requested function isn't available, lets say why
-        raise salt.exceptions.CommandExecutionError(lazy_obj.missing_fun_string(fun))
+        raise hubblestack.utils.exceptions.CommandExecutionError(lazy_obj.missing_fun_string(fun))
 
 
 class LazyDict(collections.MutableMapping):
@@ -875,7 +875,7 @@ def grains(opts, force_refresh=False, proxy=None):
     if opts.get('grains_cache', False):
         cumask = os.umask(0o77)
         try:
-            if salt.utils.platform.is_windows():
+            if hubblestack.utils.platform.is_windows():
                 # Make sure cache file isn't read-only
                 __salt__['cmd.run']('attrib -R "{0}"'.format(cfn))
             with salt.utils.fopen(cfn, 'w+b') as fp_:

hubblestack/extmods/modules/pulsar.py

@@ -21,12 +21,12 @@
 import re
 import yaml
 import time
-from salt.exceptions import CommandExecutionError
+from hubblestack.utils.exceptions import CommandExecutionError
 
 # Import salt libs
 import salt.ext.six
 import salt.loader
-import salt.utils.platform
+import hubblestack.utils.platform
 
 # Import third party libs
 try:
@@ -57,7 +57,7 @@ class pyinotify:
 hubble_status = HubbleStatus(__name__, 'top', 'process')
 
 def __virtual__():
-    if salt.utils.platform.is_windows():
+    if hubblestack.utils.platform.is_windows():
         return False, 'This module only works on Linux'
     return True
 

hubblestack/extmods/modules/safecommand.py

@@ -17,7 +17,7 @@
 
 import logging
 
-from salt.exceptions import CommandExecutionError
+from hubblestack.utils.exceptions import CommandExecutionError
 
 log = logging.getLogger(__name__)
 

hubblestack/extmods/modules/win_pulsar.py

@@ -15,9 +15,9 @@
 
 import salt.ext.six
 import salt.loader
-import salt.utils.platform
+import hubblestack.utils.platform
 
-from salt.exceptions import CommandExecutionError
+from hubblestack.utils.exceptions import CommandExecutionError
 
 log = logging.getLogger(__name__)
 DEFAULT_MASK = ['File create', 'File delete', 'Hard link change', 'Data extend',
@@ -32,7 +32,7 @@
 
 
 def __virtual__():
-    if not salt.utils.platform.is_windows():
+    if not hubblestack.utils.platform.is_windows():
         return False, 'This module only works on windows'
     win_version = __grains__['osfullname']
     if '2008' in win_version or '2003' in win_version:

hubblestack/extmods/modules/win_pulsar_winaudit.py

@@ -16,9 +16,9 @@
 
 import salt.ext.six
 import salt.loader
-import salt.utils.platform
+import hubblestack.utils.platform
 
-from salt.exceptions import CommandExecutionError
+from hubblestack.utils.exceptions import CommandExecutionError
 
 log = logging.getLogger(__name__)
 DEFAULT_MASK = ['ExecuteFile', 'Write', 'Delete', 'DeleteSubdirectoriesAndFiles',
@@ -33,7 +33,7 @@
 
 
 def __virtual__():
-    if not salt.utils.platform.is_windows():
+    if not hubblestack.utils.platform.is_windows():
         return False, 'This module only works on windows'
     return __virtualname__
 

hubblestack/extmods/utils/gitfs.py

@@ -30,7 +30,7 @@
 import salt.utils.hashutils
 import salt.utils.itertools
 import salt.utils.path
-import salt.utils.platform
+import hubblestack.utils.platform
 import salt.utils.stringutils
 import salt.utils.url
 import salt.utils.user
@@ -39,7 +39,7 @@
 from salt.config import DEFAULT_MASTER_OPTS as _DEFAULT_MASTER_OPTS
 from salt.utils.odict import OrderedDict
 from salt.utils.process import os_is_running as pid_exists
-from salt.exceptions import (
+from hubblestack.utils.exceptions import (
     FileserverConfigError,
     GitLockError,
     get_error_message
@@ -595,7 +595,7 @@ def clean_stale_refs(self):
 
         cmd = subprocess.Popen(
             shlex.split(cmd_str),
-            close_fds=not salt.utils.platform.is_windows(),
+            close_fds=not hubblestack.utils.platform.is_windows(),
             cwd=os.path.dirname(self.gitdir),
             env=env,
             stdout=subprocess.PIPE,
@@ -3086,7 +3086,7 @@ def link_mountpoint(self, repo):
                                 # is remove the symlink and let it be created
                                 # below.
                                 try:
-                                    if salt.utils.platform.is_windows() \
+                                    if hubblestack.utils.platform.is_windows() \
                                             and not ldest.startswith('\\\\') \
                                             and os.path.isdir(ldest):
                                         # On Windows, symlinks to directories