From 7d266d733c67ecf2ddeea7f0e21afcc96157bcbf Mon Sep 17 00:00:00 2001 From: root Date: Tue, 1 Aug 2017 20:41:03 +0000 Subject: [PATCH 01/50] added changes in pulsar.py for top.pulsar --- hubblestack/extmods/modules/pulsar.py | 37 ++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/hubblestack/extmods/modules/pulsar.py b/hubblestack/extmods/modules/pulsar.py index 5c9e15e09..050c723fd 100644 --- a/hubblestack/extmods/modules/pulsar.py +++ b/hubblestack/extmods/modules/pulsar.py @@ -76,7 +76,7 @@ def _get_notifier(): return __context__['pulsar.notifier'] -def process(configfile='salt://hubblestack_pulsar/hubblestack_pulsar_config.yaml', +def process(configfile, verbose=False): ''' Watch the configured files @@ -395,3 +395,38 @@ def _dict_update(dest, upd, recursive_update=True, merge_lists=False): for k in upd: dest[k] = upd[k] return dest + +def top(topfile='/root/myhubblefiles/top.pulsar', + debug=None): + + results = [] + + # Get a list of yaml to run + top_data = _get_top_data(topfile) + + data_by_tag = {} + for data in top_data: + results.extend(process(configfile=data[0],verbose=False)) + + return results + +def _get_top_data(topfile): + ''' + Helper method to retrieve and parse the nova topfile + ''' + try: + with open(topfile) as handle: + topdata = yaml.safe_load(handle) + except Exception as e: + raise CommandExecutionError('Could not load topfile: {0}'.format(e)) + + if not isinstance(topdata, dict) or 'pulsar' not in topdata or \ + not(isinstance(topdata['pulsar'], dict)): + raise CommandExecutionError('pulsar topfile not formatted correctly') + + topdata = topdata['pulsar'] + + ret = topdata.values() + + return ret + From 254753af259f9c6094646b2ed1cf21b2db2a87e1 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Wed, 2 Aug 2017 09:49:06 -0600 Subject: [PATCH 02/50] Add **kwargs everywhere https://github.com/hubblestack/hubble-salt/pull/99 --- hubblestack/files/hubblestack_nova/cve_scan.py | 2 +- hubblestack/files/hubblestack_nova/cve_scan_v2.py | 2 +- hubblestack/files/hubblestack_nova/firewall.py | 2 +- hubblestack/files/hubblestack_nova/grep.py | 2 +- hubblestack/files/hubblestack_nova/misc.py | 2 +- hubblestack/files/hubblestack_nova/netstat.py | 2 +- hubblestack/files/hubblestack_nova/openssl.py | 2 +- hubblestack/files/hubblestack_nova/pkg.py | 2 +- hubblestack/files/hubblestack_nova/pkgng_audit.py | 2 +- hubblestack/files/hubblestack_nova/service.py | 2 +- hubblestack/files/hubblestack_nova/stat_nova.py | 2 +- hubblestack/files/hubblestack_nova/sysctl.py | 2 +- hubblestack/files/hubblestack_nova/vulners_scanner.py | 2 +- hubblestack/files/hubblestack_nova/win_auditpol.py | 2 +- hubblestack/files/hubblestack_nova/win_firewall.py | 2 +- hubblestack/files/hubblestack_nova/win_gp.py | 2 +- hubblestack/files/hubblestack_nova/win_pkg.py | 2 +- hubblestack/files/hubblestack_nova/win_reg.py | 2 +- hubblestack/files/hubblestack_nova/win_secedit.py | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/hubblestack/files/hubblestack_nova/cve_scan.py b/hubblestack/files/hubblestack_nova/cve_scan.py index cf5bc9147..1814c6b82 100644 --- a/hubblestack/files/hubblestack_nova/cve_scan.py +++ b/hubblestack/files/hubblestack_nova/cve_scan.py @@ -21,7 +21,7 @@ def __virtual__(): return False, 'This module requires Linux and the oscap binary' -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Run the network.netstat command ''' diff --git a/hubblestack/files/hubblestack_nova/cve_scan_v2.py b/hubblestack/files/hubblestack_nova/cve_scan_v2.py index 76d74a85f..a7e172ff2 100644 --- a/hubblestack/files/hubblestack_nova/cve_scan_v2.py +++ b/hubblestack/files/hubblestack_nova/cve_scan_v2.py @@ -92,7 +92,7 @@ def __virtual__(): return not salt.utils.is_windows() -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Main audit function. See module docstring for more information on usage. ''' diff --git a/hubblestack/files/hubblestack_nova/firewall.py b/hubblestack/files/hubblestack_nova/firewall.py index 618b707b8..a4563b6fe 100644 --- a/hubblestack/files/hubblestack_nova/firewall.py +++ b/hubblestack/files/hubblestack_nova/firewall.py @@ -99,7 +99,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): __data__ = {} for profile, data in data_list: _merge_yaml(__data__, data, profile) diff --git a/hubblestack/files/hubblestack_nova/grep.py b/hubblestack/files/hubblestack_nova/grep.py index 243a1a0ba..f757cf67e 100644 --- a/hubblestack/files/hubblestack_nova/grep.py +++ b/hubblestack/files/hubblestack_nova/grep.py @@ -73,7 +73,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Run the grep audits contained in the YAML files processed by __virtual__ ''' diff --git a/hubblestack/files/hubblestack_nova/misc.py b/hubblestack/files/hubblestack_nova/misc.py index 72a8ff9f8..09341a9d9 100644 --- a/hubblestack/files/hubblestack_nova/misc.py +++ b/hubblestack/files/hubblestack_nova/misc.py @@ -59,7 +59,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Run the misc audits contained in the data_list ''' diff --git a/hubblestack/files/hubblestack_nova/netstat.py b/hubblestack/files/hubblestack_nova/netstat.py index 88ad8837c..197176773 100644 --- a/hubblestack/files/hubblestack_nova/netstat.py +++ b/hubblestack/files/hubblestack_nova/netstat.py @@ -36,7 +36,7 @@ def __virtual__(): return False, 'No network.netstat function found' -def audit(data_list, tags, debug=True): +def audit(data_list, tags, debug=True, **kwargs): ''' Run the network.netstat command ''' diff --git a/hubblestack/files/hubblestack_nova/openssl.py b/hubblestack/files/hubblestack_nova/openssl.py index 953951a58..03cee7e74 100644 --- a/hubblestack/files/hubblestack_nova/openssl.py +++ b/hubblestack/files/hubblestack_nova/openssl.py @@ -99,7 +99,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=True): +def audit(data_list, tags, debug=True, **kwargs): __data__ = {} for profile, data in data_list: _merge_yaml(__data__, data, profile) diff --git a/hubblestack/files/hubblestack_nova/pkg.py b/hubblestack/files/hubblestack_nova/pkg.py index c6cf19833..5247e4977 100644 --- a/hubblestack/files/hubblestack_nova/pkg.py +++ b/hubblestack/files/hubblestack_nova/pkg.py @@ -79,7 +79,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Run the pkg audits contained in the YAML files processed by __virtual__ ''' diff --git a/hubblestack/files/hubblestack_nova/pkgng_audit.py b/hubblestack/files/hubblestack_nova/pkgng_audit.py index ebc212a0d..547795626 100644 --- a/hubblestack/files/hubblestack_nova/pkgng_audit.py +++ b/hubblestack/files/hubblestack_nova/pkgng_audit.py @@ -20,7 +20,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Run the pkg.audit command ''' diff --git a/hubblestack/files/hubblestack_nova/service.py b/hubblestack/files/hubblestack_nova/service.py index aaa20f577..9418e3c33 100644 --- a/hubblestack/files/hubblestack_nova/service.py +++ b/hubblestack/files/hubblestack_nova/service.py @@ -72,7 +72,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Run the service audits contained in the YAML files processed by __virtual__ ''' diff --git a/hubblestack/files/hubblestack_nova/stat_nova.py b/hubblestack/files/hubblestack_nova/stat_nova.py index 3babcb332..995d792af 100644 --- a/hubblestack/files/hubblestack_nova/stat_nova.py +++ b/hubblestack/files/hubblestack_nova/stat_nova.py @@ -58,7 +58,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Run the stat audits contained in the YAML files processed by __virtual__ ''' diff --git a/hubblestack/files/hubblestack_nova/sysctl.py b/hubblestack/files/hubblestack_nova/sysctl.py index a58cab569..03b35bfcc 100644 --- a/hubblestack/files/hubblestack_nova/sysctl.py +++ b/hubblestack/files/hubblestack_nova/sysctl.py @@ -49,7 +49,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Run the sysctl audits contained in the YAML files processed by __virtual__ ''' diff --git a/hubblestack/files/hubblestack_nova/vulners_scanner.py b/hubblestack/files/hubblestack_nova/vulners_scanner.py index 5f98962ea..82ac977a4 100644 --- a/hubblestack/files/hubblestack_nova/vulners_scanner.py +++ b/hubblestack/files/hubblestack_nova/vulners_scanner.py @@ -35,7 +35,7 @@ def __virtual__(): return not sys.platform.startswith('win') -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): os_name = __grains__.get('os').lower() os_version = __grains__.get('osmajorrelease') diff --git a/hubblestack/files/hubblestack_nova/win_auditpol.py b/hubblestack/files/hubblestack_nova/win_auditpol.py index 08a0b3235..3953f3c5e 100644 --- a/hubblestack/files/hubblestack_nova/win_auditpol.py +++ b/hubblestack/files/hubblestack_nova/win_auditpol.py @@ -25,7 +25,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Runs auditpol on the local machine and audits the return data with the CIS yaml processed by __virtual__ diff --git a/hubblestack/files/hubblestack_nova/win_firewall.py b/hubblestack/files/hubblestack_nova/win_firewall.py index fd149df7f..2ac80b3d2 100644 --- a/hubblestack/files/hubblestack_nova/win_firewall.py +++ b/hubblestack/files/hubblestack_nova/win_firewall.py @@ -25,7 +25,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Runs auditpol on the local machine and audits the return data with the CIS yaml processed by __virtual__ diff --git a/hubblestack/files/hubblestack_nova/win_gp.py b/hubblestack/files/hubblestack_nova/win_gp.py index 009419f8c..29e5408b4 100644 --- a/hubblestack/files/hubblestack_nova/win_gp.py +++ b/hubblestack/files/hubblestack_nova/win_gp.py @@ -25,7 +25,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Runs auditpol on the local machine and audits the return data with the CIS yaml processed by __virtual__ diff --git a/hubblestack/files/hubblestack_nova/win_pkg.py b/hubblestack/files/hubblestack_nova/win_pkg.py index 8bc333603..dd540f8a2 100644 --- a/hubblestack/files/hubblestack_nova/win_pkg.py +++ b/hubblestack/files/hubblestack_nova/win_pkg.py @@ -25,7 +25,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Runs auditpol on the local machine and audits the return data with the CIS yaml processed by __virtual__ diff --git a/hubblestack/files/hubblestack_nova/win_reg.py b/hubblestack/files/hubblestack_nova/win_reg.py index 7f5e38745..5ae5d007b 100644 --- a/hubblestack/files/hubblestack_nova/win_reg.py +++ b/hubblestack/files/hubblestack_nova/win_reg.py @@ -24,7 +24,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Runs salt reg query on the local machine and audits the return data with the CIS yaml processed by __virtual__ diff --git a/hubblestack/files/hubblestack_nova/win_secedit.py b/hubblestack/files/hubblestack_nova/win_secedit.py index 31b5348a9..70bd1f5f3 100644 --- a/hubblestack/files/hubblestack_nova/win_secedit.py +++ b/hubblestack/files/hubblestack_nova/win_secedit.py @@ -30,7 +30,7 @@ def __virtual__(): return True -def audit(data_list, tags, debug=False): +def audit(data_list, tags, debug=False, **kwargs): ''' Runs secedit on the local machine and audits the return data with the CIS yaml processed by __virtual__ From ffdc38c5e59b3ee989f89ff458381ad441e4d48e Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Wed, 2 Aug 2017 09:57:36 -0600 Subject: [PATCH 03/50] Add fields module for custom fields --- hubblestack/extmods/modules/fields.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 hubblestack/extmods/modules/fields.py diff --git a/hubblestack/extmods/modules/fields.py b/hubblestack/extmods/modules/fields.py new file mode 100644 index 000000000..7ebcd3fb6 --- /dev/null +++ b/hubblestack/extmods/modules/fields.py @@ -0,0 +1,27 @@ +# -*- encoding: utf-8 -*- +''' +Module dealing with sending custom fields to splunk +''' +from __future__ import absolute_import +import logging + +log = logging.getLogger(__name__) + + +def fields(custom_fields=None): + ''' + Use config.get to retrieve custom data based on the keys in the `fields` + list. + + Arguments: + + fields + List of keys to retrieve + ''' + if custom_fields is None or not isinstance(fields, list): + log.error('custom_fields argument must be formed as a list of strings') + return {} + ret = {} + for field in custom_fields: + ret[field] = __salt__['config.get'](field) + return ret From b98f605e10fc273c39a6b8702d2b0c863c11c1fc Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Wed, 2 Aug 2017 10:05:58 -0600 Subject: [PATCH 04/50] Move fields() into nebula, return as nebula data --- hubblestack/extmods/modules/fields.py | 27 ------------------- hubblestack/extmods/modules/nebula_osquery.py | 23 ++++++++++++++++ 2 files changed, 23 insertions(+), 27 deletions(-) delete mode 100644 hubblestack/extmods/modules/fields.py diff --git a/hubblestack/extmods/modules/fields.py b/hubblestack/extmods/modules/fields.py deleted file mode 100644 index 7ebcd3fb6..000000000 --- a/hubblestack/extmods/modules/fields.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- encoding: utf-8 -*- -''' -Module dealing with sending custom fields to splunk -''' -from __future__ import absolute_import -import logging - -log = logging.getLogger(__name__) - - -def fields(custom_fields=None): - ''' - Use config.get to retrieve custom data based on the keys in the `fields` - list. - - Arguments: - - fields - List of keys to retrieve - ''' - if custom_fields is None or not isinstance(fields, list): - log.error('custom_fields argument must be formed as a list of strings') - return {} - ret = {} - for field in custom_fields: - ret[field] = __salt__['config.get'](field) - return ret diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index 915ffd5bd..c8602cb5a 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -198,6 +198,29 @@ def queries(query_group, return ret +def fields(custom_fields=None): + ''' + Use config.get to retrieve custom data based on the keys in the `custom_fields` + list. + + Arguments: + + custom_fields + List of keys to retrieve + ''' + if custom_fields is None or not isinstance(fields, list): + log.error('custom_fields argument must be formed as a list of strings') + return [] + ret = {} + for field in custom_fields: + ret[field] = __salt__['config.get'](field) + # Return it as nebula data + return [{'custom_fields': { + 'data': [ret], + 'result': True + }}] + + def version(): ''' Report version of this module From 1f34f0935f86993d5a6f1470b754554561ddc2ed Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Wed, 2 Aug 2017 10:09:03 -0600 Subject: [PATCH 05/50] Use *args instead --- hubblestack/extmods/modules/nebula_osquery.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index c8602cb5a..ea4e4049b 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -198,27 +198,26 @@ def queries(query_group, return ret -def fields(custom_fields=None): +def fields(*args): ''' - Use config.get to retrieve custom data based on the keys in the `custom_fields` + Use config.get to retrieve custom data based on the keys in the `*args` list. Arguments: - custom_fields + *args List of keys to retrieve ''' - if custom_fields is None or not isinstance(fields, list): - log.error('custom_fields argument must be formed as a list of strings') - return [] ret = {} - for field in custom_fields: + for field in args: ret[field] = __salt__['config.get'](field) # Return it as nebula data - return [{'custom_fields': { - 'data': [ret], - 'result': True - }}] + if ret: + return [{'custom_fields': { + 'data': [ret], + 'result': True + }}] + return [] def version(): From f003cf54a283e2669ca6a43e0c2fb4ec13b318b4 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Wed, 2 Aug 2017 10:25:36 -0600 Subject: [PATCH 06/50] Add `custom_` prefix to nebula.fields --- hubblestack/extmods/modules/nebula_osquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index ea4e4049b..c6a34bc04 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -210,7 +210,7 @@ def fields(*args): ''' ret = {} for field in args: - ret[field] = __salt__['config.get'](field) + ret['custom_{0}'.format(field)] = __salt__['config.get'](field) # Return it as nebula data if ret: return [{'custom_fields': { From a1366d388c6fdd26e47d34383fe8aa0069d9f301 Mon Sep 17 00:00:00 2001 From: Chandler Newby Date: Thu, 3 Aug 2017 19:12:26 -0600 Subject: [PATCH 07/50] Add option to extract fields at index time in Splunk returners --- .../extmods/returners/splunk_nebula_return.py | 18 ++++++++++ .../extmods/returners/splunk_nova_return.py | 36 +++++++++++++++++++ .../extmods/returners/splunk_pulsar_return.py | 20 ++++++++++- 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/hubblestack/extmods/returners/splunk_nebula_return.py b/hubblestack/extmods/returners/splunk_nebula_return.py index 355124df2..7dc3ea377 100644 --- a/hubblestack/extmods/returners/splunk_nebula_return.py +++ b/hubblestack/extmods/returners/splunk_nebula_return.py @@ -75,6 +75,14 @@ def returner(ret): timeout = opts['timeout'] custom_fields = opts['custom_fields'] + # Set up the fields to be extracted at index time. The field values must be strings. + # Note that these fields will also still be available in the event data + index_extracted_fields = ['aws_instance_id', 'aws_account_id', 'azure_vmId'] + try: + index_extracted_fields.extend(opts['index_extracted_fields']) + except TypeError: + pass + # Set up the collector hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_port=http_event_collector_port, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout) @@ -132,6 +140,14 @@ def returner(ret): payload.update({'sourcetype': opts['sourcetype']}) payload.update({'event': event}) + # Potentially add metadata fields: + fields = {} + for item in index_extracted_fields: + if item in payload['event'] and not isinstance(payload['event'], (list, dict, tuple)): + fields[item] = str(payload['event'][item]) + if fields: + payload.update({'fields': fields}) + # If the osquery query includes a field called 'time' it will be checked. # If it's within the last year, it will be used as the eventtime. event_time = query_result.get('time', '') @@ -165,6 +181,7 @@ def _get_options(): processed['http_event_server_ssl'] = opt.get('hec_ssl', True) processed['proxy'] = opt.get('proxy', {}) processed['timeout'] = opt.get('timeout', 9.05) + processed['index_extracted_fields'] = opt.get('index_extracted_fields', []) splunk_opts.append(processed) return splunk_opts else: @@ -182,6 +199,7 @@ def _get_options(): splunk_opts['http_event_server_ssl'] = hec_ssl splunk_opts['proxy'] = __salt__['config.get']('hubblestack:nebula:returner:splunk:proxy', {}) splunk_opts['timeout'] = __salt__['config.get']('hubblestack:nebula:returner:splunk:timeout', 9.05) + splunk_opts['index_extracted_fields'] = __salt__['config.get']('hubblestack:nebula:returner:splunk:index_extracted_fields', []) return [splunk_opts] diff --git a/hubblestack/extmods/returners/splunk_nova_return.py b/hubblestack/extmods/returners/splunk_nova_return.py index 7cd72745d..90e2429d0 100644 --- a/hubblestack/extmods/returners/splunk_nova_return.py +++ b/hubblestack/extmods/returners/splunk_nova_return.py @@ -74,6 +74,13 @@ def returner(ret): timeout = opts['timeout'] custom_fields = opts['custom_fields'] + # Set up the fields to be extracted at index time. The field values must be strings. + # Note that these fields will also still be available in the event data + index_extracted_fields = ['aws_instance_id', 'aws_account_id', 'azure_vmId'] + try: + index_extracted_fields.extend(opts['index_extracted_fields']) + except TypeError: + pass # Set up the collector hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_port=http_event_collector_port, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout) @@ -139,6 +146,15 @@ def returner(ret): payload.update({'index': opts['index']}) payload.update({'sourcetype': opts['sourcetype']}) payload.update({'event': event}) + + # Potentially add metadata fields: + fields = {} + for item in index_extracted_fields: + if item in payload['event'] and not isinstance(payload['event'], (list, dict, tuple)): + fields[item] = str(payload['event'][item]) + if fields: + payload.update({'fields': fields}) + hec.batchEvent(payload) for suc in data.get('Success', []): @@ -175,6 +191,15 @@ def returner(ret): payload.update({'sourcetype': opts['sourcetype']}) payload.update({'index': opts['index']}) payload.update({'event': event}) + + # Potentially add metadata fields: + fields = {} + for item in index_extracted_fields: + if item in payload['event'] and not isinstance(payload['event'], (list, dict, tuple)): + fields[item] = str(payload['event'][item]) + if fields: + payload.update({'fields': fields}) + hec.batchEvent(payload) if data.get('Compliance', None): @@ -203,6 +228,15 @@ def returner(ret): payload.update({'sourcetype': opts['sourcetype']}) payload.update({'index': opts['index']}) payload.update({'event': event}) + + # Potentially add metadata fields: + fields = {} + for item in index_extracted_fields: + if item in payload['event'] and not isinstance(payload['event'], (list, dict, tuple)): + fields[item] = str(payload['event'][item]) + if fields: + payload.update({'fields': fields}) + hec.batchEvent(payload) hec.flushBatch() @@ -245,6 +279,7 @@ def _get_options(): processed['http_event_server_ssl'] = opt.get('hec_ssl', True) processed['proxy'] = opt.get('proxy', {}) processed['timeout'] = opt.get('timeout', 9.05) + processed['index_extracted_fields'] = opt.get('index_extracted_fields', []) splunk_opts.append(processed) return splunk_opts else: @@ -262,6 +297,7 @@ def _get_options(): splunk_opts['http_event_server_ssl'] = hec_ssl splunk_opts['proxy'] = __salt__['config.get']('hubblestack:nova:returner:splunk:proxy', {}) splunk_opts['timeout'] = __salt__['config.get']('hubblestack:nova:returner:splunk:timeout', 9.05) + splunk_opts['index_extracted_fields'] = __salt__['config.get']('hubblestack:nova:returner:splunk:index_extracted_fields', []) return [splunk_opts] diff --git a/hubblestack/extmods/returners/splunk_pulsar_return.py b/hubblestack/extmods/returners/splunk_pulsar_return.py index 2336192da..86071cf7e 100644 --- a/hubblestack/extmods/returners/splunk_pulsar_return.py +++ b/hubblestack/extmods/returners/splunk_pulsar_return.py @@ -38,7 +38,6 @@ - site - product_group ''' - import socket # Import cloud details from cloud_details import get_cloud_details @@ -80,6 +79,14 @@ def returner(ret): timeout = opts['timeout'] custom_fields = opts['custom_fields'] + # Set up the fields to be extracted at index time. The field values must be strings. + # Note that these fields will also still be available in the event data + index_extracted_fields = ['aws_instance_id', 'aws_account_id', 'azure_vmId'] + try: + index_extracted_fields.extend(opts['index_extracted_fields']) + except TypeError: + pass + # Set up the collector hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_port=http_event_collector_port, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout) # Check whether or not data is batched: @@ -223,6 +230,15 @@ def returner(ret): payload.update({'index': opts['index']}) payload.update({'sourcetype': opts['sourcetype']}) payload.update({'event': event}) + + # Potentially add metadata fields: + fields = {} + for item in index_extracted_fields: + if item in payload['event'] and not isinstance(payload['event'], (list, dict, tuple)): + fields[item] = str(payload['event'][item]) + if fields: + payload.update({'fields': fields}) + hec.batchEvent(payload) hec.flushBatch() @@ -254,6 +270,7 @@ def _get_options(): processed['http_event_server_ssl'] = opt.get('hec_ssl', True) processed['proxy'] = opt.get('proxy', {}) processed['timeout'] = opt.get('timeout', 9.05) + processed['index_extracted_fields'] = opt.get('index_extracted_fields', []) splunk_opts.append(processed) return splunk_opts else: @@ -271,6 +288,7 @@ def _get_options(): splunk_opts['http_event_server_ssl'] = hec_ssl splunk_opts['proxy'] = __salt__['config.get']('hubblestack:pulsar:returner:splunk:proxy', {}) splunk_opts['timeout'] = __salt__['config.get']('hubblestack:pulsar:returner:splunk:timeout', 9.05) + splunk_opts['index_extracted_fields'] = __salt__['config.get']('hubblestack:pulsar:returner:splunk:index_extracted_fields', []) return [splunk_opts] From f20acf2efd8d387c3ab70beed752c10a666eea61 Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Fri, 4 Aug 2017 06:49:17 -0600 Subject: [PATCH 08/50] absolute paths in init.d and systemd service file this would avoid depending on correct path environmant variable and symlinking under /usr in order to appear in the path. --- pkg/hubble | 2 +- pkg/hubble.service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/hubble b/pkg/hubble index fe940e84b..d7e7451a4 100755 --- a/pkg/hubble +++ b/pkg/hubble @@ -10,7 +10,7 @@ ### END INIT INFO dir="/etc/hubble" -cmd="hubble -d" +cmd="/opt/hubble/hubble -d" name=`basename $0` pid_file="/var/run/$name.pid" diff --git a/pkg/hubble.service b/pkg/hubble.service index ae2a1b14b..f31adad0d 100644 --- a/pkg/hubble.service +++ b/pkg/hubble.service @@ -4,7 +4,7 @@ Description=Hubblestack [Service] Type=forking PIDFile=/var/run/hubble.pid -ExecStart=/usr/bin/hubble -d +ExecStart=/opt/hubble/hubble -d [Install] WantedBy=multi-user.target From c3597e9d89d5d8094a11cece55197061f70cc22f Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Fri, 4 Aug 2017 06:54:15 -0600 Subject: [PATCH 09/50] coreos dockerfile --- pkg/coreos/Dockerfile | 111 ++++++++++++++++++++++++ pkg/coreos/pyinstaller-requirements.txt | 16 ++++ 2 files changed, 127 insertions(+) create mode 100644 pkg/coreos/Dockerfile create mode 100644 pkg/coreos/pyinstaller-requirements.txt diff --git a/pkg/coreos/Dockerfile b/pkg/coreos/Dockerfile new file mode 100644 index 000000000..08933985d --- /dev/null +++ b/pkg/coreos/Dockerfile @@ -0,0 +1,111 @@ +# This Dockerfile aims to make building Hubble v2 packages easier. +# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile +# 2. docker build -t . +# The resulting image is ready to run the pyinstaller on container start and drop hubble-coreos.tar.gz +# in the /data directory. Mount /data volume into a directory on the host to access the package. +# To run the container: docker run -it --rm -v `pwd`:/data + +FROM debian:testing-slim + +RUN apt-get update \ + && apt-get -y upgrade + +#paths that hubble or hubble parts need in the package +RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/osquery + +#osquery build start +#osquery should be built first since requirements for other packages can interfere with osquery dependencies +#to build, osquery scripts want sudo and a user to sudo with. +#to pin to a different version change the following envirnment variable +ENV OSQUERY_SRC_VERSION=2.6.0 +ENV OSQUERY_BUILD_USER=osquerybuilder +ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git +RUN apt-get -y install git make python ruby sudo +RUN useradd --shell /bin/bash --create-home --user-group --groups sudo "$OSQUERY_BUILD_USER" \ + && sed -i 's/^%sudo.*/%sudo\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers +RUN mkdir -p /usr/local/osquery/ \ + && chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ +USER $OSQUERY_BUILD_USER +ENV SKIP_TESTS=1 +RUN cd /home/"$OSQUERY_BUILD_USER" \ + && git clone "$OSQUERY_GIT_URL" \ + && cd osquery/ \ + && git checkout "$OSQUERY_SRC_VERSION" \ +#these homebrew hashes need to be current. hashes in osquery git repo are often out of date for the tags we check out and try to build. +#this is a problem and they are aware of it. let the magic hashes commence: + && sed -i 's,^\(HOMEBREW_CORE=\).*,\1'941ca36839ea354031846d73ad538e1e44e673f4',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_CORE=\).*,\1'abc5c5782c5850f2deff1f3d463945f90f2feaac',' tools/provision.sh \ + && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ + && make sysprep \ + && make deps \ + && make \ + && make strip +USER root +RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ + && chown -R root. /opt/osquery \ + && chmod -R 500 /opt/osquery/* \ + && mkdir -p /opt/osquery/lenses \ + && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ + && chmod -R 400 /opt/osquery/lenses/* +RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version + +#install packages that should be needed for ligbit2 compilation and successful pyinstaller run +RUN apt-get -y install \ + python-dev libffi-dev libssl-dev libyaml-dev \ + libxml2-dev libxslt1-dev python-cffi \ + zlib1g-dev cmake python-setuptools \ + gcc wget python-pip openssl \ + && apt-get clean + +#libgit2 install start +#must precede pyinstaller requirements +ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz +ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 +ENV LIBGIT2_SRC_VERSION=0.26.0 +ENV LIBGIT2TEMP=/tmp/libgit2temp +RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ + && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ + && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ + && tar xzf libgit2.tar.gz \ + && cd libgit2-"$LIBGIT2_SRC_VERSION"/ \ + && export LIBGIT2=/usr/local/ \ + && cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \ + && make \ + && make install + +#pyinstaller requirements start +#must be preceded by libgit2 install +COPY pyinstaller-requirements.txt / +RUN pip install --upgrade pip \ + && pip -v install -r pyinstaller-requirements.txt + +#pyinstaller start +#commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built +#use the following variables to choose the version of hubble +ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git +ENV HUBBLE_SRC_PATH=/hubble_src +ENV _HOOK_DIR="./pkg/" +ENV _BINARY_LOG_LEVEL="INFO" +ENV _INCLUDE_PATH="" +RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ + && cd "$HUBBLE_SRC_PATH" \ + && git checkout "$HUBBLE_SRC_VERSION" \ + && cp -rf "$HUBBLE_SRC_PATH" /hubble_build +RUN mkdir /data +VOLUME /data +WORKDIR /hubble_build +ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ] +CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} hubble.py \ + && cp /hubble_build/conf/osquery.conf /etc/osquery/ \ + && cp /hubble_build/conf/osquery.flags /etc/osquery/ \ +# hubble default configuration file + && cp -rf /hubble_build/conf/hubble /etc/hubble/ \ + && cp -rf /hubble_build/conf/hubble-profile.sh /etc/profile.d/ \ + && cp -pr /hubble_build/dist/hubble /opt/hubble/hubble-libs \ + && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \ + # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes + && rm -rf /opt/hubble/hubble-libs/librpm* \ + && tar -cPvzf /data/hubble-${HUBBLE_SRC_VERSION}-coreos.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh" ] + diff --git a/pkg/coreos/pyinstaller-requirements.txt b/pkg/coreos/pyinstaller-requirements.txt new file mode 100644 index 000000000..3010621f1 --- /dev/null +++ b/pkg/coreos/pyinstaller-requirements.txt @@ -0,0 +1,16 @@ +pyinstaller==3.2 # currently 3.2.1 version is not supported because of botocore exception +Crypto +pyopenssl>=16.2.0 +argparse +requests>=2.13.0 +logging +pprint +daemon +boto3 +botocore +pygit2 +salt-ssh +gitpython +pyinotify +cffi +azure From 86fd8d0afb6ee2c9d95aaa9f8c2185bcff912da9 Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Fri, 4 Aug 2017 06:57:24 -0600 Subject: [PATCH 10/50] debian 9 dockerfile --- pkg/debian9/Dockerfile | 134 +++++++++++++++++++++++ pkg/debian9/pyinstaller-requirements.txt | 16 +++ 2 files changed, 150 insertions(+) create mode 100644 pkg/debian9/Dockerfile create mode 100644 pkg/debian9/pyinstaller-requirements.txt diff --git a/pkg/debian9/Dockerfile b/pkg/debian9/Dockerfile new file mode 100644 index 000000000..2321be0fa --- /dev/null +++ b/pkg/debian9/Dockerfile @@ -0,0 +1,134 @@ +# This Dockerfile aims to make building Hubble v2 packages easier. +# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile +# 2. docker build -t . +# The resulting image is ready to run the pyinstaller on container start and drop hubble-coreos.tar.gz +# in the /data directory. Mount /data volume into a directory on the host to access the package. +# To run the container: docker run -it --rm -v `pwd`:/data + +FROM debian:9 + +RUN apt-get update \ + && apt-get -y upgrade + +#paths that hubble or hubble parts need in the package +RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/osquery + +#osquery build start +#osquery should be built first since requirements for other packages can interfere with osquery dependencies +#to build, osquery scripts want sudo and a user to sudo with. +#to pin to a different version change the following envirnment variable +ENV OSQUERY_SRC_VERSION=2.6.0 +ENV OSQUERY_BUILD_USER=osquerybuilder +ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git +RUN apt-get -y install git make python ruby sudo +RUN useradd --shell /bin/bash --create-home --user-group --groups sudo "$OSQUERY_BUILD_USER" \ + && sed -i 's/^%sudo.*/%sudo\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers +RUN mkdir -p /usr/local/osquery/ \ + && chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ +USER $OSQUERY_BUILD_USER +ENV SKIP_TESTS=1 +RUN cd /home/"$OSQUERY_BUILD_USER" \ + && git clone "$OSQUERY_GIT_URL" \ + && cd osquery/ \ + && git checkout "$OSQUERY_SRC_VERSION" \ +#these homebrew hashes need to be current. hashes in osquery git repo are often out of date for the tags we check out and try to build. +#this is a problem and they are aware of it. let the magic hashes commence: + && sed -i 's,^\(HOMEBREW_CORE=\).*,\1'941ca36839ea354031846d73ad538e1e44e673f4',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_CORE=\).*,\1'abc5c5782c5850f2deff1f3d463945f90f2feaac',' tools/provision.sh \ + && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ + && make sysprep \ + && make deps \ + && make \ + && make strip +USER root +RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ + && chown -R root. /opt/osquery \ + && chmod -R 500 /opt/osquery/* \ + && mkdir -p /opt/osquery/lenses \ + && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ + && chmod -R 400 /opt/osquery/lenses/* +RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version + +#install packages that should be needed for ligbit2 compilation and successful pyinstaller run +RUN apt-get -y install \ + python-dev libffi-dev libssl-dev libyaml-dev \ + libxml2-dev libxslt1-dev python-cffi \ + zlib1g-dev cmake python-setuptools \ + gcc wget python-pip openssl \ + && apt-get clean + +#libgit2 install start +#must precede pyinstaller requirements +ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz +ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 +ENV LIBGIT2_SRC_VERSION=0.26.0 +ENV LIBGIT2TEMP=/tmp/libgit2temp +RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ + && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ + && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ + && tar xzf libgit2.tar.gz \ + && cd libgit2-"$LIBGIT2_SRC_VERSION"/ \ + && export LIBGIT2=/usr/local/ \ + && cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \ + && make \ + && make install + +#pyinstaller requirements start +#must be preceded by libgit2 install +COPY pyinstaller-requirements.txt / +RUN pip install --upgrade pip \ + && pip -v install -r pyinstaller-requirements.txt + +#deb package making requirements start +RUN apt-get install -y ruby ruby-dev rubygems gcc make \ + && gem install --no-ri --no-rdoc fpm + +#pyinstaller start +#commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built +#use the following variables to choose the version of hubble +ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git +ENV HUBBLE_SRC_PATH=/hubble_src +ENV _HOOK_DIR="./pkg/" +ENV _BINARY_LOG_LEVEL="INFO" +ENV _INCLUDE_PATH="" +RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ + && cd "$HUBBLE_SRC_PATH" \ + && git checkout "$HUBBLE_SRC_VERSION" \ + && cp -rf "$HUBBLE_SRC_PATH" /hubble_build +RUN mkdir /data +VOLUME /data +WORKDIR /hubble_build +ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ] +CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} hubble.py \ + && cp /hubble_build/conf/osquery.conf /etc/osquery/ \ + && cp /hubble_build/conf/osquery.flags /etc/osquery/ \ +# hubble default configuration file + && cp -rf /hubble_build/conf/hubble /etc/hubble/ \ + && cp -rf /hubble_build/conf/hubble-profile.sh /etc/profile.d/ \ + && cp -pr /hubble_build/dist/hubble /opt/hubble/hubble-libs \ + && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \ + # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes + && rm -rf /opt/hubble/hubble-libs/librpm* \ +#debian pkg start + && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d/ \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ + && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ +#fpm start + && fpm -s dir -t deb \ + -n hubblestack \ + -v ${HUBBLE_SRC_VERSION}-1 \ + #todo: get rid of the git dependency with static bin in the future + -d 'git' \ + --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ + --deb-no-default-config-files \ + etc/hubble etc/osquery etc/init.d opt usr \ + && cp hubblestack_${HUBBLE_SRC_VERSION}-1_amd64.deb /data/hubblestack_${HUBBLE_SRC_VERSION}-1_debian9_amd64.deb" ] + diff --git a/pkg/debian9/pyinstaller-requirements.txt b/pkg/debian9/pyinstaller-requirements.txt new file mode 100644 index 000000000..3010621f1 --- /dev/null +++ b/pkg/debian9/pyinstaller-requirements.txt @@ -0,0 +1,16 @@ +pyinstaller==3.2 # currently 3.2.1 version is not supported because of botocore exception +Crypto +pyopenssl>=16.2.0 +argparse +requests>=2.13.0 +logging +pprint +daemon +boto3 +botocore +pygit2 +salt-ssh +gitpython +pyinotify +cffi +azure From 7d279a2b98a4fef48b68db91f0b7f0980878d7de Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Fri, 4 Aug 2017 06:59:16 -0600 Subject: [PATCH 11/50] debian 8 dockerfile --- pkg/debian8/Dockerfile | 139 +++++++++++++++++++++++ pkg/debian8/pyinstaller-requirements.txt | 16 +++ 2 files changed, 155 insertions(+) create mode 100644 pkg/debian8/Dockerfile create mode 100644 pkg/debian8/pyinstaller-requirements.txt diff --git a/pkg/debian8/Dockerfile b/pkg/debian8/Dockerfile new file mode 100644 index 000000000..812b76fc5 --- /dev/null +++ b/pkg/debian8/Dockerfile @@ -0,0 +1,139 @@ +# This Dockerfile aims to make building Hubble v2 packages easier. +# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile +# 2. docker build -t . +# The resulting image is ready to run the pyinstaller on container start and drop hubble-coreos.tar.gz +# in the /data directory. Mount /data volume into a directory on the host to access the package. +# To run the container: docker run -it --rm -v `pwd`:/data + +FROM debian:8 + +RUN apt-get update \ + && apt-get -y upgrade + +#paths that hubble or hubble parts need in the package +RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/osquery + +#osquery build start +#osquery should be built first since requirements for other packages can interfere with osquery dependencies +#to build, osquery scripts want sudo and a user to sudo with. +#to pin to a different version change the following envirnment variable +ENV OSQUERY_SRC_VERSION=2.6.0 +ENV OSQUERY_BUILD_USER=osquerybuilder +ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git +RUN apt-get -y install git make python ruby sudo locales +RUN useradd --shell /bin/bash --create-home --user-group --groups sudo "$OSQUERY_BUILD_USER" \ + && sed -i 's/^%sudo.*/%sudo\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers +RUN mkdir -p /usr/local/osquery/ \ + && chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ \ + && echo "LC_ALL=en_US.UTF-8" >> /etc/default/locale \ + && sed -i '/en_US.UTF-8\ UTF-8/s/^#//' /etc/locale.gen \ + && locale-gen +USER $OSQUERY_BUILD_USER +ENV SKIP_TESTS=1 +RUN cd /home/"$OSQUERY_BUILD_USER" \ + && git clone "$OSQUERY_GIT_URL" \ + && cd osquery/ \ + && git checkout "$OSQUERY_SRC_VERSION" \ +#these homebrew hashes need to be current. hashes in osquery git repo are often out of date for the tags we check out and try to build. +#this is a problem and they are aware of it. let the magic hashes commence: + && sed -i 's,^\(HOMEBREW_CORE=\).*,\1'941ca36839ea354031846d73ad538e1e44e673f4',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_CORE=\).*,\1'abc5c5782c5850f2deff1f3d463945f90f2feaac',' tools/provision.sh \ + && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ + && make sysprep \ + && make deps \ + && make \ + && make strip +USER root +RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ + && chown -R root. /opt/osquery \ + && chmod -R 500 /opt/osquery/* \ + && mkdir -p /opt/osquery/lenses \ + && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ + && chmod -R 400 /opt/osquery/lenses/* +RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version + +#install packages that should be needed for ligbit2 compilation and successful pyinstaller run +RUN apt-get -y install \ + python-dev libffi-dev libssl-dev libyaml-dev \ + libxml2-dev libxslt1-dev python-cffi \ + zlib1g-dev cmake python-setuptools \ + gcc wget openssl \ + && apt-get clean + +#libgit2 install start +#must precede pyinstaller requirements +ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz +ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 +ENV LIBGIT2_SRC_VERSION=0.26.0 +ENV LIBGIT2TEMP=/tmp/libgit2temp +RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ + && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ + && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ + && tar xzf libgit2.tar.gz \ + && cd libgit2-"$LIBGIT2_SRC_VERSION"/ \ + && export LIBGIT2=/usr/local/ \ + && cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \ + && make \ + && make install + +#pyinstaller requirements start +#must be preceded by libgit2 install +COPY pyinstaller-requirements.txt / +RUN ldconfig +RUN wget -c https://bootstrap.pypa.io/get-pip.py \ + && python get-pip.py \ + && pip -v install -r pyinstaller-requirements.txt + +#deb package making requirements start +RUN apt-get install -y ruby ruby-dev rubygems gcc make \ + && gem install --no-ri --no-rdoc fpm + +#pyinstaller start +#commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built +#use the following variables to choose the version of hubble +ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git +ENV HUBBLE_SRC_PATH=/hubble_src +ENV _HOOK_DIR="./pkg/" +ENV _BINARY_LOG_LEVEL="INFO" +ENV _INCLUDE_PATH="" +RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ + && cd "$HUBBLE_SRC_PATH" \ + && git checkout "$HUBBLE_SRC_VERSION" \ + && cp -rf "$HUBBLE_SRC_PATH" /hubble_build +RUN mkdir /data +VOLUME /data +WORKDIR /hubble_build +ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ] +CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} hubble.py \ + && cp /hubble_build/conf/osquery.conf /etc/osquery/ \ + && cp /hubble_build/conf/osquery.flags /etc/osquery/ \ +# hubble default configuration file + && cp -rf /hubble_build/conf/hubble /etc/hubble/ \ + && cp -rf /hubble_build/conf/hubble-profile.sh /etc/profile.d/ \ + && cp -pr /hubble_build/dist/hubble /opt/hubble/hubble-libs \ + && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \ + # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes + && rm -rf /opt/hubble/hubble-libs/librpm* \ +#debian pkg start + && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d/ \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ + && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ +#fpm start + && fpm -s dir -t deb \ + -n hubblestack \ + -v ${HUBBLE_SRC_VERSION}-1 \ + #todo: get rid of the git dependency with static bin in the future + -d 'git' \ + --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ + --deb-no-default-config-files \ + etc/hubble etc/osquery etc/init.d opt usr \ + && cp hubblestack_${HUBBLE_SRC_VERSION}-1_amd64.deb /data/hubblestack_${HUBBLE_SRC_VERSION}-1_debian8_amd64.deb" ] + diff --git a/pkg/debian8/pyinstaller-requirements.txt b/pkg/debian8/pyinstaller-requirements.txt new file mode 100644 index 000000000..3010621f1 --- /dev/null +++ b/pkg/debian8/pyinstaller-requirements.txt @@ -0,0 +1,16 @@ +pyinstaller==3.2 # currently 3.2.1 version is not supported because of botocore exception +Crypto +pyopenssl>=16.2.0 +argparse +requests>=2.13.0 +logging +pprint +daemon +boto3 +botocore +pygit2 +salt-ssh +gitpython +pyinotify +cffi +azure From cc609682362912f5fe1fff3fb02a30421af4d27b Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Fri, 4 Aug 2017 07:01:21 -0600 Subject: [PATCH 12/50] centos 7 dockerfile --- pkg/centos7/Dockerfile | 138 +++++++++++++++++++++++ pkg/centos7/pyinstaller-requirements.txt | 16 +++ 2 files changed, 154 insertions(+) create mode 100644 pkg/centos7/Dockerfile create mode 100644 pkg/centos7/pyinstaller-requirements.txt diff --git a/pkg/centos7/Dockerfile b/pkg/centos7/Dockerfile new file mode 100644 index 000000000..1fbc9892e --- /dev/null +++ b/pkg/centos7/Dockerfile @@ -0,0 +1,138 @@ +# This Dockerfile aims to make building Hubble v2 packages easier. +# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile +# 2. docker build -t . +# The resulting image is ready to run the pyinstaller on container start and drop hubble-coreos.tar.gz +# in the /data directory. Mount /data volume into a directory on the host to access the package. +# To run the container: docker run -it --rm -v `pwd`:/data + +FROM centos:7 + +RUN yum -y update + +#paths that hubble or hubble parts need in the package +RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/osquery + +#osquery build start +#osquery should be built first since requirements for other packages can interfere with osquery dependencies +#to build, osquery scripts want sudo and a user to sudo with. +#to pin to a different version change the following envirnment variable +ENV OSQUERY_SRC_VERSION=2.6.0 +ENV OSQUERY_BUILD_USER=osquerybuilder +ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git +RUN yum -y install git make python ruby sudo which +RUN useradd --shell /bin/bash --create-home --user-group --groups wheel "$OSQUERY_BUILD_USER" \ + && sed -i 's/^%wheel.*/%wheel\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers +RUN mkdir -p /usr/local/osquery/ \ + && chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ +USER $OSQUERY_BUILD_USER +ENV SKIP_TESTS=1 +RUN cd /home/"$OSQUERY_BUILD_USER" \ + && git clone "$OSQUERY_GIT_URL" \ + && cd osquery/ \ + && git checkout "$OSQUERY_SRC_VERSION" \ +#these homebrew hashes need to be current. hashes in osquery git repo are often out of date for the tags we check out and try to build. +#this is a problem and they are aware of it. let the magic hashes commence: + && sed -i 's,^\(HOMEBREW_CORE=\).*,\1'941ca36839ea354031846d73ad538e1e44e673f4',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_CORE=\).*,\1'abc5c5782c5850f2deff1f3d463945f90f2feaac',' tools/provision.sh \ + && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ + && make sysprep \ + && make deps \ + && make \ + && make strip +USER root +RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ + && chown -R root. /opt/osquery \ + && chmod -R 500 /opt/osquery/* \ + && mkdir -p /opt/osquery/lenses \ + && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ + && chmod -R 400 /opt/osquery/lenses/* +RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version + +#install packages that should be needed for ligbit2 compilation and successful pyinstaller run +RUN yum -y install \ + python27-devel libffi-devel openssl-devel \ + libxml2-devel libxslt-devel libjpeg-devel \ + zlib-devel make cmake python27-setuptools \ + python27-python-setuptools gcc python-devel \ + python-setuptools wget + # python-dev libffi-dev libssl-dev libyaml-dev \ + # libxml2-dev libxslt1-dev python-cffi \ + # zlib1g-dev cmake python-setuptools \ + # gcc wget python-pip openssl \ + #&& apt-get clean + +#libgit2 install start +#must precede pyinstaller requirements +ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz +ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 +ENV LIBGIT2_SRC_VERSION=0.26.0 +ENV LIBGIT2TEMP=/tmp/libgit2temp +RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ + && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ + && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ + && tar xzf libgit2.tar.gz \ + && cd libgit2-"$LIBGIT2_SRC_VERSION"/ \ + && export LIBGIT2=/usr/local/ \ + && cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \ + && make \ + && make install + +#pyinstaller requirements start +#must be preceded by libgit2 install +COPY pyinstaller-requirements.txt / +RUN wget -c https://bootstrap.pypa.io/get-pip.py \ + && python get-pip.py \ + && pip -v install -r pyinstaller-requirements.txt + +#deb package making requirements start +RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ + && gem install --no-ri --no-rdoc fpm + +#pyinstaller start +#commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built +#use the following variables to choose the version of hubble +ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git +ENV HUBBLE_SRC_PATH=/hubble_src +ENV _HOOK_DIR="./pkg/" +ENV _BINARY_LOG_LEVEL="INFO" +ENV _INCLUDE_PATH="" +RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ + && cd "$HUBBLE_SRC_PATH" \ + && git checkout "$HUBBLE_SRC_VERSION" \ + && cp -rf "$HUBBLE_SRC_PATH" /hubble_build +RUN mkdir /data +VOLUME /data +WORKDIR /hubble_build +ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ] +CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} hubble.py \ + && cp /hubble_build/conf/osquery.conf /etc/osquery/ \ + && cp /hubble_build/conf/osquery.flags /etc/osquery/ \ +# hubble default configuration file + && cp -rf /hubble_build/conf/hubble /etc/hubble/ \ + && cp -rf /hubble_build/conf/hubble-profile.sh /etc/profile.d/ \ + && cp -pr /hubble_build/dist/hubble /opt/hubble/hubble-libs \ + && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \ + # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes + && rm -rf /opt/hubble/hubble-libs/librpm* \ +#debian pkg start + && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d/ \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ + && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ +#fpm start + && fpm -s dir -t rpm \ + -n hubblestack \ + -v ${HUBBLE_SRC_VERSION}-1 \ + #todo: get rid of the git dependency with static bin in the future + -d 'git' \ + --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ + --deb-no-default-config-files \ + etc/hubble etc/osquery etc/init.d opt usr \ + && cp hubblestack-${HUBBLE_SRC_VERSION}_1-1.x86_64.rpm /data/hubblestack_${HUBBLE_SRC_VERSION}-1_centos7_x86_64.rpm" ] diff --git a/pkg/centos7/pyinstaller-requirements.txt b/pkg/centos7/pyinstaller-requirements.txt new file mode 100644 index 000000000..3010621f1 --- /dev/null +++ b/pkg/centos7/pyinstaller-requirements.txt @@ -0,0 +1,16 @@ +pyinstaller==3.2 # currently 3.2.1 version is not supported because of botocore exception +Crypto +pyopenssl>=16.2.0 +argparse +requests>=2.13.0 +logging +pprint +daemon +boto3 +botocore +pygit2 +salt-ssh +gitpython +pyinotify +cffi +azure From 7cf66977ad52f23334e81d5de844e4e4586279e7 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Fri, 4 Aug 2017 13:50:31 -0600 Subject: [PATCH 13/50] Fix nebula to prefer our bundled osqueryi if present --- hubblestack/extmods/modules/nebula_osquery.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index c6a34bc04..484971683 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -80,7 +80,7 @@ def queries(query_group, query_file = 'salt://hubblestack_nebula/hubblestack_nebula_win_queries.yaml' else: query_file = 'salt://hubblestack_nebula/hubblestack_nebula_queries.yaml' - if not salt.utils.which('osqueryi'): + if not salt.utils.which('osqueryi') and not salt.utils.which('/opt/osquery/osqueryi'): if query_group == 'day': log.warning('osquery not installed on this host. Returning baseline data') # Match the formatting of normal osquery results. Not super @@ -170,7 +170,10 @@ def queries(query_group, 'result': True, } - cmd = ['osqueryi', '--read_max', MAX_FILE_SIZE, '--json', query_sql] + if salt.utils.which('/opt/osquery/osqueryi'): + cmd = ['/opt/osquery/osqueryi', '--read_max', MAX_FILE_SIZE, '--json', query_sql] + else: + cmd = ['osqueryi', '--read_max', MAX_FILE_SIZE, '--json', query_sql] res = __salt__['cmd.run_all'](cmd) if res['retcode'] == 0: query_ret['data'] = json.loads(res['stdout']) From a2eff32c36c427b07272c009d8fc38aa91354f8c Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Mon, 7 Aug 2017 10:03:42 -0600 Subject: [PATCH 14/50] amazonlinux 2017.03 dockerfile --- pkg/amazonlinux2017.03/Dockerfile | 141 ++++++++++++++++++ .../pyinstaller-requirements.txt | 16 ++ 2 files changed, 157 insertions(+) create mode 100644 pkg/amazonlinux2017.03/Dockerfile create mode 100644 pkg/amazonlinux2017.03/pyinstaller-requirements.txt diff --git a/pkg/amazonlinux2017.03/Dockerfile b/pkg/amazonlinux2017.03/Dockerfile new file mode 100644 index 000000000..d6945c50e --- /dev/null +++ b/pkg/amazonlinux2017.03/Dockerfile @@ -0,0 +1,141 @@ +# This Dockerfile aims to make building Hubble v2 packages easier. +# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile +# 2. docker build -t . +# The resulting image is ready to run the pyinstaller on container start and drop hubble-coreos.tar.gz +# in the /data directory. Mount /data volume into a directory on the host to access the package. +# To run the container: docker run -it --rm -v `pwd`:/data + +FROM amazonlinux:2017.03 + +RUN yum -y update + +#paths that hubble or hubble parts need in the package +RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/osquery + +#osquery build start +#osquery should be built first since requirements for other packages can interfere with osquery dependencies +#to build, osquery scripts want sudo and a user to sudo with. +#to pin to a different version change the following envirnment variable +ENV OSQUERY_SRC_VERSION=2.6.0 +ENV OSQUERY_BUILD_USER=osquerybuilder +ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git +RUN yum -y install git make python ruby sudo which +RUN useradd --shell /bin/bash --create-home --user-group --groups wheel "$OSQUERY_BUILD_USER" \ + && sed -i '0,/^#\ %wheel/s/^#\ %wheel.*/%wheel\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers +RUN mkdir -p /usr/local/osquery/ \ + && chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ +USER $OSQUERY_BUILD_USER +ENV SKIP_TESTS=1 +RUN cd /home/"$OSQUERY_BUILD_USER" \ + && git clone "$OSQUERY_GIT_URL" \ + && cd osquery/ \ + && git checkout "$OSQUERY_SRC_VERSION" \ +#these homebrew hashes need to be current. hashes in osquery git repo are often out of date for the tags we check out and try to build. +#this is a problem and they are aware of it. let the magic hashes commence: + && sed -i 's,^\(HOMEBREW_CORE=\).*,\1'941ca36839ea354031846d73ad538e1e44e673f4',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_CORE=\).*,\1'abc5c5782c5850f2deff1f3d463945f90f2feaac',' tools/provision.sh \ + && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ + && make sysprep \ + && make deps \ + && make \ + && make strip +USER root +RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ + && chown -R root. /opt/osquery \ + && chmod -R 500 /opt/osquery/* \ + && mkdir -p /opt/osquery/lenses \ + && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ + && chmod -R 400 /opt/osquery/lenses/* +RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version + +#install packages that should be needed for ligbit2 compilation and successful pyinstaller run +RUN yum -y install \ + python27-devel libffi-devel openssl-devel \ + libxml2-devel libxslt-devel libjpeg-devel \ + zlib-devel make cmake python27-setuptools \ + python27-python-setuptools gcc python-devel \ + python-setuptools wget + +#libgit2 install start +#must precede pyinstaller requirements +ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz +ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 +ENV LIBGIT2_SRC_VERSION=0.26.0 +ENV LIBGIT2TEMP=/tmp/libgit2temp +RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ + && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ + && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ + && tar xzf libgit2.tar.gz \ + && cd libgit2-"$LIBGIT2_SRC_VERSION"/ \ + && export LIBGIT2=/usr/local/ \ + && cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \ + && make \ + && make install + +#pyinstaller requirements start +#must be preceded by libgit2 install +COPY pyinstaller-requirements.txt / +#default python-pip from yum does not like upgrading itself from pip. looking for better options other than wget. +RUN wget -c https://bootstrap.pypa.io/get-pip.py \ + && python get-pip.py \ + && pip -v install -r pyinstaller-requirements.txt + +#deb package making requirements start +RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ + && gem install --no-ri --no-rdoc fpm + +#pyinstaller start +#commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built +#use the following variables to choose the version of hubble +ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git +ENV HUBBLE_SRC_PATH=/hubble_src +ENV _HOOK_DIR="./pkg/" +ENV _BINARY_LOG_LEVEL="INFO" +ENV _INCLUDE_PATH="" +RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ + && cd "$HUBBLE_SRC_PATH" \ + && git checkout "$HUBBLE_SRC_VERSION" \ + && cp -rf "$HUBBLE_SRC_PATH" /hubble_build +RUN mkdir /data +VOLUME /data +WORKDIR /hubble_build +ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ] +CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} hubble.py \ + && cp /hubble_build/conf/osquery.conf /etc/osquery/ \ + && cp /hubble_build/conf/osquery.flags /etc/osquery/ \ +# hubble default configuration file + && cp -rf /hubble_build/conf/hubble /etc/hubble/ \ + && cp -rf /hubble_build/conf/hubble-profile.sh /etc/profile.d/ \ + && cp -pr /hubble_build/dist/hubble /opt/hubble/hubble-libs \ + && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \ + # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes + && rm -rf /opt/hubble/hubble-libs/librpm* \ +#rpm pkg start + && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d/ \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ + && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ +#during container run, if a configuration file exists in a /data copy it over the existing one so it would be +#possile to optionally include a custom one with the package + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ ; fi \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && mkdir -p usr/bin \ +#symlink to have hubble binary in path + && ln -s /opt/hubble/hubble usr/bin/hubble \ +#fpm start + && fpm -s dir -t rpm \ + -n hubblestack \ + -v ${HUBBLE_SRC_VERSION} \ + #todo: get rid of the git dependency with static bin in the future + -d 'git' \ + --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ + --deb-no-default-config-files \ + etc/hubble etc/osquery etc/init.d opt usr \ +#edit to change iteration number, if necessary + && cp hubblestack-${HUBBLE_SRC_VERSION}-1.x86_64.rpm /data/hubblestack_${HUBBLE_SRC_VERSION}-1_al2017.03_x86_64.rpm" ] diff --git a/pkg/amazonlinux2017.03/pyinstaller-requirements.txt b/pkg/amazonlinux2017.03/pyinstaller-requirements.txt new file mode 100644 index 000000000..3010621f1 --- /dev/null +++ b/pkg/amazonlinux2017.03/pyinstaller-requirements.txt @@ -0,0 +1,16 @@ +pyinstaller==3.2 # currently 3.2.1 version is not supported because of botocore exception +Crypto +pyopenssl>=16.2.0 +argparse +requests>=2.13.0 +logging +pprint +daemon +boto3 +botocore +pygit2 +salt-ssh +gitpython +pyinotify +cffi +azure From 7638a0b122b44532c2e8317388dfe0e07884487c Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Mon, 7 Aug 2017 10:06:00 -0600 Subject: [PATCH 15/50] amazonlinux 2016.09 dockerfile --- pkg/amazonlinux2016.09/Dockerfile | 141 ++++++++++++++++++ .../pyinstaller-requirements.txt | 16 ++ 2 files changed, 157 insertions(+) create mode 100644 pkg/amazonlinux2016.09/Dockerfile create mode 100644 pkg/amazonlinux2016.09/pyinstaller-requirements.txt diff --git a/pkg/amazonlinux2016.09/Dockerfile b/pkg/amazonlinux2016.09/Dockerfile new file mode 100644 index 000000000..6b198e9a6 --- /dev/null +++ b/pkg/amazonlinux2016.09/Dockerfile @@ -0,0 +1,141 @@ +# This Dockerfile aims to make building Hubble v2 packages easier. +# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile +# 2. docker build -t . +# The resulting image is ready to run the pyinstaller on container start and drop hubble-coreos.tar.gz +# in the /data directory. Mount /data volume into a directory on the host to access the package. +# To run the container: docker run -it --rm -v `pwd`:/data + +FROM amazonlinux:2016.09 + +RUN yum -y update + +#paths that hubble or hubble parts need in the package +RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/osquery + +#osquery build start +#osquery should be built first since requirements for other packages can interfere with osquery dependencies +#to build, osquery scripts want sudo and a user to sudo with. +#to pin to a different version change the following envirnment variable +ENV OSQUERY_SRC_VERSION=2.6.0 +ENV OSQUERY_BUILD_USER=osquerybuilder +ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git +RUN yum -y install git make python ruby sudo which +RUN useradd --shell /bin/bash --create-home --user-group --groups wheel "$OSQUERY_BUILD_USER" \ + && sed -i '0,/^#\ %wheel/s/^#\ %wheel.*/%wheel\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers +RUN mkdir -p /usr/local/osquery/ \ + && chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ +USER $OSQUERY_BUILD_USER +ENV SKIP_TESTS=1 +RUN cd /home/"$OSQUERY_BUILD_USER" \ + && git clone "$OSQUERY_GIT_URL" \ + && cd osquery/ \ + && git checkout "$OSQUERY_SRC_VERSION" \ +#these homebrew hashes need to be current. hashes in osquery git repo are often out of date for the tags we check out and try to build. +#this is a problem and they are aware of it. let the magic hashes commence: + && sed -i 's,^\(HOMEBREW_CORE=\).*,\1'941ca36839ea354031846d73ad538e1e44e673f4',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_CORE=\).*,\1'abc5c5782c5850f2deff1f3d463945f90f2feaac',' tools/provision.sh \ + && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ + && make sysprep \ + && make deps \ + && make \ + && make strip +USER root +RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ + && chown -R root. /opt/osquery \ + && chmod -R 500 /opt/osquery/* \ + && mkdir -p /opt/osquery/lenses \ + && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ + && chmod -R 400 /opt/osquery/lenses/* +RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version + +#install packages that should be needed for ligbit2 compilation and successful pyinstaller run +RUN yum -y install \ + python27-devel libffi-devel openssl-devel \ + libxml2-devel libxslt-devel libjpeg-devel \ + zlib-devel make cmake python27-setuptools \ + python27-python-setuptools gcc python-devel \ + python-setuptools wget + +#libgit2 install start +#must precede pyinstaller requirements +ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz +ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 +ENV LIBGIT2_SRC_VERSION=0.26.0 +ENV LIBGIT2TEMP=/tmp/libgit2temp +RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ + && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ + && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ + && tar xzf libgit2.tar.gz \ + && cd libgit2-"$LIBGIT2_SRC_VERSION"/ \ + && export LIBGIT2=/usr/local/ \ + && cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \ + && make \ + && make install + +#pyinstaller requirements start +#must be preceded by libgit2 install +COPY pyinstaller-requirements.txt / +#default python-pip from yum does not like upgrading itself from pip. looking for better options other than wget. +RUN wget -c https://bootstrap.pypa.io/get-pip.py \ + && python get-pip.py \ + && pip -v install -r pyinstaller-requirements.txt + +#deb package making requirements start +RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ + && gem install --no-ri --no-rdoc fpm + +#pyinstaller start +#commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built +#use the following variables to choose the version of hubble +ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git +ENV HUBBLE_SRC_PATH=/hubble_src +ENV _HOOK_DIR="./pkg/" +ENV _BINARY_LOG_LEVEL="INFO" +ENV _INCLUDE_PATH="" +RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ + && cd "$HUBBLE_SRC_PATH" \ + && git checkout "$HUBBLE_SRC_VERSION" \ + && cp -rf "$HUBBLE_SRC_PATH" /hubble_build +RUN mkdir /data +VOLUME /data +WORKDIR /hubble_build +ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ] +CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} hubble.py \ + && cp /hubble_build/conf/osquery.conf /etc/osquery/ \ + && cp /hubble_build/conf/osquery.flags /etc/osquery/ \ +# hubble default configuration file + && cp -rf /hubble_build/conf/hubble /etc/hubble/ \ + && cp -rf /hubble_build/conf/hubble-profile.sh /etc/profile.d/ \ + && cp -pr /hubble_build/dist/hubble /opt/hubble/hubble-libs \ + && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \ + # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes + && rm -rf /opt/hubble/hubble-libs/librpm* \ +#rpm pkg start + && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d/ \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ + && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ +#during container run, if a configuration file exists in a /data copy it over the existing one so it would be +#possile to optionally include a custom one with the package + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ ; fi \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && mkdir -p usr/bin \ +#symlink to have hubble binary in path + && ln -s /opt/hubble/hubble usr/bin/hubble \ +#fpm start + && fpm -s dir -t rpm \ + -n hubblestack \ + -v ${HUBBLE_SRC_VERSION} \ + #todo: get rid of the git dependency with static bin in the future + -d 'git' \ + --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ + --deb-no-default-config-files \ + etc/hubble etc/osquery etc/init.d opt usr \ +#edit to change iteration number, if necessary + && cp hubblestack-${HUBBLE_SRC_VERSION}-1.x86_64.rpm /data/hubblestack_${HUBBLE_SRC_VERSION}-1_al2016.09_x86_64.rpm" ] diff --git a/pkg/amazonlinux2016.09/pyinstaller-requirements.txt b/pkg/amazonlinux2016.09/pyinstaller-requirements.txt new file mode 100644 index 000000000..3010621f1 --- /dev/null +++ b/pkg/amazonlinux2016.09/pyinstaller-requirements.txt @@ -0,0 +1,16 @@ +pyinstaller==3.2 # currently 3.2.1 version is not supported because of botocore exception +Crypto +pyopenssl>=16.2.0 +argparse +requests>=2.13.0 +logging +pprint +daemon +boto3 +botocore +pygit2 +salt-ssh +gitpython +pyinotify +cffi +azure From 5ead745f1ee3197adeed77e9e157a5947851e2ae Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 7 Aug 2017 11:26:34 -0600 Subject: [PATCH 16/50] Use the osquerybinpath grain in nebula --- hubblestack/extmods/grains/osqueryinfo.py | 6 ++++-- hubblestack/extmods/modules/nebula_osquery.py | 7 ++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/hubblestack/extmods/grains/osqueryinfo.py b/hubblestack/extmods/grains/osqueryinfo.py index 5bad795c4..c4bdced14 100644 --- a/hubblestack/extmods/grains/osqueryinfo.py +++ b/hubblestack/extmods/grains/osqueryinfo.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import salt.utils +import salt.modules.cmdmod __salt__ = { 'cmd.run': salt.modules.cmdmod._run_quiet } @@ -14,13 +15,14 @@ def osquerygrain(): grains = {} option = '--version' - osqueryipaths = ('osqueryi', '/usr/bin/osqueryi', '/opt/osquery/osqueryi') + # Prefer our /opt/osquery/osqueryi if present + osqueryipaths = ('/opt/osquery/osqueryi', 'osqueryi', '/usr/bin/osqueryi') for path in osqueryipaths: if salt.utils.which(path): for item in __salt__['cmd.run']('{0} {1}'.format(path, option)).split(): if item[:1].isdigit(): grains['osqueryversion'] = item - grains['osquerybinpath'] = path + grains['osquerybinpath'] = salt.utils.which(path) break break return grains diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index 484971683..4bc2bda82 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -80,7 +80,7 @@ def queries(query_group, query_file = 'salt://hubblestack_nebula/hubblestack_nebula_win_queries.yaml' else: query_file = 'salt://hubblestack_nebula/hubblestack_nebula_queries.yaml' - if not salt.utils.which('osqueryi') and not salt.utils.which('/opt/osquery/osqueryi'): + if 'osquerybinpath' not in __grains__: if query_group == 'day': log.warning('osquery not installed on this host. Returning baseline data') # Match the formatting of normal osquery results. Not super @@ -170,10 +170,7 @@ def queries(query_group, 'result': True, } - if salt.utils.which('/opt/osquery/osqueryi'): - cmd = ['/opt/osquery/osqueryi', '--read_max', MAX_FILE_SIZE, '--json', query_sql] - else: - cmd = ['osqueryi', '--read_max', MAX_FILE_SIZE, '--json', query_sql] + cmd = [__grains__['osquerybinpath'], '--read_max', MAX_FILE_SIZE, '--json', query_sql] res = __salt__['cmd.run_all'](cmd) if res['retcode'] == 0: query_ret['data'] = json.loads(res['stdout']) From 2f547b5244825905d82f64f5bda2592d78e96614 Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Mon, 7 Aug 2017 11:38:02 -0600 Subject: [PATCH 17/50] sync changes between all dockerfiles These changes introduce symlink from /usr/bin to /opt/hubble/hubble to have hubble binary in path, but only in cases where rpm or deb package is made. Also, if /data/hubble exist in container, this file will be copied in place of the default configuration. This makes possible to include a custom configuration file at container run time. --- pkg/centos7/Dockerfile | 21 ++++++++++++--------- pkg/coreos/Dockerfile | 3 +++ pkg/debian8/Dockerfile | 7 +++++++ pkg/debian9/Dockerfile | 6 ++++++ 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/pkg/centos7/Dockerfile b/pkg/centos7/Dockerfile index 1fbc9892e..edef011ee 100644 --- a/pkg/centos7/Dockerfile +++ b/pkg/centos7/Dockerfile @@ -21,7 +21,7 @@ ENV OSQUERY_BUILD_USER=osquerybuilder ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git RUN yum -y install git make python ruby sudo which RUN useradd --shell /bin/bash --create-home --user-group --groups wheel "$OSQUERY_BUILD_USER" \ - && sed -i 's/^%wheel.*/%wheel\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers + && sed -i '0,/^#\ %wheel/s/^#\ %wheel.*/%wheel\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers RUN mkdir -p /usr/local/osquery/ \ && chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ USER $OSQUERY_BUILD_USER @@ -56,11 +56,6 @@ RUN yum -y install \ zlib-devel make cmake python27-setuptools \ python27-python-setuptools gcc python-devel \ python-setuptools wget - # python-dev libffi-dev libssl-dev libyaml-dev \ - # libxml2-dev libxslt1-dev python-cffi \ - # zlib1g-dev cmake python-setuptools \ - # gcc wget python-pip openssl \ - #&& apt-get clean #libgit2 install start #must precede pyinstaller requirements @@ -81,6 +76,7 @@ RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ #pyinstaller requirements start #must be preceded by libgit2 install COPY pyinstaller-requirements.txt / +#default python-pip from yum does not like upgrading itself from pip. looking for better options other than wget. RUN wget -c https://bootstrap.pypa.io/get-pip.py \ && python get-pip.py \ && pip -v install -r pyinstaller-requirements.txt @@ -116,7 +112,7 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \ # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes && rm -rf /opt/hubble/hubble-libs/librpm* \ -#debian pkg start +#rpm pkg start && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ @@ -125,14 +121,21 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ +#during container run, if a configuration file exists in a /data copy it over the existing one so it would be +#possile to optionally include a custom one with the package + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ ; fi \ && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && mkdir -p usr/bin \ +#symlink to have hubble binary in path + && ln -s /opt/hubble/hubble usr/bin/hubble \ #fpm start && fpm -s dir -t rpm \ -n hubblestack \ - -v ${HUBBLE_SRC_VERSION}-1 \ + -v ${HUBBLE_SRC_VERSION} \ #todo: get rid of the git dependency with static bin in the future -d 'git' \ --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr \ - && cp hubblestack-${HUBBLE_SRC_VERSION}_1-1.x86_64.rpm /data/hubblestack_${HUBBLE_SRC_VERSION}-1_centos7_x86_64.rpm" ] +#edit to change iteration number, if necessary + && cp hubblestack-${HUBBLE_SRC_VERSION}-1.x86_64.rpm /data/hubblestack_${HUBBLE_SRC_VERSION}-1_centos7_x86_64.rpm" ] diff --git a/pkg/coreos/Dockerfile b/pkg/coreos/Dockerfile index 08933985d..579b2b4cf 100644 --- a/pkg/coreos/Dockerfile +++ b/pkg/coreos/Dockerfile @@ -102,6 +102,9 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit && cp /hubble_build/conf/osquery.flags /etc/osquery/ \ # hubble default configuration file && cp -rf /hubble_build/conf/hubble /etc/hubble/ \ +#during container run, if a configuration file exists in a /data copy it over the existing one so it would be +#possile to optionally include a custom one with the package + && if [ -f /data/hubble ] ; then cp /data/hubble /etc/hubble/ ; fi \ && cp -rf /hubble_build/conf/hubble-profile.sh /etc/profile.d/ \ && cp -pr /hubble_build/dist/hubble /opt/hubble/hubble-libs \ && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \ diff --git a/pkg/debian8/Dockerfile b/pkg/debian8/Dockerfile index 812b76fc5..85c00a661 100644 --- a/pkg/debian8/Dockerfile +++ b/pkg/debian8/Dockerfile @@ -81,6 +81,7 @@ RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ #must be preceded by libgit2 install COPY pyinstaller-requirements.txt / RUN ldconfig +#default python-pip from yum does not like upgrading itself from pip. looking for better options other than wget. RUN wget -c https://bootstrap.pypa.io/get-pip.py \ && python get-pip.py \ && pip -v install -r pyinstaller-requirements.txt @@ -125,7 +126,13 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ +#during container run, if a configuration file exists in a /data copy it over the existing one so it would be +#possile to optionally include a custom one with the package + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ ; fi \ && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && mkdir -p usr/bin \ +#symlink to have hubble binary in path + && ln -s /opt/hubble/hubble usr/bin/hubble \ #fpm start && fpm -s dir -t deb \ -n hubblestack \ diff --git a/pkg/debian9/Dockerfile b/pkg/debian9/Dockerfile index 2321be0fa..343f9f89e 100644 --- a/pkg/debian9/Dockerfile +++ b/pkg/debian9/Dockerfile @@ -120,7 +120,13 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ +#during container run, if a configuration file exists in a /data copy it over the existing one so it would be +#possile to optionally include a custom one with the package + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ ; fi \ && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && mkdir -p usr/bin \ +#symlink to have hubble binary in path + && ln -s /opt/hubble/hubble usr/bin/hubble \ #fpm start && fpm -s dir -t deb \ -n hubblestack \ From 8b7e100068f138d669f47be2613638622fe75f0f Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 7 Aug 2017 12:17:39 -0600 Subject: [PATCH 18/50] Fix extmods handling, add grains/__init__.py --- hubblestack/daemon.py | 10 ++++++++-- hubblestack/extmods/grains/__init__.py | 0 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 hubblestack/extmods/grains/__init__.py diff --git a/hubblestack/daemon.py b/hubblestack/daemon.py index a7f918b2a..3ff524284 100644 --- a/hubblestack/daemon.py +++ b/hubblestack/daemon.py @@ -308,10 +308,16 @@ def load_config(): elif __opts__['verbose'] >= 3: __opts__['log_level'] = 'debug' - # Setup module dirs + # Setup module/grain/returner dirs module_dirs = __opts__.get('module_dirs', []) - module_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods')) + module_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'modules')) __opts__['module_dirs'] = module_dirs + grains_dirs = __opts__.get('grains_dirs', []) + grains_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'grains')) + __opts__['grains_dirs'] = grains_dirs + returner_dirs = __opts__.get('returner_dirs', []) + returner_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'returners')) + __opts__['returner_dirs'] = returner_dirs __opts__['file_roots']['base'].insert(0, os.path.join(os.path.dirname(__file__), 'files')) if 'roots' not in __opts__['fileserver_backend']: __opts__['fileserver_backend'].append('roots') diff --git a/hubblestack/extmods/grains/__init__.py b/hubblestack/extmods/grains/__init__.py new file mode 100644 index 000000000..e69de29bb From db03f433441374bbc2a381721b625f1c61a5b8f4 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 7 Aug 2017 13:47:28 -0600 Subject: [PATCH 19/50] Add fileserver loading as well --- hubblestack/daemon.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hubblestack/daemon.py b/hubblestack/daemon.py index 3ff524284..233bab346 100644 --- a/hubblestack/daemon.py +++ b/hubblestack/daemon.py @@ -318,6 +318,9 @@ def load_config(): returner_dirs = __opts__.get('returner_dirs', []) returner_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'returners')) __opts__['returner_dirs'] = returner_dirs + fileserver_dirs = __opts__.get('fileserver_dirs', []) + fileserver_dirs.append(os.path.join(os.path.dirname(__file__), 'extmods', 'fileserver')) + __opts__['fileserver_dirs'] = fileserver_dirs __opts__['file_roots']['base'].insert(0, os.path.join(os.path.dirname(__file__), 'files')) if 'roots' not in __opts__['fileserver_backend']: __opts__['fileserver_backend'].append('roots') From 3981f92a7fb9864cb83f160408898c619d3d9b1a Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 7 Aug 2017 14:01:30 -0600 Subject: [PATCH 20/50] Rev to 2.2.0 in dockerfiles --- pkg/amazonlinux2016.09/Dockerfile | 4 ++-- pkg/amazonlinux2017.03/Dockerfile | 4 ++-- pkg/centos7/Dockerfile | 4 ++-- pkg/coreos/Dockerfile | 4 ++-- pkg/debian8/Dockerfile | 6 +++--- pkg/debian9/Dockerfile | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/amazonlinux2016.09/Dockerfile b/pkg/amazonlinux2016.09/Dockerfile index 6b198e9a6..07db2a603 100644 --- a/pkg/amazonlinux2016.09/Dockerfile +++ b/pkg/amazonlinux2016.09/Dockerfile @@ -88,7 +88,7 @@ RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_SRC_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -96,7 +96,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "$HUBBLE_SRC_VERSION" \ + && git checkout "v$HUBBLE_SRC_VERSION" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data diff --git a/pkg/amazonlinux2017.03/Dockerfile b/pkg/amazonlinux2017.03/Dockerfile index d6945c50e..b59550715 100644 --- a/pkg/amazonlinux2017.03/Dockerfile +++ b/pkg/amazonlinux2017.03/Dockerfile @@ -88,7 +88,7 @@ RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_SRC_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -96,7 +96,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "$HUBBLE_SRC_VERSION" \ + && git checkout "v$HUBBLE_SRC_VERSION" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data diff --git a/pkg/centos7/Dockerfile b/pkg/centos7/Dockerfile index edef011ee..ce7078653 100644 --- a/pkg/centos7/Dockerfile +++ b/pkg/centos7/Dockerfile @@ -88,7 +88,7 @@ RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_SRC_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -96,7 +96,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "$HUBBLE_SRC_VERSION" \ + && git checkout "v$HUBBLE_SRC_VERSION" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data diff --git a/pkg/coreos/Dockerfile b/pkg/coreos/Dockerfile index 579b2b4cf..7f91ae69a 100644 --- a/pkg/coreos/Dockerfile +++ b/pkg/coreos/Dockerfile @@ -83,7 +83,7 @@ RUN pip install --upgrade pip \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_SRC_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -91,7 +91,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "$HUBBLE_SRC_VERSION" \ + && git checkout "v$HUBBLE_SRC_VERSION" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data diff --git a/pkg/debian8/Dockerfile b/pkg/debian8/Dockerfile index 85c00a661..a01aae3bb 100644 --- a/pkg/debian8/Dockerfile +++ b/pkg/debian8/Dockerfile @@ -80,7 +80,7 @@ RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ #pyinstaller requirements start #must be preceded by libgit2 install COPY pyinstaller-requirements.txt / -RUN ldconfig +RUN ldconfig #default python-pip from yum does not like upgrading itself from pip. looking for better options other than wget. RUN wget -c https://bootstrap.pypa.io/get-pip.py \ && python get-pip.py \ @@ -93,7 +93,7 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_SRC_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -101,7 +101,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "$HUBBLE_SRC_VERSION" \ + && git checkout "v$HUBBLE_SRC_VERSION" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data diff --git a/pkg/debian9/Dockerfile b/pkg/debian9/Dockerfile index 343f9f89e..00859c4df 100644 --- a/pkg/debian9/Dockerfile +++ b/pkg/debian9/Dockerfile @@ -87,7 +87,7 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_SRC_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -95,7 +95,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "$HUBBLE_SRC_VERSION" \ + && git checkout "v$HUBBLE_SRC_VERSION" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data From d52ce3f73408517174d6728bc2c1cabd5e974ebf Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 7 Aug 2017 14:02:10 -0600 Subject: [PATCH 21/50] Rev to 2.2.0 --- hubblestack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hubblestack/__init__.py b/hubblestack/__init__.py index bc6379c18..04188a16d 100644 --- a/hubblestack/__init__.py +++ b/hubblestack/__init__.py @@ -1 +1 @@ -__version__ = '2.1.7' +__version__ = '2.2.0' From e1bd30f9ae8d03c43bd6a9c63b4051b57e56bc40 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 7 Aug 2017 14:03:07 -0600 Subject: [PATCH 22/50] Rev build scripts and specs We don't use these anymore, but just in case... --- pkg/build_debs.sh | 20 ++++++++++---------- pkg/build_rpms.sh | 22 +++++++++++----------- pkg/specs/hubblestack-el6.spec | 2 +- pkg/specs/hubblestack-el7.spec | 2 +- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/pkg/build_debs.sh b/pkg/build_debs.sh index 585f6e58f..531752908 100755 --- a/pkg/build_debs.sh +++ b/pkg/build_debs.sh @@ -22,14 +22,14 @@ mkdir -p dist bash ./init_pkg.sh -y cp ../hubble.tar.gz dist/hubble.tar.gz mv ../hubble.tar.gz build/hubble.tar.gz -mkdir build/hubblestack-2.1.7 -tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.1.7 -mkdir -p build/hubblestack-2.1.7/etc/init.d -cp ./hubble build/hubblestack-2.1.7/etc/init.d -mkdir -p build/hubblestack-2.1.7/usr/lib/systemd/system -cp ./hubble.service build/hubblestack-2.1.7/usr/lib/systemd/system -cp -f ../conf/hubble build/hubblestack-2.1.7/etc/hubble/hubble -cd build/hubblestack-2.1.7 +mkdir build/hubblestack-2.2.0 +tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.2.0 +mkdir -p build/hubblestack-2.2.0/etc/init.d +cp ./hubble build/hubblestack-2.2.0/etc/init.d +mkdir -p build/hubblestack-2.2.0/usr/lib/systemd/system +cp ./hubble.service build/hubblestack-2.2.0/usr/lib/systemd/system +cp -f ../conf/hubble build/hubblestack-2.2.0/etc/hubble/hubble +cd build/hubblestack-2.2.0 sudo apt-get install -y ruby ruby-dev rubygems gcc make sudo gem install --no-ri --no-rdoc fpm @@ -39,9 +39,9 @@ ln -s /opt/osquery/osqueryd usr/bin/osqueryd ln -s /opt/osquery/osqueryi usr/bin/osqueryi fpm -s dir -t deb \ -n hubblestack \ - -v 2.1.7-1 \ + -v 2.2.0-1 \ -d 'git' \ --config-files /etc/hubble/hubble --config-files /etc/osquery/osquery.conf \ --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr/bin -cp hubblestack_2.1.7-1_amd64.deb ../../dist/ +cp hubblestack_2.2.0-1_amd64.deb ../../dist/ diff --git a/pkg/build_rpms.sh b/pkg/build_rpms.sh index 93e1cdabe..6f9dc70f3 100755 --- a/pkg/build_rpms.sh +++ b/pkg/build_rpms.sh @@ -22,23 +22,23 @@ mkdir -p dist bash ./init_pkg.sh -y cp ../hubble.tar.gz dist/hubble.tar.gz mv ../hubble.tar.gz build/hubble.tar.gz -mkdir build/hubblestack-2.1.7 -tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.1.7 -mkdir -p build/hubblestack-2.1.7/etc/init.d -cp ./hubble build/hubblestack-2.1.7/etc/init.d -mkdir -p build/hubblestack-2.1.7/usr/lib/systemd/system -cp ./hubble.service build/hubblestack-2.1.7/usr/lib/systemd/system -cp -f ../conf/hubble build/hubblestack-2.1.7/etc/hubble/hubble +mkdir build/hubblestack-2.2.0 +tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.2.0 +mkdir -p build/hubblestack-2.2.0/etc/init.d +cp ./hubble build/hubblestack-2.2.0/etc/init.d +mkdir -p build/hubblestack-2.2.0/usr/lib/systemd/system +cp ./hubble.service build/hubblestack-2.2.0/usr/lib/systemd/system +cp -f ../conf/hubble build/hubblestack-2.2.0/etc/hubble/hubble cd build -tar -czvf hubblestack-2.1.7.tar.gz hubblestack-2.1.7/ +tar -czvf hubblestack-2.2.0.tar.gz hubblestack-2.2.0/ mkdir -p rpmbuild/{RPMS,SRPMS,BUILD,SOURCES,SPECS,tmp} -cp hubblestack-2.1.7.tar.gz rpmbuild/SOURCES/ +cp hubblestack-2.2.0.tar.gz rpmbuild/SOURCES/ cd rpmbuild cp ../../specs/* SPECS/ rpmbuild --define "_topdir $(pwd)" --define "_tmppath %{_topdir}/tmp" -ba SPECS/hubblestack-el6.spec -cp RPMS/x86_64/hubblestack-2.1.7-1.x86_64.rpm ../../dist/hubblestack-2.1.7-1.el6.x86_64.rpm +cp RPMS/x86_64/hubblestack-2.2.0-1.x86_64.rpm ../../dist/hubblestack-2.2.0-1.el6.x86_64.rpm rpmbuild --define "_topdir $(pwd)" --define "_tmppath %{_topdir}/tmp" -ba SPECS/hubblestack-el7.spec -cp RPMS/x86_64/hubblestack-2.1.7-1.x86_64.rpm ../../dist/hubblestack-2.1.7-1.el7.x86_64.rpm +cp RPMS/x86_64/hubblestack-2.2.0-1.x86_64.rpm ../../dist/hubblestack-2.2.0-1.el7.x86_64.rpm diff --git a/pkg/specs/hubblestack-el6.spec b/pkg/specs/hubblestack-el6.spec index 65c4ba210..1190eccec 100644 --- a/pkg/specs/hubblestack-el6.spec +++ b/pkg/specs/hubblestack-el6.spec @@ -9,7 +9,7 @@ Summary: Hubblestack is a module, open-source security compliance framework Name: hubblestack -Version: 2.1.7 +Version: 2.2.0 Release: 1 License: Apache 2.0 Group: Development/Tools diff --git a/pkg/specs/hubblestack-el7.spec b/pkg/specs/hubblestack-el7.spec index e2e6a9bfb..f33996631 100644 --- a/pkg/specs/hubblestack-el7.spec +++ b/pkg/specs/hubblestack-el7.spec @@ -9,7 +9,7 @@ Summary: Hubblestack is a module, open-source security compliance framework Name: hubblestack -Version: 2.1.7 +Version: 2.2.0 Release: 1 License: Apache 2.0 Group: Development/Tools From 701e99ddfeb5642f9305c37cc6c9ea579a065740 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 7 Aug 2017 16:28:14 -0600 Subject: [PATCH 23/50] Separate HUBBLE_VERSION and HUBBLE_CHECKOUT --- pkg/amazonlinux2016.09/Dockerfile | 29 +++++++++++++++-------------- pkg/amazonlinux2017.03/Dockerfile | 29 +++++++++++++++-------------- pkg/centos7/Dockerfile | 29 +++++++++++++++-------------- pkg/coreos/Dockerfile | 7 ++++--- pkg/debian8/Dockerfile | 29 +++++++++++++++-------------- pkg/debian9/Dockerfile | 29 +++++++++++++++-------------- 6 files changed, 79 insertions(+), 73 deletions(-) diff --git a/pkg/amazonlinux2016.09/Dockerfile b/pkg/amazonlinux2016.09/Dockerfile index 07db2a603..5fad59c49 100644 --- a/pkg/amazonlinux2016.09/Dockerfile +++ b/pkg/amazonlinux2016.09/Dockerfile @@ -88,7 +88,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.0 +ENV HUBBLE_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -96,7 +97,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "v$HUBBLE_SRC_VERSION" \ + && git checkout "$HUBBLE_CHECKOUT" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data @@ -113,29 +114,29 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes && rm -rf /opt/hubble/hubble-libs/librpm* \ #rpm pkg start - && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ - && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d \ - && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d/ \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ - && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ - && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ + && tar -cPvzf /data/hubblestack-${HUBBLE_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system \ + && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \ #during container run, if a configuration file exists in a /data copy it over the existing one so it would be #possile to optionally include a custom one with the package - && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ ; fi \ - && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ ; fi \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ && mkdir -p usr/bin \ #symlink to have hubble binary in path && ln -s /opt/hubble/hubble usr/bin/hubble \ #fpm start && fpm -s dir -t rpm \ -n hubblestack \ - -v ${HUBBLE_SRC_VERSION} \ + -v ${HUBBLE_VERSION} \ #todo: get rid of the git dependency with static bin in the future -d 'git' \ --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr \ #edit to change iteration number, if necessary - && cp hubblestack-${HUBBLE_SRC_VERSION}-1.x86_64.rpm /data/hubblestack_${HUBBLE_SRC_VERSION}-1_al2016.09_x86_64.rpm" ] + && cp hubblestack-${HUBBLE_VERSION}-1.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-1.al1609.x86_64.rpm" ] diff --git a/pkg/amazonlinux2017.03/Dockerfile b/pkg/amazonlinux2017.03/Dockerfile index b59550715..14acb6d0f 100644 --- a/pkg/amazonlinux2017.03/Dockerfile +++ b/pkg/amazonlinux2017.03/Dockerfile @@ -88,7 +88,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.0 +ENV HUBBLE_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -96,7 +97,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "v$HUBBLE_SRC_VERSION" \ + && git checkout "$HUBBLE_CHECKOUT" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data @@ -113,29 +114,29 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes && rm -rf /opt/hubble/hubble-libs/librpm* \ #rpm pkg start - && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ - && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d \ - && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d/ \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ - && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ - && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ + && tar -cPvzf /data/hubblestack-${HUBBLE_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system \ + && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \ #during container run, if a configuration file exists in a /data copy it over the existing one so it would be #possile to optionally include a custom one with the package - && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ ; fi \ - && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ ; fi \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ && mkdir -p usr/bin \ #symlink to have hubble binary in path && ln -s /opt/hubble/hubble usr/bin/hubble \ #fpm start && fpm -s dir -t rpm \ -n hubblestack \ - -v ${HUBBLE_SRC_VERSION} \ + -v ${HUBBLE_VERSION} \ #todo: get rid of the git dependency with static bin in the future -d 'git' \ --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr \ #edit to change iteration number, if necessary - && cp hubblestack-${HUBBLE_SRC_VERSION}-1.x86_64.rpm /data/hubblestack_${HUBBLE_SRC_VERSION}-1_al2017.03_x86_64.rpm" ] + && cp hubblestack-${HUBBLE_VERSION}-1.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-1.al1703.x86_64.rpm" ] diff --git a/pkg/centos7/Dockerfile b/pkg/centos7/Dockerfile index ce7078653..c791f9c6f 100644 --- a/pkg/centos7/Dockerfile +++ b/pkg/centos7/Dockerfile @@ -88,7 +88,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.0 +ENV HUBBLE_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -96,7 +97,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "v$HUBBLE_SRC_VERSION" \ + && git checkout "$HUBBLE_CHECKOUT" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data @@ -113,29 +114,29 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes && rm -rf /opt/hubble/hubble-libs/librpm* \ #rpm pkg start - && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ - && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d \ - && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d/ \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ - && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ - && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ + && tar -cPvzf /data/hubblestack-${HUBBLE_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system \ + && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \ #during container run, if a configuration file exists in a /data copy it over the existing one so it would be #possile to optionally include a custom one with the package - && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ ; fi \ - && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ ; fi \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ && mkdir -p usr/bin \ #symlink to have hubble binary in path && ln -s /opt/hubble/hubble usr/bin/hubble \ #fpm start && fpm -s dir -t rpm \ -n hubblestack \ - -v ${HUBBLE_SRC_VERSION} \ + -v ${HUBBLE_VERSION} \ #todo: get rid of the git dependency with static bin in the future -d 'git' \ --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr \ #edit to change iteration number, if necessary - && cp hubblestack-${HUBBLE_SRC_VERSION}-1.x86_64.rpm /data/hubblestack_${HUBBLE_SRC_VERSION}-1_centos7_x86_64.rpm" ] + && cp hubblestack-${HUBBLE_VERSION}-1.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-1.el7.x86_64.rpm" ] diff --git a/pkg/coreos/Dockerfile b/pkg/coreos/Dockerfile index 7f91ae69a..f04b2cead 100644 --- a/pkg/coreos/Dockerfile +++ b/pkg/coreos/Dockerfile @@ -83,7 +83,8 @@ RUN pip install --upgrade pip \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.0 +ENV HUBBLE_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -91,7 +92,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "v$HUBBLE_SRC_VERSION" \ + && git checkout "$HUBBLE_CHECKOUT" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data @@ -110,5 +111,5 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \ # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes && rm -rf /opt/hubble/hubble-libs/librpm* \ - && tar -cPvzf /data/hubble-${HUBBLE_SRC_VERSION}-coreos.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh" ] + && tar -cPvzf /data/hubblestack-${HUBBLE_VERSION}-coreos.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh" ] diff --git a/pkg/debian8/Dockerfile b/pkg/debian8/Dockerfile index a01aae3bb..704ddd909 100644 --- a/pkg/debian8/Dockerfile +++ b/pkg/debian8/Dockerfile @@ -93,7 +93,8 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.0 +ENV HUBBLE_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -101,7 +102,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "v$HUBBLE_SRC_VERSION" \ + && git checkout "$HUBBLE_CHECKOUT" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data @@ -118,29 +119,29 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes && rm -rf /opt/hubble/hubble-libs/librpm* \ #debian pkg start - && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ - && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d \ - && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d/ \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ - && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ - && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ + && tar -cPvzf /data/hubblestack-${HUBBLE_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSIOHUBBLE_VERSION}/usr/lib/systemd/system \ + && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \ #during container run, if a configuration file exists in a /data copy it over the existing one so it would be #possile to optionally include a custom one with the package - && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ ; fi \ - && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ ; fi \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ && mkdir -p usr/bin \ #symlink to have hubble binary in path && ln -s /opt/hubble/hubble usr/bin/hubble \ #fpm start && fpm -s dir -t deb \ -n hubblestack \ - -v ${HUBBLE_SRC_VERSION}-1 \ + -v ${HUBBLE_VERSION}-1 \ #todo: get rid of the git dependency with static bin in the future -d 'git' \ --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr \ - && cp hubblestack_${HUBBLE_SRC_VERSION}-1_amd64.deb /data/hubblestack_${HUBBLE_SRC_VERSION}-1_debian8_amd64.deb" ] + && cp hubblestack_${HUBBLE_VERSION}-1_amd64.deb /data/hubblestack_${HUBBLE_VERSION}-1deb8_amd64.deb" ] diff --git a/pkg/debian9/Dockerfile b/pkg/debian9/Dockerfile index 00859c4df..3c17e6f68 100644 --- a/pkg/debian9/Dockerfile +++ b/pkg/debian9/Dockerfile @@ -87,7 +87,8 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.0 +ENV HUBBLE_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -95,7 +96,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "v$HUBBLE_SRC_VERSION" \ + && git checkout "$HUBBLE_CHECKOUT" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data @@ -112,29 +113,29 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes && rm -rf /opt/hubble/hubble-libs/librpm* \ #debian pkg start - && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ - && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d \ - && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d/ \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ - && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ - && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ + && tar -cPvzf /data/hubblestack-${HUBBLE_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system \ + && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \ #during container run, if a configuration file exists in a /data copy it over the existing one so it would be #possile to optionally include a custom one with the package - && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ ; fi \ - && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ ; fi \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ && mkdir -p usr/bin \ #symlink to have hubble binary in path && ln -s /opt/hubble/hubble usr/bin/hubble \ #fpm start && fpm -s dir -t deb \ -n hubblestack \ - -v ${HUBBLE_SRC_VERSION}-1 \ + -v ${HUBBLE_VERSION}-1 \ #todo: get rid of the git dependency with static bin in the future -d 'git' \ --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr \ - && cp hubblestack_${HUBBLE_SRC_VERSION}-1_amd64.deb /data/hubblestack_${HUBBLE_SRC_VERSION}-1_debian9_amd64.deb" ] + && cp hubblestack_${HUBBLE_VERSION}-1_amd64.deb /data/hubblestack_${HUBBLE_VERSION}-1deb9_amd64.deb" ] From af739a24b69846d4dcf699a9cfb0cde2a428e252 Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Mon, 7 Aug 2017 17:03:42 -0600 Subject: [PATCH 24/50] centos 6 dockerfile --- pkg/centos6/Dockerfile | 146 +++++++++++++++++++++++ pkg/centos6/pyinstaller-requirements.txt | 16 +++ 2 files changed, 162 insertions(+) create mode 100644 pkg/centos6/Dockerfile create mode 100644 pkg/centos6/pyinstaller-requirements.txt diff --git a/pkg/centos6/Dockerfile b/pkg/centos6/Dockerfile new file mode 100644 index 000000000..850dfc245 --- /dev/null +++ b/pkg/centos6/Dockerfile @@ -0,0 +1,146 @@ +# This Dockerfile aims to make building Hubble v2 packages easier. +# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile +# 2. docker build -t . +# The resulting image is ready to run the pyinstaller on container start and drop hubble-coreos.tar.gz +# in the /data directory. Mount /data volume into a directory on the host to access the package. +# To run the container: docker run -it --rm -v `pwd`:/data + +FROM centos:6 + +RUN yum -y update + +#paths that hubble or hubble parts need in the package +RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/osquery + +#osquery build start +#osquery should be built first since requirements for other packages can interfere with osquery dependencies +#to build, osquery scripts want sudo and a user to sudo with. +#to pin to a different version change the following envirnment variable +ENV OSQUERY_SRC_VERSION=2.6.0 +ENV OSQUERY_BUILD_USER=osquerybuilder +ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git +RUN yum -y install git make python ruby sudo which python-argparse +RUN useradd --shell /bin/bash --create-home --user-group --groups wheel "$OSQUERY_BUILD_USER" \ + && sed -i '0,/^#\ %wheel/s/^#\ %wheel.*/%wheel\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers +RUN mkdir -p /usr/local/osquery/ \ + && chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ +USER $OSQUERY_BUILD_USER +ENV SKIP_TESTS=1 +RUN cd /home/"$OSQUERY_BUILD_USER" \ + && git clone "$OSQUERY_GIT_URL" \ + && cd osquery/ \ + && git checkout "$OSQUERY_SRC_VERSION" \ +#these homebrew hashes need to be current. hashes in osquery git repo are often out of date for the tags we check out and try to build. +#this is a problem and they are aware of it. let the magic hashes commence: + && sed -i 's,^\(HOMEBREW_CORE=\).*,\1'941ca36839ea354031846d73ad538e1e44e673f4',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_CORE=\).*,\1'abc5c5782c5850f2deff1f3d463945f90f2feaac',' tools/provision.sh \ + && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ + && make sysprep \ + && make deps \ + && make \ + && make strip +USER root +RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ + && chown -R root. /opt/osquery \ + && chmod -R 500 /opt/osquery/* \ + && mkdir -p /opt/osquery/lenses \ + && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ + && chmod -R 400 /opt/osquery/lenses/* +RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version + +#install packages that should be needed for ligbit2 compilation and successful pyinstaller run +RUN yum -y install \ + python27-devel libffi-devel openssl-devel \ + libxml2-devel libxslt-devel libjpeg-devel \ + zlib-devel make cmake python27-setuptools \ + python27-python-setuptools gcc python-devel \ + python-setuptools wget + +#libgit2 install start +#must precede pyinstaller requirements +ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz +ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 +ENV LIBGIT2_SRC_VERSION=0.26.0 +ENV LIBGIT2TEMP=/tmp/libgit2temp +RUN mkdir -p "$LIBGIT2TEMP" \ + && cd "$LIBGIT2TEMP" \ + && echo "$LIBGIT2_SRC_URL" \ + && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ + && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ + && tar xzf libgit2.tar.gz \ + && cd libgit2-"$LIBGIT2_SRC_VERSION"/ \ + && export LIBGIT2=/usr/local/ \ + && cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \ + && make \ + && make install + +#pyinstaller requirements start +#must be preceded by libgit2 install +COPY pyinstaller-requirements.txt / +#default python-pip from yum does not like upgrading itself from pip. looking for better options other than wget. +RUN wget -c https://bootstrap.pypa.io/get-pip.py \ + && yum -y install centos-release-scl \ + && yum -y install python27 \ + && chmod u+x ./get-pip.py \ + && scl enable python27 "./get-pip.py" \ + && scl enable python27 "pip -v install -r pyinstaller-requirements.txt" + +#fpm package making requirements start +RUN yum install -y rpmbuild gcc make rh-ruby23 rh-ruby23-ruby-devel \ + && scl enable rh-ruby23 "gem install --no-ri --no-rdoc fpm" + +#pyinstaller start +#commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built +#use the following variables to choose the version of hubble +ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git +ENV HUBBLE_SRC_PATH=/hubble_src +ENV _HOOK_DIR="./pkg/" +ENV _BINARY_LOG_LEVEL="INFO" +ENV _INCLUDE_PATH="" +RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ + && cd "$HUBBLE_SRC_PATH" \ + && git checkout "$HUBBLE_SRC_VERSION" \ + && cp -rf "$HUBBLE_SRC_PATH" /hubble_build +RUN mkdir /data +VOLUME /data +WORKDIR /hubble_build +ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ] +CMD [ "scl enable python27 'pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} hubble.py' \ + && cp /hubble_build/conf/osquery.conf /etc/osquery/ \ + && cp /hubble_build/conf/osquery.flags /etc/osquery/ \ +# hubble default configuration file + && cp -rf /hubble_build/conf/hubble /etc/hubble/ \ + && cp -rf /hubble_build/conf/hubble-profile.sh /etc/profile.d/ \ + && cp -pr /hubble_build/dist/hubble /opt/hubble/hubble-libs \ + && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \ + # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes + && rm -rf /opt/hubble/hubble-libs/librpm* \ +#rpm pkg start + && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d/ \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ + && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ +#during container run, if a configuration file exists in a /data copy it over the existing one so it would be +#possile to optionally include a custom one with the package + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ ; fi \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && mkdir -p usr/bin \ +#symlink to have hubble binary in path + && ln -s /opt/hubble/hubble usr/bin/hubble \ +#fpm start + && scl enable rh-ruby23 'fpm -s dir -t rpm \ + -n hubblestack \ + -v ${HUBBLE_SRC_VERSION} \ + #todo: get rid of the git dependency with static bin in the future + -d git \ + --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ + --deb-no-default-config-files \ + etc/hubble etc/osquery etc/init.d opt usr' \ +#edit to change iteration number, if necessary + && cp hubblestack-${HUBBLE_SRC_VERSION}-1.x86_64.rpm /data/hubblestack_${HUBBLE_SRC_VERSION}-1_centos6_x86_64.rpm" ] diff --git a/pkg/centos6/pyinstaller-requirements.txt b/pkg/centos6/pyinstaller-requirements.txt new file mode 100644 index 000000000..3010621f1 --- /dev/null +++ b/pkg/centos6/pyinstaller-requirements.txt @@ -0,0 +1,16 @@ +pyinstaller==3.2 # currently 3.2.1 version is not supported because of botocore exception +Crypto +pyopenssl>=16.2.0 +argparse +requests>=2.13.0 +logging +pprint +daemon +boto3 +botocore +pygit2 +salt-ssh +gitpython +pyinotify +cffi +azure From ae7bdac1de330c39a108e5e703df0829b1e020be Mon Sep 17 00:00:00 2001 From: Chandler Newby Date: Mon, 7 Aug 2017 17:29:18 -0600 Subject: [PATCH 25/50] Fix index-time extraction logic --- hubblestack/extmods/returners/splunk_nebula_return.py | 2 +- hubblestack/extmods/returners/splunk_nova_return.py | 6 +++--- hubblestack/extmods/returners/splunk_pulsar_return.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hubblestack/extmods/returners/splunk_nebula_return.py b/hubblestack/extmods/returners/splunk_nebula_return.py index 7dc3ea377..a4e8618db 100644 --- a/hubblestack/extmods/returners/splunk_nebula_return.py +++ b/hubblestack/extmods/returners/splunk_nebula_return.py @@ -143,7 +143,7 @@ def returner(ret): # Potentially add metadata fields: fields = {} for item in index_extracted_fields: - if item in payload['event'] and not isinstance(payload['event'], (list, dict, tuple)): + if item in payload['event'] and not isinstance(payload['event'][item], (list, dict, tuple)): fields[item] = str(payload['event'][item]) if fields: payload.update({'fields': fields}) diff --git a/hubblestack/extmods/returners/splunk_nova_return.py b/hubblestack/extmods/returners/splunk_nova_return.py index 90e2429d0..486aaa309 100644 --- a/hubblestack/extmods/returners/splunk_nova_return.py +++ b/hubblestack/extmods/returners/splunk_nova_return.py @@ -150,7 +150,7 @@ def returner(ret): # Potentially add metadata fields: fields = {} for item in index_extracted_fields: - if item in payload['event'] and not isinstance(payload['event'], (list, dict, tuple)): + if item in payload['event'] and not isinstance(payload['event'][item], (list, dict, tuple)): fields[item] = str(payload['event'][item]) if fields: payload.update({'fields': fields}) @@ -195,7 +195,7 @@ def returner(ret): # Potentially add metadata fields: fields = {} for item in index_extracted_fields: - if item in payload['event'] and not isinstance(payload['event'], (list, dict, tuple)): + if item in payload['event'] and not isinstance(payload['event'][item], (list, dict, tuple)): fields[item] = str(payload['event'][item]) if fields: payload.update({'fields': fields}) @@ -232,7 +232,7 @@ def returner(ret): # Potentially add metadata fields: fields = {} for item in index_extracted_fields: - if item in payload['event'] and not isinstance(payload['event'], (list, dict, tuple)): + if item in payload['event'] and not isinstance(payload['event'][item], (list, dict, tuple)): fields[item] = str(payload['event'][item]) if fields: payload.update({'fields': fields}) diff --git a/hubblestack/extmods/returners/splunk_pulsar_return.py b/hubblestack/extmods/returners/splunk_pulsar_return.py index 86071cf7e..07d94b80f 100644 --- a/hubblestack/extmods/returners/splunk_pulsar_return.py +++ b/hubblestack/extmods/returners/splunk_pulsar_return.py @@ -234,7 +234,7 @@ def returner(ret): # Potentially add metadata fields: fields = {} for item in index_extracted_fields: - if item in payload['event'] and not isinstance(payload['event'], (list, dict, tuple)): + if item in payload['event'] and not isinstance(payload['event'][item], (list, dict, tuple)): fields[item] = str(payload['event'][item]) if fields: payload.update({'fields': fields}) From 956d505734d40876366cb63a69cf3267d00ee705 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 8 Aug 2017 09:19:48 -0600 Subject: [PATCH 26/50] Fix centos6 Dockerfile to match others --- pkg/centos6/Dockerfile | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/pkg/centos6/Dockerfile b/pkg/centos6/Dockerfile index 850dfc245..3591b036c 100644 --- a/pkg/centos6/Dockerfile +++ b/pkg/centos6/Dockerfile @@ -93,7 +93,8 @@ RUN yum install -y rpmbuild gcc make rh-ruby23 rh-ruby23-ruby-devel \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_SRC_VERSION=develop +ENV HUBBLE_CHECKOUT=v2.2.0 +ENV HUBBLE_VERSION=2.2.0 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" @@ -101,7 +102,7 @@ ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ - && git checkout "$HUBBLE_SRC_VERSION" \ + && git checkout "$HUBBLE_CHECKOUT" \ && cp -rf "$HUBBLE_SRC_PATH" /hubble_build RUN mkdir /data VOLUME /data @@ -118,29 +119,29 @@ CMD [ "scl enable python27 'pyinstaller --onedir --noconfirm --log-level ${_BINA # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes && rm -rf /opt/hubble/hubble-libs/librpm* \ #rpm pkg start - && tar -cPvzf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ - && tar -xzvf /data/hubblestack-${HUBBLE_SRC_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d \ - && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/init.d/ \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system \ - && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/usr/lib/systemd/system/ \ - && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ \ + && tar -cPvzf /data/hubblestack-${HUBBLE_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system \ + && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \ #during container run, if a configuration file exists in a /data copy it over the existing one so it would be #possile to optionally include a custom one with the package - && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION}/etc/hubble/ ; fi \ - && cd /hubble_build/debbuild/hubblestack-${HUBBLE_SRC_VERSION} \ + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ ; fi \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ && mkdir -p usr/bin \ #symlink to have hubble binary in path && ln -s /opt/hubble/hubble usr/bin/hubble \ #fpm start && scl enable rh-ruby23 'fpm -s dir -t rpm \ -n hubblestack \ - -v ${HUBBLE_SRC_VERSION} \ + -v ${HUBBLE_VERSION} \ #todo: get rid of the git dependency with static bin in the future -d git \ --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr' \ #edit to change iteration number, if necessary - && cp hubblestack-${HUBBLE_SRC_VERSION}-1.x86_64.rpm /data/hubblestack_${HUBBLE_SRC_VERSION}-1_centos6_x86_64.rpm" ] + && cp hubblestack-${HUBBLE_VERSION}-1.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-1.el6.x86_64.rpm" ] From 552941e8f09f9dffc140ad843da89d5bda103d22 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 8 Aug 2017 09:22:44 -0600 Subject: [PATCH 27/50] Rev to v2.2.1 --- hubblestack/__init__.py | 2 +- pkg/amazonlinux2016.09/Dockerfile | 4 ++-- pkg/amazonlinux2017.03/Dockerfile | 4 ++-- pkg/build_debs.sh | 20 ++++++++++---------- pkg/build_rpms.sh | 22 +++++++++++----------- pkg/centos6/Dockerfile | 4 ++-- pkg/centos7/Dockerfile | 4 ++-- pkg/coreos/Dockerfile | 4 ++-- pkg/debian8/Dockerfile | 4 ++-- pkg/debian9/Dockerfile | 4 ++-- pkg/specs/hubblestack-el6.spec | 2 +- pkg/specs/hubblestack-el7.spec | 2 +- 12 files changed, 38 insertions(+), 38 deletions(-) diff --git a/hubblestack/__init__.py b/hubblestack/__init__.py index 04188a16d..36a511eca 100644 --- a/hubblestack/__init__.py +++ b/hubblestack/__init__.py @@ -1 +1 @@ -__version__ = '2.2.0' +__version__ = '2.2.1' diff --git a/pkg/amazonlinux2016.09/Dockerfile b/pkg/amazonlinux2016.09/Dockerfile index 5fad59c49..4e2ba2a30 100644 --- a/pkg/amazonlinux2016.09/Dockerfile +++ b/pkg/amazonlinux2016.09/Dockerfile @@ -88,8 +88,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.2.0 -ENV HUBBLE_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.1 +ENV HUBBLE_VERSION=2.2.1 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/amazonlinux2017.03/Dockerfile b/pkg/amazonlinux2017.03/Dockerfile index 14acb6d0f..ddff94759 100644 --- a/pkg/amazonlinux2017.03/Dockerfile +++ b/pkg/amazonlinux2017.03/Dockerfile @@ -88,8 +88,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.2.0 -ENV HUBBLE_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.1 +ENV HUBBLE_VERSION=2.2.1 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/build_debs.sh b/pkg/build_debs.sh index 531752908..926052aa8 100755 --- a/pkg/build_debs.sh +++ b/pkg/build_debs.sh @@ -22,14 +22,14 @@ mkdir -p dist bash ./init_pkg.sh -y cp ../hubble.tar.gz dist/hubble.tar.gz mv ../hubble.tar.gz build/hubble.tar.gz -mkdir build/hubblestack-2.2.0 -tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.2.0 -mkdir -p build/hubblestack-2.2.0/etc/init.d -cp ./hubble build/hubblestack-2.2.0/etc/init.d -mkdir -p build/hubblestack-2.2.0/usr/lib/systemd/system -cp ./hubble.service build/hubblestack-2.2.0/usr/lib/systemd/system -cp -f ../conf/hubble build/hubblestack-2.2.0/etc/hubble/hubble -cd build/hubblestack-2.2.0 +mkdir build/hubblestack-2.2.1 +tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.2.1 +mkdir -p build/hubblestack-2.2.1/etc/init.d +cp ./hubble build/hubblestack-2.2.1/etc/init.d +mkdir -p build/hubblestack-2.2.1/usr/lib/systemd/system +cp ./hubble.service build/hubblestack-2.2.1/usr/lib/systemd/system +cp -f ../conf/hubble build/hubblestack-2.2.1/etc/hubble/hubble +cd build/hubblestack-2.2.1 sudo apt-get install -y ruby ruby-dev rubygems gcc make sudo gem install --no-ri --no-rdoc fpm @@ -39,9 +39,9 @@ ln -s /opt/osquery/osqueryd usr/bin/osqueryd ln -s /opt/osquery/osqueryi usr/bin/osqueryi fpm -s dir -t deb \ -n hubblestack \ - -v 2.2.0-1 \ + -v 2.2.1-1 \ -d 'git' \ --config-files /etc/hubble/hubble --config-files /etc/osquery/osquery.conf \ --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr/bin -cp hubblestack_2.2.0-1_amd64.deb ../../dist/ +cp hubblestack_2.2.1-1_amd64.deb ../../dist/ diff --git a/pkg/build_rpms.sh b/pkg/build_rpms.sh index 6f9dc70f3..47f29d2ea 100755 --- a/pkg/build_rpms.sh +++ b/pkg/build_rpms.sh @@ -22,23 +22,23 @@ mkdir -p dist bash ./init_pkg.sh -y cp ../hubble.tar.gz dist/hubble.tar.gz mv ../hubble.tar.gz build/hubble.tar.gz -mkdir build/hubblestack-2.2.0 -tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.2.0 -mkdir -p build/hubblestack-2.2.0/etc/init.d -cp ./hubble build/hubblestack-2.2.0/etc/init.d -mkdir -p build/hubblestack-2.2.0/usr/lib/systemd/system -cp ./hubble.service build/hubblestack-2.2.0/usr/lib/systemd/system -cp -f ../conf/hubble build/hubblestack-2.2.0/etc/hubble/hubble +mkdir build/hubblestack-2.2.1 +tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.2.1 +mkdir -p build/hubblestack-2.2.1/etc/init.d +cp ./hubble build/hubblestack-2.2.1/etc/init.d +mkdir -p build/hubblestack-2.2.1/usr/lib/systemd/system +cp ./hubble.service build/hubblestack-2.2.1/usr/lib/systemd/system +cp -f ../conf/hubble build/hubblestack-2.2.1/etc/hubble/hubble cd build -tar -czvf hubblestack-2.2.0.tar.gz hubblestack-2.2.0/ +tar -czvf hubblestack-2.2.1.tar.gz hubblestack-2.2.1/ mkdir -p rpmbuild/{RPMS,SRPMS,BUILD,SOURCES,SPECS,tmp} -cp hubblestack-2.2.0.tar.gz rpmbuild/SOURCES/ +cp hubblestack-2.2.1.tar.gz rpmbuild/SOURCES/ cd rpmbuild cp ../../specs/* SPECS/ rpmbuild --define "_topdir $(pwd)" --define "_tmppath %{_topdir}/tmp" -ba SPECS/hubblestack-el6.spec -cp RPMS/x86_64/hubblestack-2.2.0-1.x86_64.rpm ../../dist/hubblestack-2.2.0-1.el6.x86_64.rpm +cp RPMS/x86_64/hubblestack-2.2.1-1.x86_64.rpm ../../dist/hubblestack-2.2.1-1.el6.x86_64.rpm rpmbuild --define "_topdir $(pwd)" --define "_tmppath %{_topdir}/tmp" -ba SPECS/hubblestack-el7.spec -cp RPMS/x86_64/hubblestack-2.2.0-1.x86_64.rpm ../../dist/hubblestack-2.2.0-1.el7.x86_64.rpm +cp RPMS/x86_64/hubblestack-2.2.1-1.x86_64.rpm ../../dist/hubblestack-2.2.1-1.el7.x86_64.rpm diff --git a/pkg/centos6/Dockerfile b/pkg/centos6/Dockerfile index 3591b036c..96cea6b1f 100644 --- a/pkg/centos6/Dockerfile +++ b/pkg/centos6/Dockerfile @@ -93,8 +93,8 @@ RUN yum install -y rpmbuild gcc make rh-ruby23 rh-ruby23-ruby-devel \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.2.0 -ENV HUBBLE_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.1 +ENV HUBBLE_VERSION=2.2.1 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/centos7/Dockerfile b/pkg/centos7/Dockerfile index c791f9c6f..2aec9ada4 100644 --- a/pkg/centos7/Dockerfile +++ b/pkg/centos7/Dockerfile @@ -88,8 +88,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.2.0 -ENV HUBBLE_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.1 +ENV HUBBLE_VERSION=2.2.1 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/coreos/Dockerfile b/pkg/coreos/Dockerfile index f04b2cead..1cedbafbb 100644 --- a/pkg/coreos/Dockerfile +++ b/pkg/coreos/Dockerfile @@ -83,8 +83,8 @@ RUN pip install --upgrade pip \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.2.0 -ENV HUBBLE_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.1 +ENV HUBBLE_VERSION=2.2.1 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/debian8/Dockerfile b/pkg/debian8/Dockerfile index 704ddd909..fa88227ee 100644 --- a/pkg/debian8/Dockerfile +++ b/pkg/debian8/Dockerfile @@ -93,8 +93,8 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.2.0 -ENV HUBBLE_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.1 +ENV HUBBLE_VERSION=2.2.1 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/debian9/Dockerfile b/pkg/debian9/Dockerfile index 3c17e6f68..3403e3042 100644 --- a/pkg/debian9/Dockerfile +++ b/pkg/debian9/Dockerfile @@ -87,8 +87,8 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.2.0 -ENV HUBBLE_VERSION=2.2.0 +ENV HUBBLE_CHECKOUT=v2.2.1 +ENV HUBBLE_VERSION=2.2.1 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/specs/hubblestack-el6.spec b/pkg/specs/hubblestack-el6.spec index 1190eccec..f5d0411a6 100644 --- a/pkg/specs/hubblestack-el6.spec +++ b/pkg/specs/hubblestack-el6.spec @@ -9,7 +9,7 @@ Summary: Hubblestack is a module, open-source security compliance framework Name: hubblestack -Version: 2.2.0 +Version: 2.2.1 Release: 1 License: Apache 2.0 Group: Development/Tools diff --git a/pkg/specs/hubblestack-el7.spec b/pkg/specs/hubblestack-el7.spec index f33996631..9453536f6 100644 --- a/pkg/specs/hubblestack-el7.spec +++ b/pkg/specs/hubblestack-el7.spec @@ -9,7 +9,7 @@ Summary: Hubblestack is a module, open-source security compliance framework Name: hubblestack -Version: 2.2.0 +Version: 2.2.1 Release: 1 License: Apache 2.0 Group: Development/Tools From 25b1c4a0684f1eece64c909be879ba1b1677426d Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 8 Aug 2017 11:29:25 -0600 Subject: [PATCH 28/50] Fix typo in debian 8 dockerfile --- pkg/debian8/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/debian8/Dockerfile b/pkg/debian8/Dockerfile index fa88227ee..0e2ef9e84 100644 --- a/pkg/debian8/Dockerfile +++ b/pkg/debian8/Dockerfile @@ -124,7 +124,7 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \ && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSIOHUBBLE_VERSION}/usr/lib/systemd/system \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system \ && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system/ \ && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \ #during container run, if a configuration file exists in a /data copy it over the existing one so it would be From db362741cc88f4bf0c4ffe223c8acdea9a1976dc Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 8 Aug 2017 11:48:39 -0600 Subject: [PATCH 29/50] Fix grep module match_output_regex Default to re.search with multiline Set `match_output_multiline: False` and add ^ to front of regex to get old behavior --- hubblestack/files/hubblestack_nova/grep.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hubblestack/files/hubblestack_nova/grep.py b/hubblestack/files/hubblestack_nova/grep.py index f757cf67e..eda782fa3 100644 --- a/hubblestack/files/hubblestack_nova/grep.py +++ b/hubblestack/files/hubblestack_nova/grep.py @@ -122,8 +122,12 @@ def audit(data_list, tags, debug=False, **kwargs): if tag_data['match_output'] not in grep_ret: found = False else: # match with regex - if not re.match(tag_data['match_output'], grep_ret): - found = False + if tag_data.get('match_output_multiline', True): + if not re.search(tag_data['match_output'], grep_ret, re.MULTILINE): + found = False + else: + if not re.search(tag_data['match_output'], grep_ret): + found = False if not os.path.exists(name) and 'match_on_file_missing' in tag_data: if tag_data['match_on_file_missing']: From e42e669f8989d2b96f8f3679ff413b079b3c2f33 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 8 Aug 2017 11:50:14 -0600 Subject: [PATCH 30/50] Add match_output_multiline to grep docs --- hubblestack/files/hubblestack_nova/grep.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hubblestack/files/hubblestack_nova/grep.py b/hubblestack/files/hubblestack_nova/grep.py index eda782fa3..63d3e035c 100644 --- a/hubblestack/files/hubblestack_nova/grep.py +++ b/hubblestack/files/hubblestack_nova/grep.py @@ -28,6 +28,7 @@ pattern: '/tmp' # grep pattern match_output: 'nodev' # string to check for in output of grep command (optional) match_output_regex: True # whether to use regex when matching output (default: False) + match_output_multiline: False # whether to use multiline flag for regex matching (default: True) grep_args: # extra args to grep - '-E' - '-i' From bcdfe9e5a5607fb4266ddda0317e4d9975ed9b09 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 8 Aug 2017 14:53:49 -0600 Subject: [PATCH 31/50] Fix old cffi version in debian8 --- pkg/debian8/pyinstaller-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/debian8/pyinstaller-requirements.txt b/pkg/debian8/pyinstaller-requirements.txt index 3010621f1..8a51dfda2 100644 --- a/pkg/debian8/pyinstaller-requirements.txt +++ b/pkg/debian8/pyinstaller-requirements.txt @@ -12,5 +12,5 @@ pygit2 salt-ssh gitpython pyinotify -cffi +cffi>=1.10.0 azure From bb216839f350f72ff888e98c99b623349e9b2a76 Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Tue, 8 Aug 2017 16:29:24 -0600 Subject: [PATCH 32/50] augeas lenses default now in /opt/osquery/lenses, also systemd service file removed for centos6 --- pkg/amazonlinux2016.09/Dockerfile | 6 +++++- pkg/amazonlinux2017.03/Dockerfile | 6 +++++- pkg/centos6/Dockerfile | 6 +++--- pkg/centos7/Dockerfile | 6 +++++- pkg/coreos/Dockerfile | 6 +++++- pkg/debian8/Dockerfile | 6 +++++- pkg/debian9/Dockerfile | 6 +++++- 7 files changed, 33 insertions(+), 9 deletions(-) diff --git a/pkg/amazonlinux2016.09/Dockerfile b/pkg/amazonlinux2016.09/Dockerfile index 4e2ba2a30..ae4b6ad14 100644 --- a/pkg/amazonlinux2016.09/Dockerfile +++ b/pkg/amazonlinux2016.09/Dockerfile @@ -37,6 +37,8 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \ && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ && make sysprep \ +#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep + && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \ && make deps \ && make \ && make strip @@ -44,6 +46,7 @@ USER root RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ && chown -R root. /opt/osquery \ && chmod -R 500 /opt/osquery/* \ +#put augeas lenses into the default directory that we changed earlier && mkdir -p /opt/osquery/lenses \ && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ && chmod -R 400 /opt/osquery/lenses/* @@ -63,7 +66,8 @@ ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 ENV LIBGIT2_SRC_VERSION=0.26.0 ENV LIBGIT2TEMP=/tmp/libgit2temp -RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ +RUN mkdir -p "$LIBGIT2TEMP" \ + && cd "$LIBGIT2TEMP" \ && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ && tar xzf libgit2.tar.gz \ diff --git a/pkg/amazonlinux2017.03/Dockerfile b/pkg/amazonlinux2017.03/Dockerfile index ddff94759..67b9df278 100644 --- a/pkg/amazonlinux2017.03/Dockerfile +++ b/pkg/amazonlinux2017.03/Dockerfile @@ -37,6 +37,8 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \ && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ && make sysprep \ +#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep + && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \ && make deps \ && make \ && make strip @@ -44,6 +46,7 @@ USER root RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ && chown -R root. /opt/osquery \ && chmod -R 500 /opt/osquery/* \ +#put augeas lenses into the default directory that we changed earlier && mkdir -p /opt/osquery/lenses \ && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ && chmod -R 400 /opt/osquery/lenses/* @@ -63,7 +66,8 @@ ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 ENV LIBGIT2_SRC_VERSION=0.26.0 ENV LIBGIT2TEMP=/tmp/libgit2temp -RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ +RUN mkdir -p "$LIBGIT2TEMP" \ + && cd "$LIBGIT2TEMP" \ && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ && tar xzf libgit2.tar.gz \ diff --git a/pkg/centos6/Dockerfile b/pkg/centos6/Dockerfile index 96cea6b1f..fb375ee1f 100644 --- a/pkg/centos6/Dockerfile +++ b/pkg/centos6/Dockerfile @@ -37,6 +37,8 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \ && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ && make sysprep \ +#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep + && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \ && make deps \ && make \ && make strip @@ -44,6 +46,7 @@ USER root RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ && chown -R root. /opt/osquery \ && chmod -R 500 /opt/osquery/* \ +#put augeas lenses into the default directory that we changed earlier && mkdir -p /opt/osquery/lenses \ && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ && chmod -R 400 /opt/osquery/lenses/* @@ -65,7 +68,6 @@ ENV LIBGIT2_SRC_VERSION=0.26.0 ENV LIBGIT2TEMP=/tmp/libgit2temp RUN mkdir -p "$LIBGIT2TEMP" \ && cd "$LIBGIT2TEMP" \ - && echo "$LIBGIT2_SRC_URL" \ && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ && tar xzf libgit2.tar.gz \ @@ -124,8 +126,6 @@ CMD [ "scl enable python27 'pyinstaller --onedir --noconfirm --log-level ${_BINA && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \ && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \ - && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system \ - && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system/ \ && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \ #during container run, if a configuration file exists in a /data copy it over the existing one so it would be #possile to optionally include a custom one with the package diff --git a/pkg/centos7/Dockerfile b/pkg/centos7/Dockerfile index 2aec9ada4..45e7d60be 100644 --- a/pkg/centos7/Dockerfile +++ b/pkg/centos7/Dockerfile @@ -37,6 +37,8 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \ && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ && make sysprep \ +#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep +&& sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \ && make deps \ && make \ && make strip @@ -44,6 +46,7 @@ USER root RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ && chown -R root. /opt/osquery \ && chmod -R 500 /opt/osquery/* \ +#put augeas lenses into the default directory that we changed earlier && mkdir -p /opt/osquery/lenses \ && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ && chmod -R 400 /opt/osquery/lenses/* @@ -63,7 +66,8 @@ ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 ENV LIBGIT2_SRC_VERSION=0.26.0 ENV LIBGIT2TEMP=/tmp/libgit2temp -RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ +RUN mkdir -p "$LIBGIT2TEMP" \ + && cd "$LIBGIT2TEMP" \ && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ && tar xzf libgit2.tar.gz \ diff --git a/pkg/coreos/Dockerfile b/pkg/coreos/Dockerfile index 1cedbafbb..582614730 100644 --- a/pkg/coreos/Dockerfile +++ b/pkg/coreos/Dockerfile @@ -38,6 +38,8 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \ && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ && make sysprep \ +#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep + && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \ && make deps \ && make \ && make strip @@ -45,6 +47,7 @@ USER root RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ && chown -R root. /opt/osquery \ && chmod -R 500 /opt/osquery/* \ +#put augeas lenses into the default directory that we changed earlier && mkdir -p /opt/osquery/lenses \ && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ && chmod -R 400 /opt/osquery/lenses/* @@ -64,7 +67,8 @@ ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 ENV LIBGIT2_SRC_VERSION=0.26.0 ENV LIBGIT2TEMP=/tmp/libgit2temp -RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ +RUN mkdir -p "$LIBGIT2TEMP" \ + && cd "$LIBGIT2TEMP" \ && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ && tar xzf libgit2.tar.gz \ diff --git a/pkg/debian8/Dockerfile b/pkg/debian8/Dockerfile index 0e2ef9e84..17e79cf67 100644 --- a/pkg/debian8/Dockerfile +++ b/pkg/debian8/Dockerfile @@ -41,6 +41,8 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \ && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ && make sysprep \ +#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep + && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \ && make deps \ && make \ && make strip @@ -48,6 +50,7 @@ USER root RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ && chown -R root. /opt/osquery \ && chmod -R 500 /opt/osquery/* \ +#put augeas lenses into the default directory that we changed earlier && mkdir -p /opt/osquery/lenses \ && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ && chmod -R 400 /opt/osquery/lenses/* @@ -67,7 +70,8 @@ ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 ENV LIBGIT2_SRC_VERSION=0.26.0 ENV LIBGIT2TEMP=/tmp/libgit2temp -RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ +RUN mkdir -p "$LIBGIT2TEMP" \ + && cd "$LIBGIT2TEMP" \ && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ && tar xzf libgit2.tar.gz \ diff --git a/pkg/debian9/Dockerfile b/pkg/debian9/Dockerfile index 3403e3042..694d66b62 100644 --- a/pkg/debian9/Dockerfile +++ b/pkg/debian9/Dockerfile @@ -38,6 +38,8 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \ && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ && make sysprep \ +#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep + && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \ && make deps \ && make \ && make strip @@ -45,6 +47,7 @@ USER root RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ && chown -R root. /opt/osquery \ && chmod -R 500 /opt/osquery/* \ +#put augeas lenses into the default directory that we changed earlier && mkdir -p /opt/osquery/lenses \ && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ && chmod -R 400 /opt/osquery/lenses/* @@ -64,7 +67,8 @@ ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 ENV LIBGIT2_SRC_VERSION=0.26.0 ENV LIBGIT2TEMP=/tmp/libgit2temp -RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \ +RUN mkdir -p "$LIBGIT2TEMP" \ + && cd "$LIBGIT2TEMP" \ && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ && tar xzf libgit2.tar.gz \ From d7bec7e67dd4d039ce37b11a5d0fb21c540095fe Mon Sep 17 00:00:00 2001 From: yassingh Date: Thu, 10 Aug 2017 16:48:04 +0530 Subject: [PATCH 33/50] Adding miscellaneous tests for CoreOS (#2) Adding more functions to misc.py --- hubblestack/files/hubblestack_nova/misc.py | 256 ++++++++++++++++++++- 1 file changed, 248 insertions(+), 8 deletions(-) diff --git a/hubblestack/files/hubblestack_nova/misc.py b/hubblestack/files/hubblestack_nova/misc.py index 09341a9d9..bed64a857 100644 --- a/hubblestack/files/hubblestack_nova/misc.py +++ b/hubblestack/files/hubblestack_nova/misc.py @@ -51,6 +51,7 @@ import re import salt.utils from salt.ext import six +from collections import Counter log = logging.getLogger(__name__) @@ -75,7 +76,6 @@ def audit(data_list, tags, debug=False, **kwargs): log.debug(__tags__) ret = {'Success': [], 'Failure': [], 'Controlled': []} - for tag in __tags__: if fnmatch.fnmatch(tag, tags): for tag_data in __tags__[tag]: @@ -171,19 +171,27 @@ def _execute_shell_command(cmd): ''' This function will execute passed command in /bin/shell ''' - return __salt__['cmd.run'](cmd, python_shell=True, shell='/bin/bash') + return __salt__['cmd.run'](cmd, python_shell=True, shell='/bin/bash', ignore_retcode=True) def check_all_ports_firewall_rules(reason=''): ''' Ensure firewall rule for all open ports ''' - end_open_ports = _execute_shell_command('netstat -ln | grep "Active UNIX domain sockets (only servers)" -n | cut -d ":" -f1') - start_open_ports = _execute_shell_command('netstat -ln | grep "Active Internet connections (only servers)" -n | cut -d ":" -f1') - open_ports = _execute_shell_command('netstat -ln | awk \'FNR > ' + start_open_ports + ' && FNR < ' + end_open_ports + ' && $6 == "LISTEN" {print $4}\' | sed -e "s/.*://"') - firewall_ports = _execute_shell_command('iptables -L INPUT -v -n | awk \'FNR > 2 {print $11}\' | sed -e "s/.*://"') - if set(open_ports).issubset(set(firewall_ports)): + start_open_ports = (_execute_shell_command('netstat -ln | grep "Active Internet connections (only servers)" -n | cut -d ":" -f1')).strip() + end_open_ports = (_execute_shell_command('netstat -ln | grep "Active UNIX domain sockets (only servers)" -n | cut -d ":" -f1')).strip() + open_ports = (_execute_shell_command('netstat -ln | awk \'FNR > ' + start_open_ports + ' && FNR < ' + end_open_ports + ' && $6 == "LISTEN" && $4 !~ /127.0.0.1/ {print $4}\' | sed -e "s/.*://"')).strip() + open_ports = open_ports.split('\n') if open_ports != "" else [] + firewall_ports = (_execute_shell_command('iptables -L INPUT -v -n | awk \'FNR > 2 && $11 != "" && $11 ~ /^dpt:/ {print $11}\' | sed -e "s/.*://"')).strip() + firewall_ports = firewall_ports.split('\n') if firewall_ports != "" else [] + no_firewall_ports = [] + + for open_port in open_ports: + if open_port not in firewall_ports: + no_firewall_ports.append(open_port) + + if len(no_firewall_ports) == 0: return True - return False + return str(no_firewall_ports) def check_password_fields_not_empty(reason=''): ''' @@ -278,6 +286,227 @@ def test_failure_reason(reason): ''' return reason +def test_mount_attrs(mount_name,attribute,check_type='hard'): + ''' + Ensure that a given directory is mounted with appropriate attributes + If check_type is soft, then in absence of volume, True will be returned + If check_type is hard, then in absence of volume, False will be returned + ''' + #check that the path exists on system + command = 'test -e ' + mount_name + ' ; echo $?' + output = _execute_shell_command( command) + if output.strip() == '1': + return True if check_type == "soft" else (mount_name + " folder does not exist") + + #if the path exits, proceed with following code + output = _execute_shell_command('mount | grep ' + mount_name) + if output.strip() == '': + return True if check_type == "soft" else (mount_name + " is not mounted") + elif attribute not in output: + return str(output) + else: + return True + +def check_time_synchronization(): + ''' + Ensure that some service is running to synchronize the system clock + ''' + command = 'systemctl status systemd-timesyncd ntpd | grep "Active: active (running)"' + output = _execute_shell_command( command ) + if output.strip() == '': + return "neither ntpd nor timesyncd is running" + else: + return True + + +def restrict_permissions(path,permission): + ''' + Ensure that the file permissions on path are equal or more strict than the pemissions given in argument + ''' + path_details = __salt__['file.stats'](path) + given_permission = path_details.get('mode') + given_permission = given_permission[-3:] + max_permission = str(permission) + if (_is_permission_in_limit(max_permission[0],given_permission[0]) and _is_permission_in_limit(max_permission[1],given_permission[1]) and _is_permission_in_limit(max_permission[2],given_permission[2])): + return True + return given_permission + +def _is_permission_in_limit(max_permission,given_permission): + ''' + Return true only if given_permission is not more linient that max_permission. In other words, if + r or w or x is present in given_permission but absent in max_permission, it should return False + Takes input two integer values from 0 to 7. + ''' + max_permission = int(max_permission) + given_permission = int(given_permission) + allowed_r = False + allowed_w = False + allowed_x = False + given_r = False + given_w = False + given_x = False + + if max_permission >= 4: + allowed_r = True + max_permission = max_permission - 4 + if max_permission >= 2: + allowed_w = True + max_permission = max_permission - 2 + if max_permission >= 1: + allowed_x = True + + if given_permission >= 4: + given_r = True + given_permission = given_permission - 4 + if given_permission >= 2: + given_w = True + given_permission = given_permission - 2 + if given_permission >= 1: + given_x = True + + if given_r and ( not allowed_r ): + return False + if given_w and ( not allowed_w ): + return False + if given_x and ( not allowed_x ): + return False + + return True + + +def check_path_integrity(): + ''' + Ensure that system PATH variable is not malformed. + ''' + + script = """ + if [ "`echo $PATH | grep ::`" != "" ]; then + echo "Empty Directory in PATH (::)" + fi + + if [ "`echo $PATH | grep :$`" != "" ]; then + echo "Trailing : in PATH" + fi + + p=`echo $PATH | sed -e 's/::/:/' -e 's/:$//' -e 's/:/ /g'` + set -- $p + while [ "$1" != "" ]; do + if [ "$1" = "." ]; then + echo "PATH contains ." + shift + continue + fi + + if [ -d $1 ]; then + dirperm=`ls -ldH $1 | cut -f1 -d" "` + if [ `echo $dirperm | cut -c6` != "-" ]; then + echo "Group Write permission set on directory $1" + fi + if [ `echo $dirperm | cut -c9` != "-" ]; then + echo "Other Write permission set on directory $1" + fi + dirown=`ls -ldH $1 | awk '{print $3}'` + if [ "$dirown" != "root" ] ; then + echo $1 is not owned by root + fi + else + echo $1 is not a directory + fi + shift + done + + """ + output = _execute_shell_command(script) + if output.strip() == '': + return True + else: + return output + + +def check_duplicate_uids(reason=''): + ''' + Return False if any duplicate user id exist in /etc/group file, else return True + ''' + uids = _execute_shell_command("cat /etc/passwd | cut -f3 -d\":\"").strip() + uids = uids.split('\n') if uids != "" else [] + duplicate_uids = [k for k,v in Counter(uids).items() if v>1] + if duplicate_uids is None or duplicate_uids == []: + return True + + return str(duplicate_uids) + + +def check_duplicate_gids(reason=''): + ''' + Return False if any duplicate group id exist in /etc/group file, else return True + ''' + gids = _execute_shell_command("cat /etc/group | cut -f3 -d\":\"").strip() + gids = gids.split('\n') if gids != "" else [] + duplicate_gids = [k for k,v in Counter(gids).items() if v>1] + if duplicate_gids is None or duplicate_gids == []: + return True + + return str(duplicate_gids) + + +def check_duplicate_unames(reason=''): + ''' + Return False if any duplicate user names exist in /etc/group file, else return True + ''' + unames = _execute_shell_command("cat /etc/passwd | cut -f1 -d\":\"").strip() + unames = unames.split('\n') if unames != "" else [] + duplicate_unames = [k for k,v in Counter(unames).items() if v>1] + if duplicate_unames is None or duplicate_unames == []: + return True + + return str(duplicate_unames) + + +def check_duplicate_gnames(reason=''): + ''' + Return False if any duplicate group names exist in /etc/group file, else return True + ''' + gnames = _execute_shell_command("cat /etc/group | cut -f1 -d\":\"").strip() + gnames = gnames.split('\n') if gnames != "" else [] + duplicate_gnames = [k for k,v in Counter(gnames).items() if v>1] + if duplicate_gnames is None or duplicate_gnames == []: + return True + + return str(duplicate_gnames) + + +def check_directory_files_permission(path,permission): + ''' + Check all files permission inside a directory + ''' + files_list = _execute_shell_command("find /var/log -type f").strip() + files_list = files_list.split('\n') if files_list != "" else [] + bad_permission_files = [] + for file_in_directory in files_list: + per = restrict_permissions(file_in_directory, permission) + if per is not True: + bad_permission_files += [file_in_directory + ": Bad Permission - " + per + ":"] + + if bad_permission_files == []: + return True + + return str(bad_permission_files) + + +def check_core_dumps(reason=''): + ''' + Ensure core dumps are restricted + ''' + hard_core_dump_value = _execute_shell_command("grep -R -E \"hard +core\" /etc/security/limits.conf /etc/security/limits.d/ | awk '{print $4}'").strip() + hard_core_dump_value = hard_core_dump_value.split('\n') if hard_core_dump_value != "" else [] + if '0' in hard_core_dump_value: + return True + + if hard_core_dump_value is None or hard_core_dump_value == [] or hard_core_dump_value == "": + return "'hard core' not found in any file" + + return str(hard_core_dump_value) + FUNCTION_MAP = { 'check_all_ports_firewall_rules': check_all_ports_firewall_rules, @@ -292,4 +521,15 @@ def test_failure_reason(reason): 'test_success': test_success, 'test_failure': test_failure, 'test_failure_reason': test_failure_reason, + 'test_mount_attrs' : test_mount_attrs, + 'check_path_integrity' : check_path_integrity, + 'restrict_permissions' : restrict_permissions, + 'check_time_synchronization' : check_time_synchronization, + 'check_core_dumps': check_core_dumps, + 'check_directory_files_permission': check_directory_files_permission, + 'check_duplicate_gnames': check_duplicate_gnames, + 'check_duplicate_unames': check_duplicate_unames, + 'check_duplicate_gids': check_duplicate_gids, + 'check_duplicate_uids': check_duplicate_uids, } + From dc5ebd8c5f9e1ac9127b21f109e64964805e8746 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Thu, 10 Aug 2017 11:55:39 -0600 Subject: [PATCH 34/50] Add exception logging for yaml loads in the nova_loader --- hubblestack/extmods/modules/nova_loader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hubblestack/extmods/modules/nova_loader.py b/hubblestack/extmods/modules/nova_loader.py index b43559e43..e28603bc9 100644 --- a/hubblestack/extmods/modules/nova_loader.py +++ b/hubblestack/extmods/modules/nova_loader.py @@ -1911,6 +1911,7 @@ def _load_module(self, name): data = yaml.safe_load(fh_) except Exception as exc: self.__missing_data__[name] = str(exc) + log.exception('Error loading yaml {0}'.format(fh_)) return False self.__data__[name] = data From 07994cc5e9fb74aa18871a4ec5b23c487ddf09e5 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Wed, 9 Aug 2017 10:49:14 -0600 Subject: [PATCH 35/50] Remove deb options from rpm fpm commands --- pkg/amazonlinux2016.09/Dockerfile | 1 - pkg/amazonlinux2017.03/Dockerfile | 1 - pkg/centos6/Dockerfile | 1 - pkg/centos7/Dockerfile | 1 - 4 files changed, 4 deletions(-) diff --git a/pkg/amazonlinux2016.09/Dockerfile b/pkg/amazonlinux2016.09/Dockerfile index ae4b6ad14..185f952d4 100644 --- a/pkg/amazonlinux2016.09/Dockerfile +++ b/pkg/amazonlinux2016.09/Dockerfile @@ -140,7 +140,6 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit #todo: get rid of the git dependency with static bin in the future -d 'git' \ --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ - --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr \ #edit to change iteration number, if necessary && cp hubblestack-${HUBBLE_VERSION}-1.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-1.al1609.x86_64.rpm" ] diff --git a/pkg/amazonlinux2017.03/Dockerfile b/pkg/amazonlinux2017.03/Dockerfile index 67b9df278..b0df6d4e6 100644 --- a/pkg/amazonlinux2017.03/Dockerfile +++ b/pkg/amazonlinux2017.03/Dockerfile @@ -140,7 +140,6 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit #todo: get rid of the git dependency with static bin in the future -d 'git' \ --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ - --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr \ #edit to change iteration number, if necessary && cp hubblestack-${HUBBLE_VERSION}-1.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-1.al1703.x86_64.rpm" ] diff --git a/pkg/centos6/Dockerfile b/pkg/centos6/Dockerfile index fb375ee1f..f60fedf3d 100644 --- a/pkg/centos6/Dockerfile +++ b/pkg/centos6/Dockerfile @@ -141,7 +141,6 @@ CMD [ "scl enable python27 'pyinstaller --onedir --noconfirm --log-level ${_BINA #todo: get rid of the git dependency with static bin in the future -d git \ --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ - --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr' \ #edit to change iteration number, if necessary && cp hubblestack-${HUBBLE_VERSION}-1.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-1.el6.x86_64.rpm" ] diff --git a/pkg/centos7/Dockerfile b/pkg/centos7/Dockerfile index 45e7d60be..7c55235a1 100644 --- a/pkg/centos7/Dockerfile +++ b/pkg/centos7/Dockerfile @@ -140,7 +140,6 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit #todo: get rid of the git dependency with static bin in the future -d 'git' \ --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ - --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr \ #edit to change iteration number, if necessary && cp hubblestack-${HUBBLE_VERSION}-1.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-1.el7.x86_64.rpm" ] From 22af7533fd4311a801e87304df250329f9c30671 Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Fri, 11 Aug 2017 09:32:03 -0600 Subject: [PATCH 36/50] Dockerfile for debian 7 --- pkg/debian7/Dockerfile | 166 +++++++++++++++++++++++ pkg/debian7/pyinstaller-requirements.txt | 16 +++ 2 files changed, 182 insertions(+) create mode 100644 pkg/debian7/Dockerfile create mode 100644 pkg/debian7/pyinstaller-requirements.txt diff --git a/pkg/debian7/Dockerfile b/pkg/debian7/Dockerfile new file mode 100644 index 000000000..30102c974 --- /dev/null +++ b/pkg/debian7/Dockerfile @@ -0,0 +1,166 @@ +# This Dockerfile aims to make building Hubble v2 packages easier. +# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile +# 2. docker build -t . +# The resulting image is ready to run the pyinstaller on container start and drop hubble-coreos.tar.gz +# in the /data directory. Mount /data volume into a directory on the host to access the package. +# To run the container: docker run -it --rm -v `pwd`:/data + +FROM debian:7 + +RUN apt-get update \ + && apt-get -y upgrade + +#paths that hubble or hubble parts need in the package +RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/osquery + +#osquery build start +#osquery should be built first since requirements for other packages can interfere with osquery dependencies +#to build, osquery scripts want sudo and a user to sudo with. +#to pin to a different version change the following envirnment variable +ENV OSQUERY_SRC_VERSION=2.6.0 +ENV OSQUERY_BUILD_USER=osquerybuilder +ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git +RUN apt-get -y install git make python ruby sudo locales +RUN useradd --shell /bin/bash --create-home --user-group --groups sudo "$OSQUERY_BUILD_USER" \ + && sed -i 's/^%sudo.*/%sudo\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers +RUN mkdir -p /usr/local/osquery/ \ + && chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ \ + && echo "LC_ALL=en_US.UTF-8" >> /etc/default/locale \ + && sed -i '/en_US.UTF-8\ UTF-8/s/^#//' /etc/locale.gen \ + && locale-gen +USER $OSQUERY_BUILD_USER +ENV SKIP_TESTS=1 +RUN cd /home/"$OSQUERY_BUILD_USER" \ + && git clone "$OSQUERY_GIT_URL" \ + && cd osquery/ \ + && git checkout "$OSQUERY_SRC_VERSION" \ +#these homebrew hashes need to be current. hashes in osquery git repo are often out of date for the tags we check out and try to build. +#this is a problem and they are aware of it. let the magic hashes commence: + && sed -i 's,^\(HOMEBREW_CORE=\).*,\1'941ca36839ea354031846d73ad538e1e44e673f4',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_CORE=\).*,\1'abc5c5782c5850f2deff1f3d463945f90f2feaac',' tools/provision.sh \ + && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \ + && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ + && make sysprep \ +#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep + && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \ + && make deps \ + && make \ + && make strip +USER root +RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \ + && chown -R root. /opt/osquery \ + && chmod -R 500 /opt/osquery/* \ +#put augeas lenses into the default directory that we changed earlier + && mkdir -p /opt/osquery/lenses \ + && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \ + && chmod -R 400 /opt/osquery/lenses/* +RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version + +#install packages that should be needed for ligbit2 compilation and successful pyinstaller run +RUN apt-get update \ + && apt-get -y install \ + python-dev libffi-dev libssl-dev libyaml-dev \ + libxml2-dev libxslt1-dev zlib1g-dev python-setuptools \ + gcc wget openssl \ + && apt-get clean + +#cmake install start +#must precede libgit2 due to old version of cmake included with debian7 that has openssl version parsing bug +#https://cmake.org/gitweb?p=cmake.git;a=commitdiff;h=c5d9a8283cfac15b4a5a07f18d5eb10c1f388505 +ENV CMAKE_SRC_URL=https://cmake.org/files/v3.9/cmake-3.9.1.tar.gz +ENV CMAKE_SRC_SHA256=d768ee83d217f91bb597b3ca2ac663da7a8603c97e1f1a5184bc01e0ad2b12bb +ENV CMAKE_SRC_VERSION=3.9.1 +ENV CMAKE_TEMP=/tmp/cmaketemp +RUN mkdir -p "$CMAKE_TEMP" \ + && cd "$CMAKE_TEMP" \ + && wget -q "$CMAKE_SRC_URL" -O cmake.src.tar.gz \ + && echo "$CMAKE_SRC_SHA256 cmake.src.tar.gz" | sha256sum -c - \ + && tar xzf cmake.src.tar.gz \ + && cd cmake-"$CMAKE_SRC_VERSION" \ + && ./bootstrap \ + && make \ + && make install + +#libgit2 install start +#must be preceded by cmake install +#must precede pyinstaller requirements +ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz +ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9 +ENV LIBGIT2_SRC_VERSION=0.26.0 +ENV LIBGIT2TEMP=/tmp/libgit2temp +RUN mkdir -p "$LIBGIT2TEMP" \ + && cd "$LIBGIT2TEMP" \ + && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \ + && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ + && tar xzf libgit2.tar.gz \ + && cd libgit2-"$LIBGIT2_SRC_VERSION"/ \ + && export LIBGIT2=/usr/local/ \ + && cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \ + && make \ + && make install + +#pyinstaller requirements start +#must be preceded by libgit2 install +COPY pyinstaller-requirements.txt / +RUN wget -c https://bootstrap.pypa.io/get-pip.py \ + && python get-pip.py \ + && pip install --upgrade pycparser \ + && pip -v install -r pyinstaller-requirements.txt + +#deb package making requirements start +RUN apt-get install -y ruby ruby-dev rubygems gcc make \ + && gem install --no-ri --no-rdoc fpm + +#pyinstaller start +#commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built +#use the following variables to choose the version of hubble +ENV HUBBLE_CHECKOUT=v2.2.1 +ENV HUBBLE_VERSION=2.2.1 +ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git +ENV HUBBLE_SRC_PATH=/hubble_src +ENV _HOOK_DIR="./pkg/" +ENV _BINARY_LOG_LEVEL="INFO" +ENV _INCLUDE_PATH="" +RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ + && cd "$HUBBLE_SRC_PATH" \ + && git checkout "$HUBBLE_CHECKOUT" \ + && cp -rf "$HUBBLE_SRC_PATH" /hubble_build +RUN mkdir /data +VOLUME /data +WORKDIR /hubble_build +ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ] +CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} hubble.py \ + && cp /hubble_build/conf/osquery.conf /etc/osquery/ \ + && cp /hubble_build/conf/osquery.flags /etc/osquery/ \ +# hubble default configuration file + && cp -rf /hubble_build/conf/hubble /etc/hubble/ \ + && cp -rf /hubble_build/conf/hubble-profile.sh /etc/profile.d/ \ + && cp -pr /hubble_build/dist/hubble /opt/hubble/hubble-libs \ + && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \ + # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes + && rm -rf /opt/hubble/hubble-libs/librpm* \ +#debian pkg start + && tar -cPvzf /data/hubblestack-${HUBBLE_VERSION}.tar.gz /etc/hubble /etc/osquery /opt/hubble /opt/osquery /var/log/osquery /etc/profile.d/hubble-profile.sh \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \ + && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \ + && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \ +#during container run, if a configuration file exists in a /data copy it over the existing one so it would be +#possile to optionally include a custom one with the package + && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ ; fi \ + && cd /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \ + && mkdir -p usr/bin \ +#symlink to have hubble binary in path + && ln -s /opt/hubble/hubble usr/bin/hubble \ +#fpm start + && fpm -s dir -t deb \ + -n hubblestack \ + -v ${HUBBLE_VERSION}-1 \ + #todo: get rid of the git dependency with static bin in the future + -d 'git' \ + --config-files /etc/hubble --config-files /etc/osquery/osquery.conf \ + --deb-no-default-config-files \ + etc/hubble etc/osquery etc/init.d opt usr \ + && cp hubblestack_${HUBBLE_VERSION}-1_amd64.deb /data/hubblestack_${HUBBLE_VERSION}-1deb7_amd64.deb" ] + diff --git a/pkg/debian7/pyinstaller-requirements.txt b/pkg/debian7/pyinstaller-requirements.txt new file mode 100644 index 000000000..3010621f1 --- /dev/null +++ b/pkg/debian7/pyinstaller-requirements.txt @@ -0,0 +1,16 @@ +pyinstaller==3.2 # currently 3.2.1 version is not supported because of botocore exception +Crypto +pyopenssl>=16.2.0 +argparse +requests>=2.13.0 +logging +pprint +daemon +boto3 +botocore +pygit2 +salt-ssh +gitpython +pyinotify +cffi +azure From ddf064a3464090eac8201ed869163b7c8d0ab6a0 Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Fri, 11 Aug 2017 09:35:47 -0600 Subject: [PATCH 37/50] fix pygit2 in debian 8 --- pkg/debian8/Dockerfile | 4 +--- pkg/debian8/pyinstaller-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/debian8/Dockerfile b/pkg/debian8/Dockerfile index 17e79cf67..c89b5e727 100644 --- a/pkg/debian8/Dockerfile +++ b/pkg/debian8/Dockerfile @@ -59,8 +59,7 @@ RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version #install packages that should be needed for ligbit2 compilation and successful pyinstaller run RUN apt-get -y install \ python-dev libffi-dev libssl-dev libyaml-dev \ - libxml2-dev libxslt1-dev python-cffi \ - zlib1g-dev cmake python-setuptools \ + libxml2-dev libxslt1-dev zlib1g-dev cmake python-setuptools \ gcc wget openssl \ && apt-get clean @@ -84,7 +83,6 @@ RUN mkdir -p "$LIBGIT2TEMP" \ #pyinstaller requirements start #must be preceded by libgit2 install COPY pyinstaller-requirements.txt / -RUN ldconfig #default python-pip from yum does not like upgrading itself from pip. looking for better options other than wget. RUN wget -c https://bootstrap.pypa.io/get-pip.py \ && python get-pip.py \ diff --git a/pkg/debian8/pyinstaller-requirements.txt b/pkg/debian8/pyinstaller-requirements.txt index 8a51dfda2..3010621f1 100644 --- a/pkg/debian8/pyinstaller-requirements.txt +++ b/pkg/debian8/pyinstaller-requirements.txt @@ -12,5 +12,5 @@ pygit2 salt-ssh gitpython pyinotify -cffi>=1.10.0 +cffi azure From aa628401f85ed0e44d71d3bf4c4bc3d761262306 Mon Sep 17 00:00:00 2001 From: Yuan Liu Date: Sat, 12 Aug 2017 08:02:14 -0700 Subject: [PATCH 38/50] Add instructions to use local files as profile repo On branch documentation modified: conf/hubble --- conf/hubble | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conf/hubble b/conf/hubble index 6055f5ebb..a28e71098 100644 --- a/conf/hubble +++ b/conf/hubble @@ -9,6 +9,12 @@ gitfs_remotes: - https://github.com/hubblestack/hubblestack_data.git +## To use locally stored hubblestack_data, comment out gitfs_remote section, +## then uncomment the following +# file_roots: +# base: +# - /srv/salt/hubblestack_data +## In the above example, hubblestack_data is cloned into /srv/salt/. fileserver_backend: - roots - git From 72106e66dbd54299b14ca40224b6570be5f3c813 Mon Sep 17 00:00:00 2001 From: Andres Martinson Date: Mon, 14 Aug 2017 09:20:40 -0600 Subject: [PATCH 39/50] pygit2 fixes, minor comment wording changes --- pkg/amazonlinux2016.09/Dockerfile | 9 ++++----- pkg/amazonlinux2017.03/Dockerfile | 9 ++++----- pkg/centos6/Dockerfile | 10 ++++------ pkg/centos7/Dockerfile | 16 +++++++--------- 4 files changed, 19 insertions(+), 25 deletions(-) diff --git a/pkg/amazonlinux2016.09/Dockerfile b/pkg/amazonlinux2016.09/Dockerfile index 185f952d4..a0d38c8d6 100644 --- a/pkg/amazonlinux2016.09/Dockerfile +++ b/pkg/amazonlinux2016.09/Dockerfile @@ -57,8 +57,7 @@ RUN yum -y install \ python27-devel libffi-devel openssl-devel \ libxml2-devel libxslt-devel libjpeg-devel \ zlib-devel make cmake python27-setuptools \ - python27-python-setuptools gcc python-devel \ - python-setuptools wget + gcc python-devel python-setuptools wget #libgit2 install start #must precede pyinstaller requirements @@ -72,8 +71,7 @@ RUN mkdir -p "$LIBGIT2TEMP" \ && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ && tar xzf libgit2.tar.gz \ && cd libgit2-"$LIBGIT2_SRC_VERSION"/ \ - && export LIBGIT2=/usr/local/ \ - && cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \ + && cmake . -DCMAKE_INSTALL_PREFIX=/usr/local/ \ && make \ && make install @@ -85,7 +83,7 @@ RUN wget -c https://bootstrap.pypa.io/get-pip.py \ && python get-pip.py \ && pip -v install -r pyinstaller-requirements.txt -#deb package making requirements start +#fpm package making requirements start RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ && gem install --no-ri --no-rdoc fpm @@ -99,6 +97,7 @@ ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" +ENV LD_LIBRARY_PATH=/lib:/lib64:/usr/lib:/usr/lib64:/usr/local/lib:/usr/local/lib64 RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ && git checkout "$HUBBLE_CHECKOUT" \ diff --git a/pkg/amazonlinux2017.03/Dockerfile b/pkg/amazonlinux2017.03/Dockerfile index b0df6d4e6..61790935c 100644 --- a/pkg/amazonlinux2017.03/Dockerfile +++ b/pkg/amazonlinux2017.03/Dockerfile @@ -57,8 +57,7 @@ RUN yum -y install \ python27-devel libffi-devel openssl-devel \ libxml2-devel libxslt-devel libjpeg-devel \ zlib-devel make cmake python27-setuptools \ - python27-python-setuptools gcc python-devel \ - python-setuptools wget + gcc python-devel python-setuptools wget #libgit2 install start #must precede pyinstaller requirements @@ -72,8 +71,7 @@ RUN mkdir -p "$LIBGIT2TEMP" \ && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ && tar xzf libgit2.tar.gz \ && cd libgit2-"$LIBGIT2_SRC_VERSION"/ \ - && export LIBGIT2=/usr/local/ \ - && cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \ + && cmake . -DCMAKE_INSTALL_PREFIX=/usr/local/ \ && make \ && make install @@ -85,7 +83,7 @@ RUN wget -c https://bootstrap.pypa.io/get-pip.py \ && python get-pip.py \ && pip -v install -r pyinstaller-requirements.txt -#deb package making requirements start +#fpm package making requirements start RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ && gem install --no-ri --no-rdoc fpm @@ -99,6 +97,7 @@ ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" +ENV LD_LIBRARY_PATH=/lib:/lib64:/usr/lib:/usr/lib64:/usr/local/lib:/usr/local/lib64 RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ && git checkout "$HUBBLE_CHECKOUT" \ diff --git a/pkg/centos6/Dockerfile b/pkg/centos6/Dockerfile index f60fedf3d..327d2582d 100644 --- a/pkg/centos6/Dockerfile +++ b/pkg/centos6/Dockerfile @@ -54,10 +54,8 @@ RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version #install packages that should be needed for ligbit2 compilation and successful pyinstaller run RUN yum -y install \ - python27-devel libffi-devel openssl-devel \ - libxml2-devel libxslt-devel libjpeg-devel \ - zlib-devel make cmake python27-setuptools \ - python27-python-setuptools gcc python-devel \ + libffi-devel openssl-devel libxml2-devel libxslt-devel \ + libjpeg-devel zlib-devel python-devel make cmake gcc \ python-setuptools wget #libgit2 install start @@ -72,8 +70,7 @@ RUN mkdir -p "$LIBGIT2TEMP" \ && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ && tar xzf libgit2.tar.gz \ && cd libgit2-"$LIBGIT2_SRC_VERSION"/ \ - && export LIBGIT2=/usr/local/ \ - && cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \ + && cmake . -DCMAKE_INSTALL_PREFIX=/usr/local/ \ && make \ && make install @@ -102,6 +99,7 @@ ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" +ENV LD_LIBRARY_PATH=/lib:/lib64:/usr/lib:/usr/lib64:/usr/local/lib:/usr/local/lib64 RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ && git checkout "$HUBBLE_CHECKOUT" \ diff --git a/pkg/centos7/Dockerfile b/pkg/centos7/Dockerfile index 7c55235a1..e61b6d84c 100644 --- a/pkg/centos7/Dockerfile +++ b/pkg/centos7/Dockerfile @@ -38,7 +38,7 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \ && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \ && make sysprep \ #have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep -&& sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \ + && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \ && make deps \ && make \ && make strip @@ -54,11 +54,9 @@ RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version #install packages that should be needed for ligbit2 compilation and successful pyinstaller run RUN yum -y install \ - python27-devel libffi-devel openssl-devel \ - libxml2-devel libxslt-devel libjpeg-devel \ - zlib-devel make cmake python27-setuptools \ - python27-python-setuptools gcc python-devel \ - python-setuptools wget + libffi-devel openssl-devel libffi \ + libxml2-devel libxslt-devel libjpeg-devel zlib-devel \ + make cmake gcc python-devel python-setuptools wget #libgit2 install start #must precede pyinstaller requirements @@ -72,8 +70,7 @@ RUN mkdir -p "$LIBGIT2TEMP" \ && echo "$LIBGIT2_SRC_SHA256 libgit2.tar.gz" | sha256sum -c - \ && tar xzf libgit2.tar.gz \ && cd libgit2-"$LIBGIT2_SRC_VERSION"/ \ - && export LIBGIT2=/usr/local/ \ - && cmake . -DCMAKE_INSTALL_PREFIX=$LIBGIT2 \ + && cmake . -DCMAKE_INSTALL_PREFIX=/usr/local/ \ && make \ && make install @@ -85,7 +82,7 @@ RUN wget -c https://bootstrap.pypa.io/get-pip.py \ && python get-pip.py \ && pip -v install -r pyinstaller-requirements.txt -#deb package making requirements start +#fpm package making requirements start RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \ && gem install --no-ri --no-rdoc fpm @@ -99,6 +96,7 @@ ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" ENV _BINARY_LOG_LEVEL="INFO" ENV _INCLUDE_PATH="" +ENV LD_LIBRARY_PATH=/lib:/lib64:/usr/lib:/usr/lib64:/usr/local/lib:/usr/local/lib64 RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \ && cd "$HUBBLE_SRC_PATH" \ && git checkout "$HUBBLE_CHECKOUT" \ From 5515a6a00cfa28c4cb4c01f2bec5a0789499f1db Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 14 Aug 2017 11:20:24 -0600 Subject: [PATCH 40/50] Add `continue` to prevent errors from preventing misc.py executions --- hubblestack/files/hubblestack_nova/misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hubblestack/files/hubblestack_nova/misc.py b/hubblestack/files/hubblestack_nova/misc.py index bed64a857..dc645625c 100644 --- a/hubblestack/files/hubblestack_nova/misc.py +++ b/hubblestack/files/hubblestack_nova/misc.py @@ -91,6 +91,7 @@ def audit(data_list, tags, debug=False, **kwargs): ret['Errors'] = [] ret['Errors'].append({tag: 'No function {0} found' .format(tag_data['function'])}) + continue args = tag_data.get('args', []) kwargs = tag_data.get('kwargs', {}) From 45f9c929d07ae7a27c066a044d19d23e3b6af526 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 14 Aug 2017 14:37:24 -0600 Subject: [PATCH 41/50] Disable all the troublesome boto modules --- hubblestack/daemon.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/hubblestack/daemon.py b/hubblestack/daemon.py index 233bab346..6843d3a48 100644 --- a/hubblestack/daemon.py +++ b/hubblestack/daemon.py @@ -281,6 +281,38 @@ def load_config(): salt.config.DEFAULT_MINION_OPTS['fileserver_update_frequency'] = 43200 # 12 hours salt.config.DEFAULT_MINION_OPTS['scheduler_sleep_frequency'] = 0.5 salt.config.DEFAULT_MINION_OPTS['default_include'] = 'hubble.d/*.conf' + salt.config.DEFAULT_MINION_OPTS['disable_modules'] = [ + 'boto3_elasticache', + 'boto3_route53', + 'boto_apigateway', + 'boto_asg', + 'boto_cfn', + 'boto_cloudtrail', + 'boto_cloudwatch_event', + 'boto_cloudwatch', + 'boto_cognitoidentity', + 'boto_datapipeline', + 'boto_dynamodb', + 'boto_ec2', + 'boto_efs', + 'boto_elasticache', + 'boto_elasticsearch_domain', + 'boto_elb', + 'boto_elbv2', + 'boto_iam', + 'boto_iot', + 'boto_kinesis', + 'boto_kms', + 'boto_lambda', + 'boto_rds', + 'boto_route53', + 'boto_s3_bucket', + 'boto_secgroup', + 'boto_sns', + 'boto_sqs', + 'boto_vpc', + ] + global __opts__ global __grains__ From 0e2d21486711dc3d5e906e12a7eea7d88ba2e0b1 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 14 Aug 2017 14:42:07 -0600 Subject: [PATCH 42/50] Use extend to play well with user config --- hubblestack/daemon.py | 66 ++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/hubblestack/daemon.py b/hubblestack/daemon.py index 6843d3a48..ac6af9f8c 100644 --- a/hubblestack/daemon.py +++ b/hubblestack/daemon.py @@ -281,38 +281,6 @@ def load_config(): salt.config.DEFAULT_MINION_OPTS['fileserver_update_frequency'] = 43200 # 12 hours salt.config.DEFAULT_MINION_OPTS['scheduler_sleep_frequency'] = 0.5 salt.config.DEFAULT_MINION_OPTS['default_include'] = 'hubble.d/*.conf' - salt.config.DEFAULT_MINION_OPTS['disable_modules'] = [ - 'boto3_elasticache', - 'boto3_route53', - 'boto_apigateway', - 'boto_asg', - 'boto_cfn', - 'boto_cloudtrail', - 'boto_cloudwatch_event', - 'boto_cloudwatch', - 'boto_cognitoidentity', - 'boto_datapipeline', - 'boto_dynamodb', - 'boto_ec2', - 'boto_efs', - 'boto_elasticache', - 'boto_elasticsearch_domain', - 'boto_elb', - 'boto_elbv2', - 'boto_iam', - 'boto_iot', - 'boto_kinesis', - 'boto_kms', - 'boto_lambda', - 'boto_rds', - 'boto_route53', - 'boto_s3_bucket', - 'boto_secgroup', - 'boto_sns', - 'boto_sqs', - 'boto_vpc', - ] - global __opts__ global __grains__ @@ -357,6 +325,40 @@ def load_config(): if 'roots' not in __opts__['fileserver_backend']: __opts__['fileserver_backend'].append('roots') + disable_modules = __opts__.get('disable_modules', []) + disable_modules.extend([ + 'boto3_elasticache', + 'boto3_route53', + 'boto_apigateway', + 'boto_asg', + 'boto_cfn', + 'boto_cloudtrail', + 'boto_cloudwatch_event', + 'boto_cloudwatch', + 'boto_cognitoidentity', + 'boto_datapipeline', + 'boto_dynamodb', + 'boto_ec2', + 'boto_efs', + 'boto_elasticache', + 'boto_elasticsearch_domain', + 'boto_elb', + 'boto_elbv2', + 'boto_iam', + 'boto_iot', + 'boto_kinesis', + 'boto_kms', + 'boto_lambda', + 'boto_rds', + 'boto_route53', + 'boto_s3_bucket', + 'boto_secgroup', + 'boto_sns', + 'boto_sqs', + 'boto_vpc', + ]) + __opts__['disable_modules'] = disable_modules + # Setup logging salt.log.setup.setup_console_logger(__opts__['log_level']) salt.log.setup.setup_logfile_logger(__opts__['log_file'], From 46157ce6d6af471b15b871078b936dd89615cea4 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 14 Aug 2017 14:42:56 -0600 Subject: [PATCH 43/50] Add a comment --- hubblestack/daemon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hubblestack/daemon.py b/hubblestack/daemon.py index ac6af9f8c..cbe2a1399 100644 --- a/hubblestack/daemon.py +++ b/hubblestack/daemon.py @@ -325,6 +325,7 @@ def load_config(): if 'roots' not in __opts__['fileserver_backend']: __opts__['fileserver_backend'].append('roots') + # Disable all of salt's boto modules, they give nothing but trouble to the loader disable_modules = __opts__.get('disable_modules', []) disable_modules.extend([ 'boto3_elasticache', From ce77563a41d229f29d907bd6c7b5a7decd594439 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 14 Aug 2017 21:33:00 +0000 Subject: [PATCH 44/50] changes added for top() in pulsar.py --- hubblestack/extmods/modules/pulsar.py | 34 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/hubblestack/extmods/modules/pulsar.py b/hubblestack/extmods/modules/pulsar.py index 050c723fd..14c3a0b98 100644 --- a/hubblestack/extmods/modules/pulsar.py +++ b/hubblestack/extmods/modules/pulsar.py @@ -396,24 +396,26 @@ def _dict_update(dest, upd, recursive_update=True, merge_lists=False): dest[k] = upd[k] return dest -def top(topfile='/root/myhubblefiles/top.pulsar', +#TODO:change paths for top.pulsar and config yaml to 'salt://hubblestack_pulsar/top.file' + +def top(topfile='/root/myhubblefiles/topfile/top.pulsar', debug=None): - results = [] + results_pulsar = [] - # Get a list of yaml to run - top_data = _get_top_data(topfile) + configs = get_top_data(topfile) - data_by_tag = {} - for data in top_data: - results.extend(process(configfile=data[0],verbose=False)) + if not isinstance(configs, list): + configs = configs.split(',') - return results + configs = ['/root/myhubblefiles/topfile/' + config + '.yaml' + for config in configs] -def _get_top_data(topfile): - ''' - Helper method to retrieve and parse the nova topfile - ''' + results_pulsar.extend(process(configs, verbose=False)) + + return results_pulsar + +def get_top_data(topfile): try: with open(topfile) as handle: topdata = yaml.safe_load(handle) @@ -422,11 +424,15 @@ def _get_top_data(topfile): if not isinstance(topdata, dict) or 'pulsar' not in topdata or \ not(isinstance(topdata['pulsar'], dict)): - raise CommandExecutionError('pulsar topfile not formatted correctly') + raise CommandExecutionError('Pulsar topfile not formatted correctly') topdata = topdata['pulsar'] - ret = topdata.values() + ret = [] + + for match, data in topdata.iteritems(): + if __salt__['match.compound'](match): + ret.extend(data) return ret From d721499d04e422b01abfe01b763d6d7d62c9b2b0 Mon Sep 17 00:00:00 2001 From: Yuan Liu Date: Tue, 15 Aug 2017 08:44:00 -0700 Subject: [PATCH 45/50] Add instruction to find pre-built packages On branch documentation modified: README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8ce8902ae..ab3943215 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,10 @@ cd hubble/pkg Package will be in the `hubble/pkg/dist/` directory. There's no guarantee of glibc compatibility. +## Using released packages + +Various pre-built packages targeting several popular operating systems can be found under [Releases](/hubblestack/hubble/releases). + # Usage A config template has been placed in `/etc/hubble/hubble`. Modify it to your From da97fef07b70810a5d7769fb168d0c56560b8eff Mon Sep 17 00:00:00 2001 From: root Date: Wed, 16 Aug 2017 17:00:06 +0000 Subject: [PATCH 46/50] changed the default paths for topfile --- hubblestack/extmods/modules/pulsar.py | 24 +++++++---------- hubblestack/extmods/modules/win_pulsar.py | 32 +++++++++++++++++++++++ 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/hubblestack/extmods/modules/pulsar.py b/hubblestack/extmods/modules/pulsar.py index 14c3a0b98..4432fa0f5 100644 --- a/hubblestack/extmods/modules/pulsar.py +++ b/hubblestack/extmods/modules/pulsar.py @@ -76,7 +76,7 @@ def _get_notifier(): return __context__['pulsar.notifier'] -def process(configfile, +def process(configfile='salt://hubblestack_pulsar/hubblestack_pulsar_config.yaml', verbose=False): ''' Watch the configured files @@ -396,26 +396,21 @@ def _dict_update(dest, upd, recursive_update=True, merge_lists=False): dest[k] = upd[k] return dest -#TODO:change paths for top.pulsar and config yaml to 'salt://hubblestack_pulsar/top.file' - -def top(topfile='/root/myhubblefiles/topfile/top.pulsar', - debug=None): - - results_pulsar = [] +def top(topfile='salt://hubblestack_pulsar/top.file', + verbose=False): configs = get_top_data(topfile) - if not isinstance(configs, list): - configs = configs.split(',') - - configs = ['/root/myhubblefiles/topfile/' + config + '.yaml' + configs = ['salt://hubblestack_pulsar/' + config + '.yaml' for config in configs] - results_pulsar.extend(process(configs, verbose=False)) - - return results_pulsar + return process(configs, verbose=verbose) def get_top_data(topfile): + + + topfile = os.path.join(_hubble_dir()[1], topfile) + try: with open(topfile) as handle: topdata = yaml.safe_load(handle) @@ -435,4 +430,3 @@ def get_top_data(topfile): ret.extend(data) return ret - diff --git a/hubblestack/extmods/modules/win_pulsar.py b/hubblestack/extmods/modules/win_pulsar.py index c9c0c08ef..df6c41a47 100644 --- a/hubblestack/extmods/modules/win_pulsar.py +++ b/hubblestack/extmods/modules/win_pulsar.py @@ -546,3 +546,35 @@ def _dict_update(dest, upd, recursive_update=True, merge_lists=False): for k in upd: dest[k] = upd[k] return dest + +def top(topfile='salt://hubblestack_pulsar/win_top.pulsar', + verbose=False): + + configs = get_top_data(topfile) + + configs = ['salt://hubblestack_pulsar/' + config + '.yaml' + for config in configs] + + return process(configs, verbose=verbose) + +def get_top_data(topfile): + try: + with open(topfile) as handle: + topdata = yaml.safe_load(handle) + except Exception as e: + raise CommandExecutionError('Could not load topfile: {0}'.format(e)) + + if not isinstance(topdata, dict) or 'pulsar' not in topdata or \ + not(isinstance(topdata['pulsar'], dict)): + raise CommandExecutionError('Pulsar topfile not formatted correctly') + + topdata = topdata['pulsar'] + + ret = [] + + for match, data in topdata.iteritems(): + if __salt__['match.compound'](match): + ret.extend(data) + + return ret + From 46ffb8d19aa083ab5913c415a2df192a056a3f30 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 16 Aug 2017 19:41:48 +0000 Subject: [PATCH 47/50] changes added to replace '.' by '/' to support subdirectories --- hubblestack/extmods/modules/pulsar.py | 9 +++++---- hubblestack/extmods/modules/win_pulsar.py | 7 ++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/hubblestack/extmods/modules/pulsar.py b/hubblestack/extmods/modules/pulsar.py index 4432fa0f5..ddfbfbd13 100644 --- a/hubblestack/extmods/modules/pulsar.py +++ b/hubblestack/extmods/modules/pulsar.py @@ -396,20 +396,21 @@ def _dict_update(dest, upd, recursive_update=True, merge_lists=False): dest[k] = upd[k] return dest -def top(topfile='salt://hubblestack_pulsar/top.file', + +def top(topfile='salt://hubblestack_pulsar/top.pulsar', verbose=False): configs = get_top_data(topfile) - configs = ['salt://hubblestack_pulsar/' + config + '.yaml' + configs = ['salt://hubblestack_pulsar/' + config.replace('.', '/') + '.yaml' for config in configs] return process(configs, verbose=verbose) -def get_top_data(topfile): +def get_top_data(topfile): - topfile = os.path.join(_hubble_dir()[1], topfile) + topfile = __salt['cp.cache_file'](topfile) try: with open(topfile) as handle: diff --git a/hubblestack/extmods/modules/win_pulsar.py b/hubblestack/extmods/modules/win_pulsar.py index df6c41a47..14ff52b22 100644 --- a/hubblestack/extmods/modules/win_pulsar.py +++ b/hubblestack/extmods/modules/win_pulsar.py @@ -547,17 +547,22 @@ def _dict_update(dest, upd, recursive_update=True, merge_lists=False): dest[k] = upd[k] return dest + def top(topfile='salt://hubblestack_pulsar/win_top.pulsar', verbose=False): configs = get_top_data(topfile) - configs = ['salt://hubblestack_pulsar/' + config + '.yaml' + configs = ['salt://hubblestack_pulsar/' + config.replace('.','/') + '.yaml' for config in configs] return process(configs, verbose=verbose) + def get_top_data(topfile): + + topfile = __salt__['cp.cache_file'](topfile) + try: with open(topfile) as handle: topdata = yaml.safe_load(handle) From 19b5d3c1e0553ccdb9e8252a14200fd180341a2d Mon Sep 17 00:00:00 2001 From: root Date: Wed, 16 Aug 2017 19:45:56 +0000 Subject: [PATCH 48/50] fixed typo --- hubblestack/extmods/modules/pulsar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hubblestack/extmods/modules/pulsar.py b/hubblestack/extmods/modules/pulsar.py index ddfbfbd13..464bff541 100644 --- a/hubblestack/extmods/modules/pulsar.py +++ b/hubblestack/extmods/modules/pulsar.py @@ -410,7 +410,7 @@ def top(topfile='salt://hubblestack_pulsar/top.pulsar', def get_top_data(topfile): - topfile = __salt['cp.cache_file'](topfile) + topfile = __salt__['cp.cache_file'](topfile) try: with open(topfile) as handle: From fce9dcf43a1fef06784eedbeda0f00ffe88e3c87 Mon Sep 17 00:00:00 2001 From: Vishal Srivastav Date: Thu, 17 Aug 2017 15:23:27 +0530 Subject: [PATCH 49/50] Adding new functions in misc module * Adding new functions in misc module --- hubblestack/files/hubblestack_nova/misc.py | 297 ++++++++++++++++++++- 1 file changed, 292 insertions(+), 5 deletions(-) diff --git a/hubblestack/files/hubblestack_nova/misc.py b/hubblestack/files/hubblestack_nova/misc.py index dc645625c..bb50ecfed 100644 --- a/hubblestack/files/hubblestack_nova/misc.py +++ b/hubblestack/files/hubblestack_nova/misc.py @@ -91,7 +91,7 @@ def audit(data_list, tags, debug=False, **kwargs): ret['Errors'] = [] ret['Errors'].append({tag: 'No function {0} found' .format(tag_data['function'])}) - continue + continue args = tag_data.get('args', []) kwargs = tag_data.get('kwargs', {}) @@ -168,6 +168,13 @@ def _get_tags(data): # Begin function definitions ############################ +def _is_valid_home_directory(directory_path): + directory_path = None if directory_path is None else directory_path.strip() + if directory_path is not None and directory_path != "" and os.path.isdir(directory_path) and directory_path != "/": + return True + + return False + def _execute_shell_command(cmd): ''' This function will execute passed command in /bin/shell @@ -308,7 +315,7 @@ def test_mount_attrs(mount_name,attribute,check_type='hard'): else: return True -def check_time_synchronization(): +def check_time_synchronization(reason=''): ''' Ensure that some service is running to synchronize the system clock ''' @@ -332,6 +339,7 @@ def restrict_permissions(path,permission): return True return given_permission + def _is_permission_in_limit(max_permission,given_permission): ''' Return true only if given_permission is not more linient that max_permission. In other words, if @@ -375,7 +383,7 @@ def _is_permission_in_limit(max_permission,given_permission): return True -def check_path_integrity(): +def check_path_integrity(reason=''): ''' Ensure that system PATH variable is not malformed. ''' @@ -509,6 +517,274 @@ def check_core_dumps(reason=''): return str(hard_core_dump_value) +def check_service_status(service_name, state): + ''' + Ensure that the given service is in the required state. Return False if it is not in desired state + Return True otherwise + state can be enabled or disabled. + ''' + output = _execute_shell_command('systemctl is-enabled ' + service_name + ' >/dev/null 2>&1; echo $?') + if (state == "disabled" and output.strip() == "1") or (state == "enabled" and output.strip() == "0"): + return True + else: + return _execute_shell_command('systemctl is-enabled ' + service_name + ' 2>/dev/null') + +def check_ssh_timeout_config(reason=''): + ''' + Ensure SSH Idle Timeout Interval is configured + ''' + + client_alive_interval = _execute_shell_command("grep \"^ClientAliveInterval\" /etc/ssh/sshd_config | awk '{print $NF}'").strip() + if client_alive_interval != '' and int(client_alive_interval) <= 300: + client_alive_count_max = _execute_shell_command("grep \"^ClientAliveCountMax\" /etc/ssh/sshd_config | awk '{print $NF}'").strip() + if client_alive_count_max != '' and int(client_alive_count_max) <= 3: + return True + else: + return "ClientAliveCountMax value should be less than equal to 3" + else: + return "ClientAliveInterval value should be less than equal to 300" + + +def check_unowned_files(reason=''): + ''' + Ensure no unowned files or directories exist + ''' + + unowned_files = _execute_shell_command("df --local -P | awk 'NR!=1 {print $6}' | xargs -I '{}' find '{}' -xdev -nouser 2>/dev/null").strip() + unowned_files = unowned_files.split('\n') if unowned_files != "" else [] + # The command above only searches local filesystems, there may still be compromised items on network mounted partitions. + # Following command will check each partition for unowned files + unowned_partition_files = _execute_shell_command("mount | awk '{print $3}' | xargs -I '{}' find '{}' -xdev -nouser 2>/dev/null").strip() + unowned_partition_files = unowned_partition_files.split('\n') if unowned_partition_files != "" else [] + unowned_files = unowned_files + unowned_partition_files + if unowned_files == []: + return True + + return str(list(set(unowned_files))) + + +def check_ungrouped_files(reason=''): + ''' + Ensure no ungrouped files or directories exist + ''' + + ungrouped_files = _execute_shell_command("df --local -P | awk 'NR!=1 {print $6}' | xargs -I '{}' find '{}' -xdev -nogroup 2>/dev/null").strip() + ungrouped_files = ungrouped_files.split('\n') if ungrouped_files != "" else [] + # The command above only searches local filesystems, there may still be compromised items on network mounted partitions. + # Following command will check each partition for unowned files + ungrouped_partition_files = _execute_shell_command("mount | awk '{print $3}' | xargs -I '{}' find '{}' -xdev -nogroup 2>/dev/null").strip() + ungrouped_partition_files = ungrouped_partition_files.split('\n') if ungrouped_partition_files != "" else [] + ungrouped_files = ungrouped_files + ungrouped_partition_files + if ungrouped_files == []: + return True + + return str(list(set(ungrouped_files))) + + +def check_all_users_home_directory(max_system_uid): + ''' + Ensure all users' home directories exist + ''' + + max_system_uid = int(max_system_uid) + users_uids_dirs = _execute_shell_command("cat /etc/passwd | awk -F: '{ print $1 \" \" $3 \" \" $6 }'").strip() + users_uids_dirs = users_uids_dirs.split('\n') if users_uids_dirs != "" else [] + error = [] + for user_data in users_uids_dirs: + user_uid_dir = user_data.strip().split(" ") + if len(user_uid_dir) < 3: + user_uid_dir = user_uid_dir + ['']*(3-len(user_uid_dir)) + if user_uid_dir[1].isdigit(): + if not _is_valid_home_directory(user_uid_dir[2]) and int(user_uid_dir[1]) >= max_system_uid and user_uid_dir[0] is not "nfsnobody": + error += ["Either home directory " + user_uid_dir[2] + " of user " + user_uid_dir[0] + " is invalid or does not exist."] + else: + error += ["User " + user_uid_dir[0] + " has invalid uid " + user_uid_dir[1]] + if error == []: + return True + + return str(error) + + +def check_users_home_directory_permissions(reason=''): + ''' + Ensure users' home directories permissions are 750 or more restrictive + ''' + + users_dirs = _execute_shell_command("cat /etc/passwd | egrep -v '(root|halt|sync|shutdown)' | awk -F: '($7 != \"/sbin/nologin\") {print $1\" \"$6}'").strip() + users_dirs = users_dirs.split('\n') if users_dirs != "" else [] + error = [] + for user_dir in users_dirs: + user_dir = user_dir.split(" ") + if len(user_dir) < 2: + user_dir = user_dir + ['']*(2-len(user_dir)) + if not _is_valid_home_directory(user_dir[1]): + error += ["Either home directory " + user_dir[1] + " of user " + user_dir[0] + " is invalid or does not exist."] + else: + result = restrict_permissions(user_dir[1], "750") + if result is not True: + error += ["permission on home directory " + user_dir[1] + " of user " + user_dir[0] + " is wrong: " + result] + + if error == []: + return True + + return str(error) + + +def check_users_own_their_home(max_system_uid): + ''' + Ensure users own their home directories + ''' + + max_system_uid = int(max_system_uid) + + users_uids_dirs = _execute_shell_command("cat /etc/passwd | awk -F: '{ print $1 \" \" $3 \" \" $6 }'").strip() + users_uids_dirs = users_uids_dirs.split('\n') if users_uids_dirs != "" else [] + error = [] + for user_data in users_uids_dirs: + user_uid_dir = user_data.strip().split(" ") + if len(user_uid_dir) < 3: + user_uid_dir = user_uid_dir + ['']*(3-len(user_uid_dir)) + if user_uid_dir[1].isdigit(): + if not _is_valid_home_directory(user_uid_dir[2]): + error += ["Either home directory " + user_uid_dir[2] + " of user " + user_uid_dir[0] + " is invalid or does not exist."] + elif int(user_uid_dir[1]) >= max_system_uid and user_uid_dir[0] is not "nfsnobody": + owner = _execute_shell_command("stat -L -c \"%U\" \"" + user_uid_dir[2] + "\"") + if owner is not user_uid_dir[0]: + error += ["The home directory " + user_uid_dir[2] + " of user " + user_uid_dir[0] + " is owned by " + owner] + else: + error += ["User " + user_uid_dir[0] + " has invalid uid " + user_uid_dir[1]] + + if error == []: + return True + + return str(error) + + +def check_users_dot_files(reason=''): + ''' + Ensure users' dot files are not group or world writable + ''' + + users_dirs = _execute_shell_command("cat /etc/passwd | egrep -v '(root|halt|sync|shutdown)' | awk -F: '($7 != \"/sbin/nologin\") {print $1\" \"$6}'").strip() + users_dirs = users_dirs.split('\n') if users_dirs != "" else [] + error = [] + for user_dir in users_dirs: + user_dir = user_dir.split() + if len(user_dir) < 2: + user_dir = user_dir + ['']*(2-len(user_dir)) + if not _is_valid_home_directory(user_dir[1]): + error += ["Either home directory " + user_dir[1] + " of user " + user_dir[0] + " is invalid or does not exist."] + else: + dot_files = _execute_shell_command("find " + user_dir[1] + " -name \".*\"").strip() + dot_files = dot_files.split('\n') if dot_files != "" else [] + for dot_file in dot_files: + if os.path.isfile(dot_file): + file_permission = _execute_shell_command("ls -ld " + dot_file + " | cut -f1 -d\" \"").strip() + if file_permission[5] is "w": + error += ["Group Write permission set on file " + dot_file + " for user " + user_dir[0]] + if file_permission[8] is "w": + error += ["Other Write permission set on file " + dot_file + " for user " + user_dir[0]] + + if error == []: + return True + + return str(error) + + +def check_users_forward_files(reason=''): + ''' + Ensure no users have .forward files + ''' + + users_dirs = _execute_shell_command("cat /etc/passwd | awk -F: '{ print $1\" \"$6 }'").strip() + users_dirs = users_dirs.split('\n') if users_dirs != "" else [] + error = [] + for user_dir in users_dirs: + user_dir = user_dir.split() + if len(user_dir) < 2: + user_dir = user_dir + ['']*(2-len(user_dir)) + if not _is_valid_home_directory(user_dir[1]): + error += ["Either home directory " + user_dir[1] + " of user " + user_dir[0] + " is invalid or does not exist."] + else: + forward_file = _execute_shell_command("find " + user_dir[1] + " -name \".forward\"").strip() + if forward_file is not None and os.path.isfile(forward_file): + error += ["Home directory: " + user_dir[1] + ", for user: " + user_dir[0] + " has " + forward_file + " file"] + + if error == []: + return True + + return str(error) + + +def check_users_netrc_files(reason=''): + ''' + Ensure no users have .netrc files + ''' + + users_dirs = _execute_shell_command("cat /etc/passwd | awk -F: '{ print $1\" \"$6 }'").strip() + users_dirs = users_dirs.split('\n') if users_dirs != "" else [] + error = [] + for user_dir in users_dirs: + user_dir = user_dir.split() + if len(user_dir) < 2: + user_dir = user_dir + ['']*(2-len(user_dir)) + if not _is_valid_home_directory(user_dir[1]): + error += ["Either home directory " + user_dir[1] + " of user " + user_dir[0] + " is invalid or does not exist."] + else: + netrc_file = _execute_shell_command("find " + user_dir[1] + " -name \".netrc\"").strip() + if netrc_file is not None and os.path.isfile(netrc_file): + error += ["Home directory: " + user_dir[1] + ", for user: " + user_dir[0] + " has .netrc file"] + + if error == []: + return True + + return str(error) + + +def check_groups_validity(reason=''): + ''' + Ensure all groups in /etc/passwd exist in /etc/group + ''' + + group_ids_in_passwd = _execute_shell_command("cut -s -d: -f4 /etc/passwd 2>/dev/null").strip() + group_ids_in_passwd = group_ids_in_passwd.split('\n') if group_ids_in_passwd != "" else [] + group_ids_in_group = _execute_shell_command("cut -s -d: -f3 /etc/group 2>/dev/null").strip() + group_ids_in_group = group_ids_in_group.split('\n') if group_ids_in_group != "" else [] + invalid_group_ids = list(set(group_ids_in_passwd) - set(group_ids_in_group)) + + if invalid_group_ids == []: + return True + + return "Groups which are referenced by /etc/passwd but does not exist in /etc/group: " + str(invalid_group_ids) + + +def ensure_reverse_path_filtering(reason=''): + ''' + Ensure Reverse Path Filtering is enabled + ''' + error_list = [] + command = "sysctl net.ipv4.conf.all.rp_filter 2> /dev/null" + output = _execute_shell_command(command) + if output.strip() == '': + error_list.append( "net.ipv4.conf.all.rp_filter not found") + search_results = re.findall("rp_filter = (\d+)",output) + result = int(search_results[0]) + if( result < 1): + error_list.append( "net.ipv4.conf.all.rp_filter value set to " + str(result)) + command = "sysctl net.ipv4.conf.default.rp_filter 2> /dev/null" + output = _execute_shell_command(command) + if output.strip() == '': + error_list.append( "net.ipv4.conf.default.rp_filter not found") + search_results = re.findall("rp_filter = (\d+)",output) + result = int(search_results[0]) + if( result < 1): + error_list.append( "net.ipv4.conf.default.rp_filter value set to " + str(result)) + if len(error_list) > 0 : + return str(error_list) + else: + return True + + FUNCTION_MAP = { 'check_all_ports_firewall_rules': check_all_ports_firewall_rules, 'check_password_fields_not_empty': check_password_fields_not_empty, @@ -529,8 +805,19 @@ def check_core_dumps(reason=''): 'check_core_dumps': check_core_dumps, 'check_directory_files_permission': check_directory_files_permission, 'check_duplicate_gnames': check_duplicate_gnames, - 'check_duplicate_unames': check_duplicate_unames, + 'check_duplicate_unames': check_duplicate_unames, 'check_duplicate_gids': check_duplicate_gids, 'check_duplicate_uids': check_duplicate_uids, + 'check_service_status' : check_service_status, + 'check_ssh_timeout_config': check_ssh_timeout_config, + 'check_unowned_files': check_unowned_files, + 'check_ungrouped_files': check_ungrouped_files, + 'check_all_users_home_directory': check_all_users_home_directory, + 'check_users_home_directory_permissions': check_users_home_directory_permissions, + 'check_users_own_their_home': check_users_own_their_home, + 'check_users_dot_files': check_users_dot_files, + 'check_users_forward_files': check_users_forward_files, + 'check_users_netrc_files': check_users_netrc_files, + 'check_groups_validity': check_groups_validity, + 'ensure_reverse_path_filtering': ensure_reverse_path_filtering, } - From fefd551dd0e744acdc6db72d4e3e583b85bd2e75 Mon Sep 17 00:00:00 2001 From: Anurag Paliwal Date: Thu, 17 Aug 2017 23:04:40 +0530 Subject: [PATCH 50/50] Correcting indentation in misc.py --- hubblestack/files/hubblestack_nova/misc.py | 81 +++++++++++----------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/hubblestack/files/hubblestack_nova/misc.py b/hubblestack/files/hubblestack_nova/misc.py index bb50ecfed..e01bbe109 100644 --- a/hubblestack/files/hubblestack_nova/misc.py +++ b/hubblestack/files/hubblestack_nova/misc.py @@ -59,7 +59,6 @@ def __virtual__(): return True - def audit(data_list, tags, debug=False, **kwargs): ''' Run the misc audits contained in the data_list @@ -91,7 +90,7 @@ def audit(data_list, tags, debug=False, **kwargs): ret['Errors'] = [] ret['Errors'].append({tag: 'No function {0} found' .format(tag_data['function'])}) - continue + continue args = tag_data.get('args', []) kwargs = tag_data.get('kwargs', {}) @@ -342,10 +341,10 @@ def restrict_permissions(path,permission): def _is_permission_in_limit(max_permission,given_permission): ''' - Return true only if given_permission is not more linient that max_permission. In other words, if - r or w or x is present in given_permission but absent in max_permission, it should return False + Return true only if given_permission is not more linient that max_permission. In other words, if + r or w or x is present in given_permission but absent in max_permission, it should return False Takes input two integer values from 0 to 7. - ''' + ''' max_permission = int(max_permission) given_permission = int(given_permission) allowed_r = False @@ -381,47 +380,47 @@ def _is_permission_in_limit(max_permission,given_permission): return False return True - + def check_path_integrity(reason=''): ''' Ensure that system PATH variable is not malformed. - ''' + ''' script = """ - if [ "`echo $PATH | grep ::`" != "" ]; then - echo "Empty Directory in PATH (::)" - fi - - if [ "`echo $PATH | grep :$`" != "" ]; then - echo "Trailing : in PATH" - fi - - p=`echo $PATH | sed -e 's/::/:/' -e 's/:$//' -e 's/:/ /g'` - set -- $p - while [ "$1" != "" ]; do - if [ "$1" = "." ]; then - echo "PATH contains ." - shift - continue - fi - - if [ -d $1 ]; then - dirperm=`ls -ldH $1 | cut -f1 -d" "` - if [ `echo $dirperm | cut -c6` != "-" ]; then - echo "Group Write permission set on directory $1" - fi - if [ `echo $dirperm | cut -c9` != "-" ]; then - echo "Other Write permission set on directory $1" - fi - dirown=`ls -ldH $1 | awk '{print $3}'` - if [ "$dirown" != "root" ] ; then + if [ "`echo $PATH | grep ::`" != "" ]; then + echo "Empty Directory in PATH (::)" + fi + + if [ "`echo $PATH | grep :$`" != "" ]; then + echo "Trailing : in PATH" + fi + + p=`echo $PATH | sed -e 's/::/:/' -e 's/:$//' -e 's/:/ /g'` + set -- $p + while [ "$1" != "" ]; do + if [ "$1" = "." ]; then + echo "PATH contains ." + shift + continue + fi + + if [ -d $1 ]; then + dirperm=`ls -ldH $1 | cut -f1 -d" "` + if [ `echo $dirperm | cut -c6` != "-" ]; then + echo "Group Write permission set on directory $1" + fi + if [ `echo $dirperm | cut -c9` != "-" ]; then + echo "Other Write permission set on directory $1" + fi + dirown=`ls -ldH $1 | awk '{print $3}'` + if [ "$dirown" != "root" ] ; then echo $1 is not owned by root - fi - else - echo $1 is not a directory - fi - shift + fi + else + echo $1 is not a directory + fi + shift done """ @@ -510,7 +509,7 @@ def check_core_dumps(reason=''): hard_core_dump_value = hard_core_dump_value.split('\n') if hard_core_dump_value != "" else [] if '0' in hard_core_dump_value: return True - + if hard_core_dump_value is None or hard_core_dump_value == [] or hard_core_dump_value == "": return "'hard core' not found in any file" @@ -771,7 +770,7 @@ def ensure_reverse_path_filtering(reason=''): result = int(search_results[0]) if( result < 1): error_list.append( "net.ipv4.conf.all.rp_filter value set to " + str(result)) - command = "sysctl net.ipv4.conf.default.rp_filter 2> /dev/null" + command = "sysctl net.ipv4.conf.default.rp_filter 2> /dev/null" output = _execute_shell_command(command) if output.strip() == '': error_list.append( "net.ipv4.conf.default.rp_filter not found")