diff --git a/.gitignore b/.gitignore index adaf55ca9..02b52b7c9 100644 --- a/.gitignore +++ b/.gitignore @@ -61,7 +61,7 @@ coverage.xml .venv venv/ ENV/ -virtualenv/ +virtualenv*/ # mkdocs documentation /site diff --git a/README.md b/README.md index cd2d783ac..68169e06f 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ The recommended method for installing the STIX-shifter is via pip. Two prerequis 1. Main stix-shifter package: `pip install stix-shifter` -2. stix-shifter-utility package: `pip install stix-shifter-utils` +2. Stix-shifter Utility package: `pip install stix-shifter-utils` 3. Desired stix-shifter connector module package: `pip install stix-shifter-modules- ` Example: `pip install stix-shifter-modules-qradar` @@ -40,7 +40,7 @@ The recommended method for installing the STIX-shifter is via pip. Two prerequis ## Usage -### As A Script +### As A Command Line Utility The STIX-Shifter comes with a bundled script which you can use to translate STIX Pattern to a native datasource query. It can also be used to translate a JSON data source query result to a STIX bundle of observable objects. You can also send query to a datasource by using a transmission option. @@ -54,12 +54,30 @@ Example: $ stix-shifter translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} ``` -**Note:** In order to build `stix-shifter` packages from source follow the below prerequisite steps: +In order to build `stix-shifter` packages from source follow the below prerequisite steps: 1. Go to the stix-shifter parent directory - 2. Generate latest requirements.txt: `python3 generate_requirements.py` - 3. Install the dependencies in your python 3 environment: `pip install -r requirements.txt` - 4. Alternatively you can create a Python 3 virtual environemnt: - `virtualenv -p python3 virtualenv && source virtualenv/bin/activate && pip install -r requirements-dev.txt` + 2. Optionally, you can create a Python 3 virtual environemnt: + `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` + 3. Run setup: `python3 setup.py install` + + +### Running From the Source + +You may also use `python3 main.py` script. All the options are the same as "As a command line utility" usage above. + +Example: + +``` +python3 main.py translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} +``` + +In order to run `python3 main.py` from the source follow the below prerequisite steps: + 1. Go to the stix-shifter parent directory + 2. Optionally, you can create a Python 3 virtual environemnt: + `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` + 3. Run setup to install dependancies: `INSTALL_REQUIREMENTS_ONLY=1 python3 setup.py install`. + +**Note:** setup.py only installs dependencies when INSTALL_REQUIREMENTS_ONLY=1 directive is used. This option is similar to `python3 generate_requirements.py && pip install -r requirements.txt` ### As A Library diff --git a/generate_requirements.py b/generate_requirements.py index 1a7242f54..88b367183 100644 --- a/generate_requirements.py +++ b/generate_requirements.py @@ -1,25 +1,30 @@ import os -src_folders = ["stix_shifter_utils", "stix_shifter", "stix_shifter_modules"] -install_requires = set() -requirements_files = [] -for src_folder in src_folders: - for r, d, f in os.walk(src_folder): - for file in f: - if 'requirements.txt'==file and not os.path.isfile(os.path.join(r, 'SKIP.ME')): - requirements_files.append(os.path.join(r, file)) -print('requirements_files: %s' % requirements_files) -for requirements_file in requirements_files: - with open(requirements_file) as f: - lines = f.readlines() - lines = [x.strip() for x in lines] - lines = list(filter(lambda s: len(s)>0, lines)) - install_requires.update(lines) -install_requires = list(install_requires) -install_requires.sort() -print('install_requires: %s' % install_requires) +def generate_requirements(): + src_folders = ["stix_shifter_utils", "stix_shifter", "stix_shifter_modules"] + install_requires = set() + requirements_files = [] + for src_folder in src_folders: + for r, d, f in os.walk(src_folder): + for file in f: + if 'requirements.txt'==file and not os.path.isfile(os.path.join(r, 'SKIP.ME')): + requirements_files.append(os.path.join(r, file)) + print('requirements_files: %s' % requirements_files) + for requirements_file in requirements_files: + with open(requirements_file) as f: + lines = f.readlines() + lines = [x.strip() for x in lines] + lines = list(filter(lambda s: len(s)>0, lines)) + install_requires.update(lines) + install_requires = list(install_requires) + install_requires.sort() + print('install_requires: %s' % install_requires) -with open('requirements.txt', 'w') as out_file: - for item in install_requires: - out_file.write(item) - out_file.write('\n') + with open('requirements.txt', 'w') as out_file: + for item in install_requires: + out_file.write(item) + out_file.write('\n') + + +if __name__ == "__main__": + generate_requirements() \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 044e726b2..15411d64b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,3 +9,6 @@ description-file = README.md # need to generate separate wheels for each Python version that you # support. universal=1 + +[options] +zip_safe = False diff --git a/setup.py b/setup.py index 6c580c69e..79bc93a53 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,32 @@ +import os +import subprocess +import sys + +if sys.version_info.major == 3 and sys.version_info.minor > 5: + # good + print(sys.version) +else: + print("Error: stix-shifter requires python version at least or greater than 3.6") + exit(1) + + +from generate_requirements import generate_requirements +generate_requirements() + +subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "requirements-dev.txt"]) + + + +if os.getenv('INSTALL_REQUIREMENTS_ONLY', None) == '1': + exit(0) + + from setuptools import find_packages # To use a consistent encoding from codecs import open -import sys import shutil -import subprocess import json import io -import os from jsonmerge import merge import tempfile import importlib @@ -61,18 +81,12 @@ def fill_connectors(projects, modules_path): 'stix_shifter_utils', 'stix_shifter', 'stix_shifter_modules' - ] - } -elif mode == '3': - projects = { - "stix_shifter_utils": ["stix_shifter_utils"], - "stix_shifter": ["stix_shifter"], - "stix_shifter_modules": ["stix_shifter_modules"], + ] } elif mode == 'N': projects = { "stix_shifter_utils": ["stix_shifter_utils"], - "stix_shifter": ["stix_shifter"], + "stix_shifter": ["stix_shifter"] } fill_connectors(projects, "stix_shifter_modules") else: @@ -88,7 +102,7 @@ def fill_connectors(projects, modules_path): for project_name in projects.keys(): cleanup_file_list = [] - temp_dir = None + temp_dir_list = [] module_dir = None src_folders = projects[project_name] @@ -210,16 +224,21 @@ def fill_connectors(projects, modules_path): with open(os.path.join(conf_path, 'dialects.json'), 'w', encoding="utf-8") as f: f.write(json.dumps(dialects_full, indent=4, sort_keys=False)) temp_dir = tempfile.TemporaryDirectory() + temp_dir_list.append([temp_dir, module_dir]) shutil.move(configuration_path, temp_dir.name) os.rename(conf_path, configuration_path) cleanup_file_list.append(configuration_path) # Inject util files - for util_src, util_dest in utils_include_list.items(): - util_dest = util_dest % module_dir - if not shutil.os.path.exists(util_dest): - shutil.copyfile(util_src, util_dest) - cleanup_file_list.append(util_dest) + if mode != "1": + for util_src, util_dest in utils_include_list.items(): + util_dest = util_dest % module_dir + if shutil.os.path.exists(util_src) and not shutil.os.path.exists(util_dest): + try: + shutil.copyfile(util_src, util_dest) + cleanup_file_list.append(util_dest) + except Exception as e: + pass for r, d, f in os.walk(module_dir): r_split = r.split(os.sep) @@ -248,8 +267,9 @@ def fill_connectors(projects, modules_path): shutil.rmtree(cleanup_file) else: os.remove(cleanup_file) - if temp_dir is not None: - shutil.move(os.path.join(temp_dir.name, 'configuration'), module_dir) - temp_dir = None + for temp_dir, module_dir in temp_dir_list: + if temp_dir is not None: + shutil.move(os.path.join(temp_dir.name, 'configuration'), module_dir) + temp_dir.cleanup() print('---------------------------------') shutil.rmtree(TMP_MAPPING_DIR) diff --git a/stix_shifter_utils/utils/module_discovery.py b/stix_shifter_utils/utils/module_discovery.py index 074d8b8a9..c090df4e9 100644 --- a/stix_shifter_utils/utils/module_discovery.py +++ b/stix_shifter_utils/utils/module_discovery.py @@ -1,6 +1,7 @@ import os from importlib import import_module from pathlib import Path +from .param_validator import choose_module_path def process_dialects(cli_module, options): @@ -43,7 +44,7 @@ def dialect_list(module): if '__file__' in dir(modules) and modules.__file__ is not None: modules_path = Path(modules.__file__).parent else: - modules_path = modules.__path__._path[0] + modules_path = choose_module_path(module, modules.__path__._path) dialects_path = os.path.join(modules_path, f'{module}/stix_translation/json') ENDING = '_from_stix_map.json' dialects = [] diff --git a/stix_shifter_utils/utils/param_validator.py b/stix_shifter_utils/utils/param_validator.py index 30ad324b3..2ebb3d810 100644 --- a/stix_shifter_utils/utils/param_validator.py +++ b/stix_shifter_utils/utils/param_validator.py @@ -8,9 +8,9 @@ def get_merged_config(module): ss_modules_path = importlib.import_module('stix_shifter_modules') if isinstance(ss_modules_path.__path__, list): - base_path = ss_modules_path.__path__[0] + base_path = choose_module_path(module, ss_modules_path.__path__) else: - base_path = ss_modules_path.__path__._path[0] + base_path = choose_module_path(module, ss_modules_path.__path__._path) module_config_path = path.join(base_path, module, 'configuration', 'config.json') base_config_path = path.join(base_path, 'config.json') with open(module_config_path) as mapping_file: @@ -21,6 +21,13 @@ def get_merged_config(module): module_configs = merge(base_configs, module_configs) return module_configs +def choose_module_path(module, path_list): + path = path_list[0] + module_name = 'stix_shifter_modules_' + module + for p in path_list: + if module_name in p: + return p + return path def modernize_objects(module, params): expected_configs = get_merged_config(module)