Fix local build and install, originally by jiyongj (#779)

.gitignore

@@ -61,7 +61,7 @@ coverage.xml
 .venv
 venv/
 ENV/
-virtualenv/
+virtualenv*/
 
 # mkdocs documentation
 /site

README.md

@@ -32,15 +32,15 @@ The recommended method for installing the STIX-shifter is via pip. Two prerequis
 
 1. Main stix-shifter package:  `pip install stix-shifter`
 
-2. stix-shifter-utility package:  `pip install stix-shifter-utils`
+2. Stix-shifter Utility package:  `pip install stix-shifter-utils`
 
 3. Desired stix-shifter connector module package:  `pip install stix-shifter-modules-<module name> `
    Example:  `pip install stix-shifter-modules-qradar`
 
 ## Usage
 
 
-### As A Script
+### As A Command Line Utility
 
 The STIX-Shifter comes with a bundled script which you can use to translate STIX Pattern to a native datasource query. It can also be used to translate a JSON data source query result to a STIX bundle of observable objects. You can also send query to a datasource by using a transmission option. 
 
@@ -54,12 +54,30 @@ Example:
 $ stix-shifter translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {}
 ```
 
-**Note:** In order to build `stix-shifter` packages from source follow the below prerequisite steps:
+In order to build `stix-shifter` packages from source follow the below prerequisite steps:
    1. Go to the stix-shifter parent directory
-   2. Generate latest requirements.txt: `python3 generate_requirements.py`
-   3. Install the dependencies in your python 3 environment: `pip install -r requirements.txt` 
-   4. Alternatively you can create a Python 3 virtual environemnt:
-       `virtualenv -p python3 virtualenv && source virtualenv/bin/activate && pip install -r requirements-dev.txt`
+   2. Optionally, you can create a Python 3 virtual environemnt:
+       `virtualenv -p python3 virtualenv && source virtualenv/bin/activate`
+   3. Run setup: `python3 setup.py install`
+
+
+### Running From the Source
+
+You may also use `python3 main.py` script. All the options are the same as "As a command line utility" usage above.
+
+Example:
+
+```
+python3 main.py translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {}
+```
+
+In order to run `python3 main.py` from the source follow the below prerequisite steps:
+   1. Go to the stix-shifter parent directory
+   2. Optionally, you can create a Python 3 virtual environemnt:
+       `virtualenv -p python3 virtualenv && source virtualenv/bin/activate`
+   3. Run setup to install dependancies: `INSTALL_REQUIREMENTS_ONLY=1 python3 setup.py install`. 
+
+**Note:** setup.py only installs dependencies when INSTALL_REQUIREMENTS_ONLY=1 directive is used. This option is similar to `python3 generate_requirements.py && pip install -r requirements.txt`
 
 ### As A Library
 

generate_requirements.py

@@ -1,25 +1,30 @@
 import os
 
-src_folders = ["stix_shifter_utils", "stix_shifter", "stix_shifter_modules"]
-install_requires = set()
-requirements_files = []
-for src_folder in src_folders:
-    for r, d, f in os.walk(src_folder):
-        for file in f:
-            if 'requirements.txt'==file and not os.path.isfile(os.path.join(r, 'SKIP.ME')):
-                requirements_files.append(os.path.join(r, file))
-print('requirements_files: %s' % requirements_files)
-for requirements_file in requirements_files:
-    with open(requirements_file) as f:
-        lines = f.readlines()
-    lines = [x.strip() for x in lines]
-    lines = list(filter(lambda s: len(s)>0, lines))
-    install_requires.update(lines)
-install_requires = list(install_requires)
-install_requires.sort()
-print('install_requires: %s' % install_requires)
+def generate_requirements():
+    src_folders = ["stix_shifter_utils", "stix_shifter", "stix_shifter_modules"]
+    install_requires = set()
+    requirements_files = []
+    for src_folder in src_folders:
+        for r, d, f in os.walk(src_folder):
+            for file in f:
+                if 'requirements.txt'==file and not os.path.isfile(os.path.join(r, 'SKIP.ME')):
+                    requirements_files.append(os.path.join(r, file))
+    print('requirements_files: %s' % requirements_files)
+    for requirements_file in requirements_files:
+        with open(requirements_file) as f:
+            lines = f.readlines()
+        lines = [x.strip() for x in lines]
+        lines = list(filter(lambda s: len(s)>0, lines))
+        install_requires.update(lines)
+    install_requires = list(install_requires)
+    install_requires.sort()
+    print('install_requires: %s' % install_requires)
 
-with open('requirements.txt', 'w') as out_file:
-    for item in install_requires:
-        out_file.write(item)
-        out_file.write('\n')
+    with open('requirements.txt', 'w') as out_file:
+        for item in install_requires:
+            out_file.write(item)
+            out_file.write('\n')
+
+
+if __name__ == "__main__":
+    generate_requirements()

setup.cfg

@@ -9,3 +9,6 @@ description-file = README.md
 # need to generate separate wheels for each Python version that you
 # support.
 universal=1
+
+[options]
+zip_safe = False

setup.py

@@ -1,12 +1,32 @@
+import os
+import subprocess
+import sys
+
+if sys.version_info.major == 3 and sys.version_info.minor > 5:
+    # good
+    print(sys.version)
+else:
+    print("Error: stix-shifter requires python version at least or greater than 3.6")
+    exit(1)
+
+
+from generate_requirements import generate_requirements
+generate_requirements()
+
+subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "requirements-dev.txt"])
+
+
+
+if os.getenv('INSTALL_REQUIREMENTS_ONLY', None) == '1':
+    exit(0)
+
+
 from setuptools import find_packages
 # To use a consistent encoding
 from codecs import open
-import sys
 import shutil
-import subprocess
 import json
 import io
-import os
 from jsonmerge import merge
 import tempfile
 import importlib
@@ -61,18 +81,12 @@ def fill_connectors(projects, modules_path):
             'stix_shifter_utils',
             'stix_shifter',
             'stix_shifter_modules'
-            ]
-    }
-elif mode == '3':
-    projects = {
-        "stix_shifter_utils": ["stix_shifter_utils"],
-        "stix_shifter": ["stix_shifter"],
-        "stix_shifter_modules": ["stix_shifter_modules"],
+        ]
     }
 elif mode == 'N':
     projects = {
         "stix_shifter_utils": ["stix_shifter_utils"],
-        "stix_shifter": ["stix_shifter"],
+        "stix_shifter": ["stix_shifter"]
     }
     fill_connectors(projects, "stix_shifter_modules")
 else:
@@ -88,7 +102,7 @@ def fill_connectors(projects, modules_path):
 
 for project_name in projects.keys():
     cleanup_file_list = []
-    temp_dir = None
+    temp_dir_list = []
     module_dir = None
 
     src_folders = projects[project_name]
@@ -210,16 +224,21 @@ def fill_connectors(projects, modules_path):
             with open(os.path.join(conf_path, 'dialects.json'), 'w', encoding="utf-8") as f:
                 f.write(json.dumps(dialects_full, indent=4, sort_keys=False))
             temp_dir = tempfile.TemporaryDirectory()
+            temp_dir_list.append([temp_dir, module_dir])
             shutil.move(configuration_path, temp_dir.name)
             os.rename(conf_path, configuration_path)
             cleanup_file_list.append(configuration_path)
 
         # Inject util files 
-        for util_src, util_dest in utils_include_list.items():
-            util_dest = util_dest % module_dir
-            if not shutil.os.path.exists(util_dest):
-                shutil.copyfile(util_src, util_dest)
-                cleanup_file_list.append(util_dest)
+        if mode != "1":
+            for util_src, util_dest in utils_include_list.items():
+                util_dest = util_dest % module_dir
+                if shutil.os.path.exists(util_src) and not shutil.os.path.exists(util_dest):
+                    try: 
+                        shutil.copyfile(util_src, util_dest)
+                        cleanup_file_list.append(util_dest)
+                    except Exception as e:
+                        pass
 
         for r, d, f in os.walk(module_dir):
             r_split = r.split(os.sep)
@@ -248,8 +267,9 @@ def fill_connectors(projects, modules_path):
                 shutil.rmtree(cleanup_file)
             else:
                 os.remove(cleanup_file)
-    if temp_dir is not None:
-        shutil.move(os.path.join(temp_dir.name, 'configuration'), module_dir)
-        temp_dir = None
+    for temp_dir, module_dir in temp_dir_list:
+        if temp_dir is not None:
+            shutil.move(os.path.join(temp_dir.name, 'configuration'), module_dir)
+            temp_dir.cleanup()
     print('---------------------------------')
 shutil.rmtree(TMP_MAPPING_DIR)

stix_shifter_utils/utils/module_discovery.py

@@ -1,6 +1,7 @@
 import os
 from importlib import import_module
 from pathlib import Path
+from .param_validator import choose_module_path
 
 
 def process_dialects(cli_module, options):
@@ -43,7 +44,7 @@ def dialect_list(module):
     if '__file__' in dir(modules) and modules.__file__ is not None:
         modules_path = Path(modules.__file__).parent
     else:
-        modules_path = modules.__path__._path[0]
+        modules_path = choose_module_path(module, modules.__path__._path)
     dialects_path = os.path.join(modules_path, f'{module}/stix_translation/json')
     ENDING = '_from_stix_map.json'
     dialects = []

stix_shifter_utils/utils/param_validator.py

@@ -8,9 +8,9 @@
 def get_merged_config(module):
     ss_modules_path = importlib.import_module('stix_shifter_modules')
     if isinstance(ss_modules_path.__path__, list):
-        base_path = ss_modules_path.__path__[0]
+        base_path = choose_module_path(module, ss_modules_path.__path__)
     else:
-        base_path = ss_modules_path.__path__._path[0]
+        base_path = choose_module_path(module, ss_modules_path.__path__._path)
     module_config_path = path.join(base_path, module, 'configuration', 'config.json')
     base_config_path = path.join(base_path, 'config.json')
     with open(module_config_path) as mapping_file:
@@ -21,6 +21,13 @@ def get_merged_config(module):
         module_configs = merge(base_configs, module_configs)
     return module_configs
 
+def choose_module_path(module, path_list):
+    path = path_list[0]
+    module_name = 'stix_shifter_modules_' + module
+    for p in path_list:
+        if module_name in p:
+            return p
+    return path
 
 def modernize_objects(module, params):
     expected_configs = get_merged_config(module)