diff --git a/defaults/main.yaml b/defaults/main.yaml index 64af3c2b..44fcf03a 100644 --- a/defaults/main.yaml +++ b/defaults/main.yaml @@ -2,9 +2,9 @@ __INSTL_VERSION__: - 2 # major version e.g. python-batch - - 1 # new major feature e.g. new command - - 11 # new minor feature e.g. new option to command, new python batch class - - 10 # bug fix + - 2 # new major feature e.g. new command + - 0 # new minor feature e.g. new option to command, new python batch class + - 0 # bug fix __INSTL_VERSION_STR_SHORT__: $(__INSTL_VERSION__[0]).$(__INSTL_VERSION__[1]).$(__INSTL_VERSION__[2]).$(__INSTL_VERSION__[3]) # __INSTL_VERSION_STR_SHORT__ will be defined at run time by InstlInstanceBase.get_version_str diff --git a/instl.spec b/instl.spec index e4ab8d78..4b3e9efe 100644 --- a/instl.spec +++ b/instl.spec @@ -9,6 +9,7 @@ import socket import datetime import re from subprocess import check_output +from pathlib import Path block_cipher = None @@ -16,7 +17,7 @@ a = Analysis(['instl'], pathex=['instl'], binaries=None, datas=None, - hiddenimports=['distutils', 'six','packaging', 'packaging.version', 'packaging.specifiers', 'packaging.requirements', 'xmltodict'], + hiddenimports=['distutils', 'packaging', 'packaging.version', 'packaging.specifiers', 'packaging.requirements'], hookspath=None, runtime_hooks=None, excludes=['PyQt4', 'matplotlib', "PIL", "numpy", "wx", "tornado", "networkx", @@ -30,6 +31,8 @@ instl_defaults_path = os.path.join("defaults") for defaults_file in os.listdir(instl_defaults_path): if fnmatch.fnmatch(defaults_file, '*.yaml') or fnmatch.fnmatch(defaults_file, '*.ddl'): a.datas += [("defaults/"+defaults_file, os.path.join(instl_defaults_path, defaults_file), "DATA")] +pip_info_path = os.path.join("defaults", "pip-freeze.txt") +a.datas += [("defaults/pip-freeze.txt", pip_info_path, "DATA")] git_branch = check_output(["git", "rev-parse", "--symbolic-full-name", "--abbrev-ref", "HEAD"]).decode('utf-8') @@ -49,6 +52,7 @@ __GITHUB_BRANCH__: {} """.format(str(datetime.datetime.now()), socket.gethostname(), platform.node(), PyInstallerVersion, git_branch)) a.datas += [("defaults/compile-info.yaml", compile_info_path, "DATA")] + instl_help_path = os.path.join("help") for help_file in os.listdir(instl_help_path): if fnmatch.fnmatch(help_file, '*.yaml'): @@ -79,4 +83,4 @@ coll = COLLECT(exe, app = BUNDLE(coll, name='instl.bundle', icon=None, - bundle_identifier=None) \ No newline at end of file + bundle_identifier=None) diff --git a/pybatch/fileSystemBatchCommands.py b/pybatch/fileSystemBatchCommands.py index a1cd6c14..d2cf3572 100644 --- a/pybatch/fileSystemBatchCommands.py +++ b/pybatch/fileSystemBatchCommands.py @@ -971,6 +971,10 @@ def __init__(self, glob_pattern, class_to_run, target_param_name, *argv_for_glob self.class_to_run = class_to_run self.argv_for_glob_handler = list(argv_for_glob_handler) self.kwargs_for_glob_handler = kwargs_for_glob_handler + if "excludes" in kwargs_for_glob_handler: + self.excludes = kwargs_for_glob_handler["excludes"] + else: + self.excludes = [] pass def repr_own_args(self, all_args: List[str]) -> None: @@ -989,6 +993,7 @@ def repr_own_args(self, all_args: List[str]) -> None: def progress_msg_self(self): return f"""{self.__class__.__name__} '{self.glob_pattern}'""" + def __call__(self, *args, **kwargs): PythonBatchCommandBase.__call__(self, *args, **kwargs) self.doing = f"""running {self.class_to_run} on glob pattern'{self.glob_pattern}'""" @@ -997,7 +1002,13 @@ def __call__(self, *args, **kwargs): if not self.target_param_name: self.argv_for_glob_handler.insert(0, None) - for a_path in glob.glob(self.glob_pattern): + excluded_items = [] + for excluded_item in self.excludes: + excluded_item = str(Path(self.glob_pattern).parent) + "/" + excluded_item + excluded_items.extend(glob.glob(excluded_item)) + paths = set(glob.glob(self.glob_pattern)) - set(excluded_items) + + for a_path in paths: if self.target_param_name: self.kwargs_for_glob_handler[self.target_param_name] = a_path with self.class_to_run(*self.argv_for_glob_handler, **self.kwargs_for_glob_handler) as handler: diff --git a/pybatch/reportingBatchCommands.py b/pybatch/reportingBatchCommands.py index ceacbd8a..f9fb5539 100644 --- a/pybatch/reportingBatchCommands.py +++ b/pybatch/reportingBatchCommands.py @@ -329,7 +329,7 @@ def __init__(self, unresolved_file, resolved_file=None, config_files=None, raise :param config_files: additional files to read config_vars definitions from :param raise_if_unresolved: when True, will raise exception if any unresolved $(...) references are left :param avoid_resolve_marker: config vars marked with this char (if != '$') will be ignored and after - the resolve will be replacfed by '$'. This will enable to mark certain config vars so they are not resolved + the resolve will be replaced by '$'. This will enable to mark certain config vars so they are not resolved """ super().__init__(**kwargs) self.unresolved_file = unresolved_file @@ -401,6 +401,9 @@ def __call__(self, *args, **kwargs) -> None: class ReadConfigVarValueFromTextFile(pybatch.PythonBatchCommandBase, essential=True): + """ + Read a file and assign the file's whole contents to a configVar + """ def __init__(self, file_path_to_read, var_name, **kwargs): super().__init__(**kwargs) self.file_path_to_read = file_path_to_read diff --git a/pybatch/subprocessBatchCommands.py b/pybatch/subprocessBatchCommands.py index 049d9a5f..1a153c9b 100644 --- a/pybatch/subprocessBatchCommands.py +++ b/pybatch/subprocessBatchCommands.py @@ -77,7 +77,10 @@ def __call__(self, *args, **kwargs): need_to_close_out_file = False if self.out_file: if isinstance(self.out_file, (str, os.PathLike, bytes)): - out_stream = utils.utf8_open_for_write(self.out_file, "w") + out_file = Path(self.out_file).resolve() + out_file.parent.mkdir(parents=True, exist_ok=True) + out_stream = utils.utf8_open_for_write(out_file, "w") + log.info(f"output will be written to {out_file}") need_to_close_out_file = True elif hasattr(self.out_file, "write"): # out_file is already an open file out_stream = self.out_file @@ -183,7 +186,6 @@ def get_run_args(self, run_args) -> None: class ShellCommand(RunProcessBase): """ run a single command in a shell """ - def __init__(self, shell_command, message=None, ignore_specific_exit_codes=(), **kwargs): kwargs["shell"] = True super().__init__(ignore_specific_exit_codes=ignore_specific_exit_codes, **kwargs) diff --git a/pyinstl/instlMisc.py b/pyinstl/instlMisc.py index 5d0badd2..00c896fe 100644 --- a/pyinstl/instlMisc.py +++ b/pyinstl/instlMisc.py @@ -93,7 +93,7 @@ def do_test_import(self): import importlib bad_modules = list() - for module in ("yaml", "appdirs", "configVar", "utils", "svnTree", "aYaml", "xmltodict"): + for module in ("yaml", "appdirs", "configVar", "utils", "svnTree", "aYaml"): try: importlib.import_module(module) except ImportError: diff --git a/requirements.txt b/requirements.txt index 2bdff54a..dd163a72 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,9 @@ -appdirs==1.4.3 -future==0.17.1 -networkx==2.2 -packaging==18.0 -psutil==5.6.6 -PyInstaller==4.3 -PyYAML==5.4 -requests==2.21.0 -six==1.12.0 -xmltodict==0.12.0 -redis==2.10.6 +appdirs +future +networkx +packaging +psutil +PyInstaller +PyYAML +requests +redis diff --git a/requirements_win_only.txt b/requirements_win_only.txt index 5dd4e168..b4e2d81f 100644 --- a/requirements_win_only.txt +++ b/requirements_win_only.txt @@ -1,4 +1,4 @@ # Windows only -pypiwin32==220 -pywin32==224 -pywin32-ctypes==0.2.0 +pypiwin32 +pywin32 +pywin32-ctypes