diff --git a/com.ibm.streamsx.topology/CHANGELOG.md b/com.ibm.streamsx.topology/CHANGELOG.md index 8c50edf18..a3aeee3e8 100644 --- a/com.ibm.streamsx.topology/CHANGELOG.md +++ b/com.ibm.streamsx.topology/CHANGELOG.md @@ -3,6 +3,7 @@ ## branch latest: targeted for v2.1.0 +* [#2649](https://github.com/IBMStreams/streamsx.topology/issues/2649) Python scripts: Print diagnostic info in verbose mode when spl-python-extract skips python modules * [#2646](https://github.com/IBMStreams/streamsx.topology/issues/2646) Python: Support new consumingReads parameter of spl.endpoint::EndpointSink * [#2640](https://github.com/IBMStreams/streamsx.topology/issues/2640) Python: Support SPL annotation @catch * [#2590](https://github.com/IBMStreams/streamsx.topology/issues/2590) Python: Support definition of event-time streams diff --git a/com.ibm.streamsx.topology/opt/python/packages/streamsx/scripts/extract.py b/com.ibm.streamsx.topology/opt/python/packages/streamsx/scripts/extract.py index 6a33852af..3d02fd4b2 100644 --- a/com.ibm.streamsx.topology/opt/python/packages/streamsx/scripts/extract.py +++ b/com.ibm.streamsx.topology/opt/python/packages/streamsx/scripts/extract.py @@ -155,6 +155,8 @@ def _parse_cmd_args(self, args): help='Toolkit directory') cmd_parser.add_argument('--make-toolkit', action='store_true', help='Index toolkit using spl-make-toolkit') + cmd_parser.add_argument('-f', '--force', action='store_true', + help='Force SPL operator extraction ignoring the modification times of toolkit index and Python modules') cmd_parser.add_argument('-v', '--verbose', action='store_true', help='Print more diagnostics') return cmd_parser.parse_args(args) @@ -503,6 +505,8 @@ def _extract_from_toolkit(args): tk_streams = os.path.join(tk_dir, 'opt', 'python', 'streams') if not os.path.isdir(tk_streams) or not fnmatch.filter(os.listdir(tk_streams), '*.py'): # Nothing to do for Python extraction + if extractor._cmd_args.verbose: + print('directory ' + str(tk_streams) + ' is not present or it does not contains Python files. Nothing to do.') extractor._make_toolkit() return @@ -511,13 +515,24 @@ def _extract_from_toolkit(args): fcntl.flock(lfno, fcntl.LOCK_EX) tk_idx = os.path.join(tk_dir, 'toolkit.xml') - tk_time = os.path.getmtime(tk_idx) if os.path.exists(tk_idx) else None + if extractor._cmd_args.force: + tk_time = None + else: + tk_time = os.path.getmtime(tk_idx) if os.path.exists(tk_idx) else None changed = False if tk_time else True if tk_time: for mf in glob.glob(os.path.join(tk_streams, '*.py')): - if os.path.getmtime(mf) >= tk_time: - changed = True - break + if os.path.getmtime(mf) >= tk_time: + changed = True + break + else: + if extractor._cmd_args.verbose: + print(str(mf) + " is older than toolkit.xml") + + if extractor._cmd_args.verbose and not changed: + print('The toolkit.xml (the toolkit index) is newer than all python modules in opt/python/streams.') + print('Assuming the toolkit index already contains the Python operators. Skipping operator extraction.') + print('Consider to use the -f | --force option to force SPL operator extraction.') if changed: path_items = _setup_path(tk_dir, tk_streams) @@ -538,7 +553,7 @@ def _extract_from_toolkit(args): extractor._make_toolkit() _reset_path(path_items) - fcntl.flock(lfno, fcntl.LOCK_UN) + fcntl.flock(lfno, fcntl.LOCK_UN) def _setup_path(tk_dir, tk_streams): sys.path.insert(1, tk_streams)