From f46bc48f7b056670f87cbc25e51a0979c7fff8db Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 24 Jan 2021 12:32:27 +0100 Subject: [PATCH 1/6] Default to 64 bits windows on CI Signed-off-by: Philippe Ombredanne --- etc/ci/azure-win.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml index 6220857..afe1686 100644 --- a/etc/ci/azure-win.yml +++ b/etc/ci/azure-win.yml @@ -3,7 +3,7 @@ parameters: image_name: '' python_versions: [] test_suites: {} - python_architecture: x86 + python_architecture: x64 jobs: - job: ${{ parameters.job_name }} From 51510cbdb2f2d066d6652695aed40175a37d88a4 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Thu, 11 Feb 2021 15:56:55 -0500 Subject: [PATCH 2/6] Fix .gitattributes Signed-off-by: Steven Esser --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index c446d38..b79df5c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,2 @@ # Ignore all Git auto CR/LF line endings conversions -* binary +* -text From 5257bca6743443efc710acc1378da4cf5c033131 Mon Sep 17 00:00:00 2001 From: Pratik Dey Date: Mon, 5 Apr 2021 01:06:07 +0530 Subject: [PATCH 3/6] Fixed Full Root scans for Virtual Codebase Signed-off-by: Pratik Dey --- src/commoncode/resource.py | 2 +- .../virtual_codebase/path_full_root.json | 36 +++++++++++++++++++ tests/test_resource.py | 7 ++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 tests/data/resource/virtual_codebase/path_full_root.json diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py index cbecaed..efc766a 100644 --- a/src/commoncode/resource.py +++ b/src/commoncode/resource.py @@ -1511,7 +1511,7 @@ def _get_or_create_parent(self, path, parent_by_path): Note: the root path and root Resource must already be in `parent_by_path` or else this function does not work. """ - parent_path = parent_directory(path).rstrip('/').rstrip('\\') + parent_path = parent_directory(path).rstrip('/').rstrip('\\').lstrip("/") existing_parent = parent_by_path.get(parent_path) if existing_parent: return existing_parent diff --git a/tests/data/resource/virtual_codebase/path_full_root.json b/tests/data/resource/virtual_codebase/path_full_root.json new file mode 100644 index 0000000..dc768ec --- /dev/null +++ b/tests/data/resource/virtual_codebase/path_full_root.json @@ -0,0 +1,36 @@ +{ + "scancode_notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "scancode_version": "2.2.1.post12.6d07756e", + "scancode_options": { + "--info": true, + "--license-score": 0, + "--full-root": true, + "--format": "json-pp" + }, + "files_count": 1, + "files": [ + { + "path": "/Users/sesser/code/nexb/scancode-toolkit/samples/README", + "type": "file", + "name": "README", + "base_name": "README", + "extension": "", + "date": "2017-09-22", + "size": 236, + "sha1": "2e07e32c52d607204fad196052d70e3d18fb8636", + "md5": "effc6856ef85a9250fb1a470792b3f38", + "files_count": null, + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "scan_errors": [] + } + ] + } + \ No newline at end of file diff --git a/tests/test_resource.py b/tests/test_resource.py index 697646c..4611720 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -1323,6 +1323,13 @@ def test_VirtualCodebase_create_from_multiple_scans(self): ] assert expected == results + def test_VirtualCodebase_scanning_full_root(self): + test_file = self.get_test_loc("resource/virtual_codebase/path_full_root.json") + codebase = VirtualCodebase(test_file) + resource = sorted(r for r in codebase.walk())[0] + assert resource.path == "/Users/sesser/code/nexb/scancode-toolkit/samples/README" + assert codebase.compute_counts()[0] == 1 + class TestResource(FileBasedTesting): test_data_dir = join(dirname(__file__), 'data') From 38f1db6b8c0bd4b62450d364fa34762b82f17b63 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 28 Apr 2021 15:42:29 +0200 Subject: [PATCH 4/6] Remove unused import Signed-off-by: Philippe Ombredanne --- src/commoncode/system.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/commoncode/system.py b/src/commoncode/system.py index 3496bc8..48b777b 100644 --- a/src/commoncode/system.py +++ b/src/commoncode/system.py @@ -20,7 +20,6 @@ import getpass import os -import subprocess import sys From b4c147688d83b36cae1303e4ebf30559f4f212c6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 28 Apr 2021 15:45:00 +0200 Subject: [PATCH 5/6] Add Resource.get_path(strip_root) function This functsion allows to strip the leading path segment of a Resource path and is useful when dealing with paths in ScanCode.io and ScanCode while using a VirtualCodebase Signed-off-by: Philippe Ombredanne --- src/commoncode/resource.py | 81 ++++++++++++++++--- .../stripped-and-skipped-root.json | 19 +++++ tests/test_resource.py | 7 ++ 3 files changed, 95 insertions(+), 12 deletions(-) create mode 100644 tests/data/resource/virtual_codebase/stripped-and-skipped-root.json diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py index cbecaed..6bd25e5 100644 --- a/src/commoncode/resource.py +++ b/src/commoncode/resource.py @@ -60,6 +60,7 @@ from commoncode.fileutils import splitext_name from commoncode import ignore +from commoncode import paths """ This module provides Codebase and Resource objects as an abstraction for files @@ -1054,6 +1055,12 @@ def type(self, value): else: self.is_file = False + def get_path(self, strip_root=False): + if strip_root: + return strip_first_path_segment(self.path) + else: + return self.path + @property def is_dir(self): # note: we only store is_file @@ -1354,6 +1361,29 @@ def get_path(root_location, location, full_root=False, strip_root=False): return posix_loc.replace(posix_root_loc, '', 1) +def strip_first_path_segment(path): + """ + Return a POSIX path stripped from its first path segment. + + For example:: + >>> strip_first_path_segment('') + '' + >>> strip_first_path_segment('foo') + 'foo' + >>> strip_first_path_segment('foo/bar/baz') + 'bar/baz' + >>> strip_first_path_segment('/foo/bar/baz/') + 'bar/baz' + >>> strip_first_path_segment('foo/') + 'foo/' + """ + segments = paths.split(path) + if not segments or len(segments) == 1: + return path + stripped = segments[1:] + return '/'.join(stripped) + + def get_codebase_cache_dir(temp_dir): """ Return a new, created and unique per-run cache storage directory path rooted @@ -1506,10 +1536,11 @@ def _get_or_create_parent(self, path, parent_by_path): """ Return a parent resource for a given `path` from `parent_by_path`. - If a parent resource for a `path` does not exist in `parent_by_path`, it is created recursively. + If a parent resource for a `path` does not exist in `parent_by_path`, it + is created recursively. - Note: the root path and root Resource must already be in `parent_by_path` or else this - function does not work. + Note: the root path and root Resource must already be in + `parent_by_path` or else this function does not work. """ parent_path = parent_directory(path).rstrip('/').rstrip('\\') existing_parent = parent_by_path.get(parent_path) @@ -1519,7 +1550,13 @@ def _get_or_create_parent(self, path, parent_by_path): parent_name = file_base_name(parent_path) parent_is_file = False parent_resource_data = self._create_empty_resource_data() - parent_resource = self._create_resource(parent_name, parent_parent, parent_is_file, parent_path, parent_resource_data) + parent_resource = self._create_resource( + parent_name, + parent_parent, + parent_is_file, + parent_path, + parent_resource_data, + ) parent_by_path[parent_path] = parent_resource return parent_resource @@ -1620,7 +1657,12 @@ def _populate(self, scan_data): root_name = root_path root_is_file = False root_data = self._create_empty_resource_data() - root_resource = self._create_root_resource(root_name, root_path, root_is_file, root_data) + root_resource = self._create_root_resource( + name=root_name, + path=root_path, + is_file=root_is_file, + root_data=root_data, + ) # To help recreate the resource tree we keep a mapping by path of any # parent resource @@ -1630,24 +1672,32 @@ def _populate(self, scan_data): path = resource_data.get('path') # Append virtual_root path to imported Resource path if we are merging multiple scans if multiple_input: - path = os.path.join(root_path, path) + path = posixpath.join(root_path, path) + name = resource_data.get('name', None) if not name: name = file_name(path) + is_file = resource_data.get('type', 'file') == 'file' existing_parent = parent_by_path.get(path) if existing_parent: - # We update the empty parent Resouorce we in _get_or_create_parent() with the data - # from the scan + # We update the empty parent Resouorce we in + # _get_or_create_parent() with the data from the scan for k, v in resource_data.items(): setattr(existing_parent, k, v) self.save_resource(existing_parent) else: - # Note: `root_path`: `root_resource` must be in `parent_by_path` in order for - # `_get_or_create_parent` to work + # Note: `root_path`: `root_resource` must be in `parent_by_path` + # in order for `_get_or_create_parent` to work parent = self._get_or_create_parent(path, parent_by_path) - resource = self._create_resource(name, parent, is_file, path, resource_data) + resource = self._create_resource( + name=name, + parent=parent, + is_file=is_file, + path=path, + resource_data=resource_data, + ) # Files are not parents (for now), so we do not need to add this # to the parent_by_path mapping @@ -1665,7 +1715,14 @@ def _create_root_resource(self, name, path, is_file, root_data): if root_data: root_data = remove_properties_and_basics(root_data) root = self.resource_class( - name=name, location=None, path=path, rid=0, pid=None, is_file=is_file, **root_data) + name=name, + location=None, + path=path, + rid=0, + pid=None, + is_file=is_file, + **root_data, + ) self.resource_ids.add(0) self.resources[0] = root diff --git a/tests/data/resource/virtual_codebase/stripped-and-skipped-root.json b/tests/data/resource/virtual_codebase/stripped-and-skipped-root.json new file mode 100644 index 0000000..9d6acdd --- /dev/null +++ b/tests/data/resource/virtual_codebase/stripped-and-skipped-root.json @@ -0,0 +1,19 @@ +{ + "files": [ + { + "path": "samples", + "type": "directory", + "scan_errors": [] + }, + { + "path": "samples/README", + "type": "file", + "scan_errors": [] + }, + { + "path": "samples/screenshot.png", + "type": "file", + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/test_resource.py b/tests/test_resource.py index 697646c..862274a 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -802,6 +802,13 @@ def test_virtual_codebase_walk_skip_root_basic(self): ] assert expected == [(r.name, r.is_file) for r in results] + def test_virtual_codebase_get_path_with_strip_root_and_walk_with_skip_root(self): + scan_data = self.get_test_loc('resource/virtual_codebase/stripped-and-skipped-root.json') + virtual_codebase = VirtualCodebase(location=scan_data) + results = [r.get_path(strip_root=True) for r in virtual_codebase.walk(skip_root=True)] + expected = ['README', 'screenshot.png'] + assert results == expected + def test_virtual_codebase_walk_filtered_with_filtered_root(self): scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) From 53866eb3ad69e36c686a72164906ae92987f5481 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 28 Apr 2021 15:47:19 +0200 Subject: [PATCH 6/6] Update CHANGELOG Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1217186..b396dc7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,14 @@ Release notes vNext ----- + +Version 21.4.28 +--------------- + +*2020-04-28* +- Add new function to get a Resource path stripped from its root path segment + + Version 21.1.21 ---------------