Merge pull request #42 from nexB/use-paths-in-codebase

Use paths in codebase

.github/workflows/pypi-release.yml

@@ -1,27 +1,83 @@
-name: Release library as a PyPI wheel and sdist on GH release creation
+name: Create library release archives, create a GH release and publish PyPI wheel and sdist on tag in main branch
+
+
+# This is executed automatically on a tag in the main branch
+
+# Summary of the steps:
+# - build wheels and sdist
+# - upload wheels and sdist to PyPI
+# - create gh-release and upload wheels and dists there
+# TODO: smoke test wheels and sdist
+# TODO: add changelog to release text body
+
+# WARNING: this is designed only for packages building as pure Python wheels
 
 on:
-  release:
-    types: [created]
+  workflow_dispatch:
+  push:
+    tags:
+      - "v*.*.*"
 
 jobs:
-  build-and-publish-to-pypi:
+  build-pypi-distribs:
     name: Build and publish library to PyPI
     runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@master
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.9
+
+      - name: Install pypa/build
+        run: python -m pip install build --user
+
+      - name: Build a binary wheel and a source tarball
+        run: python -m build --sdist --wheel --outdir dist/
+
+      - name: Upload built archives
+        uses: actions/upload-artifact@v3
+        with:
+          name: pypi_archives
+          path: dist/*
+
+
+  create-gh-release:
+    name: Create GH release
+    needs:
+      - build-pypi-distribs
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Download built archives
+        uses: actions/download-artifact@v3
+        with:
+          name: pypi_archives
+          path: dist
+
+      - name: Create GH release
+        uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: dist/*
+
+
+  create-pypi-release:
+    name: Create PyPI release
+    needs:
+      - create-gh-release
+    runs-on: ubuntu-20.04
+
     steps:
-     - uses: actions/checkout@master
-     - name: Set up Python
-       uses: actions/setup-python@v1
-       with:
-         python-version: 3.9
-     - name: Install pypa/build
-       run: python -m pip install build --user
-     - name: Build a binary wheel and a source tarball
-       run: python -m build --sdist --wheel --outdir dist/
-        .
-     - name: Publish distribution  to PyPI
-       if: startsWith(github.ref, 'refs/tags')
-       uses: pypa/gh-action-pypi-publish@master
-       with:
-         password: ${{ secrets.PYPI_API_TOKEN }}
-
+      - name: Download built archives
+        uses: actions/download-artifact@v3
+        with:
+          name: pypi_archives
+          path: dist
+
+      - name: Publish to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

CHANGELOG.rst

@@ -4,7 +4,85 @@ Release notes
 Version (next) 
 ------------------------------
 
+TBD.
+
+Version 31.0.0 - (2022-05-16)
+------------------------------
+
+This is a major version with API-breaking changes in the resource module.
+
+- The Resource has no rid (resource id) and no pid (parent id). Instead
+  we now use internally a simpler mapping of {path: Resource} object.
+  As a result the iteration on a Codebase is faster but this requires more
+  memory.
+
+- The Codebase and VirtualCodebase accepts a new "paths" argument that is list
+  of paths. When provided, the Codebase will only contain Resources with these
+  paths and no other resources. This handy to create a Codebase with only a
+  subset of paths of interest. When we create a Codebase or VirtualCodebase
+  with paths, we also always create any intermediate directories. So if you
+  ask for a path of "root/dir/file", we create three resources: "root",
+  "root/dir" and "root/dir/file". We accumulate codebase errors if the paths
+  does not exists in the Codebase or VirtualCodebase. The paths must start with
+  the root path segment and must be POSIX paths.
+
+- When you create a VirtualCodebase with multiple scans, we now prefix each
+  scan path with a codebase-1/, codebase-2/, etc. directory in addition to the
+  "virtual_root" shared root directory. Otherwise files data was overwritten
+  and inconsistent when each location "files" were sharing leading path
+  segments. So if you provide to JSON inputs with that each contain the path
+  "root/dir/file", the VirtualCodebase will contain these paths:
+
+    - "virtual_root/codebase-1/root/dir/file"
+    - "virtual_root/codebase-2/root/dir/file"
+
+  It is otherwise practically impossible to correctly merge file data from
+  multiple codebases reliably, so adding this prefix ensures that we are doing
+  the right thing
+
+- The Resource.path now never contains leading or trailing slash. We also
+  normalize the path everywhere. In particular this behaviour is visible when
+  you create a Codebase with a "full_root" argument. Previously, the paths of a
+  "full_root" Codebase were prefixed with a slash "/".
+
+- When you create a VirtualCodebase with more than one Resource, we now recreate
+  the directory tree for any intermediary directory used in a path that is
+  otherwise missing from files path list.
+  In particular this behaviour changed when you create a VirtualCodebase from
+  a previous Codebase created with a "full_root" argument. Previously, the
+  missing paths of a "full_root" Codebase were kept unchanged. 
+  Note that the VirtualCodebase has always ignored the "full_root" argument.
+
+- The Codebase and VirtualCodebase are now iterable. Iterating on a codebase
+  is the same as a top-down walk.
+
+- The "Codebase.original_location" attributed has been removed.
+  No known users of commoncode used this.
+
+- The Codebase and VirtualCodebase no longer have a "full_root" and
+  "strip_root" constructor arguments and attributes. These can still be
+  passed but they will be ignored.
+
+  - Resource.path is now always the plain path where the first segment
+    is the last segment of the root location, e.g. the root fiename.
+
+  - The Resource now has new "full_root_path" and "strip_root_path"
+    properties that return the corresponding paths.
+
+  - The Resource.to_dict and the new Codebase.to_list both have a new
+    "full_root" and "strip_root" arguments
+
+  - The Resource.get_path() method accepts "full_root" and "strip_root" arguments.
+
+- The Resource.create_child() method has been removed.
+
+Other changes:
+
 - Remove Python upper version limit.
+- Merge latest skeleton
+- fileutils.parent_directory() now accepts a "with_trail" argument. 
+  The returned directory has a trailing path separator unless with_trail is False.
+  The default is True and the default behaviour is unchanged.
 
 
 Version 30.2.0 - (2022-05-02)
@@ -39,7 +117,7 @@ Version 30.1.0 (2022-04-05)
 Version 30.0.0 (2021-09-24)
 ------------------------------
 
-- Switch back from clamver to semver.
+- Switch back from calver to semver.
 - Adopt latest skeleton. The default virtualenv directory is now venv and no
   longer tmp
 - Fix issue with Click progressbar API #23 that prohibited to use all supported

CODE_OF_CONDUCT.rst

@@ -0,0 +1,86 @@
+Contributor Covenant Code of Conduct
+====================================
+
+Our Pledge
+----------
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our
+project and our community a harassment-free experience for everyone,
+regardless of age, body size, disability, ethnicity, gender identity and
+expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+Our Standards
+-------------
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+-  Using welcoming and inclusive language
+-  Being respectful of differing viewpoints and experiences
+-  Gracefully accepting constructive criticism
+-  Focusing on what is best for the community
+-  Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+-  The use of sexualized language or imagery and unwelcome sexual
+   attention or advances
+-  Trolling, insulting/derogatory comments, and personal or political
+   attacks
+-  Public or private harassment
+-  Publishing others’ private information, such as a physical or
+   electronic address, without explicit permission
+-  Other conduct which could reasonably be considered inappropriate in a
+   professional setting
+
+Our Responsibilities
+--------------------
+
+Project maintainers are responsible for clarifying the standards of
+acceptable behavior and are expected to take appropriate and fair
+corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit,
+or reject comments, commits, code, wiki edits, issues, and other
+contributions that are not aligned to this Code of Conduct, or to ban
+temporarily or permanently any contributor for other behaviors that they
+deem inappropriate, threatening, offensive, or harmful.
+
+Scope
+-----
+
+This Code of Conduct applies both within project spaces and in public
+spaces when an individual is representing the project or its community.
+Examples of representing a project or community include using an
+official project e-mail address, posting via an official social media
+account, or acting as an appointed representative at an online or
+offline event. Representation of a project may be further defined and
+clarified by project maintainers.
+
+Enforcement
+-----------
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may
+be reported by contacting the project team at pombredanne@gmail.com
+or on the Gitter chat channel at https://gitter.im/aboutcode-org/discuss .
+All complaints will be reviewed and investigated and will result in a
+response that is deemed necessary and appropriate to the circumstances.
+The project team is obligated to maintain confidentiality with regard to
+the reporter of an incident. Further details of specific enforcement
+policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in
+good faith may face temporary or permanent repercussions as determined
+by other members of the project’s leadership.
+
+Attribution
+-----------
+
+This Code of Conduct is adapted from the `Contributor Covenant`_ ,
+version 1.4, available at
+https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+.. _Contributor Covenant: https://www.contributor-covenant.org

MANIFEST.in

@@ -9,7 +9,7 @@ include *.rst
 include setup.*
 include configure*
 include requirements*
-include .git*
+include .giti*
 
 global-exclude *.py[co] __pycache__ *.*~
 

configure

@@ -54,11 +54,10 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
 ################################
 # Thirdparty package locations and index handling
-# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
+# Find packages from the local thirdparty directory
 if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then
     PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty"
 fi
-PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html"
 
 
 ################################

configure.bat

@@ -52,11 +52,10 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts"
 
 @rem ################################
 @rem # Thirdparty package locations and index handling
-@rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
+@rem # Find packages from the local thirdparty directory
 if exist "%CFG_ROOT_DIR%\thirdparty" (
     set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty"
 )
-set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html"
 
 
 @rem ################################
@@ -69,7 +68,6 @@ if not defined CFG_QUIET (
 @rem ################################
 @rem # Main command line entry point
 set "CFG_REQUIREMENTS=%REQUIREMENTS%"
-set "NO_INDEX=--no-index"
 
 :again
 if not "%1" == "" (

docs/source/conf.py

@@ -28,16 +28,16 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-'sphinx.ext.intersphinx',
+    "sphinx.ext.intersphinx",
 ]
 
 # This points to aboutcode.readthedocs.io
 # In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot
 # Link was created at commit - https://github.com/nexB/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83
 
 intersphinx_mapping = {
-    'aboutcode': ('https://aboutcode.readthedocs.io/en/latest/', None),
-    'scancode-workbench': ('https://scancode-workbench.readthedocs.io/en/develop/', None),
+    "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None),
+    "scancode-workbench": ("https://scancode-workbench.readthedocs.io/en/develop/", None),
 }
 
 
@@ -62,7 +62,7 @@
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
 
-master_doc = 'index'
+master_doc = "index"
 
 html_context = {
     "display_github": True,
@@ -72,9 +72,7 @@
     "conf_py_path": "/docs/source/",  # path in the checkout to the docs root
 }
 
-html_css_files = [
-    '_static/theme_overrides.css'
-    ]
+html_css_files = ["_static/theme_overrides.css"]
 
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.