Devops: Move the source directory into src/ (#6238)

This package directory structure is fast becoming the standard in the
Python packaging community. Even though the PyPA is not yet officially
recommending it as the standard, there are various articles out there
outlining the advantages of this structure over having the source folder
at the root of the repository.

.github/system_tests/test_daemon.py

@@ -15,6 +15,16 @@
 import tempfile
 import time
 
+from aiida.common import StashMode, exceptions
+from aiida.engine import run, submit
+from aiida.engine.daemon.client import get_daemon_client
+from aiida.engine.persistence import ObjectLoader
+from aiida.engine.processes import CalcJob, Process
+from aiida.manage.caching import enable_caching
+from aiida.orm import CalcJobNode, Dict, Int, List, Str, load_code, load_node
+from aiida.orm.nodes.caching import NodeCaching
+from aiida.plugins import CalculationFactory, WorkflowFactory
+from aiida.workflows.arithmetic.add_multiply import add, add_multiply
 from workchains import (
     ArithmeticAddBaseWorkChain,
     CalcFunctionRunnerWorkChain,
@@ -28,16 +38,6 @@
     WorkFunctionRunnerWorkChain,
 )
 
-from aiida.common import StashMode, exceptions
-from aiida.engine import run, submit
-from aiida.engine.daemon.client import get_daemon_client
-from aiida.engine.persistence import ObjectLoader
-from aiida.engine.processes import CalcJob, Process
-from aiida.manage.caching import enable_caching
-from aiida.orm import CalcJobNode, Dict, Int, List, Str, load_code, load_node
-from aiida.orm.nodes.caching import NodeCaching
-from aiida.plugins import CalculationFactory, WorkflowFactory
-from aiida.workflows.arithmetic.add_multiply import add, add_multiply
 from tests.utils.memory import get_instances
 
 CODENAME_ADD = 'add@localhost'

.github/workflows/check_release_tag.py

@@ -29,5 +29,5 @@ def get_version_from_module(content: str) -> str:
     args = parser.parse_args()
     assert args.GITHUB_REF.startswith('refs/tags/v'), f'GITHUB_REF should start with "refs/tags/v": {args.GITHUB_REF}'
     tag_version = args.GITHUB_REF[11:]
-    pypi_version = get_version_from_module(Path('aiida/__init__.py').read_text(encoding='utf-8'))
+    pypi_version = get_version_from_module(Path('src/aiida/__init__.py').read_text(encoding='utf-8'))
     assert tag_version == pypi_version, f'The tag version {tag_version} != {pypi_version} specified in `pyproject.toml`'

.molecule/default/files/polish/cli.py

@@ -13,7 +13,6 @@
 import time
 
 import click
-
 from aiida.cmdline.params import options, types
 from aiida.cmdline.utils import decorators
 

.pre-commit-config.yaml

@@ -66,7 +66,7 @@ repos:
     types: [python]
     require_serial: true
     pass_filenames: false
-    files: aiida/.*py
+    files: src/aiida/.*py
 
   - id: mypy
     name: mypy
@@ -84,109 +84,109 @@ repos:
         docs/.*|
         utils/.*|
 
-        aiida/calculations/arithmetic/add.py|
-        aiida/calculations/diff_tutorial/calculations.py|
-        aiida/calculations/templatereplacer.py|
-        aiida/calculations/transfer.py|
-        aiida/cmdline/commands/cmd_archive.py|
-        aiida/cmdline/commands/cmd_calcjob.py|
-        aiida/cmdline/commands/cmd_code.py|
-        aiida/cmdline/commands/cmd_computer.py|
-        aiida/cmdline/commands/cmd_data/cmd_list.py|
-        aiida/cmdline/commands/cmd_data/cmd_upf.py|
-        aiida/cmdline/commands/cmd_devel.py|
-        aiida/cmdline/commands/cmd_group.py|
-        aiida/cmdline/commands/cmd_node.py|
-        aiida/cmdline/commands/cmd_shell.py|
-        aiida/cmdline/commands/cmd_storage.py|
-        aiida/cmdline/params/options/commands/setup.py|
-        aiida/cmdline/params/options/interactive.py|
-        aiida/cmdline/params/options/main.py|
-        aiida/cmdline/params/options/multivalue.py|
-        aiida/cmdline/params/types/group.py|
-        aiida/cmdline/utils/ascii_vis.py|
-        aiida/cmdline/utils/common.py|
-        aiida/cmdline/utils/echo.py|
-        aiida/common/extendeddicts.py|
-        aiida/common/hashing.py|
-        aiida/common/utils.py|
-        aiida/engine/daemon/execmanager.py|
-        aiida/engine/processes/calcjobs/manager.py|
-        aiida/engine/processes/calcjobs/monitors.py|
-        aiida/engine/processes/calcjobs/tasks.py|
-        aiida/engine/processes/control.py|
-        aiida/engine/processes/ports.py|
-        aiida/manage/configuration/__init__.py|
-        aiida/manage/configuration/config.py|
-        aiida/manage/configuration/profile.py|
-        aiida/manage/external/rmq/launcher.py|
-        aiida/manage/tests/main.py|
-        aiida/manage/tests/pytest_fixtures.py|
-        aiida/orm/comments.py|
-        aiida/orm/computers.py|
-        aiida/orm/implementation/storage_backend.py|
-        aiida/orm/nodes/caching.py|
-        aiida/orm/nodes/comments.py|
-        aiida/orm/nodes/data/array/bands.py|
-        aiida/orm/nodes/data/array/trajectory.py|
-        aiida/orm/nodes/data/cif.py|
-        aiida/orm/nodes/data/remote/base.py|
-        aiida/orm/nodes/data/structure.py|
-        aiida/orm/nodes/data/upf.py|
-        aiida/orm/nodes/process/calculation/calcjob.py|
-        aiida/orm/nodes/process/process.py|
-        aiida/orm/utils/builders/code.py|
-        aiida/orm/utils/builders/computer.py|
-        aiida/orm/utils/calcjob.py|
-        aiida/orm/utils/node.py|
-        aiida/orm/utils/remote.py|
-        aiida/repository/backend/disk_object_store.py|
-        aiida/repository/backend/sandbox.py|
-        aiida/restapi/common/utils.py|
-        aiida/restapi/resources.py|
-        aiida/restapi/run_api.py|
-        aiida/restapi/translator/base.py|
-        aiida/restapi/translator/computer.py|
-        aiida/restapi/translator/group.py|
-        aiida/restapi/translator/nodes/.*|
-        aiida/restapi/translator/user.py|
-        aiida/schedulers/plugins/direct.py|
-        aiida/schedulers/plugins/lsf.py|
-        aiida/schedulers/plugins/pbsbaseclasses.py|
-        aiida/schedulers/plugins/sge.py|
-        aiida/schedulers/plugins/slurm.py|
-        aiida/storage/psql_dos/migrations/utils/integrity.py|
-        aiida/storage/psql_dos/migrations/utils/legacy_workflows.py|
-        aiida/storage/psql_dos/migrations/utils/migrate_repository.py|
-        aiida/storage/psql_dos/migrations/utils/parity.py|
-        aiida/storage/psql_dos/migrations/utils/reflect.py|
-        aiida/storage/psql_dos/migrations/utils/utils.py|
-        aiida/storage/psql_dos/migrations/versions/1de112340b16_django_parity_1.py|
-        aiida/storage/psql_dos/migrator.py|
-        aiida/storage/psql_dos/models/.*|
-        aiida/storage/psql_dos/orm/.*|
-        aiida/storage/sqlite_temp/backend.py|
-        aiida/storage/sqlite_zip/backend.py|
-        aiida/storage/sqlite_zip/migrations/legacy_to_main.py|
-        aiida/storage/sqlite_zip/migrator.py|
-        aiida/storage/sqlite_zip/models.py|
-        aiida/storage/sqlite_zip/orm.py|
-        aiida/tools/data/array/kpoints/legacy.py|
-        aiida/tools/data/array/kpoints/seekpath.py|
-        aiida/tools/data/orbital/orbital.py|
-        aiida/tools/data/orbital/realhydrogen.py|
-        aiida/tools/dbimporters/plugins/.*|
-        aiida/tools/graph/age_entities.py|
-        aiida/tools/graph/age_rules.py|
-        aiida/tools/graph/deletions.py|
-        aiida/tools/graph/graph_traversers.py|
-        aiida/tools/groups/paths.py|
-        aiida/tools/query/calculation.py|
-        aiida/tools/query/mapping.py|
-        aiida/transports/cli.py|
-        aiida/transports/plugins/local.py|
-        aiida/transports/plugins/ssh.py|
-        aiida/workflows/arithmetic/multiply_add.py|
+        src/aiida/calculations/arithmetic/add.py|
+        src/aiida/calculations/diff_tutorial/calculations.py|
+        src/aiida/calculations/templatereplacer.py|
+        src/aiida/calculations/transfer.py|
+        src/aiida/cmdline/commands/cmd_archive.py|
+        src/aiida/cmdline/commands/cmd_calcjob.py|
+        src/aiida/cmdline/commands/cmd_code.py|
+        src/aiida/cmdline/commands/cmd_computer.py|
+        src/aiida/cmdline/commands/cmd_data/cmd_list.py|
+        src/aiida/cmdline/commands/cmd_data/cmd_upf.py|
+        src/aiida/cmdline/commands/cmd_devel.py|
+        src/aiida/cmdline/commands/cmd_group.py|
+        src/aiida/cmdline/commands/cmd_node.py|
+        src/aiida/cmdline/commands/cmd_shell.py|
+        src/aiida/cmdline/commands/cmd_storage.py|
+        src/aiida/cmdline/params/options/commands/setup.py|
+        src/aiida/cmdline/params/options/interactive.py|
+        src/aiida/cmdline/params/options/main.py|
+        src/aiida/cmdline/params/options/multivalue.py|
+        src/aiida/cmdline/params/types/group.py|
+        src/aiida/cmdline/utils/ascii_vis.py|
+        src/aiida/cmdline/utils/common.py|
+        src/aiida/cmdline/utils/echo.py|
+        src/aiida/common/extendeddicts.py|
+        src/aiida/common/hashing.py|
+        src/aiida/common/utils.py|
+        src/aiida/engine/daemon/execmanager.py|
+        src/aiida/engine/processes/calcjobs/manager.py|
+        src/aiida/engine/processes/calcjobs/monitors.py|
+        src/aiida/engine/processes/calcjobs/tasks.py|
+        src/aiida/engine/processes/control.py|
+        src/aiida/engine/processes/ports.py|
+        src/aiida/manage/configuration/__init__.py|
+        src/aiida/manage/configuration/config.py|
+        src/aiida/manage/configuration/profile.py|
+        src/aiida/manage/external/rmq/launcher.py|
+        src/aiida/manage/tests/main.py|
+        src/aiida/manage/tests/pytest_fixtures.py|
+        src/aiida/orm/comments.py|
+        src/aiida/orm/computers.py|
+        src/aiida/orm/implementation/storage_backend.py|
+        src/aiida/orm/nodes/caching.py|
+        src/aiida/orm/nodes/comments.py|
+        src/aiida/orm/nodes/data/array/bands.py|
+        src/aiida/orm/nodes/data/array/trajectory.py|
+        src/aiida/orm/nodes/data/cif.py|
+        src/aiida/orm/nodes/data/remote/base.py|
+        src/aiida/orm/nodes/data/structure.py|
+        src/aiida/orm/nodes/data/upf.py|
+        src/aiida/orm/nodes/process/calculation/calcjob.py|
+        src/aiida/orm/nodes/process/process.py|
+        src/aiida/orm/utils/builders/code.py|
+        src/aiida/orm/utils/builders/computer.py|
+        src/aiida/orm/utils/calcjob.py|
+        src/aiida/orm/utils/node.py|
+        src/aiida/orm/utils/remote.py|
+        src/aiida/repository/backend/disk_object_store.py|
+        src/aiida/repository/backend/sandbox.py|
+        src/aiida/restapi/common/utils.py|
+        src/aiida/restapi/resources.py|
+        src/aiida/restapi/run_api.py|
+        src/aiida/restapi/translator/base.py|
+        src/aiida/restapi/translator/computer.py|
+        src/aiida/restapi/translator/group.py|
+        src/aiida/restapi/translator/nodes/.*|
+        src/aiida/restapi/translator/user.py|
+        src/aiida/schedulers/plugins/direct.py|
+        src/aiida/schedulers/plugins/lsf.py|
+        src/aiida/schedulers/plugins/pbsbaseclasses.py|
+        src/aiida/schedulers/plugins/sge.py|
+        src/aiida/schedulers/plugins/slurm.py|
+        src/aiida/storage/psql_dos/migrations/utils/integrity.py|
+        src/aiida/storage/psql_dos/migrations/utils/legacy_workflows.py|
+        src/aiida/storage/psql_dos/migrations/utils/migrate_repository.py|
+        src/aiida/storage/psql_dos/migrations/utils/parity.py|
+        src/aiida/storage/psql_dos/migrations/utils/reflect.py|
+        src/aiida/storage/psql_dos/migrations/utils/utils.py|
+        src/aiida/storage/psql_dos/migrations/versions/1de112340b16_django_parity_1.py|
+        src/aiida/storage/psql_dos/migrator.py|
+        src/aiida/storage/psql_dos/models/.*|
+        src/aiida/storage/psql_dos/orm/.*|
+        src/aiida/storage/sqlite_temp/backend.py|
+        src/aiida/storage/sqlite_zip/backend.py|
+        src/aiida/storage/sqlite_zip/migrations/legacy_to_main.py|
+        src/aiida/storage/sqlite_zip/migrator.py|
+        src/aiida/storage/sqlite_zip/models.py|
+        src/aiida/storage/sqlite_zip/orm.py|
+        src/aiida/tools/data/array/kpoints/legacy.py|
+        src/aiida/tools/data/array/kpoints/seekpath.py|
+        src/aiida/tools/data/orbital/orbital.py|
+        src/aiida/tools/data/orbital/realhydrogen.py|
+        src/aiida/tools/dbimporters/plugins/.*|
+        src/aiida/tools/graph/age_entities.py|
+        src/aiida/tools/graph/age_rules.py|
+        src/aiida/tools/graph/deletions.py|
+        src/aiida/tools/graph/graph_traversers.py|
+        src/aiida/tools/groups/paths.py|
+        src/aiida/tools/query/calculation.py|
+        src/aiida/tools/query/mapping.py|
+        src/aiida/transports/cli.py|
+        src/aiida/transports/plugins/local.py|
+        src/aiida/transports/plugins/ssh.py|
+        src/aiida/workflows/arithmetic/multiply_add.py|
 
         tests/conftest.py|
         tests/repository/conftest.py|
@@ -230,8 +230,8 @@ repos:
     pass_filenames: false
     files: >-
       (?x)^(
-        aiida/cmdline/commands/.*|
-        aiida/cmdline/params/.*|
-        aiida/cmdline/params/types/.*|
+        src/aiida/cmdline/commands/.*|
+        src/aiida/cmdline/params/.*|
+        src/aiida/cmdline/params/types/.*|
         utils/validate_consistency.py|
       )$

docs/source/conf.py

@@ -205,7 +205,7 @@ def run_apidoc(_):
     """
     source_dir = os.path.abspath(os.path.dirname(__file__))
     apidoc_dir = os.path.join(source_dir, 'reference', 'apidoc')
-    package_dir = os.path.join(source_dir, os.pardir, os.pardir, 'aiida')
+    package_dir = os.path.join(source_dir, os.pardir, os.pardir, 'src', 'aiida')
     exclude_api_patterns = [
         os.path.join(package_dir, 'storage', 'psql_dos', 'migrations', 'versions'),
     ]

docs/source/howto/include/scripts/performance_benchmark_base.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 """Script to benchmark the performance of the AiiDA workflow engine on a given installation."""
 import click
-
 from aiida.cmdline.params import options
 from aiida.cmdline.utils import decorators, echo
 

docs/source/howto/plugin_codes.rst

@@ -85,7 +85,7 @@ The |define| method tells AiiDA which inputs the |CalcJob| expects and which out
 This is done through an instance of the :py:class:`~aiida.engine.processes.process_spec.CalcJobProcessSpec` class, which is passed as the |spec| argument to the |define| method.
 For example:
 
-.. literalinclude:: ../../../aiida/calculations/diff_tutorial/calculations.py
+.. literalinclude:: ../../../src/aiida/calculations/diff_tutorial/calculations.py
     :language: python
     :pyobject: DiffCalculation.define
 
@@ -139,7 +139,7 @@ The :py:meth:`~aiida.engine.processes.calcjobs.calcjob.CalcJob.prepare_for_submi
 Creating the input files in the format the external code expects and returning a :py:class:`~aiida.common.datastructures.CalcInfo` object that contains instructions for the AiiDA engine on how the code should be run.
 For example:
 
-.. literalinclude:: ../../../aiida/calculations/diff_tutorial/calculations.py
+.. literalinclude:: ../../../src/aiida/calculations/diff_tutorial/calculations.py
     :language: python
     :pyobject: DiffCalculation.prepare_for_submission
 
@@ -203,7 +203,7 @@ Parsing the output files produced by a code into AiiDA nodes is optional, but it
 
 To create a parser plugin, subclass the |Parser| class in a file called ``parsers.py``.
 
-.. literalinclude::  ../../../aiida/parsers/plugins/diff_tutorial/parsers.py
+.. literalinclude::  ../../../src/aiida/parsers/plugins/diff_tutorial/parsers.py
     :language: python
     :start-after: # START PARSER HEAD
     :end-before: # END PARSER HEAD
@@ -216,7 +216,7 @@ Before the ``parse()`` method is called, two important attributes are set on the
 
 Now implement its :py:meth:`~aiida.parsers.parser.Parser.parse` method as
 
-.. literalinclude:: ../../../aiida/parsers/plugins/diff_tutorial/parsers.py
+.. literalinclude:: ../../../src/aiida/parsers/plugins/diff_tutorial/parsers.py
     :language: python
     :pyobject: DiffParserSimple.parse
 
@@ -283,7 +283,7 @@ An ``exit_code`` defines:
 In order to inform AiiDA about a failed calculation, simply return from the ``parse`` method the exit code that corresponds to the detected issue.
 Here is a more complete version of the example |Parser| presented in the previous section:
 
-.. literalinclude:: ../../../aiida/parsers/plugins/diff_tutorial/parsers.py
+.. literalinclude:: ../../../src/aiida/parsers/plugins/diff_tutorial/parsers.py
     :language: python
     :pyobject: DiffParser.parse