From 9659aa152eaa4b39aa26792d07dfef03d9718fe1 Mon Sep 17 00:00:00 2001 From: Thomas Rausch Date: Thu, 9 May 2024 16:24:13 +0200 Subject: [PATCH] configure project to work with src layout (#10800) --- .circleci/config.yml | 17 +++++++++-------- .dockerignore | 1 + .github/workflows/tests-pro-integration.yml | 6 +++--- Dockerfile | 3 ++- Dockerfile.s3 | 3 ++- Makefile | 7 ++++--- localstack-core/localstack/aws/scaffold.py | 6 ++++-- localstack-core/localstack/dev/run/paths.py | 4 ++-- .../testing/pytest/metric_collection.py | 2 +- pyproject.toml | 15 ++++++++++----- scripts/generate_minimal_boto3stubs_install.py | 4 +++- setup.py | 2 +- 12 files changed, 42 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cde3d127c506f..5697ea29b266c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -134,7 +134,7 @@ jobs: circleci-agent step halt else source .venv/bin/activate - python -m localstack.testing.testselection.scripts.generate_test_selection_from_pr /tmp/workspace/repo $CI_PULL_REQUEST target/testselection/test-selection.txt + PYTHONPATH=localstack-core python -m localstack.testing.testselection.scripts.generate_test_selection_from_pr /tmp/workspace/repo $CI_PULL_REQUEST target/testselection/test-selection.txt cat target/testselection/test-selection.txt fi @@ -402,7 +402,7 @@ jobs: if [ -z $TESTSELECTION_PYTEST_ARGS ] ; then TEST_FILES=$(circleci tests glob "tests/aws/**/test_*.py" "tests/integration/**/test_*.py" | circleci tests split --verbose --split-by=timings | tr '\n' ' ') else - TEST_FILES=$(circleci tests glob "tests/aws/**/test_*.py" "tests/integration/**/test_*.py" | python -m localstack.testing.testselection.scripts.filter_by_test_selection target/testselection/test-selection.txt | circleci tests split --verbose --split-by=timings | tr '\n' ' ') + TEST_FILES=$(circleci tests glob "tests/aws/**/test_*.py" "tests/integration/**/test_*.py" | PYTHONPATH=localstack-core python -m localstack.testing.testselection.scripts.filter_by_test_selection target/testselection/test-selection.txt | circleci tests split --verbose --split-by=timings | tr '\n' ' ') fi echo $TEST_FILES PYTEST_ARGS="${TINYBIRD_PYTEST_ARGS}${TESTSELECTION_PYTEST_ARGS}-o junit_family=legacy --junitxml=target/reports/test-report-<< parameters.platform >>-${CIRCLE_NODE_INDEX}.xml" \ @@ -465,8 +465,9 @@ jobs: - run: name: Run localstack command: | - DEBUG=1 DISABLE_EVENTS="1" IMAGE_NAME="localstack/localstack:latest" bin/localstack start -d - bin/localstack wait -t 120 || (bin/localstack logs && false) + source .venv/bin/activate + DEBUG=1 DISABLE_EVENTS="1" IMAGE_NAME="localstack/localstack:latest" localstack start -d + localstack wait -t 120 || (python -m localstack.cli.main logs && false) - run: name: Run capture-not-implemented command: | @@ -556,10 +557,10 @@ jobs: command: | source .venv/bin/activate pip install pycobertura - coverage xml --data-file=.coverage -o all.coverage.report.xml --include="localstack/services/*/**" --omit="*/**/__init__.py" - coverage xml --data-file=.coverage.acceptance -o acceptance.coverage.report.xml --include="localstack/services/*/**" --omit="*/**/__init__.py" - pycobertura show --format html -s localstack/ acceptance.coverage.report.xml -o coverage-acceptance.html - bash -c "pycobertura diff --format html -s1 localstack/ -s2 localstack/ all.coverage.report.xml acceptance.coverage.report.xml -o coverage-diff.html; if [[ \$? -eq 1 ]] ; then exit 1 ; else exit 0 ; fi" + coverage xml --data-file=.coverage -o all.coverage.report.xml --include="localstack-core/localstack/services/*/**" --omit="*/**/__init__.py" + coverage xml --data-file=.coverage.acceptance -o acceptance.coverage.report.xml --include="localstack-core/localstack/services/*/**" --omit="*/**/__init__.py" + pycobertura show --format html -s localstack-core/localstack acceptance.coverage.report.xml -o coverage-acceptance.html + bash -c "pycobertura diff --format html -s1 localstack-core/localstack/ -s2 localstack-core/localstack/ all.coverage.report.xml acceptance.coverage.report.xml -o coverage-diff.html; if [[ \$? -eq 1 ]] ; then exit 1 ; else exit 0 ; fi" - run: name: Create Metric Coverage Diff (API Coverage) environment: diff --git a/.dockerignore b/.dockerignore index 1ad806c4cbb23..db8b1c5a344a7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,7 @@ .venv* .filesystem +**/.filesystem # ignore files generated in CI build tests/aws/**/node_modules diff --git a/.github/workflows/tests-pro-integration.yml b/.github/workflows/tests-pro-integration.yml index 81923ef7aa9dd..6e2a11eec25fa 100644 --- a/.github/workflows/tests-pro-integration.yml +++ b/.github/workflows/tests-pro-integration.yml @@ -291,14 +291,14 @@ jobs: # Entrypoints need to be generated _after_ the community edition has been linked into the venv run: | VENV_DIR="../localstack-ext/.venv" make entrypoints - cat localstack_core.egg-info/entry_points.txt + ../localstack-ext/.venv/bin/python -m plux show - name: Create Pro Entrypoints working-directory: localstack-ext # Entrypoints need to be generated _after_ the community edition has been linked into the venv run: | make entrypoints - cat localstack_ext.egg-info/entry_points.txt + .venv/bin/python -m plux show - name: Test Pro Startup env: @@ -334,7 +334,7 @@ jobs: working-directory: localstack-ext run: | # Remove the host tmp folder (might contain remnant files with different permissions) - sudo rm -rf ../localstack/.filesystem/var/lib/localstack/tmp + sudo rm -rf ../localstack/localstack-core/.filesystem/var/lib/localstack/tmp make test - name: Archive Test Durations diff --git a/Dockerfile b/Dockerfile index 855a145427efb..ceefff63c42ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -99,6 +99,7 @@ ENV PATH "${PATH}:${JAVA_HOME}/bin" # set workdir RUN mkdir -p /opt/code/localstack +RUN mkdir /opt/code/localstack/localstack-core WORKDIR /opt/code/localstack/ # create localstack user and filesystem hierarchy, perform some permission fixes @@ -176,7 +177,7 @@ ADD Makefile pyproject.toml VERSION ./ ADD bin/localstack bin/localstack.bat bin/localstack-supervisor bin/ # add the code as late as possible -ADD localstack/ localstack/ +ADD localstack-core/ /opt/code/localstack/localstack-core # Generate the plugin entrypoints RUN make entrypoints diff --git a/Dockerfile.s3 b/Dockerfile.s3 index 5bb0aeec1fcd7..78cd4ba6176dd 100644 --- a/Dockerfile.s3 +++ b/Dockerfile.s3 @@ -4,6 +4,7 @@ ARG TARGETARCH # set workdir RUN mkdir -p /opt/code/localstack +RUN mkdir /opt/code/localstack/localstack-core WORKDIR /opt/code/localstack/ # Install runtime OS package dependencies @@ -81,7 +82,7 @@ ADD Makefile pyproject.toml VERSION requirements-base-runtime.txt ./ ADD bin/localstack bin/localstack.bat bin/localstack-supervisor bin/ # add the code as late as possible -ADD localstack/ localstack/ +ADD localstack-core/ /opt/code/localstack/localstack-core # Generate the plugin entrypoints RUN make entrypoints diff --git a/Makefile b/Makefile index bcae11f0e6f80..bd0c107f058f0 100644 --- a/Makefile +++ b/Makefile @@ -73,14 +73,14 @@ install: install-dev entrypoints ## Install full dependencies into venv entrypoints: ## Run plux to build entry points $(VENV_RUN); python -m plux entrypoints @# make sure that the entrypoints were correctly created and are non-empty - @test -s localstack_core.egg-info/entry_points.txt || (echo "Entrypoints were not correctly created! Aborting!" && exit 1) + @test -s localstack-core/localstack_core.egg-info/entry_points.txt || (echo "Entrypoints were not correctly created! Aborting!" && exit 1) dist: entrypoints ## Build source and built (wheel) distributions of the current version $(VENV_RUN); pip install --upgrade twine; python -m build publish: clean-dist dist ## Publish the library to the central PyPi repository # make sure the dist archive contains a non-empty entry_points.txt file before uploading - tar --wildcards --to-stdout -xf dist/localstack?core*.tar.gz "localstack?core*/localstack_core.egg-info/entry_points.txt" | grep . > /dev/null 2>&1 || (echo "Refusing upload, localstack-core dist does not contain entrypoints." && exit 1) + tar --wildcards --to-stdout -xf dist/localstack?core*.tar.gz "localstack?core*/localstack-core/localstack_core.egg-info/entry_points.txt" | grep . > /dev/null 2>&1 || (echo "Refusing upload, localstack-core dist does not contain entrypoints." && exit 1) $(VENV_RUN); twine upload dist/* coveralls: ## Publish coveralls metrics @@ -237,10 +237,11 @@ clean: ## Clean up (npm dependencies, downloaded infrastructure rm -rf build/ rm -rf dist/ rm -rf *.egg-info + rm -rf localstack-core/*.egg-info rm -rf $(VENV_DIR) clean-dist: ## Clean up python distribution directories rm -rf dist/ build/ - rm -rf *.egg-info + rm -rf localstack-core/*.egg-info .PHONY: usage freeze install-basic install-runtime install-test install-dev install entrypoints dist publish coveralls start docker-save-image docker-build docker-build-multiarch docker-push-master docker-create-push-manifests docker-run-tests docker-cp-coverage test test-coverage lint lint-modified format format-modified init-precommit clean clean-dist upgrade-pinned-dependencies diff --git a/localstack-core/localstack/aws/scaffold.py b/localstack-core/localstack/aws/scaffold.py index cbb8f3d74d112..bf72ac8a83b3b 100644 --- a/localstack-core/localstack/aws/scaffold.py +++ b/localstack-core/localstack/aws/scaffold.py @@ -472,7 +472,9 @@ def scaffold(): help="whether or not to save the result into the api directory", ) @click.option( - "--path", default="./localstack/aws/api", help="the path where the api should be saved" + "--path", + default="./localstack-core/localstack/aws/api", + help="the path where the api should be saved", ) def generate(service: str, doc: bool, save: bool, path: str): """ @@ -525,7 +527,7 @@ def create_code_directory(service_name: str, code: str, base_path: str): @click.option("--doc/--no-doc", default=False, help="whether or not to generate docstrings") @click.option( "--path", - default="./localstack/aws/api", + default="./localstack-core/localstack/aws/api", help="the path in which to upgrade ASF APIs", ) def upgrade(path: str, doc: bool = False): diff --git a/localstack-core/localstack/dev/run/paths.py b/localstack-core/localstack/dev/run/paths.py index a48a06a1d164e..a66e3b1fb278a 100644 --- a/localstack-core/localstack/dev/run/paths.py +++ b/localstack-core/localstack/dev/run/paths.py @@ -55,10 +55,10 @@ def dependency_source(self, name: str) -> str: class CommunityContainerPaths(ContainerPaths): - """In the community image, code is copied into /opt/code/localstack""" + """In the community image, code is copied into /opt/code/localstack/localstack-core/localstack""" def __init__(self): - self.localstack_source_dir = f"{self.project_dir}/localstack" + self.localstack_source_dir = f"{self.project_dir}/localstack-core/localstack" class ProContainerPaths(ContainerPaths): diff --git a/localstack-core/localstack/testing/pytest/metric_collection.py b/localstack-core/localstack/testing/pytest/metric_collection.py index 70c7b2d4cc2c3..c480a5330ac29 100644 --- a/localstack-core/localstack/testing/pytest/metric_collection.py +++ b/localstack-core/localstack/testing/pytest/metric_collection.py @@ -12,7 +12,7 @@ from localstack.aws.handlers.metric_handler import Metric, MetricHandler from localstack.utils.strings import short_uid -BASE_PATH = os.path.join(os.path.dirname(__file__), "../../../target/metric_reports") +BASE_PATH = os.path.join(os.path.dirname(__file__), "../../../../target/metric_reports") FNAME_RAW_DATA_CSV = os.path.join( BASE_PATH, f"metric-report-raw-data-{datetime.utcnow().strftime('%Y-%m-%d__%H_%M_%S')}-{short_uid()}.csv", diff --git a/pyproject.toml b/pyproject.toml index 744e76e259d6f..cc2c9516dd2b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -143,12 +143,14 @@ script-files = [ "bin/localstack.bat", "bin/localstack-supervisor", ] +package-dir = { "" = "localstack-core"} [tool.setuptools.dynamic] readme = { file = ["README.md"], content-type = "text/markdown"} version = { file = "VERSION" } [tool.setuptools.packages.find] +where = ["localstack-core/"] include = ["localstack*"] exclude = ["tests*"] @@ -169,6 +171,7 @@ exclude = ["tests*"] # Always generate Python 3.8-compatible code. target-version = "py38" line-length = 100 +src = ["localstack-core", "tests"] exclude = [ ".venv*", "venv*", @@ -176,9 +179,11 @@ exclude = [ "build", "target", "*.egg-info", + "localstack-core/*.egg-info", ".filesystem", + "localstack-core/.filesystem", ".git", - "localstack/services/stepfunctions/asl/antlr/runtime" + "localstack-core/localstack/services/stepfunctions/asl/antlr/runtime" ] [tool.ruff.lint] @@ -205,12 +210,12 @@ select = ["B", "C", "E", "F", "I", "W", "T", "B9"] [tool.coverage.run] relative_files = true source = [ - "localstack/" + "localstack-core/localstack" ] omit = [ - "localstack/aws/api/*", - "localstack/extensions/api/*", - "localstack/services/stepfunctions/asl/antlr/runtime/*" + "localstack-core/localstack/aws/api/*", + "localstack-core/localstack/extensions/api/*", + "localstack-core/localstack/services/stepfunctions/asl/antlr/runtime/*" ] dynamic_context = "test_function" diff --git a/scripts/generate_minimal_boto3stubs_install.py b/scripts/generate_minimal_boto3stubs_install.py index 2a36ff411c2da..176e3b28a9320 100644 --- a/scripts/generate_minimal_boto3stubs_install.py +++ b/scripts/generate_minimal_boto3stubs_install.py @@ -7,7 +7,9 @@ if __name__ == "__main__": with open( - os.path.join(os.path.dirname(__file__), "../localstack/utils/aws/client_types.py") + os.path.join( + os.path.dirname(__file__), "../localstack-core/localstack/utils/aws/client_types.py" + ) ) as fd: content = fd.read() result = re.findall(r"\smypy_boto3_([a-z0-9_]+)\s", content) diff --git a/setup.py b/setup.py index 0ac4c46692eb9..c0dd26fed2838 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def get_version(): # Set the version in the localstack/version.py file def set_version_constant(version: str): with open( - os.path.join(os.path.dirname(__file__), "localstack", "version.py"), "w" + os.path.join(os.path.dirname(__file__), "localstack-core", "localstack", "version.py"), "w" ) as version_file: version_file.write(f'__version__ = "{version}"\n')