configure project to work with src layout (#10800)

.circleci/config.yml

@@ -134,7 +134,7 @@ jobs:
                     circleci-agent step halt
                   else
                     source .venv/bin/activate
-                    python -m localstack.testing.testselection.scripts.generate_test_selection_from_pr /tmp/workspace/repo $CI_PULL_REQUEST target/testselection/test-selection.txt
+                    PYTHONPATH=localstack-core python -m localstack.testing.testselection.scripts.generate_test_selection_from_pr /tmp/workspace/repo $CI_PULL_REQUEST target/testselection/test-selection.txt
                     cat target/testselection/test-selection.txt
                   fi
 
@@ -402,7 +402,7 @@ jobs:
             if [ -z $TESTSELECTION_PYTEST_ARGS ] ; then
               TEST_FILES=$(circleci tests glob "tests/aws/**/test_*.py" "tests/integration/**/test_*.py" | circleci tests split --verbose --split-by=timings | tr '\n' ' ')
             else
-              TEST_FILES=$(circleci tests glob "tests/aws/**/test_*.py" "tests/integration/**/test_*.py" | python -m localstack.testing.testselection.scripts.filter_by_test_selection target/testselection/test-selection.txt | circleci tests split --verbose --split-by=timings | tr '\n' ' ')
+              TEST_FILES=$(circleci tests glob "tests/aws/**/test_*.py" "tests/integration/**/test_*.py" | PYTHONPATH=localstack-core python -m localstack.testing.testselection.scripts.filter_by_test_selection target/testselection/test-selection.txt | circleci tests split --verbose --split-by=timings | tr '\n' ' ')
             fi
             echo $TEST_FILES
             PYTEST_ARGS="${TINYBIRD_PYTEST_ARGS}${TESTSELECTION_PYTEST_ARGS}-o junit_family=legacy --junitxml=target/reports/test-report-<< parameters.platform >>-${CIRCLE_NODE_INDEX}.xml" \
@@ -465,8 +465,9 @@ jobs:
       - run:
           name: Run localstack
           command: |
-            DEBUG=1 DISABLE_EVENTS="1" IMAGE_NAME="localstack/localstack:latest" bin/localstack start -d
-            bin/localstack wait -t 120 || (bin/localstack logs && false)
+            source .venv/bin/activate
+            DEBUG=1 DISABLE_EVENTS="1" IMAGE_NAME="localstack/localstack:latest" localstack start -d
+            localstack wait -t 120 || (python -m localstack.cli.main logs && false)
       - run:
           name: Run capture-not-implemented
           command: |

.dockerignore

@@ -2,6 +2,7 @@
 .venv*
 
 .filesystem
+**/.filesystem
 
 # ignore files generated in CI build
 tests/aws/**/node_modules

.github/workflows/tests-pro-integration.yml

@@ -291,14 +291,14 @@ jobs:
         # Entrypoints need to be generated _after_ the community edition has been linked into the venv
         run: |
           VENV_DIR="../localstack-ext/.venv" make entrypoints
-          cat localstack_core.egg-info/entry_points.txt
+          ../localstack-ext/.venv/bin/python -m plux show
 
       - name: Create Pro Entrypoints
         working-directory: localstack-ext
         # Entrypoints need to be generated _after_ the community edition has been linked into the venv
         run: |
           make entrypoints
-          cat localstack_ext.egg-info/entry_points.txt
+          .venv/bin/python -m plux show
 
       - name: Test Pro Startup
         env:
@@ -334,7 +334,7 @@ jobs:
         working-directory: localstack-ext
         run: |
           # Remove the host tmp folder (might contain remnant files with different permissions)
-          sudo rm -rf ../localstack/.filesystem/var/lib/localstack/tmp
+          sudo rm -rf ../localstack/localstack-core/.filesystem/var/lib/localstack/tmp
           make test
 
       - name: Archive Test Results

Dockerfile

@@ -99,6 +99,7 @@ ENV PATH "${PATH}:${JAVA_HOME}/bin"
 
 # set workdir
 RUN mkdir -p /opt/code/localstack
+RUN mkdir /opt/code/localstack/localstack-core
 WORKDIR /opt/code/localstack/
 
 # create localstack user and filesystem hierarchy, perform some permission fixes
@@ -176,7 +177,7 @@ ADD Makefile pyproject.toml VERSION ./
 ADD bin/localstack bin/localstack.bat bin/localstack-supervisor bin/
 
 # add the code as late as possible
-ADD localstack/ localstack/
+ADD localstack-core/ /opt/code/localstack/localstack-core
 
 # Generate the plugin entrypoints
 RUN make entrypoints

Dockerfile.s3

@@ -4,6 +4,7 @@ ARG TARGETARCH
 
 # set workdir
 RUN mkdir -p /opt/code/localstack
+RUN mkdir /opt/code/localstack/localstack-core
 WORKDIR /opt/code/localstack/
 
 # Install runtime OS package dependencies
@@ -81,7 +82,7 @@ ADD Makefile pyproject.toml VERSION requirements-base-runtime.txt ./
 ADD bin/localstack bin/localstack.bat bin/localstack-supervisor bin/
 
 # add the code as late as possible
-ADD localstack/ localstack/
+ADD localstack-core/ /opt/code/localstack/localstack-core
 
 # Generate the plugin entrypoints
 RUN make entrypoints

Makefile

@@ -73,14 +73,14 @@ install: install-dev entrypoints  ## Install full dependencies into venv
 entrypoints:              ## Run plux to build entry points
 	$(VENV_RUN); python -m plux entrypoints
 	@# make sure that the entrypoints were correctly created and are non-empty
-	@test -s localstack_core.egg-info/entry_points.txt || (echo "Entrypoints were not correctly created! Aborting!" && exit 1)
+	@test -s localstack-core/localstack_core.egg-info/entry_points.txt || (echo "Entrypoints were not correctly created! Aborting!" && exit 1)
 
 dist: entrypoints        ## Build source and built (wheel) distributions of the current version
 	$(VENV_RUN); pip install --upgrade twine; python -m build
 
 publish: clean-dist dist  ## Publish the library to the central PyPi repository
 	# make sure the dist archive contains a non-empty entry_points.txt file before uploading
-	tar --wildcards --to-stdout -xf dist/localstack?core*.tar.gz "localstack?core*/localstack_core.egg-info/entry_points.txt" | grep . > /dev/null 2>&1 || (echo "Refusing upload, localstack-core dist does not contain entrypoints." && exit 1)
+	tar --wildcards --to-stdout -xf dist/localstack?core*.tar.gz "localstack?core*/localstack-core/localstack_core.egg-info/entry_points.txt" | grep . > /dev/null 2>&1 || (echo "Refusing upload, localstack-core dist does not contain entrypoints." && exit 1)
 	$(VENV_RUN); twine upload dist/*
 
 coveralls:         		  ## Publish coveralls metrics
@@ -236,11 +236,11 @@ clean:             		  ## Clean up (npm dependencies, downloaded infrastructure
 	rm -rf .filesystem
 	rm -rf build/
 	rm -rf dist/
-	rm -rf *.egg-info
+	rm -rf localstack-core/*.egg-info
 	rm -rf $(VENV_DIR)
 
 clean-dist:				  ## Clean up python distribution directories
 	rm -rf dist/ build/
-	rm -rf *.egg-info
+	rm -rf localstack-core/*.egg-info
 
 .PHONY: usage freeze install-basic install-runtime install-test install-dev install entrypoints dist publish coveralls start docker-save-image docker-build docker-build-multiarch docker-push-master docker-create-push-manifests docker-run-tests docker-cp-coverage test test-coverage lint lint-modified format format-modified init-precommit clean clean-dist upgrade-pinned-dependencies

localstack-core/localstack/aws/scaffold.py

@@ -472,7 +472,9 @@ def scaffold():
     help="whether or not to save the result into the api directory",
 )
 @click.option(
-    "--path", default="./localstack/aws/api", help="the path where the api should be saved"
+    "--path",
+    default="./localstack-core/localstack/aws/api",
+    help="the path where the api should be saved",
 )
 def generate(service: str, doc: bool, save: bool, path: str):
     """
@@ -525,7 +527,7 @@ def create_code_directory(service_name: str, code: str, base_path: str):
 @click.option("--doc/--no-doc", default=False, help="whether or not to generate docstrings")
 @click.option(
     "--path",
-    default="./localstack/aws/api",
+    default="./localstack-core/localstack/aws/api",
     help="the path in which to upgrade ASF APIs",
 )
 def upgrade(path: str, doc: bool = False):

localstack-core/localstack/dev/run/paths.py

@@ -55,10 +55,10 @@ def dependency_source(self, name: str) -> str:
 
 
 class CommunityContainerPaths(ContainerPaths):
-    """In the community image, code is copied into /opt/code/localstack"""
+    """In the community image, code is copied into /opt/code/localstack/localstack-core/localstack"""
 
     def __init__(self):
-        self.localstack_source_dir = f"{self.project_dir}/localstack"
+        self.localstack_source_dir = f"{self.project_dir}/localstack-core/localstack"
 
 
 class ProContainerPaths(ContainerPaths):

localstack-core/localstack/testing/pytest/metric_collection.py

@@ -12,7 +12,7 @@
 from localstack.aws.handlers.metric_handler import Metric, MetricHandler
 from localstack.utils.strings import short_uid
 
-BASE_PATH = os.path.join(os.path.dirname(__file__), "../../../target/metric_reports")
+BASE_PATH = os.path.join(os.path.dirname(__file__), "../../../../target/metric_reports")
 FNAME_RAW_DATA_CSV = os.path.join(
     BASE_PATH,
     f"metric-report-raw-data-{datetime.utcnow().strftime('%Y-%m-%d__%H_%M_%S')}-{short_uid()}.csv",

pyproject.toml

@@ -144,12 +144,14 @@ script-files = [
     "bin/localstack.bat",
     "bin/localstack-supervisor",
 ]
+package-dir = { "" = "localstack-core"}
 
 [tool.setuptools.dynamic]
 readme = { file = ["README.md"], content-type = "text/markdown"}
 version = { file = "VERSION" }
 
 [tool.setuptools.packages.find]
+where = ["localstack-core/"]
 include = ["localstack*"]
 exclude = ["tests*"]
 
@@ -170,16 +172,19 @@ exclude = ["tests*"]
 # Always generate Python 3.8-compatible code.
 target-version = "py38"
 line-length = 100
+src = ["localstack-core", "tests"]
 exclude = [
     ".venv*",
     "venv*",
     "dist",
     "build",
     "target",
     "*.egg-info",
+    "localstack-core/*.egg-info",
     ".filesystem",
+    "localstack-core/.filesystem",
     ".git",
-    "localstack/services/stepfunctions/asl/antlr/runtime"
+    "localstack-core/localstack/services/stepfunctions/asl/antlr/runtime"
     ]
 
 [tool.ruff.lint]
@@ -206,12 +211,12 @@ select = ["B", "C", "E", "F", "I", "W", "T", "B9"]
 [tool.coverage.run]
 relative_files = true
 source = [
-    "localstack/"
+    "localstack-core/"
 ]
 omit = [
-    "localstack/aws/api/*",
-    "localstack/extensions/api/*",
-    "localstack/services/stepfunctions/asl/antlr/runtime/*"
+    "localstack-core/localstack/aws/api/*",
+    "localstack-core/localstack/extensions/api/*",
+    "localstack-core/localstack/services/stepfunctions/asl/antlr/runtime/*"
 ]
 dynamic_context = "test_function"
 

scripts/generate_minimal_boto3stubs_install.py

@@ -7,7 +7,9 @@
 
 if __name__ == "__main__":
     with open(
-        os.path.join(os.path.dirname(__file__), "../localstack/utils/aws/client_types.py")
+        os.path.join(
+            os.path.dirname(__file__), "../localstack-core/localstack/utils/aws/client_types.py"
+        )
     ) as fd:
         content = fd.read()
         result = re.findall(r"\smypy_boto3_([a-z0-9_]+)\s", content)

setup.py

@@ -12,7 +12,7 @@ def get_version():
 # Set the version in the localstack/version.py file
 def set_version_constant(version: str):
     with open(
-        os.path.join(os.path.dirname(__file__), "localstack", "version.py"), "w"
+        os.path.join(os.path.dirname(__file__), "localstack-core", "localstack", "version.py"), "w"
     ) as version_file:
         version_file.write(f'__version__ = "{version}"\n')