move source code into localstack-core (#10800)

.circleci/config.yml

@@ -134,7 +134,7 @@ jobs:
                     circleci-agent step halt
                   else
                     source .venv/bin/activate
-                    python -m localstack.testing.testselection.scripts.generate_test_selection_from_pr /tmp/workspace/repo $CI_PULL_REQUEST target/testselection/test-selection.txt
+                    PYTHONPATH=localstack-core python -m localstack.testing.testselection.scripts.generate_test_selection_from_pr /tmp/workspace/repo $CI_PULL_REQUEST target/testselection/test-selection.txt
                     cat target/testselection/test-selection.txt
                   fi
 
@@ -402,7 +402,7 @@ jobs:
             if [ -z $TESTSELECTION_PYTEST_ARGS ] ; then
               TEST_FILES=$(circleci tests glob "tests/aws/**/test_*.py" "tests/integration/**/test_*.py" | circleci tests split --verbose --split-by=timings | tr '\n' ' ')
             else
-              TEST_FILES=$(circleci tests glob "tests/aws/**/test_*.py" "tests/integration/**/test_*.py" | python -m localstack.testing.testselection.scripts.filter_by_test_selection target/testselection/test-selection.txt | circleci tests split --verbose --split-by=timings | tr '\n' ' ')
+              TEST_FILES=$(circleci tests glob "tests/aws/**/test_*.py" "tests/integration/**/test_*.py" | PYTHONPATH=localstack-core python -m localstack.testing.testselection.scripts.filter_by_test_selection target/testselection/test-selection.txt | circleci tests split --verbose --split-by=timings | tr '\n' ' ')
             fi
             echo $TEST_FILES
             PYTEST_ARGS="${TINYBIRD_PYTEST_ARGS}${TESTSELECTION_PYTEST_ARGS}-o junit_family=legacy --junitxml=target/reports/test-report-<< parameters.platform >>-${CIRCLE_NODE_INDEX}.xml" \
@@ -465,8 +465,9 @@ jobs:
       - run:
           name: Run localstack
           command: |
-            DEBUG=1 DISABLE_EVENTS="1" IMAGE_NAME="localstack/localstack:latest" bin/localstack start -d
-            bin/localstack wait -t 120 || (bin/localstack logs && false)
+            source .venv/bin/activate
+            DEBUG=1 DISABLE_EVENTS="1" IMAGE_NAME="localstack/localstack:latest" localstack start -d
+            localstack wait -t 120 || (python -m localstack.cli.main logs && false)
       - run:
           name: Run capture-not-implemented
           command: |
@@ -556,10 +557,10 @@ jobs:
           command: |
             source .venv/bin/activate
             pip install pycobertura
-            coverage xml --data-file=.coverage -o all.coverage.report.xml --include="localstack/services/*/**" --omit="*/**/__init__.py"
-            coverage xml --data-file=.coverage.acceptance -o acceptance.coverage.report.xml --include="localstack/services/*/**"  --omit="*/**/__init__.py"
-            pycobertura show --format html -s localstack/ acceptance.coverage.report.xml -o coverage-acceptance.html
-            bash -c "pycobertura diff --format html -s1 localstack/ -s2 localstack/ all.coverage.report.xml acceptance.coverage.report.xml -o coverage-diff.html; if [[ \$? -eq 1 ]] ; then exit 1 ; else exit 0 ; fi"
+            coverage xml --data-file=.coverage -o all.coverage.report.xml --include="localstack-core/localstack/services/*/**" --omit="*/**/__init__.py"
+            coverage xml --data-file=.coverage.acceptance -o acceptance.coverage.report.xml --include="localstack-core/localstack/services/*/**"  --omit="*/**/__init__.py"
+            pycobertura show --format html -s localstack-core/localstack acceptance.coverage.report.xml -o coverage-acceptance.html
+            bash -c "pycobertura diff --format html -s1 localstack-core/localstack/ -s2 localstack-core/localstack/ all.coverage.report.xml acceptance.coverage.report.xml -o coverage-diff.html; if [[ \$? -eq 1 ]] ; then exit 1 ; else exit 0 ; fi"
       - run:
           name: Create Metric Coverage Diff (API Coverage)
           environment:

.dockerignore

@@ -2,6 +2,7 @@
 .venv*
 
 .filesystem
+**/.filesystem
 
 # ignore files generated in CI build
 tests/aws/**/node_modules

.github/workflows/asf-updates.yml

@@ -58,12 +58,12 @@ jobs:
           # Check against the PR branch if it exists, otherwise against the master
           # Store the result in target/diff-check.log and store the diff count in the GitHub Action output "diff-count"
           mkdir -p target
-          (git diff --name-only origin/asf-auto-updates localstack/aws/api/ 2>/dev/null || git diff --name-only origin/master localstack/aws/api/ 2>/dev/null) | tee target/diff-check.log
+          (git diff --name-only origin/asf-auto-updates localstack-core/localstack/aws/api/ 2>/dev/null || git diff --name-only origin/master localstack-core/localstack/aws/api/ 2>/dev/null) | tee target/diff-check.log
           echo "diff-count=$(cat target/diff-check.log | wc -l)" >> $GITHUB_OUTPUT
 
           # Store a (multiline-sanitized) list of changed services (compared to the master) in the GitHub Action output "changed-services"
           echo "changed-services<<EOF" >> $GITHUB_OUTPUT
-          echo "$(git diff --name-only origin/master localstack/aws/api/ | sed 's#localstack/aws/api/#- #g' | sed 's#/__init__.py##g' | sed 's/_/-/g')" >> $GITHUB_OUTPUT
+          echo "$(git diff --name-only origin/master localstack-core/localstack/aws/api/ | sed 's#localstack-core/localstack/aws/api/#- #g' | sed 's#/__init__.py##g' | sed 's/_/-/g')" >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Update botocore and transitive pins

.github/workflows/tests-pro-integration.yml

@@ -291,14 +291,14 @@ jobs:
         # Entrypoints need to be generated _after_ the community edition has been linked into the venv
         run: |
           VENV_DIR="../localstack-ext/.venv" make entrypoints
-          cat localstack_core.egg-info/entry_points.txt
+          ../localstack-ext/.venv/bin/python -m plux show
 
       - name: Create Pro Entrypoints
         working-directory: localstack-ext
         # Entrypoints need to be generated _after_ the community edition has been linked into the venv
         run: |
           make entrypoints
-          cat localstack_ext.egg-info/entry_points.txt
+          .venv/bin/python -m plux show
 
       - name: Test Pro Startup
         env:
@@ -334,7 +334,7 @@ jobs:
         working-directory: localstack-ext
         run: |
           # Remove the host tmp folder (might contain remnant files with different permissions)
-          sudo rm -rf ../localstack/.filesystem/var/lib/localstack/tmp
+          sudo rm -rf ../localstack/localstack-core/.filesystem/var/lib/localstack/tmp
           make test
 
       - name: Archive Test Durations

.github/workflows/tests-s3-image.yml

@@ -4,15 +4,15 @@ on:
   push:
     paths:
       - .github/workflows/tests-s3-image.yml
-      - localstack/aws/*.py
-      - localstack/aws/handlers/*¨
-      - localstack/aws/protocol/**
-      - localstack/aws/serving/**
-      - localstack/aws/api/s3/**
-      - localstack/http/**
-      - localstack/runtime/**
-      - localstack/services/s3/**
-      - localstack/*.py
+      - localstack-core/localstack/aws/*.py
+      - localstack-core/localstack/aws/handlers/*¨
+      - localstack-core/localstack/aws/protocol/**
+      - localstack-core/localstack/aws/serving/**
+      - localstack-core/localstack/aws/api/s3/**
+      - localstack-core/localstack/http/**
+      - localstack-core/localstack/runtime/**
+      - localstack-core/localstack/services/s3/**
+      - localstack-core/localstack/*.py
       - tests/aws/services/s3/**
       - Dockerfile.s3
       - requirements-*.txt
@@ -23,15 +23,15 @@ on:
   pull_request:
     paths:
       - .github/workflows/tests-s3-image.yml
-      - localstack/aws/*.py
-      - localstack/aws/handlers/*¨
-      - localstack/aws/protocol/**
-      - localstack/aws/serving/**
-      - localstack/aws/api/s3/**
-      - localstack/http/**
-      - localstack/runtime/**
-      - localstack/services/s3/**
-      - localstack/*.py
+      - localstack-core/localstack/aws/*.py
+      - localstack-core/localstack/aws/handlers/*¨
+      - localstack-core/localstack/aws/protocol/**
+      - localstack-core/localstack/aws/serving/**
+      - localstack-core/localstack/aws/api/s3/**
+      - localstack-core/localstack/http/**
+      - localstack-core/localstack/runtime/**
+      - localstack-core/localstack/services/s3/**
+      - localstack-core/localstack/*.py
       - tests/aws/services/s3/**
       - Dockerfile.s3
       - requirements-*.txt

Dockerfile

@@ -99,6 +99,7 @@ ENV PATH "${PATH}:${JAVA_HOME}/bin"
 
 # set workdir
 RUN mkdir -p /opt/code/localstack
+RUN mkdir /opt/code/localstack/localstack-core
 WORKDIR /opt/code/localstack/
 
 # create localstack user and filesystem hierarchy, perform some permission fixes
@@ -176,7 +177,7 @@ ADD Makefile pyproject.toml VERSION ./
 ADD bin/localstack bin/localstack.bat bin/localstack-supervisor bin/
 
 # add the code as late as possible
-ADD localstack/ localstack/
+ADD localstack-core/ /opt/code/localstack/localstack-core
 
 # Generate the plugin entrypoints
 RUN make entrypoints

Dockerfile.s3

@@ -4,6 +4,7 @@ ARG TARGETARCH
 
 # set workdir
 RUN mkdir -p /opt/code/localstack
+RUN mkdir /opt/code/localstack/localstack-core
 WORKDIR /opt/code/localstack/
 
 # Install runtime OS package dependencies
@@ -81,7 +82,7 @@ ADD Makefile pyproject.toml VERSION requirements-base-runtime.txt ./
 ADD bin/localstack bin/localstack.bat bin/localstack-supervisor bin/
 
 # add the code as late as possible
-ADD localstack/ localstack/
+ADD localstack-core/ /opt/code/localstack/localstack-core
 
 # Generate the plugin entrypoints
 RUN make entrypoints

Makefile

@@ -73,14 +73,14 @@ install: install-dev entrypoints  ## Install full dependencies into venv
 entrypoints:              ## Run plux to build entry points
 	$(VENV_RUN); python -m plux entrypoints
 	@# make sure that the entrypoints were correctly created and are non-empty
-	@test -s localstack_core.egg-info/entry_points.txt || (echo "Entrypoints were not correctly created! Aborting!" && exit 1)
+	@test -s localstack-core/localstack_core.egg-info/entry_points.txt || (echo "Entrypoints were not correctly created! Aborting!" && exit 1)
 
 dist: entrypoints        ## Build source and built (wheel) distributions of the current version
 	$(VENV_RUN); pip install --upgrade twine; python -m build
 
 publish: clean-dist dist  ## Publish the library to the central PyPi repository
 	# make sure the dist archive contains a non-empty entry_points.txt file before uploading
-	tar --wildcards --to-stdout -xf dist/localstack?core*.tar.gz "localstack?core*/localstack_core.egg-info/entry_points.txt" | grep . > /dev/null 2>&1 || (echo "Refusing upload, localstack-core dist does not contain entrypoints." && exit 1)
+	tar --wildcards --to-stdout -xf dist/localstack?core*.tar.gz "localstack?core*/localstack-core/localstack_core.egg-info/entry_points.txt" | grep . > /dev/null 2>&1 || (echo "Refusing upload, localstack-core dist does not contain entrypoints." && exit 1)
 	$(VENV_RUN); twine upload dist/*
 
 coveralls:         		  ## Publish coveralls metrics
@@ -237,10 +237,11 @@ clean:             		  ## Clean up (npm dependencies, downloaded infrastructure
 	rm -rf build/
 	rm -rf dist/
 	rm -rf *.egg-info
+	rm -rf localstack-core/*.egg-info
 	rm -rf $(VENV_DIR)
 
 clean-dist:				  ## Clean up python distribution directories
 	rm -rf dist/ build/
-	rm -rf *.egg-info
+	rm -rf localstack-core/*.egg-info
 
 .PHONY: usage freeze install-basic install-runtime install-test install-dev install entrypoints dist publish coveralls start docker-save-image docker-build docker-build-multiarch docker-push-master docker-create-push-manifests docker-run-tests docker-cp-coverage test test-coverage lint lint-modified format format-modified init-precommit clean clean-dist upgrade-pinned-dependencies

localstack/aws/scaffold.py → localstack-core/localstack/aws/scaffold.py

@@ -472,7 +472,9 @@ def scaffold():
     help="whether or not to save the result into the api directory",
 )
 @click.option(
-    "--path", default="./localstack/aws/api", help="the path where the api should be saved"
+    "--path",
+    default="./localstack-core/localstack/aws/api",
+    help="the path where the api should be saved",
 )
 def generate(service: str, doc: bool, save: bool, path: str):
     """
@@ -525,7 +527,7 @@ def create_code_directory(service_name: str, code: str, base_path: str):
 @click.option("--doc/--no-doc", default=False, help="whether or not to generate docstrings")
 @click.option(
     "--path",
-    default="./localstack/aws/api",
+    default="./localstack-core/localstack/aws/api",
     help="the path in which to upgrade ASF APIs",
 )
 def upgrade(path: str, doc: bool = False):

localstack/dev/run/paths.py → localstack-core/localstack/dev/run/paths.py

@@ -55,10 +55,10 @@ def dependency_source(self, name: str) -> str:
 
 
 class CommunityContainerPaths(ContainerPaths):
-    """In the community image, code is copied into /opt/code/localstack"""
+    """In the community image, code is copied into /opt/code/localstack/localstack-core/localstack"""
 
     def __init__(self):
-        self.localstack_source_dir = f"{self.project_dir}/localstack"
+        self.localstack_source_dir = f"{self.project_dir}/localstack-core/localstack"
 
 
 class ProContainerPaths(ContainerPaths):