diff --git a/.github/workflows/tidy3d-extras-python-client-integration-tests.yml b/.github/workflows/tidy3d-extras-python-client-integration-tests.yml new file mode 100644 index 0000000000..a6ad2e0621 --- /dev/null +++ b/.github/workflows/tidy3d-extras-python-client-integration-tests.yml @@ -0,0 +1,145 @@ +name: "public/tidy3d-extras/python-client-integration-tests" + +on: + workflow_dispatch: + inputs: + release_tag: + description: 'Release Tag (v2.10.0, v2.10.0rc1)' + required: false + type: string + default: '' + + workflow_call: + inputs: + release_tag: + description: 'Release Tag (v2.10.0, v2.10.0rc1)' + required: false + type: string + default: '' + outputs: + workflow_success: + description: 'Overall integration test workflow success status' + value: ${{ jobs.integration-tests.result == 'success' }} + +permissions: + contents: read + +jobs: + integration-tests: + name: python-${{ matrix.python-version }}-${{ matrix.platform }}-extras + runs-on: ${{ matrix.platform }} + permissions: + contents: read + concurrency: + group: tidy3d-extras-integration-${{ github.event.pull_request.number || github.sha }}-${{ matrix.platform }}-${{ matrix.python-version }} + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + python-version: ['3.10', '3.13'] + platform: [windows-latest, ubuntu-latest, macos-latest] + defaults: + run: + shell: bash + env: + PIP_ONLY_BINARY: gdstk + MPLBACKEND: agg + RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || inputs.release_tag }} + + steps: + - name: checkout-head + if: ${{ !env.RELEASE_TAG }} + uses: actions/checkout@v4 + with: + fetch-depth: 1 + submodules: false + persist-credentials: false + + - name: checkout-tag + if: ${{ env.RELEASE_TAG }} + uses: actions/checkout@v4 + with: + ref: refs/tags/${{ env.RELEASE_TAG }} + fetch-depth: 1 + submodules: false + persist-credentials: false + + - name: set-python-${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: install-poetry + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1.4.1 + with: + version: 2.1.1 + virtualenvs-create: true + virtualenvs-in-project: true + + - name: configure-aws-credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + aws-access-key-id: ${{ secrets.AWS_CODEARTIFACT_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.AWS_CODEARTIFACT_ACCESS_SECRET }} + aws-region: us-east-1 + + - name: configure-codeartifact-authentication + run: | + set -e + echo "Getting CodeArtifact token..." + CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \ + --domain flexcompute \ + --domain-owner 625554095313 \ + --query authorizationToken \ + --output text) + + echo "Configuring Poetry with CodeArtifact credentials..." + poetry config http-basic.codeartifact aws $CODEARTIFACT_AUTH_TOKEN + echo "✅ CodeArtifact authentication configured" + + - name: install-project + shell: bash + run: | + poetry --version + python --version + python -m venv .venv + if [[ "${{ runner.os }}" == "Windows" ]]; then + source .venv/Scripts/activate + python --version + else + source .venv/bin/activate + which python + fi + poetry env use python + poetry env info + poetry run pip install --upgrade pip wheel setuptools + poetry run pip install gdstk --only-binary gdstk + poetry install -E extras -E dev + + - name: verify-extras-installation + run: | + export SIMCLOUD_APIKEY=${{ secrets.TIDY3D_API_KEY }} + poetry run tidy3d configure --apikey ${{ secrets.TIDY3D_API_KEY }} + poetry run python -c "import tidy3d; print(f'tidy3d version: {tidy3d.__version__}')" + poetry run python -c "import tidy3d_extras; print(f'tidy3d-extras version: {tidy3d_extras.__version__}')" + cd tests/ + poetry run pytest tidy3d_extras_license_test.py + + - name: run-doctests + run: | + export SIMCLOUD_APIKEY=${{ secrets.TIDY3D_API_KEY }} + poetry run tidy3d configure --apikey ${{ secrets.TIDY3D_API_KEY }} + poetry run pytest -rF --tb=short tidy3d + + - name: run-tests-coverage + env: + PYTHONUNBUFFERED: "1" + run: | + export SIMCLOUD_APIKEY=${{ secrets.TIDY3D_API_KEY }} + poetry run tidy3d configure --apikey ${{ secrets.TIDY3D_API_KEY }} + poetry run pytest --cov=tidy3d -rF --tb=short tests/_test_data/_test_datasets_no_vtk.py + poetry run pytest --cov=tidy3d -rF --tb=short tests + poetry run coverage report -m + TOTAL_COVERAGE=$(poetry run coverage report --format=total) + echo "total=$TOTAL_COVERAGE" >> "$GITHUB_ENV" + echo "### Total coverage: ${TOTAL_COVERAGE}%" diff --git a/.github/workflows/tidy3d-python-client-tests.yml b/.github/workflows/tidy3d-python-client-tests.yml index c5c0566b4a..40d1d0e77f 100644 --- a/.github/workflows/tidy3d-python-client-tests.yml +++ b/.github/workflows/tidy3d-python-client-tests.yml @@ -14,15 +14,19 @@ on: type: boolean default: false cli_tests: - description: 'Run develop-cli tests' + description: 'develop-cli' type: boolean default: false submodule_tests: - description: 'Run submodule tests' + description: 'submodule-tests' type: boolean default: false version_match_tests: - description: 'Run version consistency checks' + description: 'version-consistency-checks' + type: boolean + default: false + extras_integration_tests: + description: 'integration-tidy3d-extras' type: boolean default: false release_tag: @@ -30,7 +34,7 @@ on: required: false type: string default: '' - + workflow_call: inputs: remote_tests: @@ -58,6 +62,11 @@ on: type: boolean required: false default: false + extras_integration_tests: + description: 'Run tidy3d-extras integration tests' + type: boolean + required: false + default: false release_tag: description: 'Release Tag (v2.10.0, v2.10.0rc1)' required: false @@ -95,6 +104,7 @@ jobs: cli_tests: ${{ steps.determine-test-type.outputs.cli_tests }} submodule_tests: ${{ steps.determine-test-type.outputs.submodule_tests }} version_match_tests: ${{ steps.determine-test-type.outputs.version_match_tests }} + extras_integration_tests: ${{ steps.determine-test-type.outputs.extras_integration_tests }} steps: - name: determine-test-type id: determine-test-type @@ -108,6 +118,7 @@ jobs: INPUT_CLI: ${{ github.event.inputs.cli_tests || inputs.cli_tests }} INPUT_SUBMODULE: ${{ github.event.inputs.submodule_tests || inputs.submodule_tests }} INPUT_VERSION_MATCH: ${{ github.event.inputs.version_match_tests || inputs.version_match_tests }} + INPUT_EXTRAS_INTEGRATION: ${{ github.event.inputs.extras_integration_tests || inputs.extras_integration_tests }} run: | echo "Event: $EVENT_NAME" echo "Draft: $DRAFT_STATE" @@ -118,6 +129,7 @@ jobs: echo "Input cli: $INPUT_CLI" echo "Input submodule: $INPUT_SUBMODULE" echo "Input version_match: $INPUT_VERSION_MATCH" + echo "Input extras_integration: $INPUT_EXTRAS_INTEGRATION" remote_tests=false local_tests=false @@ -126,6 +138,7 @@ jobs: version_match_tests=false code_quality_tests=false pr_review_tests=false + extras_integration_tests=false # Workflow_dispatch input override if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then @@ -150,6 +163,10 @@ jobs: if [[ "$INPUT_VERSION_MATCH" == "true" ]]; then version_match_tests=true fi + + if [[ "$INPUT_EXTRAS_INTEGRATION" == "true" ]]; then + extras_integration_tests=true + fi fi # All PRs that have been triggered need local tests (remote reserved for merge queue/manual) @@ -163,6 +180,7 @@ jobs: local_tests=true remote_tests=true code_quality_tests=true + extras_integration_tests=true fi echo "local_tests=$local_tests" >> $GITHUB_OUTPUT @@ -172,6 +190,7 @@ jobs: echo "version_match_tests=$version_match_tests" >> $GITHUB_OUTPUT echo "code_quality_tests=$code_quality_tests" >> $GITHUB_OUTPUT echo "pr_review_tests=$pr_review_tests" >> $GITHUB_OUTPUT + echo "extras_integration_tests=$extras_integration_tests" >> $GITHUB_OUTPUT echo "code_quality_tests=$code_quality_tests" echo "pr_review_tests=$pr_review_tests" echo "local_tests=$local_tests" @@ -179,6 +198,7 @@ jobs: echo "cli_tests=$cli_tests" echo "submodule_tests=$submodule_tests" echo "version_match_tests=$version_match_tests" + echo "extras_integration_tests=$extras_integration_tests" lint: needs: determine-test-scope @@ -919,7 +939,16 @@ jobs: echo "" echo "=== Submodule Checks Passed ===" - + + extras-integration-tests: + name: extras-integration-tests + needs: determine-test-scope + if: needs.determine-test-scope.outputs.extras_integration_tests == 'true' + uses: ./.github/workflows/tidy3d-extras-python-client-tests-integration.yml + with: + release_tag: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || inputs.release_tag }} + secrets: inherit # zizmor: ignore[secrets-inherit] + workflow-validation: name: workflow-validation if: always() @@ -936,6 +965,7 @@ jobs: - develop-cli-tests - verify-version-consistency - test-submodules + - extras-integration-tests runs-on: ubuntu-latest steps: - name: check-linting-result @@ -1004,6 +1034,12 @@ jobs: echo "❌ Submodule tests failed." exit 1 + - name: check-extras-integration-tests-result + if: ${{ needs.determine-test-scope.outputs.extras_integration_tests == 'true' && needs.extras-integration-tests.result != 'success' && needs.extras-integration-tests.result != 'skipped' }} + run: | + echo "❌ tidy3d-extras integration tests failed." + exit 1 + - name: all-checks-passed if: ${{ success() }} run: echo "✅ All required jobs passed!" diff --git a/dev.Dockerfile b/dev.Dockerfile index de7dce7c78..18b0c4f36a 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -8,6 +8,8 @@ RUN apt-get update && \ git \ pandoc \ xsel \ + groff \ + mandoc \ xclip RUN apt-get update && apt-get install -y zip unzip curl \ @@ -20,6 +22,7 @@ RUN apt-get update && apt-get install -y zip unzip curl \ ENV POETRY_HOME=/opt/poetry RUN curl -sSL https://install.python-poetry.org | python3 - ENV PATH="/root/.local/bin:${POETRY_HOME}/bin:${PATH}" +RUN poetry self add poetry-codeartifact-login RUN apt-get update && DEBIAN_FRONTEND="noninteractive" TZ="America/New_York" apt-get install -y curl \ && curl -LO https://github.com/neovim/neovim/releases/latest/download/nvim-linux-x86_64.tar.gz \ diff --git a/docs/development/docker.rst b/docs/development/docker.rst index b56e48f7ef..fbab5856f0 100644 --- a/docs/development/docker.rst +++ b/docs/development/docker.rst @@ -1,88 +1,949 @@ -Docker Development Image Installation -===================================== +Docker Development Environment +============================= +This guide will get you up and running with Tidy3D Python client development using Docker. The Docker workflow provides an isolated, reproducible development environment without affecting your system packages. -Quick Start Guide ------------------ +.. contents:: Quick Links + :local: + :depth: 2 + +Overview +-------- + +The Docker development environment provides: + +* **Isolation**: Clean environment separate from your system +* **Reproducibility**: Everyone uses the same base environment +* **Simplicity**: No need to manage system dependencies +* **Flexibility**: Easy to reset and start fresh + +**What's included:** -This guide is for anyone setting up the development environment for the first time or migrating from a version after ``2.9.0`` that the dev.Dockerfile image was enabled. +* Python 3.11 with ``uv`` package manager +* Poetry for dependency management +* AWS CLI for CodeArtifact access +* Neovim, git, pandoc for documentation +* Jupyter Lab for notebook development Prerequisites -^^^^^^^^^^^^^^^ +------------- + +System Requirements +^^^^^^^^^^^^^^^^^^^ -1. Install **Docker Engine**. You can find the official instructions here: `https://docs.docker.com/engine/install/ `_. - - *Note for Ubuntu users*: It's recommended to avoid installing Docker from the Snap store. -2. Follow the official `Linux post-installation steps `_ to run Docker commands without ``sudo`` and to enable the Docker daemon to start on boot. -3. **Reboot** your system for all group changes to take effect. +* **Operating System**: Ubuntu 22.04+, macOS, or Windows with WSL2 +* **Docker Engine**: Installed and running +* **Git**: With SSH keys configured for GitHub +* **Disk Space**: ~2GB for Docker image + container + +Quick Check +~~~~~~~~~~~ + +.. code-block:: bash -Setup and Usage + # Check Docker is installed and running + docker --version + docker ps + + # Check Git SSH access + ssh -T git@github.com + +Installing Docker ^^^^^^^^^^^^^^^^^ -Follow these steps in your terminal to clone the repository, build the Docker image, and launch a development session. +If you don't have Docker installed: + +**Ubuntu** + +.. code-block:: bash + + # Install Docker Engine (NOT from snap) + curl -fsSL https://get.docker.com -o get-docker.sh + sudo sh get-docker.sh + + # Add your user to docker group + sudo usermod -aG docker $USER + + # Reboot to apply group changes + sudo reboot + + # Verify installation + docker run hello-world + +See: `Docker Ubuntu Install Guide `_ + +**macOS** + +Download and install Docker Desktop from `docker.com `_ + +**Windows** + +Install Docker Desktop with WSL2 backend from `docker.com `_ + +Access Requirements +^^^^^^^^^^^^^^^^^^^ + +* **GitHub**: Access to `flexcompute/tidy3d `_ +* **Tidy3D Account**: Sign up at `tidy3d.simulation.cloud `_ +* **API Key**: Get yours from `account page `_ + +Quick Start Guide +----------------- + +This guide is for anyone setting up the development environment for the first time or migrating from a version after ``2.9.0`` when the dev.Dockerfile image was enabled. + +Step 1: Clone the Repository +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: bash - # 1. Clone the repository and navigate into it + # Clone the repository and navigate into it git clone https://github.com/flexcompute/tidy3d.git cd tidy3d - # 2. Build the development Docker image + # Checkout develop branch or your feature branch + git checkout develop + +Step 2: Build the Docker Image +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Build the development Docker image (~5 minutes): + +.. code-block:: bash + + # Build the development Docker image docker build -t tidy3d-python-client-dev -f dev.Dockerfile . - # 3. Create a persistent container with your local code mounted +The image includes: + +* Debian base with Python 3.11 +* ``uv`` for fast package installation +* Poetry for dependency management +* AWS CLI, git, pandoc, neovim +* ``flexdaemon`` user (UID 1000, GID 1000) + +Step 3: Create the Development Container +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create a persistent container with your local code mounted: + +.. code-block:: bash + + # Create a persistent container with your local code mounted # This also maps port 8888 for tools like Jupyter Lab docker container create --name=tidy3d_python_client_dev \ - --userns=host \ - -p 8888:8888 \ - -v .:/home/flexdaemon/tidy3d \ - tidy3d-python-client-dev + --userns=host \ + -p 8888:8888 \ + -v .:/home/flexdaemon/tidy3d \ + tidy3d-python-client-dev + +**Flags explained:** - # 4. Start the container +* ``--name``: Container name for easy reference +* ``--userns=host``: Use host user namespace (prevents permission issues) +* ``-p 8888:8888``: Expose port 8888 for Jupyter Lab +* ``-v .:/home/flexdaemon/tidy3d``: Mount current directory into container + +Step 4: Start the Container +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + # Start the container docker start tidy3d_python_client_dev - # 5. Open an interactive shell inside the running container +Step 5: Open an Interactive Shell +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + # Open an interactive shell inside the running container docker exec -it tidy3d_python_client_dev /bin/bash -You are now inside the container's shell. From here, you can set up the Python environment. +You're now inside the container at ``/home/flexdaemon``! + +Step 6: Set Up Python Environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Inside the container, create a virtual environment and install the package: .. code-block:: bash - # (Inside Container) 6. Create a virtual environment and activate it + # (Inside Container) Navigate to the mounted tidy3d directory + cd tidy3d + + # (Inside Container) Create a virtual environment and activate it uv venv -p 3.11 source .venv/bin/activate - # (Inside Container) 7. Install the project in editable mode with dev dependencies + # (Inside Container) Install the project in editable mode with dev dependencies uv pip install -e .[dev] -With the environment ready, you can run tests, format code, or start a Jupyter Lab session. +.. note:: + The virtual environment is created inside the mounted directory, so it persists between container restarts. + +Step 7: Configure API Key and Run Tests +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Configure your API key and verify everything is working: + +.. code-block:: bash + + # (Inside Container) Configure your API key + tidy3d configure --apikey=YOUR_API_KEY_HERE + + # (Inside Container) Test the configuration + python -c "import tidy3d.web as web; web.test()" + + # (Inside Container) Run tests + pytest tests/ -v + +If tests pass, you're ready to develop! + +Development Workflow +-------------------- + +Daily Workflow +^^^^^^^^^^^^^^ + +1. **Start the container** (if not running): + + .. code-block:: bash + + # On host + docker start tidy3d_python_client_dev + docker exec -it tidy3d_python_client_dev bash + +2. **Activate virtual environment** (inside container): + + .. code-block:: bash + + cd tidy3d + source .venv/bin/activate + +3. **Make changes** to the code on your host machine using your favorite editor (VSCode, PyCharm, etc.) + + Changes are immediately reflected in the container since the directory is mounted. + +4. **Run tests** (inside container): + + .. code-block:: bash + + # Run all tests + pytest + + # Run specific test file + pytest tests/test_simulation.py + + # Run with verbose output + pytest -v -s + +5. **Commit changes** (on host or in container): + + .. code-block:: bash + + git add . + git commit -m "Your descriptive message" + git push origin your-feature-branch + +Multiple Terminal Sessions +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can open multiple terminals into the same container: + +.. code-block:: bash + + # Terminal 1 (host) + docker exec -it tidy3d_python_client_dev bash + + # Terminal 2 (host) + docker exec -it tidy3d_python_client_dev bash + + # Terminal 3 (host) + docker exec -it tidy3d_python_client_dev bash + +Each terminal shares the same filesystem and running processes. + +Running Tests +^^^^^^^^^^^^^ -Running Github Actions Locally -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. code-block:: bash + + # Inside container with venv activated + + # Run all tests + pytest + + # Run tests in parallel (faster) + pytest -n auto + + # Run specific test module + pytest tests/test_components.py + + # Run tests matching pattern + pytest -k "test_simulation" + + # Run with coverage + pytest --cov=tidy3d --cov-report=html + # View coverage: python -m http.server 8000 -d htmlcov + +Code Quality Checks +^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + # Inside container with venv activated + + # Format code with ruff + ruff format . + + # Check for linting issues + ruff check . + + # Auto-fix linting issues + ruff check --fix . + + # Run type checking + mypy tidy3d + +Building Documentation +^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + # Inside container with venv activated + cd docs + + # Build HTML documentation + make clean + make html + + # View the docs + cd _build/html + python -m http.server 8000 + # Open http://localhost:8000 in your host browser + +Managing the Container +---------------------- -To run github actions locally install the -[act](https://github.com/nektos/gh-act) extension: -```bash -gh extension install nektos/gh-act -``` +Start and Stop +^^^^^^^^^^^^^^ -You can then run remote-tests locally: +.. code-block:: bash + + # Start container + docker start tidy3d_python_client_dev + + # Stop container (preserves state) + docker stop tidy3d_python_client_dev + + # Restart container + docker restart tidy3d_python_client_dev + +Checking Status +^^^^^^^^^^^^^^^ .. code-block:: bash - gh act --pull=false -W .github/workflows/tidy3d-python-client-tests.yml -P "ubuntu-latest=tidy3d_python_client_dev:latest" --input remote_tests=true "workflow_dispatch" + # Check if container is running + docker ps + + # Check all containers (including stopped) + docker ps -a + + # View container logs + docker logs tidy3d_python_client_dev + + # Follow container logs (live) + docker logs -f tidy3d_python_client_dev + +Removing and Recreating +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + # Stop and remove container + docker stop tidy3d_python_client_dev + docker container rm tidy3d_python_client_dev + + # Recreate container (preserves code on host) + docker container create --name=tidy3d_python_client_dev \ + --userns=host \ + -p 8888:8888 \ + -v .:/home/flexdaemon/tidy3d \ + tidy3d-python-client-dev + +.. note:: + Your code changes are safe! They're stored on your host in the mounted directory. + +Rebuilding the Image +^^^^^^^^^^^^^^^^^^^^ + +If ``dev.Dockerfile`` changes or you want to update base dependencies: + +.. code-block:: bash + + # Rebuild the image + docker build -t tidy3d-python-client-dev -f dev.Dockerfile . + + # Remove old container and create new one + docker stop tidy3d_python_client_dev + docker container rm tidy3d_python_client_dev + docker container create --name=tidy3d_python_client_dev \ + --userns=host \ + -p 8888:8888 \ + -v .:/home/flexdaemon/tidy3d \ + tidy3d-python-client-dev +Advanced Usage +-------------- Running Jupyter Lab ^^^^^^^^^^^^^^^^^^^ -To run a Jupyter Lab instance that's accessible from your host machine's web browser, execute the following command from within the container's shell: +Start Jupyter Lab inside the container to work with notebooks: .. code-block:: bash - # (Inside Container) Make sure your virtual environment is activated! - jupyter lab . --ip=0.0.0.0 + # Inside container with venv activated + cd tidy3d + jupyter lab . --ip=0.0.0.0 --port=8888 + +Jupyter Lab will print a URL like: + +.. code-block:: -After it starts, Jupyter will print a URL to the terminal containing a security token. It will look like this: -``http://127.0.0.1:8888/lab?token=...`` + http://127.0.0.1:8888/lab?token=abc123def456... Copy this complete URL and paste it into your web browser (like Firefox) on your host machine to begin your session. + +.. note:: + Port 8888 is forwarded to your host via the ``-p 8888:8888`` flag used when creating the container. + +Running GitHub Actions Locally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To run GitHub Actions locally, install the `act `_ extension: + +.. code-block:: bash + + gh extension install nektos/gh-act + +You can then run remote-tests locally using your dev container image: + +.. code-block:: bash + + gh act --pull=false \ + -W .github/workflows/tidy3d-python-client-tests.yml \ + -P "ubuntu-latest=tidy3d-python-client-dev:latest" \ + --input remote_tests=true \ + "workflow_dispatch" + +.. note:: + The image name must match the one you built: ``tidy3d-python-client-dev:latest`` + +VSCode Integration +^^^^^^^^^^^^^^^^^^ + +Attach VSCode to the running container for a full IDE experience: + +**Install Extensions** + +In VSCode, install: + +* **Remote - Containers** (ms-vscode-remote.remote-containers) +* **Docker** (ms-azuretools.vscode-docker) + +**Attach to Container** + +1. Start your container: + + .. code-block:: bash + + docker start tidy3d_python_client_dev + +2. In VSCode: + + * Open Command Palette (Cmd/Ctrl+Shift+P) + * Type "Remote-Containers: Attach to Running Container" + * Select ``tidy3d_python_client_dev`` + +3. VSCode will open a new window connected to the container + +4. Open folder: ``/home/flexdaemon/tidy3d`` + +5. Select Python interpreter: ``.venv/bin/python`` + +Now you can: + +* Edit code with full IntelliSense +* Run tests from VSCode test explorer +* Debug Python code with breakpoints +* Use integrated terminal (already inside container) + +Optional: AWS CodeArtifact Access (Internal Developers Only) +------------------------------------------------------------- + +.. note:: + **Who needs this?** + + CodeArtifact hosts internal packages like ``tidy3d-extras`` needed for: + + * Integration tests in CI/CD + * Backend-dependent development work + * Full-stack Tidy3D development + + **External contributors**: You can skip this entire section. The open-source ``tidy3d`` client works without CodeArtifact. + +.. warning:: + **Security Policy**: We use AWS SSO for authentication, which requires daily login for security compliance. + This is a simple 2-step process per development session: + + 1. ``aws sso login`` - Opens browser for authentication (~30 seconds) + 2. ``poetry aws-login`` - Automatic token management + +Why CodeArtifact? +^^^^^^^^^^^^^^^^^ + +**Q: Why can't we use PyPI?** + +A: Internal packages like ``tidy3d-extras`` contain proprietary code that cannot be published publicly. + +**Q: Why SSO with daily expiration?** + +A: Flexcompute security policy requires time-limited credentials. Long-lived tokens pose security risks. + +**Q: What if I don't want to deal with this?** + +A: Use the Docker development environment, which matches our CI setup exactly. Or if you're only working on the open-source client, you don't need CodeArtifact at all. + +Supported Path: Poetry with AWS Login Plugin +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The development container includes the ``poetry-codeartifact-login`` plugin for simplified authentication. + +.. note:: + **For non-Docker users**: If you're not using the Docker container, you'll need to install the plugin first: + + .. code-block:: bash + + # Install the Poetry plugin (one-time, outside Docker) + poetry self add poetry-codeartifact-login + + # Also ensure AWS CLI v2 is installed on your system + aws --version # Should show version 2.x + + Docker users can skip this step - the plugin is pre-installed in the dev container. + +**One-Time Setup** + +Inside the container (or on your local machine if not using Docker), configure AWS SSO (only needed once): + +.. code-block:: bash + + # Configure AWS SSO + aws configure sso + +Enter the following when prompted: + +.. list-table:: + :widths: 40 60 + :header-rows: 1 + + * - Prompt + - Value + * - SSO session name + - ``flexcompute-pypi`` + * - SSO start URL + - ``https://d-9067bfae6e.awsapps.com/start/#`` + * - SSO region + - ``us-east-1`` + * - Account + - ``625554095313`` + * - Role + - ``codeartifact-readonly`` + * - Profile name + - ``flexcompute-pypi`` + +**Daily Authentication** (each development session) + +Each day when you start development, run these two commands: + +.. code-block:: bash + + # Step 1: Login to AWS SSO (opens browser on host for authentication) + aws sso login --profile flexcompute-pypi + + # Step 2: Configure Poetry with CodeArtifact + poetry aws-login codeartifact --profile flexcompute-pypi + + # That's it! Now you can install internal packages + poetry install -E extras + +.. note:: + **How it works**: The ``poetry aws-login`` command automatically: + + * Generates a short-lived CodeArtifact authentication token + * Configures Poetry to use the token for the ``codeartifact`` source + * Stores credentials securely (no manual token passing required) + + The plugin handles all token management internally, so you never see or copy tokens manually. + +Persisting AWS Credentials +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To avoid re-configuring AWS SSO each time you recreate the container, mount your AWS credentials: + +.. code-block:: bash + + # Recreate container with AWS credentials mounted + docker container create --name=tidy3d_python_client_dev \ + --userns=host \ + -p 8888:8888 \ + -v .:/home/flexdaemon/tidy3d \ + -v ~/.aws:/home/flexdaemon/.aws \ + tidy3d-python-client-dev + +This way, you only need to run ``aws configure sso`` once on your host machine. + +Alternative Package Managers (Self-Support) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you prefer ``pip``, ``uv``, or other package managers instead of Poetry, you'll need to configure +CodeArtifact authentication yourself. + +**The officially supported development path uses Poetry.** For other package managers: + +* See `AWS CodeArtifact pip documentation `_ +* Use ``aws codeartifact login --tool pip`` for manual setup +* You are responsible for configuration and troubleshooting + +Alternatively, use the Docker development environment which has everything pre-configured. + +Quick Reference +--------------- + +Command Cheatsheet +^^^^^^^^^^^^^^^^^^ + +**Container Management:** + +.. code-block:: bash + + # Build image + docker build -t tidy3d-python-client-dev -f dev.Dockerfile . + + # Create container + docker container create --name=tidy3d_python_client_dev \ + --userns=host -p 8888:8888 \ + -v .:/home/flexdaemon/tidy3d \ + tidy3d-python-client-dev + + # Start/stop container + docker start tidy3d_python_client_dev + docker stop tidy3d_python_client_dev + + # Enter container + docker exec -it tidy3d_python_client_dev bash + +**Inside Container:** + +.. code-block:: bash + + # Setup environment + cd tidy3d + uv venv -p 3.11 + source .venv/bin/activate + uv pip install -e .[dev] + + # Testing + pytest # All tests + pytest -n auto # Parallel tests + pytest tests/test_file.py # Specific file + + # Code quality + ruff format . # Format code + ruff check . # Lint code + mypy tidy3d # Type check + + # Documentation + cd docs && make html # Build docs + + # Jupyter + jupyter lab . --ip=0.0.0.0 # Start Jupyter Lab + + # CodeArtifact (internal developers only) + aws sso login --profile flexcompute-pypi # Daily: Login to AWS + poetry aws-login codeartifact --profile flexcompute-pypi # Daily: Configure Poetry + +Directory Structure +^^^^^^^^^^^^^^^^^^^ + +.. code-block:: + + Host: + ~/flexcompute/tidy3d/ + ├── tidy3d/ # Source code (mounted to container) + ├── tests/ # Test suite (mounted) + ├── docs/ # Documentation (mounted) + ├── .venv/ # Virtual env (created inside, persists) + ├── pyproject.toml # Poetry config (mounted) + └── dev.Dockerfile # Docker image definition + + Container: + /home/flexdaemon/ + ├── tidy3d/ # Mounted from host + │ ├── .venv/ # Your virtual environment + │ ├── tidy3d/ # Package source + │ └── tests/ # Tests + └── .aws/ # AWS credentials (if mounted) + +Troubleshooting +--------------- + +Container Issues +^^^^^^^^^^^^^^^^ + +**Problem**: ``docker: permission denied`` + +**Solution**: Add your user to the docker group: + +.. code-block:: bash + + sudo usermod -aG docker $USER + # Logout and login (or reboot) + +**Problem**: Container exits immediately + +**Solution**: Check logs for errors: + +.. code-block:: bash + + docker logs tidy3d_python_client_dev + + # Try running interactively to debug + docker run --rm -it tidy3d-python-client-dev bash + +**Problem**: Port 8888 already in use + +**Solution**: Use a different port: + +.. code-block:: bash + + docker container create --name=tidy3d_python_client_dev \ + --userns=host \ + -p 8889:8888 \ + -v .:/home/flexdaemon/tidy3d \ + tidy3d-python-client-dev + + # Access Jupyter at http://localhost:8889 + +Permission Issues +^^^^^^^^^^^^^^^^^ + +**Problem**: Cannot edit files, permission denied + +**Solution**: Ensure container uses host user namespace: + +.. code-block:: bash + + # Container must be created with --userns=host flag + docker container create --name=tidy3d_python_client_dev \ + --userns=host \ + -v .:/home/flexdaemon/tidy3d \ + tidy3d-python-client-dev + +**Problem**: ``flexdaemon`` user has wrong UID + +**Solution**: The ``dev.Dockerfile`` creates user with UID 1000. If your host user has a different UID, files may have permission issues. Check your host UID: + +.. code-block:: bash + + # On host + id -u # Should output 1000 for best compatibility + + # If different, you can modify the Dockerfile or change file ownership + +Virtual Environment Issues +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Problem**: ``uv venv`` command not found + +**Solution**: ``uv`` is pre-installed in the image. Verify: + +.. code-block:: bash + + which uv + uv --version + + # If missing, rebuild the image + +**Problem**: Virtual environment activation fails + +**Solution**: Make sure you're in the right directory: + +.. code-block:: bash + + cd /home/flexdaemon/tidy3d + ls .venv # Should exist after 'uv venv' + source .venv/bin/activate + +Test Failures +^^^^^^^^^^^^^ + +**Problem**: Tests fail with ``ModuleNotFoundError`` + +**Solution**: Reinstall in editable mode: + +.. code-block:: bash + + # Inside container with venv activated + cd /home/flexdaemon/tidy3d + uv pip install -e .[dev] + +**Problem**: Import errors for optional dependencies + +**Solution**: Install all extras: + +.. code-block:: bash + + uv pip install -e .[dev,jax,vtk,trimesh,gdstk] + +CodeArtifact Authentication Issues +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Problem**: ``poetry install -E extras`` fails with "Unable to find tidy3d-extras" or authentication error + +**Solution**: Verify SSO session is active and re-authenticate: + +.. code-block:: bash + + # Check if SSO session is valid + aws sts get-caller-identity --profile flexcompute-pypi + + # If expired or fails, re-authenticate (2-step process) + aws sso login --profile flexcompute-pypi + poetry aws-login codeartifact --profile flexcompute-pypi + + # Retry installation + poetry install -E extras + +**Problem**: ``aws sso login`` doesn't work, times out, or fails + +**Solution**: This is an AWS account access issue. Contact IT support or your team lead to verify: + +* Your AWS account is active +* You have access to the correct AWS organization +* Your SSO credentials are correct +* You have the ``codeartifact-readonly`` role assigned + +**Problem**: ``poetry aws-login`` command not found + +**Solution**: The plugin is pre-installed in the dev container. If missing: + +.. code-block:: bash + + # Check if plugin is installed + poetry self show plugins + + # Reinstall if needed + poetry self add poetry-codeartifact-login + +**Problem**: I use pip/uv, not Poetry. How do I authenticate? + +**Solution**: The officially supported path is Poetry with the ``poetry aws-login`` command. + +For pip/uv users, you must self-support using AWS documentation. + +**Option 1 (for pip users):** + +.. code-block:: bash + + aws sso login --profile flexcompute-pypi + aws codeartifact login --tool pip \ + --repository pypi-releases \ + --domain flexcompute \ + --domain-owner 625554095313 \ + --region us-east-1 \ + --profile flexcompute-pypi + +**Option 2 (not recommended):** + +.. code-block:: bash + + CODEARTIFACT_AUTH_TOKEN=`aws codeartifact get-authorization-token \ + --domain flexcompute \ + --domain-owner 625554095313 \ + --query authorizationToken \ + --output text \ + --profile flexcompute-pypi` + + pip config set site.extra-index-url \ + https://aws:$CODEARTIFACT_AUTH_TOKEN@flexcompute-625554095313.d.codeartifact.us-east-1.amazonaws.com/pypi/pypi-releases/simple/ + +Note: Tokens expire after 12 hours, requiring daily re-authentication. + +Alternatively, use the Docker development environment. + +**Problem**: ``poetry lock`` or ``poetry update`` hangs + +**Solution**: Clear Poetry cache and retry: + +.. code-block:: bash + + # Clear Poetry cache + poetry cache clear pypi --all + poetry cache clear codeartifact --all + + # Retry operation + poetry update --lock + +Jupyter Issues +^^^^^^^^^^^^^^ + +**Problem**: Cannot access Jupyter Lab from host browser + +**Solution**: Ensure Jupyter is binding to 0.0.0.0: + +.. code-block:: bash + + # Inside container + jupyter lab . --ip=0.0.0.0 --port=8888 + + # Port must be forwarded when creating container + # Check with: docker inspect tidy3d_python_client_dev | grep PortBindings + +**Problem**: Jupyter kernel dies or doesn't start + +**Solution**: Check virtual environment has ipykernel: + +.. code-block:: bash + + uv pip install ipykernel + python -m ipykernel install --user --name=tidy3d + +Next Steps +---------- + +After completing this guide, you should have: + +✅ Docker development container running +✅ Python environment set up with dependencies +✅ Tests passing successfully +✅ (Optional) Jupyter Lab accessible +✅ (Optional) VSCode attached to container +✅ (Optional) AWS CodeArtifact configured + +**Continue learning:** + +* :doc:`usage` - Advanced development workflows +* :doc:`documentation` - Building and writing documentation +* :doc:`recommendations` - Best practices and code style +* :doc:`release/index` - Release process and versioning + +**More Docker resources:** + +* `Docker Documentation `_ +* `Dev Containers in VSCode `_ + +Welcome to Docker-based Tidy3D development! diff --git a/tests/tidy3d_extras_license_test.py b/tests/tidy3d_extras_license_test.py new file mode 100644 index 0000000000..f94b333340 --- /dev/null +++ b/tests/tidy3d_extras_license_test.py @@ -0,0 +1,111 @@ +"""Test that the license check raises an error if the api key is invalid.""" + +from __future__ import annotations + +import os +import subprocess +import sys + +import pytest + +import tidy3d as td + + +def _extension_can_load() -> bool: + """Return True if the tidy3d_extras extension can be loaded on this platform.""" + try: + import tidy3d_extras + + # Check if the extension module is actually available + return hasattr(tidy3d_extras, "extension") + except Exception: + return False + + +def _local_subpixel_works_with_bad_key() -> bool: + """Return True if local subpixel succeeds even with a bad API key.""" + sim = td.Simulation( + size=(1, 0, 0), + grid_spec=td.GridSpec.auto(wavelength=1), + boundary_spec=td.BoundarySpec.all_sides(td.Periodic()), + run_time=1e-30, + ) + prev_pref = td.config.simulation.use_local_subpixel + td.config.simulation.use_local_subpixel = True + try: + _ = sim.epsilon_on_grid( + grid=sim.discretize(sim.geometry), + freq=td.C_0 / 1.55, + ) + return True + except td.exceptions.Tidy3dImportError: + return False + finally: + td.config.simulation.use_local_subpixel = prev_pref + + +def test_license_check(monkeypatch, caplog): + monkeypatch.setenv("SIMCLOUD_APIKEY", "BADKEY") + + # package should still import successfully, just without .extension + result = subprocess.run( + [sys.executable, "-c", "import tidy3d_extras"], + capture_output=True, + text=True, + check=True, + cwd=os.path.dirname(__file__), + ) + assert result.returncode == 0 + print(result.stdout) + + # calling local_subpixel should fail with a clear error message when the API key is bad + with pytest.raises(subprocess.CalledProcessError) as excinfo: + subprocess.run( + [ + sys.executable, + "-c", + "import tidy3d_extras_license_test; tidy3d_extras_license_test.subpixel()", + ], + capture_output=True, + text=True, + check=True, + cwd=os.path.dirname(__file__), + ) + print(result.stdout) + print(excinfo.value.stdout) + print(excinfo.value.stderr) + + # Check if the extension can actually load on this platform + # On some platforms (e.g., macOS with certain Python versions), the extension + # may fail to load due to ABI compatibility issues before license checks can run + extension_loads = _extension_can_load() + + combined_output = excinfo.value.stdout + excinfo.value.stderr + + # Core license / auth failure - only check if extension loads properly + if extension_loads: + assert "Incorrect API Key" in combined_output, ( + "Expected 'Incorrect API Key' error when extension loads but API key is invalid" + ) + + # tidy3d-extras initialization and feature error messages should always be present + assert ( + "invalid API key" in combined_output or "did not initialize correctly" in combined_output + ), "Expected tidy3d-extras initialization error message" + assert "local_subpixel" in combined_output, ( + "Expected 'local_subpixel' to be mentioned in error message" + ) + + +def subpixel(): + sim = td.Simulation( + size=(1, 0, 0), + grid_spec=td.GridSpec.auto(wavelength=1), + boundary_spec=td.BoundarySpec.all_sides(td.Periodic()), + run_time=1e-30, + ) + td.config.simulation.use_local_subpixel = True + _ = sim.epsilon_on_grid( + grid=sim.discretize(sim.geometry), + freq=td.C_0 / 1.55, + )