2.7 branch

.github/ISSUE_TEMPLATE/autograd_bug.md

@@ -0,0 +1,20 @@
+---
+name: Autograd Bug Report
+about: Create a report to help us improve our autograd feature
+title: 'Autograd Bug'
+labels: 'autograd'
+assignees: 'tylerflex'
+
+---
+
+**Description**
+Please paste the error message displayed in your warning when running `tidy3d.web.run()` or `tidy3d.web.run_async()`
+
+**Simulation Setup**
+
+Please describe your simulation setup as specifically as you can, including code snippets or uploading the `simulation.to_file('sim.json')` file. 
+
+Also please include what options were passed to the `web` function.
+
+**Other information**
+Anything else you'd like to add to help us look into it?

.github/workflows/release.yml

@@ -16,7 +16,7 @@ jobs:
       with:
         ref: ${{ github.ref }}
     - name: Exit if any RC release
-      if: contains(${{ github.ref }}, 'rc') == false
+      if: contains(github.ref, 'rc') == false
       uses: everlytic/branch-merge@1.1.2
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
@@ -54,7 +54,7 @@ jobs:
       with:
         ref: "latest"
     - name: Exit if any RC release
-      if: contains(${{ github.ref }}, 'rc') == false
+      if: contains(github.ref, 'rc') == false
       uses: everlytic/branch-merge@1.1.2
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/run_tests.yml

@@ -11,20 +11,22 @@ on:
       - 'pre/*'
 
 jobs:
+
   pre-commit:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           submodules: 'recursive'
       - name: Test pre-commit hooks
         run: |
           python -m pip install --upgrade pip
           pip install pre-commit
           pre-commit run # this should be really more agressive
+
   test-latest-submodules:
     runs-on: ubuntu-latest
     steps:
@@ -39,12 +41,19 @@ jobs:
         id: get_branch
         run: |
           if [[ "${{ github.event_name }}" == "pull_request" ]]; then
-            echo "BRANCH_NAME=${{ github.event.pull_request.base.ref }}" >> $GITHUB_ENV
+            # Extract the base branch of the PR
+            BRANCH_NAME="${{ github.event.pull_request.base.ref }}"
+            echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
           else
-            echo "BRANCH_NAME=$(echo $GITHUB_REF | sed 's|refs/heads/||')" >> $GITHUB_ENV
+            # Assume it's a push event, extract the branch name from $GITHUB_REF
+            BRANCH_NAME=$(echo $GITHUB_REF | sed 's|refs/heads/||')
+            echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
           fi
-          echo $BRANCH_NAME
+          # Now echoing the BRANCH_NAME to verify
+          echo "BRANCH_NAME: $BRANCH_NAME"
         shell: bash
+        env:
+          GITHUB_REF: ${{ github.ref }}
 
       - name: Initialize and update submodule
         run: |
@@ -59,17 +68,18 @@ jobs:
           # Checking out Notebooks submodule with the same branch as the main project
           echo "Checking $NOTEBOOKS_PATH for updates..."
           cd $NOTEBOOKS_PATH
+          NOTEBOOKS_CURRENT_COMMIT=$(git rev-parse HEAD)
           echo $(git fetch --all --verbose)
           echo $(git remote get-url origin)
-          git checkout origin/$BRANCH_NAME
           if git show-ref --verify refs/remotes/origin/$BRANCH_NAME; then
            echo "Branch $BRANCH_NAME exists."
           else
             echo "::error::Branch $BRANCH_NAME does not exist on remote."
             exit 1
           fi
           NOTEBOOKS_LATEST_COMMIT=$(git rev-parse refs/remotes/origin/${{ env.BRANCH_NAME }})
-          NOTEBOOKS_CURRENT_COMMIT=$(git rev-parse HEAD)
+          echo "NOTEBOOKS_LATEST_COMMIT: $NOTEBOOKS_LATEST_COMMIT"
+          echo "NOTEBOOKS_CURRENT_COMMIT: $NOTEBOOKS_CURRENT_COMMIT"
 
 
           cd ../..
@@ -83,22 +93,27 @@ jobs:
           # Checking FAQs only on the develop branch.
           echo "Checking $FAQ_PATH for updates..."
           cd $FAQ_PATH
-          FAQ_LATEST_COMMIT=$(git rev-parse origin/develop)
           FAQ_CURRENT_COMMIT=$(git rev-parse HEAD)
+          echo $(git fetch --all --verbose)
+          echo $(git remote get-url origin)
+          FAQ_LATEST_COMMIT=$(git rev-parse refs/remotes/origin/develop)
+          echo "FAQ_LATEST_COMMIT: $FAQ_LATEST_COMMIT"
+          echo "FAQ_CURRENT_COMMIT: $FAQ_CURRENT_COMMIT"
           cd ../..
           if [ "$FAQ_LATEST_COMMIT" != "$FAQ_CURRENT_COMMIT" ]; then
             echo "::error ::Submodule $FAQ_PATH is not up to date. Please update it."
             exit 1
           else
             echo "Submodule $FAQ_PATH is up to date."
           fi
+
   build:
     name: test ${{ matrix.python-version }} - ${{ matrix.platform }}
     runs-on: ${{ matrix.platform }}
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11']
-        platform: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+        platform: [ubuntu-latest, windows-latest, macos-latest]
     defaults:
       run:
         shell: bash
@@ -124,13 +139,16 @@ jobs:
     - name: Install library
       run: |
         poetry --version
+        poetry run pip install --upgrade pip wheel setuptools
+        poetry run pip install gdstk --only-binary gdstk
         poetry install -E dev
+
     #----------------------------------------------
     #    add matrix specifics and run test suite
     #----------------------------------------------
     - name: Run tests
       run: |
-        poetry run black . --check --diff
+        poetry run ruff format . --check --diff
         poetry run ruff check tidy3d --fix --exit-non-zero-on-fix
         poetry run pytest -rA tests
         poetry run pytest -rA tests/_test_data/_test_datasets_no_vtk.py

.github/workflows/sync-to-readthedocs-repo.yaml

@@ -1,6 +1,5 @@
 name: "sync-to-readthedocs-repo"
-permissions:
-  contents: write
+
 on:
   workflow_dispatch:
   push:
@@ -9,43 +8,72 @@ on:
       - latest
       - develop
       - 'pre/*'
-      - 'v*'
       - 'production/test/*'
+    tags:
+      - 'v*'
+      - 'demo/*'
+
 jobs:
-  extract_branch:
+  extract_branch_or_tag:
     outputs:
-      branch: ${{ steps.extractbranch.outputs.branch }}
+      ref_name: ${{ steps.extract.outputs.ref_name }}
     runs-on: ubuntu-latest
     steps:
-      - id: extractbranch
-        name: Extract branch name
+      - id: extract
+        name: Extract branch or tag name
         shell: bash
         run: |
-          echo "::set-output name=branch::${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
-          echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
-          echo "${{ steps.extractbranch.outputs.branch }}"
+          REF_NAME="${GITHUB_REF#refs/*/}"
+          echo "::set-output name=ref_name::$REF_NAME"
+          echo "Extracted ref: $REF_NAME"
+
   build-and-deploy:
-    needs: extract_branch
+    permissions:
+      contents: write
+    needs: extract_branch_or_tag
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
+      # Conditional Checkout for Branch
+      - name: Checkout Branch
+        if: contains(github.ref, 'refs/heads/')
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          token: ${{ secrets.GH_PAT }}
+          ref: ${{ needs.extract_branch_or_tag.outputs.ref_name }}
+
+      # Conditional Checkout for Tag
+      - name: Checkout Tag
+        if: contains(github.ref, 'refs/tags/')
         uses: actions/checkout@v3
         with:
           submodules: true
-      - name: Create new branch if doesn't exist
+          token: ${{ secrets.GH_PAT }}
+          fetch-depth: 0
+          ref: ${{ needs.extract_branch_or_tag.outputs.ref_name }}
+          fetch-tags: true
+
+      - name: Create new branch or tag
         uses: GuillaumeFalourd/create-other-repo-branch-action@v1.5
+        if: contains(github.ref, 'refs/heads/')
         with:
           repository_owner: flexcompute-readthedocs
           repository_name: tidy3d-docs
-          new_branch_name: ${{ needs.extract_branch.outputs.branch }}
+          new_branch_name: ${{ needs.extract_branch_or_tag.outputs.ref_name }}
           access_token: ${{ secrets.GH_PAT }}
           new_branch_ref: default_clean_sync_branch
           ignore_branch_exists: true
+
+      - name: Tag and push
+        if: contains(github.ref, 'refs/tags/')
+        run: |
+          git push --force --tags https://${{ secrets.GH_PAT }}@github.com/flexcompute-readthedocs/tidy3d-docs.git
+
       - uses: JamesIves/github-pages-deploy-action@v4
         with:
           folder: .
           token: ${{ secrets.GH_PAT }}
           repository-name: flexcompute-readthedocs/tidy3d-docs
           target-folder: .
-          branch: ${{ needs.extract_branch.outputs.branch }}
+          branch: ${{ needs.extract_branch_or_tag.outputs.ref_name }}
           force: true

.github/workflows/update_coverage.yml

@@ -82,7 +82,7 @@ jobs:
     - name: Run tests
       run: |
         source $VENV
-        black . --check --diff
+        ruff format . --check --diff
         ruff check tidy3d --fix --exit-non-zero-on-fix
         coverage run -m pytest -rA tests
         coverage report -m

.gitignore

@@ -80,6 +80,9 @@ pip-log.txt
 .dmypy.json
 .mypy_cache/
 
+# pickle files
+*.pkl
+
 # PyInstaller
 *.manifest
 *.spec
@@ -109,6 +112,7 @@ _autosummary
 
 # Test and temporary files
 tests/tmp/*
+tests/data/invdes_history.*
 
 # Translations
 *.mo
@@ -129,3 +133,7 @@ htmlcov/
 
 # Specific file and folder exclusions
 .idea
+.vscode
+
+# cProfile output
+*.prof

.pre-commit-config.yaml

@@ -1,12 +1,7 @@
 repos:
-  - repo: https://github.com/psf/black
-    rev: 23.12.1
-    hooks:
-    - id: black
-      name: black <<< NOTE:, if you get a message stating that "All done! n file(s) reformatted", you must git add and commit the changes again to complete the commit. >>>
-
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.3.2"
+    rev: "v0.4.8"
     hooks:
      - id: ruff
-       args: [ --fix ]
+       args: [ --fix ]
+     - id: ruff-format