Merge pull request #1796 from gdsfactory/klive_handshake_docs_py2

Klive handshake docs py2
gdsfactory · Jun 22, 2023 · 13725c2 · 13725c2
2 parents 401369a + 1f87b9f
commit 13725c2
Show file tree

Hide file tree

Showing 101 changed files with 8,973 additions and 9,271 deletions.
diff --git a/.gitignore b/.gitignore
@@ -70,7 +70,6 @@ cml/
 mzi.yml
 *.out
 mpb-sim/
-data/
 extrakdown-parser/
 
 # Installer logs

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,27 +1,28 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: ca84e500209b3757213759d4d522f8ed307cd638
+    rev: f8d8c45220230434bd7440d85a7f64c67bcdb952
     hooks:
       - id: check-yaml
         exclude: ^(conda\.recipe/meta\.yaml|conda_build/templates/.*\.yaml|docs/click/meta\.yaml|conda/meta\.yaml|conda/construct.yaml|.*\.pic\.yml|conda/constructor/Miniforge3/construct.yaml)
       - id: end-of-file-fixer
       - id: trailing-whitespace
 
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: d1248ca9beaf0ba526d265f4108836d89cf551b7
     hooks:
       - id: black
+        # exclude: '^gdsfactory/samples/notebooks/|^docs/notebooks/'
 
   - repo: https://github.com/codespell-project/codespell
-    rev: ad3ff374e97e29ca87c94b5dc7eccdd29adc6296
+    rev: af69927c7d3965d3fe8f68007e12cd971847573f
     hooks:
       - id: codespell
         args: ["-L TE,TE/TM,te,ba,FPR,fpr_spacing,ro,nd,donot,schem"]
         additional_dependencies:
           - tomli
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "7c149cf0b9d2c4f7558de1ea5d2c137ca074d9d0"
+    rev: "770e3ceae230e989c8337eb434b7707d5f796e3d"
     hooks:
       - id: ruff
 

diff --git a/Makefile b/Makefile
@@ -18,18 +18,6 @@ dev: full
 	pre-commit install
 	gf install klayout-genericpdk
 
-mamba:
-	bash conda/mamba.sh
-
-patch:
-	bumpversion patch
-
-minor:
-	bumpversion minor
-
-major:
-	bumpversion major
-
 plugins:
 	conda install -c conda-forge pymeep=*=mpi_mpich_* nlopt -y
 	conda install -c conda-forge slepc4py=*=complex* -y
@@ -73,10 +61,6 @@ data-upload:
 	echo 'no need to upload'
 	# aws s3 sync data s3://gdslib
 	# gh release upload v6.90.3 data/gds/*.gds --clobber
-	# gh release upload v6.90.3 data/sp/*.npz --clobber
-	# gh release upload v6.90.3 data/sp/*.yml --clobber
-	# gh release upload v6.90.3 data/modes/*.msh --clobber
-	# gh release upload v6.90.3 data/modes/*.npz --clobber
 
 test-data:
 	git clone https://github.com/gdsfactory/gdsfactory-test-data.git -b test-data test-data
@@ -220,6 +204,9 @@ constructor:
 	conda install constructor -y
 	constructor conda
 
+notebooks:
+	jupytext gdsfactory/samples/notebooks/*.md --to ipynb notebooks/
+
 nbqa:
 	nbqa blacken-docs docs/notebooks/**/*.ipynb --nbqa-md
 	nbqa blacken-docs docs/notebooks/*.ipynb --nbqa-md
@@ -237,7 +224,8 @@ jupytext-clean:
 	jupytext docs/**/*.py --to py
 
 notebooks:
-	jupytext docs/notebooks/**/*.py --to ipynb
-	jupytext docs/notebooks/*.py --to ipynb
+	# jupytext docs/notebooks/*.py --to ipynb
+	# jupytext docs/notebooks/*.ipynb --to to
+	jupytext --pipe black docs/notebooks/*.py
 
 .PHONY: gdsdiff build conda gdslib docs doc
diff --git a/README.md b/README.md
@@ -78,17 +78,7 @@ You can also access:
 
 Use python3.10 or python3.11, as some tools like kfactory are not available for older versions of python. We recommend [VSCode](https://code.visualstudio.com/) as an IDE.
 
-If you don't have python installed on your system you can [download the gdsfactory installer](https://github.com/gdsfactory/gdsfactory/releases) that includes python3, miniconda and gdsfactory.
-
-You can also install python with mamba package manager (faster than conda):
-
-| OS      | Architecture          | Download  |
-| --------|-----------------------|-----------|
-| Linux   | x86_64 (amd64)        | [Linux-x86_64](https://github.com/gdsfactory/gdsfactory/releases/latest/download/Mambaforge-Linux-x86_64.sh) |
-| OS X    | x86_64                | [MacOSX-x86_64](https://github.com/gdsfactory/gdsfactory/releases/latest/download/Mambaforge-MacOSX-x86_64.sh) |
-| OS X    | arm64 (Apple Silicon) | [MacOSX-arm64](https://github.com/gdsfactory/gdsfactory/releases/latest/download/Mambaforge-MacOSX-arm64.sh) |
-| Windows | x86_64                | [Windows-x86_64](https://github.com/gdsfactory/gdsfactory/releases/latest/download/Mambaforge-Windows-x86_64.exe) |
-
+If you don't have python installed on your system you can [download anaconda](https://www.anaconda.com/download/)
 
 Once you have python installed, open Anaconda Prompt as Administrator and then install the latest gdsfactory using pip.
 

diff --git a/docs/_config.yml b/docs/_config.yml
@@ -18,11 +18,11 @@ execute:
     - "*01_pin_waveguide*"
     - "*1_fdtd_sparameters*"
     - "*2_interconnect*"
-    - "*optimiser*"
+    - "*ray_optimiser*"
     - "*03_numerical_implantation*"
     - "*02_model_extraction*"
-    - "*001_meep_sparameters*"
-    - "*20_schematic_driven_layout*"
+    # - "*20_schematic_driven_layout*"
+    # - "*001_meep_sparameters*"
     # - "*00_tidy3d.ipynb"
     # - "*03_cascaded_mzi*"
     # - "*jupyter_notebooks*"
@@ -79,6 +79,6 @@ sphinx:
       - https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js
     nb_execution_raise_on_error: true
     nb_custom_formats:
-        .md:
+        .py:
             - jupytext.reads
-            - fmt: md
+            - fmt: py
diff --git a/docs/_toc.yml b/docs/_toc.yml
@@ -30,7 +30,7 @@ parts:
         - file: notebooks/01_references
         - file: notebooks/02_movement
         - file: notebooks/03_cells_autoname_and_cache
-        - file: notebooks/03_waveguides_paths_crossections
+        - file: notebooks/03_Path_CrossSection
         - file: notebooks/04_components_shapes
         - file: notebooks/04_components_geometry
         - file: notebooks/04_components_pack
@@ -46,47 +46,47 @@ parts:
         - file: notebooks/21_drc
     - file: workflow
       sections:
-        - file: notebooks/workflow/jupyter_notebooks
-        - file: notebooks/workflow/0_layout_summary
-        - file: notebooks/workflow/1_mzi
-        - file: notebooks/workflow/2_ring
-        - file: notebooks/workflow/3_cascaded_mzi
+        - file: notebooks/workflow_jupyter_notebooks
+        - file: notebooks/workflow_0_layout_summary
+        - file: notebooks/workflow_1_mzi
+        - file: notebooks/workflow_2_ring
+        - file: notebooks/workflow_3_cascaded_mzi
 - caption: Plugins
   chapters:
     - file: plugins_optimization
       sections:
-        - file: notebooks/ray/optimiser
+        - file: notebooks/ray_optimiser
     - file: plugins_mesh
       sections:
         - file: notebooks/materials
-        - file: notebooks/meshing/01_intro
-        - file: notebooks/meshing/02_2D_xy_mesh
-        - file: notebooks/meshing/03_2D_uz_mesh
-        - file: notebooks/meshing/04_refinement
+        - file: notebooks/meshing_01_intro
+        - file: notebooks/meshing_02_2D_xy_mesh
+        - file: notebooks/meshing_03_2D_uz_mesh
+        - file: notebooks/meshing_04_refinement
     - file: plugins_process
       sections:
-        - file: notebooks/thermal/thermal
-        - file: notebooks/devsim/01_pin_waveguide
-        - file: notebooks/tcad/02_analytical_process
-        - file: notebooks/tcad/03_numerical_implantation
+        - file: notebooks/femwell_02_heater
+        - file: notebooks/devsim_01_pin_waveguide
+        - file: notebooks/tcad_02_analytical_process
+        - file: notebooks/tcad_03_numerical_implantation
     - file: plugins_mode_solver
       sections:
-        - file: notebooks/fem/01_mode_solving
-        - file: notebooks/tidy3d/01_tidy3d_modes
-        - file: notebooks/mpb/001_mpb_waveguide
-        - file: notebooks/eme/01_meow
+        - file: notebooks/femwell_01_modes
+        - file: notebooks/tidy3d_01_tidy3d_modes
+        - file: notebooks/mpb_001_mpb_waveguide
+        - file: notebooks/meow_01
     - file: plugins_fdtd
       sections:
-        - file: notebooks/tidy3d/00_tidy3d
-        - file: notebooks/meep/001_meep_sparameters
-        - file: notebooks/meep/002_gratings
-        - file: notebooks/lumerical/1_fdtd_sparameters
+        - file: notebooks/tidy3d_00_tidy3d
+        - file: notebooks/meep_001_meep_sparameters
+        - file: notebooks/meep_002_gratings
+        - file: notebooks/lumerical_1_fdtd_sparameters
     - file: plugins_circuits
       sections:
-        - file: notebooks/sax/01_sax
-        - file: notebooks/sax/02_model_extraction
-        - file: notebooks/sax/03_variability_analysis
-        - file: notebooks/lumerical/2_interconnect
+        - file: notebooks/sax_01_sax
+        - file: notebooks/sax_02_model_extraction
+        - file: notebooks/sax_03_variability_analysis
+        - file: notebooks/lumerical_2_interconnect
     - file: notebooks/12_database
 - caption: Reference
   chapters:

diff --git a/docs/developer.md b/docs/developer.md
@@ -11,7 +11,6 @@ The following lines will:
 - install gdsfactory locally on your computer in `-e` edit mode.
 - install pre-commit hooks for making sure your code syntax and style matches some basic rules.
 
-
 ```
 git clone git@github.com:YourUserName/gdsfactory.git
 cd gdsfactory
@@ -21,7 +20,6 @@ pip install -e .[full,dev]
 pre-commit install
 ```
 
-
 ## Style
 
 - Follow [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html) and take a look at existing gdsfactory code.
@@ -62,12 +60,23 @@ You can run tests with `pytest`. This will run 3 types of tests:
     - when running the test it will do a boolean of the `run_layout` and the `ref_layout` and raise an error for any significant differences.
     - you can check out any changes in your library with `gf gds diff ref_layouts/bbox.gds run_layouts/bbox.gds`
 
-If test failed because you modified the geometry you can regenerate the regression tests with:
+In addition to unit tests run against the library, gdsfactory has a suite of regression tests which ensure that Components are never unintentionally modified between revisions. These regression tests include
+| Test Type | Path | Format | Purpose |
+|------|------|---------|--|
+| GDS | `tests/components/test_components.py:test_gds` | GDS | Tests that GDS files have not changed either structurally (cell names and hierarchy) or geometrically (XOR). |
+| Settings | `tests/components/test_components.py:test_settings` | YAML | Tests that component settings have not changed. |
+| Netlist | `tests/test_netlists.py` | YAML | Tests that extracted netlist yaml contents have not changed. |
+| Ports | `tests/components/test_ports.py` | CSV | Tests that port locations have not changed |
+| Containers | `tests/test_containers.py` | YAML | Tests that container settings have not changed |
 
-```
+To regenerate regression reference files, you can run
+
+```shell
 pytest --force-regen -s
 ```
 
+Note that the `--force-regen` flag will regenerate textual reference files, via [pytest-regressions](https://pytest-regressions.readthedocs.io/en/latest/overview.html). When GDS file regressions are found, the `-s` flag will cause pytest to step through the failures one-by-one, so you can inspect the XOR result in Klayout (automatically loaded via klive) and debug messages in the terminal. You will be prompted if you would like to accept or reject the set of changes for each file.
+
 ## Build your own Reticles/projects/PDKs
 
 We recommend creating a separate python project for each mask and PDK.
@@ -92,7 +101,6 @@ What do we test?
 - Component Settings.
 - Port positions and ensure they are on grid.
 
-
 ```python
 import pytest
 from pytest_regressions.data_regression import DataRegressionFixture

diff --git a/gdsfactory/add_loopback.py b/gdsfactory/add_loopback.py
@@ -1,4 +1,4 @@
-"""Add reference for a grating coupler array."""
+"""Add loopback reference for a grating coupler array."""
 from __future__ import annotations
 
 from typing import List, Optional

diff --git a/gdsfactory/klive.py b/gdsfactory/klive.py
@@ -8,13 +8,15 @@
 import socket
 from pathlib import Path
 from typing import Optional, Union
+from gdsfactory import config
 
 
 def show(
     gds_filename: Union[Path, str],
     keep_position: bool = True,
     technology: Optional[str] = None,
     port: int = 8082,
+    delete: bool = True,
 ) -> None:
     """Show GDS in klayout.
 
@@ -23,6 +25,7 @@ def show(
         keep_position: keep position and active layers.
         technology: Name of the KLayout technology to use.
         port: klayout server port.
+        delete: deletes file.
     """
     if not os.path.isfile(gds_filename):
         raise ValueError(f"{gds_filename} does not exist")
@@ -36,39 +39,34 @@ def show(
     }
     data_string = json.dumps(data)
     try:
-        conn = socket.create_connection(("127.0.0.1", port), timeout=1.0)
-        data_string += "\n"
-        data_string = (
-            data_string.encode() if hasattr(data_string, "encode") else data_string
-        )
-        conn.sendall(data_string)
-        conn.close()
+        conn = socket.create_connection(("127.0.0.1", 8082), timeout=0.5)
+        data = data_string + "\n"
+        enc_data = data.encode()
+        conn.sendall(enc_data)
+        conn.settimeout(5)
     except OSError:
-        pass
-
-    # try:
-    #     from gdsfactory.plugins.web.server import host, port
+        config.logger.warning(
+            "Could not connect to klive server. Is klayout open and klive plugin installed?"
+        )
+    else:
+        msg = ""
+        try:
+            msg = conn.recv(1024).decode("utf-8")
+            config.logger.info(f"Message from klive: {msg}")
+        except OSError:
+            config.logger.warning("klive didn't send data, closing")
+        finally:
+            conn.close()
 
-    #     conn = socket.create_connection((host, port), timeout=1.0)
-    #     data = {
-    #         "gds_file": os.path.abspath(gds_filename),
-    #     }
-    #     data_string = json.dumps(data)
-    #     data_string = data_string + "\n"
-    #     data_string = (
-    #         data_string.encode() if hasattr(data_string, "encode") else data_string
-    #     )
-    #     conn.sendall(data_string)
-    #     conn.close()
-    # except OSError:
-    #     pass
+    if delete:
+        Path(gds_filename).unlink()
 
 
 if __name__ == "__main__":
     import gdsfactory as gf
 
     # c = gf.components.mzi()
-    c = gf.components.straight()
+    c = gf.components.straight(length=10)
     # gdspath = c.write_gds()
     # show(gdspath)
     c.show()