Merge pull request #860 from tvt173/fix-numpy-serialization

Fix numpy serialization
gdsfactory · Nov 14, 2022 · 4056bde · 4056bde
2 parents 2845270 + 7da266f
commit 4056bde
Show file tree

Hide file tree

Showing 11 changed files with 232 additions and 115 deletions.
diff --git a/gdsfactory/serialization.py b/gdsfactory/serialization.py
@@ -11,6 +11,9 @@
 import toolz
 from omegaconf import DictConfig, OmegaConf
 
+DEFAULT_SERIALIZATION_MAX_DIGITS = 8
+"""By default, the maximum number of digits retained when serializing float-like arrays"""
+
 
 def clean_dict(d: Dict[str, Any]) -> Dict[str, Any]:
     """Cleans dictionary recursively."""
@@ -51,8 +54,8 @@ def clean_value_json(value: Any) -> Any:
         return float(value)
 
     elif isinstance(value, np.ndarray):
-        value = np.round(value, 3)
-        return orjson.dumps(value, option=orjson.OPT_SERIALIZE_NUMPY).decode()
+        value = np.round(value, DEFAULT_SERIALIZATION_MAX_DIGITS)
+        return orjson.loads(orjson.dumps(value, option=orjson.OPT_SERIALIZE_NUMPY))
     elif callable(value) and isinstance(value, functools.partial):
         sig = inspect.signature(value.func)
         args_as_kwargs = dict(zip(sig.parameters.keys(), value.args))
@@ -83,7 +86,10 @@ def clean_value_json(value: Any) -> Any:
         value = clean_dict(OmegaConf.to_container(value))
 
     elif isinstance(value, (list, tuple, set)):
-        value = [clean_value_json(i) for i in value]
+        if len(value) > 0 and isinstance(value[0], np.ndarray):
+            value = clean_value_json(np.asarray(value))
+        else:
+            value = [clean_value_json(i) for i in value]
     else:
         try:
             value_json = orjson.dumps(

diff --git a/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_connections_.yml b/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_connections_.yml
@@ -36,5 +36,7 @@ warnings:
       - wgw,o2
       - wgn,o1
       values:
-      - '[1.5,0.0]'
-      - '[0.0,0.0]'
+      - - 1.5
+        - 0.0
+      - - 0.0
+        - 0.0
diff --git a/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_different_link_factory_.yml b/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_different_link_factory_.yml
@@ -689,10 +689,14 @@ warnings:
       - straight_6,o1
       - straight_10,o2
       values:
-      - '[50.0,0.0]'
-      - '[850.0,400.0]'
-      - '[50.0,200.0]'
-      - '[850.0,600.0]'
+      - - 50.0
+        - 0.0
+      - - 850.0
+        - 400.0
+      - - 50.0
+        - 200.0
+      - - 850.0
+        - 600.0
   vertical_dc:
     unconnected_ports:
     - message: 4 unconnected vertical_dc ports!
@@ -702,7 +706,11 @@ warnings:
       - br,pad
       - tr,pad
       values:
-      - '[0.0,0.0]'
-      - '[0.0,200.0]'
-      - '[900.0,400.0]'
-      - '[900.0,600.0]'
+      - - 0.0
+        - 0.0
+      - - 0.0
+        - 200.0
+      - - 900.0
+        - 400.0
+      - - 900.0
+        - 600.0
diff --git a/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_docstring_.yml b/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_docstring_.yml
@@ -103,7 +103,11 @@ warnings:
       - mmi_top,o1
       - mmi_top,o2
       values:
-      - '[113.0,-38.75]'
-      - '[113.0,-40.0]'
-      - '[0.0,0.0]'
-      - '[42.0,0.625]'
+      - - 113.0
+        - -38.75
+      - - 113.0
+        - -40.0
+      - - 0.0
+        - 0.0
+      - - 42.0
+        - 0.625
diff --git a/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_doe_.yml b/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_doe_.yml
@@ -168,15 +168,27 @@ warnings:
       - mmi1x2_sweep,mmi1x2_length_mmi2_width_mmi4_3_o2
       - mmi1x2_sweep,mmi1x2_length_mmi2_width_mmi4_3_o3
       values:
-      - '[50.5,55.5]'
-      - '[170.5,56.125]'
-      - '[170.5,54.875]'
-      - '[50.5,162.5]'
-      - '[170.5,163.125]'
-      - '[170.5,161.875]'
-      - '[50.5,269.5]'
-      - '[72.5,270.125]'
-      - '[72.5,268.875]'
-      - '[172.5,266.5]'
-      - '[194.5,267.125]'
-      - '[194.5,265.875]'
+      - - 50.5
+        - 55.5
+      - - 170.5
+        - 56.125
+      - - 170.5
+        - 54.875
+      - - 50.5
+        - 162.5
+      - - 170.5
+        - 163.125
+      - - 170.5
+        - 161.875
+      - - 50.5
+        - 269.5
+      - - 72.5
+        - 270.125
+      - - 72.5
+        - 268.875
+      - - 172.5
+        - 266.5
+      - - 194.5
+        - 267.125
+      - - 194.5
+        - 265.875
diff --git a/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_doe_function_.yml b/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_doe_function_.yml
@@ -1225,59 +1225,115 @@ warnings:
       - mzis,mzi_delta_length100_add_f4894d38_1_0_vertical_te_10
       - mzis,mzi_delta_length100_add_f4894d38_1_0_vertical_te_20
       values:
-      - '[189.973,24.583]'
-      - '[316.973,24.583]'
-      - '[62.973,24.583]'
-      - '[443.973,24.583]'
-      - '[189.973,398.632]'
-      - '[316.973,398.632]'
-      - '[62.973,398.632]'
-      - '[443.973,398.632]'
-      - '[189.973,772.683]'
-      - '[316.973,772.683]'
-      - '[62.973,772.683]'
-      - '[443.973,772.683]'
-      - '[189.973,1146.732]'
-      - '[316.973,1146.732]'
-      - '[62.973,1146.732]'
-      - '[443.973,1146.732]'
-      - '[606.913,24.583]'
-      - '[733.913,24.583]'
-      - '[479.913,24.583]'
-      - '[860.913,24.583]'
-      - '[606.913,378.632]'
-      - '[733.913,378.632]'
-      - '[479.913,378.632]'
-      - '[860.913,378.632]'
-      - '[606.913,732.683]'
-      - '[733.913,732.683]'
-      - '[479.913,732.683]'
-      - '[860.913,732.683]'
-      - '[606.913,1066.732]'
-      - '[733.913,1066.732]'
-      - '[479.913,1066.732]'
-      - '[860.913,1066.732]'
-      - '[1023.853,24.583]'
-      - '[1150.853,24.583]'
-      - '[896.853,24.583]'
-      - '[1277.853,24.583]'
-      - '[1023.853,358.632]'
-      - '[1150.853,358.632]'
-      - '[896.853,358.632]'
-      - '[1277.853,358.632]'
-      - '[1023.853,672.683]'
-      - '[1150.853,672.683]'
-      - '[896.853,672.683]'
-      - '[1277.853,672.683]'
-      - '[1023.853,986.733]'
-      - '[1150.853,986.733]'
-      - '[896.853,986.733]'
-      - '[1277.853,986.733]'
-      - '[1430.799,-81.568]'
-      - '[1557.799,-81.568]'
-      - '[1303.799,-81.568]'
-      - '[1684.799,-81.568]'
-      - '[1430.799,-126.568]'
-      - '[1557.799,-126.568]'
-      - '[1303.799,-126.568]'
-      - '[1684.799,-126.568]'
+      - - 189.973
+        - 24.5825
+      - - 316.973
+        - 24.5825
+      - - 62.973
+        - 24.5825
+      - - 443.973
+        - 24.5825
+      - - 189.973
+        - 398.6325
+      - - 316.973
+        - 398.6325
+      - - 62.973
+        - 398.6325
+      - - 443.973
+        - 398.6325
+      - - 189.973
+        - 772.6825
+      - - 316.973
+        - 772.6825
+      - - 62.973
+        - 772.6825
+      - - 443.973
+        - 772.6825
+      - - 189.973
+        - 1146.7325
+      - - 316.973
+        - 1146.7325
+      - - 62.973
+        - 1146.7325
+      - - 443.973
+        - 1146.7325
+      - - 606.913
+        - 24.5825
+      - - 733.913
+        - 24.5825
+      - - 479.913
+        - 24.5825
+      - - 860.913
+        - 24.5825
+      - - 606.913
+        - 378.6325
+      - - 733.913
+        - 378.6325
+      - - 479.913
+        - 378.6325
+      - - 860.913
+        - 378.6325
+      - - 606.913
+        - 732.6825
+      - - 733.913
+        - 732.6825
+      - - 479.913
+        - 732.6825
+      - - 860.913
+        - 732.6825
+      - - 606.913
+        - 1066.7325
+      - - 733.913
+        - 1066.7325
+      - - 479.913
+        - 1066.7325
+      - - 860.913
+        - 1066.7325
+      - - 1023.853
+        - 24.5825
+      - - 1150.853
+        - 24.5825
+      - - 896.853
+        - 24.5825
+      - - 1277.853
+        - 24.5825
+      - - 1023.853
+        - 358.6325
+      - - 1150.853
+        - 358.6325
+      - - 896.853
+        - 358.6325
+      - - 1277.853
+        - 358.6325
+      - - 1023.853
+        - 672.6825
+      - - 1150.853
+        - 672.6825
+      - - 896.853
+        - 672.6825
+      - - 1277.853
+        - 672.6825
+      - - 1023.853
+        - 986.7325
+      - - 1150.853
+        - 986.7325
+      - - 896.853
+        - 986.7325
+      - - 1277.853
+        - 986.7325
+      - - 1430.799
+        - -81.5685
+      - - 1557.799
+        - -81.5685
+      - - 1303.799
+        - -81.5685
+      - - 1684.799
+        - -81.5685
+      - - 1430.799
+        - -126.5685
+      - - 1557.799
+        - -126.5685
+      - - 1303.799
+        - -126.5685
+      - - 1684.799
+        - -126.5685
diff --git a/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_mirror_simple_.yml b/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_mirror_simple_.yml
@@ -35,5 +35,7 @@ warnings:
       - s,o1
       - b,o2
       values:
-      - '[0.0,0.0]'
-      - '[20.0,-10.0]'
+      - - 0.0
+        - 0.0
+      - - 20.0
+        - -10.0
diff --git a/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_mmis_.yml b/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_mmis_.yml
@@ -103,4 +103,5 @@ warnings:
       ports:
       - mmi_short,o3
       values:
-      - '[15.0,-0.625]'
+      - - 15.0
+        - -0.625
diff --git a/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_waypoints_.yml b/gdsfactory/tests/test_component_from_yaml/test_netlists_sample_waypoints_.yml
@@ -546,18 +546,30 @@ warnings:
       - b,e15
       - b,e16
       values:
-      - '[-250.0,1000.0]'
-      - '[-100.0,1000.0]'
-      - '[50.0,1000.0]'
-      - '[200.0,1000.0]'
-      - '[350.0,1000.0]'
-      - '[500.0,1000.0]'
-      - '[0.0,0.0]'
-      - '[150.0,0.0]'
-      - '[300.0,0.0]'
-      - '[450.0,0.0]'
-      - '[600.0,0.0]'
-      - '[750.0,0.0]'
+      - - -250.0
+        - 1000.0
+      - - -100.0
+        - 1000.0
+      - - 50.0
+        - 1000.0
+      - - 200.0
+        - 1000.0
+      - - 350.0
+        - 1000.0
+      - - 500.0
+        - 1000.0
+      - - 0.0
+        - 0.0
+      - - 150.0
+        - 0.0
+      - - 300.0
+        - 0.0
+      - - 450.0
+        - 0.0
+      - - 600.0
+        - 0.0
+      - - 750.0
+        - 0.0
   optical:
     unconnected_ports:
     - message: 4 unconnected optical ports!
@@ -567,7 +579,11 @@ warnings:
       - straight_6,o1
       - straight_10,o2
       values:
-      - '[0.0,0.0]'
-      - '[-250.0,1000.0]'
-      - '[150.0,0.0]'
-      - '[-100.0,1000.0]'
+      - - 0.0
+        - 0.0
+      - - -250.0
+        - 1000.0
+      - - 150.0
+        - 0.0
+      - - -100.0
+        - 1000.0