update json serialization docs + add module docs

cirq-aqt/cirq_aqt/json_resolver_cache.py

@@ -18,6 +18,6 @@
 from cirq.protocols.json_serialization import ObjectFactory
 
 
-@functools.lru_cache(maxsize=1)
+@functools.lru_cache()
 def _class_resolver_dictionary() -> Dict[str, ObjectFactory]:
     return {}

cirq-core/cirq/__init__.py

@@ -628,19 +628,16 @@
 )
 
 
-def _register_resolver() -> None:
-    """Registers the cirq module's public classes for JSON serialization."""
-    from cirq.protocols.json_serialization import _internal_register_resolver
-    from cirq.json_resolver_cache import _class_resolver_dictionary
-
-    _internal_register_resolver(_class_resolver_dictionary)
+# Registers cirq-core's public classes for JSON serialization.
+# pylint: disable=wrong-import-position
+from cirq.protocols.json_serialization import _register_resolver
+from cirq.json_resolver_cache import _class_resolver_dictionary
 
 
-_register_resolver()
+_register_resolver(_class_resolver_dictionary)
 
 # contrib's json resolver cache depends on cirq.DEFAULT_RESOLVER
 
-# pylint: disable=wrong-import-position
 from cirq import (
     contrib,
 )

cirq-core/cirq/json_resolver_cache.py

@@ -22,7 +22,7 @@
     import cirq.devices.unconstrained_device
 
 
-@functools.lru_cache(maxsize=1)
+@functools.lru_cache()
 def _class_resolver_dictionary() -> Dict[str, ObjectFactory]:
     import cirq
     from cirq.ops import raw_types

cirq-core/cirq/protocols/json_serialization.py

@@ -86,7 +86,7 @@ def my_read_json(file_or_fn, resolvers=None):
 """
 
 
-def _internal_register_resolver(dict_factory: Callable[[], Dict[str, ObjectFactory]]) -> None:
+def _register_resolver(dict_factory: Callable[[], Dict[str, ObjectFactory]]) -> None:
     """Register a resolver based on a dict factory for lazy initialization.
 
     Cirq modules are the ones referred in cirq/__init__.py. If a Cirq module

cirq-core/cirq/protocols/json_serialization_test.py

@@ -14,6 +14,7 @@
 import contextlib
 import dataclasses
 import datetime
+import importlib
 import io
 import json
 import os
@@ -598,25 +599,17 @@ def _eval_repr_data_file(path: pathlib.Path, deprecation_deadline: Optional[str]
 
     imports = {
         'cirq': cirq,
-        'datetime': datetime,
         'pd': pd,
         'sympy': sympy,
         'np': np,
         'datetime': datetime,
     }
-    try:
-        import cirq_google
-
-        imports['cirq_google'] = cirq_google
-    except ImportError:
-        pass
-
-    try:
-        import cirq_pasqal
 
-        imports['cirq_pasqal'] = cirq_pasqal
-    except ImportError:
-        pass
+    for m in TESTED_MODULES.keys():
+        try:
+            imports[m] = importlib.import_module(m)
+        except ImportError:
+            pass
 
     with contextlib.ExitStack() as stack:
         for ctx_manager in ctx_managers:

cirq-google/cirq_google/__init__.py

@@ -120,12 +120,8 @@
 from cirq_google import experimental
 
 
-def _register_resolver() -> None:
-    """Registers the cirq_google's public classes for JSON serialization."""
-    from cirq.protocols.json_serialization import _internal_register_resolver
-    from cirq_google.json_resolver_cache import _class_resolver_dictionary
+# Register cirq_google's public classes for JSON serialization.
+from cirq.protocols.json_serialization import _register_resolver
+from cirq_google.json_resolver_cache import _class_resolver_dictionary
 
-    _internal_register_resolver(_class_resolver_dictionary)
-
-
-_register_resolver()
+_register_resolver(_class_resolver_dictionary)

cirq-google/cirq_google/json_resolver_cache.py

@@ -18,7 +18,7 @@
 from cirq.protocols.json_serialization import ObjectFactory
 
 
-@functools.lru_cache(maxsize=1)
+@functools.lru_cache()
 def _class_resolver_dictionary() -> Dict[str, ObjectFactory]:
     import cirq_google
     from cirq_google.devices.known_devices import _NamedConstantXmonDevice

cirq-ionq/cirq_ionq/json_resolver_cache.py

@@ -18,6 +18,6 @@
 from cirq.protocols.json_serialization import ObjectFactory
 
 
-@functools.lru_cache(maxsize=1)
+@functools.lru_cache()
 def _class_resolver_dictionary() -> Dict[str, ObjectFactory]:
     return {}

cirq-pasqal/cirq_pasqal/__init__.py

@@ -33,12 +33,8 @@
 )
 
 
-def _register_resolver() -> None:
-    """Registers the cirq_google's public classes for JSON serialization."""
-    from cirq.protocols.json_serialization import _internal_register_resolver
-    from cirq_pasqal.json_resolver_cache import _class_resolver_dictionary
+# Register cirq_pasqal's public classes for JSON serialization.
+from cirq.protocols.json_serialization import _register_resolver
+from cirq_pasqal.json_resolver_cache import _class_resolver_dictionary
 
-    _internal_register_resolver(_class_resolver_dictionary)
-
-
-_register_resolver()
+_register_resolver(_class_resolver_dictionary)

cirq-pasqal/cirq_pasqal/json_resolver_cache.py

@@ -19,7 +19,7 @@
 from cirq.protocols.json_serialization import ObjectFactory
 
 
-@functools.lru_cache(maxsize=1)
+@functools.lru_cache()
 def _class_resolver_dictionary() -> Dict[str, ObjectFactory]:
     return {
         'PasqalDevice': cirq_pasqal.PasqalDevice,

cirq-rigetti/cirq_rigetti/json_resolver_cache.py

@@ -18,6 +18,6 @@
 from cirq.protocols.json_serialization import ObjectFactory
 
 
-@functools.lru_cache(maxsize=1)  # coverage: ignore
+@functools.lru_cache()  # coverage: ignore
 def _class_resolver_dictionary() -> Dict[str, ObjectFactory]:  # coverage: ignore
     return {}

docs/_book.yaml

@@ -86,6 +86,8 @@ upper_tabs:
         path: /cirq/dev/gates
       - title: "Plotting guidelines"
         path: /cirq/dev/plotting
+      - title: "Modules"
+        path: /cirq/dev/modules
       - title: "Serialization guidelines"
         path: /cirq/dev/serialization
       - title: "Triage process"

docs/dev/modules.md

@@ -0,0 +1,105 @@
+# Cirq modules 
+
+Cirq has a modular architecture and is organized in a monorepo, all of the modules follow the same folder structure.
+Each module is structured as follows. Let's take as example a module named `cirq-example`: 
+
+```
+cirq-example
+├── cirq_example
+│   ├── __init__.py
+│   ├── _version.py
+│   ├── json_resolver_cache.py
+│   └── json_test_data
+│       ├── __init__.py
+│       └── spec.py
+├── LICENSE
+├── README.rst
+├── requirements.txt
+├── setup.cfg
+└── setup.py
+```
+
+Note that typically there is only a single top level package, `cirq_example` - but there might be exceptions. 
+
+Additionally, there is a metapackage "cirq" that's a completely different beast and just depends on the modules. 
+This enables `pip install cirq` to have all the included modules to be installed for our users.
+
+All modules should depend on `cirq-core`, which is the central, core library for Cirq.    
+
+## Packaging 
+
+Each package gets published to PyPi as a separate package. To build all the wheel files locally, use
+
+```bash
+dev_tools/packaging/produce-package.sh ./dist `./dev_tools/packaging/generate-dev-version-id.sh`
+```
+
+Packages are versioned together, share the same version number, and are released together. 
+
+## Setting up a new module
+
+To setup a new module follow these steps: 
+
+1. Create the folder structure above, copy the files based on an existing module
+    1. LICENSE should be the same
+    2. README.rst will be the documentation that appears in PyPi
+    3. setup.py should specify an `install_requires` configuration that has `cirq-core=={module.version}` at the minimum
+2. Setup JSON serialization for each top level python package 
+
+
+### Setting up JSON serialization
+
+1. Add the `<top_level_package>/json_resolver_cache.py` file 
+    ```python
+    @functools.lru_cache()  # coverage: ignore
+    def _class_resolver_dictionary() -> Dict[str, ObjectFactory]:  # coverage: ignore
+        return {}
+    ```
+2. Register the resolver cache - at _the end_ of the `<top_level_package>/__init__.py`:
+    ```python
+
+    # Registers cirq_example's public classes for JSON serialization.
+    from cirq.protocols.json_serialization import _register_resolver
+    from cirq_example.json_resolver_cache import _class_resolver_dictionary
+    _register_resolver(_class_resolver_dictionary)
+
+    ``` 
+3. Add the `<top_level_package>/json_test_data` folder with the following content: 
+   1. `spec.py` contains the core test specification for JSON testing, that plugs into the central framework:    
+       ```python
+       import pathlib
+       import cirq_example
+       from cirq_example.json_resolver_cache import _class_resolver_dictionary
+
+       from cirq.testing.json import ModuleJsonTestSpec
+
+       TestSpec = ModuleJsonTestSpec(
+           name="cirq_example",
+           packages=[cirq_example],
+           test_data_path=pathlib.Path(__file__).parent,
+           not_yet_serializable=[],
+           should_not_be_serialized=[],
+           resolver_cache=_class_resolver_dictionary(),
+           deprecated={},
+        )
+       ```
+   2. `__init__.py` should import `TestSpec` from `spec.py`
+   3. in `cirq/protocols/json_serialization_test.py` add `'cirq_example':None` to the `TESTED_MODULES` variable. `TESTED_MODULES` is also used to prepare the test framework for deprecation warnings. 
+      With new modules, we use`None` as there is no deprecation setup. 
+
+You can run `check/pytest-changed-files` and that should execute the json_serialization_test.py as well. 
+
+That's it! Now, you can follow the [Serialization guide](./serialization.md) for adding and removing serializable objects.
+
+# Utilities 
+
+## List modules 
+
+To iterate through modules, you can list them by invoking `dev_tools/modules.py`. 
+
+```bash
+python dev_tools/modules.py --list 
+```
+
+There are different modes of listing (e.g the folder, package-path, top level package), 
+you can refer to `python dev_tools/modules.py --list --help` for the most up to date features. 

docs/dev/serialization.md

@@ -79,29 +79,29 @@ If the returned object has a `_from_json_dict_` attribute, it is called instead.
 
 ## Adding a new serializable value
 
-All of Cirq's public classes should be serializable.
+All of Cirq's public classes should be serializable. Public classes are the ones that can be found in the Cirq module top level 
+namespaces, i.e. `cirq.*`, `cirq_google.*`, `cirq_aqt.*`, etc, (see [Cirq modules](./modules.md) for setting up JSON serialization for a module).  
 This is enforced by the `test_json_test_data_coverage` test in
-`cirq/protocols/json_serialization_test.py`, which iterates over cirq's API looking for types
-with no associated json test data.
+`cirq-core/cirq/protocols/json_serialization_test.py`, which iterates over cirq's API 
+looking for types with no associated json test data.
 
-There are several steps needed to get a object serializing, deserializing, and
-passing tests:
+There are several steps needed to support an object's serialization and deserialization, 
+and pass `cirq-core/cirq/protocols/json_serialization_test.py`:
 
 1. The object should have a `_json_dict_` method that returns a dictionary
 containing a `"cirq_type"` key as well as keys for each of the value's
-attributes.
-If these keys do not match the names of the class' initializer arguments, a
-`_from_json_dict_` class method must also be defined.
+attributes. If these keys do not match the names of the class' initializer 
+arguments, a `_from_json_dict_` class method must also be defined.
 Typically the `"cirq_type"` will be the name of your class.
 
-2. Add an entry to the big hardcoded dictionary in `cirq/protocols/json.py`,
-mapping the cirq_type string you chose to the class.
-You can also map the key to a helper method that returns the class (important
-for backwards compatibility if e.g. a class is later replaced by another one).
-After doing this, `cirq.to_json` and `cirq.read_json` should start working for
-your object.
+2. In `class_resolver_dictionary` within the packages's `json_resolver_cache.py` file,
+for each serializable class, the `cirq_type` of the class should be mapped to the imported class 
+within the package. The key may also be mapped to a helper method that 
+returns the class (important for backwards compatibility if e.g. a class is later replaced 
+by another one). After doing this, `cirq.to_json` and `cirq.read_json` should start 
+working for your object.
 
-3. Add test data files to the `cirq/protocols/json_test_data` directory.
+3. Add test data files to the package's `json_test_data` directory.
 These are to ensure that the class remains deserializable in future versions.
 There should be two files: `your_class_name.repr` and `your_class_name.json`.
 `your_class_name.repr` should contain a python expression that evaluates to an
@@ -124,19 +124,28 @@ As such, "removing" a serializable value is more akin to removing it
 
 There are several steps:
 
-1. Find the object's test files in `cirq/protocols/json_test_data`.
-Change the file name extensions from `.json` to `.json_inward` and `.repr` to
-`.repr_inward`.
-This indicates that only deserialization needs to be tested, not deserialization
-and serialization.
-If `_inward` files already exist, merge into them (e.g. by ensuring they encode
-lists and then appending into those lists).
+1. Find the object's test files in relevant package's `json_test_data`
+directory. Change the file name extensions from `.json` to `.json_inward` and `.repr` to
+`.repr_inward`. This indicates that only deserialization needs to be tested, not deserialization
+and serialization. If `_inward` files already exist, merge into them (e.g. by 
+ensuring they encode lists and then appending into those lists).
 
 2. Define a parsing method to stand in for the object.
 This parsing method must return an object with the same basic behavior as the
 object being removed, but does not have to return an exactly identical object.
 For example, an X could be replaced by a PhasedX with no phasing.
-Edit the entry in the big dictionary in `cirq/protocols/json.py` to point at
-this method instead of the object being removed.
+Edit the entry in the in `cirq-<module>/json_test_data/spec.py` or in the
+ relevant package's `class_resolver_dictionary` (`cirq-<module>/cirq_module/json_resolver_cache.py`) to 
+ point at this method instead of the object being removed.
 (There will likely be debate about exactly how to do this, on a case by case
 basis.)
+
+
+## Marking a public object as non-serializable
+
+Some public objects will be exceptional and should not be serialized ever. These could be marked in the 
+given top level package's spec.py (`<module>/<top level package>/json_test_data/spec.py`) by adding its 
+name to `should_not_serialize`. 
+
+We allow for incremental introduction of new objects to serializability - if an object should be 
+serialized but is not yet serializable, it should be added to the `not_yet_serializable` list in the `spec.py` file.