Update: Add concurrent programming support (#1326)

.github/workflows/test.yml

@@ -46,6 +46,7 @@ jobs:
           sudo apt update
           . scripts/ci_install_deps
           poetry run make check-format
+
   check-style:
     name: Check style
     runs-on: ubuntu-latest
@@ -64,6 +65,7 @@ jobs:
           sudo apt update
           . scripts/ci_install_deps
           poetry run make check-style
+
   check-types:
     name: Check types
     runs-on: ubuntu-latest
@@ -82,8 +84,9 @@ jobs:
           sudo apt update
           . scripts/ci_install_deps
           poetry run make check-types
-  test-pyquil:
-    name: Test pyQuil
+
+  test-unit:
+    name: Test Unit
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -98,10 +101,34 @@ jobs:
         with:
           path: .venv
           key: poetry-${{ hashFiles('poetry.lock') }}
-      - name: Test pyQuil (Python ${{ matrix.python-version }})
+      - name: Test Unit (Python ${{ matrix.python-version }})
         run: |
           sudo apt update
           . scripts/ci_install_deps
           docker run --rm -itd -p 5555:5555 rigetti/quilc -R
           docker run --rm -itd -p 5000:5000 rigetti/qvm -S
           poetry run make test
+
+  test-e2e:
+    name: Test e2e QVM
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: actions/cache@v2
+        with:
+          path: .venv
+          key: poetry-${{ hashFiles('poetry.lock') }}
+      - name: Test e2e QVM (Python ${{ matrix.python-version }})
+        run: |
+          sudo apt update
+          . scripts/ci_install_deps
+          docker run --rm -itd -p 5555:5555 rigetti/quilc -R
+          docker run --rm -itd -p 5000:5000 rigetti/qvm -S
+          poetry run make e2e TEST_QUANTUM_PROCESSOR=2q-qvm

.gitlab-ci.yml

@@ -73,7 +73,7 @@ Test Unit (3.7):
     - . scripts/ci_install_deps
     - poetry run make test
 
-Test e2e (3.7):
+Test e2e QPU (3.7):
   stage: test
   image: python:3.7
   coverage: '/TOTAL.*?(\d+)\%/'
@@ -83,6 +83,14 @@ Test e2e (3.7):
   rules:
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "rc"'
 
+Test e2e QVM (3.7):
+  stage: test
+  image: python:3.7
+  coverage: '/TOTAL.*?(\d+)\%/'
+  script:
+    - . scripts/ci_install_deps
+    - poetry run make e2e TEST_QUANTUM_PROCESSOR=2q-qvm
+
 Test Unit (3.8):
   stage: test
   image: python:3.8
@@ -91,7 +99,7 @@ Test Unit (3.8):
     - . scripts/ci_install_deps
     - poetry run make test
 
-Test e2e (3.8):
+Test e2e QPU (3.8):
   stage: test
   image: python:3.8
   coverage: '/TOTAL.*?(\d+)\%/'
@@ -101,6 +109,14 @@ Test e2e (3.8):
   rules:
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "rc"'
 
+Test e2e QVM (3.8):
+  stage: test
+  image: python:3.8
+  coverage: '/TOTAL.*?(\d+)\%/'
+  script:
+    - . scripts/ci_install_deps
+    - poetry run make e2e TEST_QUANTUM_PROCESSOR=2q-qvm
+
 Coverage:
   stage: test
   script:

CHANGELOG.md

@@ -10,6 +10,8 @@ Changelog
 
 ### Improvements and Changes
 
+- Added support and documentation for concurrent compilation and execution (see "Advanced Usage" in docs)
+
 - `pyquil.version.__version__` has been moved to `pyquil.__version__`.
 
 - `PyquilConfig` has been replaced by `api.QCSClientConfiguration`. As a result, the only supported configuration-related

CONTRIBUTING.md

@@ -175,8 +175,7 @@ To skip [slow tests](#slow-tests), you may run:
 make test-fast
 ```
 
-If you have access to a real Rigetti QPU via QCS (as opposed to a local QVM), you can run end-to-end
-tests with:
+You can run end-to-end tests with:
 
 ```bash
 make e2e TEST_QUANTUM_PROCESSOR=<quantum processor ID>
@@ -188,6 +187,10 @@ Or you may run all tests (unit/integration/e2e) with:
 make test-all TEST_QUANTUM_PROCESSOR=<quantum processor ID>
 ```
 
+> **Note:** for `TEST_QUANTUM_PROCESSOR`, supply a value similar to what you would supply to
+> `get_qc()`. End-to-end tests are most useful against a real QPU, but they can also be run
+> against a QVM (e.g. `make e2e TEST_QUANTUM_PROCESSOR=2q-qvm`).
+
 #### Slow Tests
 
 Some tests (particularly those related to operator estimation and readout symmetrization)

Makefile

@@ -69,7 +69,7 @@ test-fast:
 
 .PHONY: e2e
 e2e:
-	pytest -n auto -v --cov=pyquil test/e2e
+	pytest -n 1 -v --cov=pyquil test/e2e
 
 .PHONY: test-all
 test-all: test e2e

docs/source/advanced_usage.rst

@@ -15,8 +15,8 @@ pyQuil Configuration
 
 :py:class:`~pyquil.api.QCSClientConfiguration` instructs pyQuil on how to connect with the
 components needed to compile and run programs (quilc, QVMs, and QCS). Any APIs that take a configuration object
-as input (e.g. :func:`get_qc()`) typically do so optionally, so that a default configuration can be loaded for
-you if one is not provided. You can override this default configuration by either instantiating your own
+as input (e.g. :py:func:`~pyquil.get_qc()`) typically do so optionally, so that a default configuration can be loaded
+for you if one is not provided. You can override this default configuration by either instantiating your own
 :py:class:`~pyquil.api.QCSClientConfiguration` object and providing it as input to the function in question,
 or by setting the ``QCS_SETTINGS_FILE_PATH`` and/or ``QCS_SECRETS_FILE_PATH`` environment variables to have
 pyQuil load its settings and secrets from specific locations. By default, configuration will be loaded from
@@ -32,6 +32,87 @@ locally):
 - QVM URL: ``http://127.0.0.1:5000``
 - quilc URL: ``tcp://127.0.0.1:5555``
 
+Concurrency
+~~~~~~~~~~~
+
+Using pyQuil for concurrent programming is as simple as calling :py:func:`~pyquil.get_qc()` from within a given thread
+or process, then using the returned :py:class:`~pyquil.api.QuantumComputer` as usual. While
+:py:class:`~pyquil.api.QuantumComputer` objects as a whole are not safe to share between threads or processes (due to
+state related to currently-running compilation or execution requests), some information they use is. Information related
+to client configuration (:py:class:`~pyquil.api.QCSClientConfiguration`) and QPU auth
+(:py:class:`~pyquil.api.EngagementManager`) can be safely extracted and shared among
+:py:class:`~pyquil.api.QuantumComputer` instances, as shown below, to save your code from redundant disk reads and
+auth-related HTTP requests.
+
+.. note::
+    QVMs process incoming requests in parallel, while QPUs process them sequentially. If you encounter timeouts while
+    trying to run large numbers of programs against a QPU, try increasing the ``timeout`` parameter on calls to
+    :py:func:`~pyquil.get_qc()` (specified in seconds).
+
+Using Multithreading
+--------------------
+
+.. code:: python
+
+    from multiprocessing.pool import ThreadPool
+
+    from pyquil import get_qc, Program
+    from pyquil.api import EngagementManager, QCSClientConfiguration
+
+
+    configuration = QCSClientConfiguration.load()
+    engagement_manager = EngagementManager(client_configuration=configuration)
+
+
+    def run(program: Program):
+        qc = get_qc("Aspen-8", client_configuration=configuration, engagement_manager=engagement_manager)
+        return qc.run(qc.compile(program))
+
+
+    programs = [Program("DECLARE ro BIT", "RX(pi) 0", "MEASURE 0 ro").wrap_in_numshots_loop(10)] * 20
+    with ThreadPool(5) as pool:
+        results = pool.map(run, programs)
+
+    for i, result in enumerate(results):
+        print(f"Results for program {i}:\n{result}\n")
+
+
+Using Multiprocessing
+---------------------
+
+.. code:: python
+
+    from multiprocessing.pool import Pool
+
+    from pyquil import get_qc, Program
+    from pyquil.api import EngagementManager, QCSClientConfiguration
+
+
+    configuration = QCSClientConfiguration.load()
+    engagement_manager = EngagementManager(client_configuration=configuration)
+
+
+    def run(program: Program):
+        qc = get_qc("Aspen-8", client_configuration=configuration, engagement_manager=engagement_manager)
+        return qc.run(qc.compile(program))
+
+
+    programs = [Program("DECLARE ro BIT", "RX(pi) 0", "MEASURE 0 ro").wrap_in_numshots_loop(10)] * 20
+    with Pool(5) as pool:
+        results = pool.map(run, programs)
+
+    for i, result in enumerate(results):
+        print(f"Results for program {i}:\n{result}\n")
+
+.. note::
+    If you encounter error messages on macOS similar to the following:
+
+    .. parsed-literal::
+        +[__NSCFConstantString initialize] may have been in progress in another thread when fork() was called.
+
+    try setting the environment variable ``OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES``.
+
+
 Using Qubit Placeholders
 ~~~~~~~~~~~~~~~~~~~~~~~~