Merge pull request #256 from simonsobs/koopman/use-pip-so3g-in-workflows

Use pip installed so3g in GH Action workflows

.github/workflows/deploy.yml

@@ -24,27 +24,19 @@ jobs:
     - name: Install ocs
       run: |
         pip3 install -r requirements.txt
-        pip3 install -e .
+        pip3 install -e .[so3g]
 
     # Test (already been run by pytest workflow, but they don't take long...)
     # Unit Tests
     - name: Run unit tests
       working-directory: ./tests
       run: |
-        COVERAGE_FILE=.coverage.unit python3 -m pytest --cov -m 'not (integtest or spt3g)'
-
-    - name: Build docker image for containerized tests
-      run: |
-        docker-compose build ocs
-
-    - name: Test with pytest within a docker container
-      run: |
-        docker run -v $PWD:/coverage --rm -w="/app/ocs/tests/" ocs sh -c "COVERAGE_FILE=/coverage/.coverage.docker python3 -m pytest --cov -m 'spt3g'"
+        COVERAGE_FILE=.coverage.unit python3 -m pytest --cov -m 'not integtest'
 
     # Integration Tests
     - name: Build images for integration tests
       run: |
-        docker-compose build ocs-fake-data-agent ocs-aggregator-agent ocs-crossbar
+        docker-compose build ocs ocs-fake-data-agent ocs-aggregator-agent ocs-crossbar
 
     - name: Run integration tests
       working-directory: ./tests
@@ -135,7 +127,7 @@ jobs:
     - name: Run unit tests
       working-directory: ./tests
       run: |
-        python3 -m pytest -m 'not (integtest or spt3g)'
+        python3 -m pytest -m 'not integtest'
 
     - name: upload to PyPI
       run: |

.github/workflows/develop.yml

@@ -24,27 +24,19 @@ jobs:
     - name: Install ocs
       run: |
         pip3 install -r requirements.txt
-        pip3 install -e .
+        pip3 install -e .[so3g]
 
     # Test (steps from pytest workflow)
     # Unit Tests
     - name: Run unit tests
       working-directory: ./tests
       run: |
-        COVERAGE_FILE=.coverage.unit python3 -m pytest --cov -m 'not (integtest or spt3g)'
-
-    - name: Build docker image for containerized tests
-      run: |
-        docker-compose build ocs
-
-    - name: Test with pytest within a docker container
-      run: |
-        docker run -v $PWD:/coverage --rm -w="/app/ocs/tests/" ocs sh -c "COVERAGE_FILE=/coverage/.coverage.docker python3 -m pytest --cov -m 'spt3g'"
+        COVERAGE_FILE=.coverage.unit python3 -m pytest --cov -m 'not integtest'
 
     # Integration Tests
     - name: Build images for integration tests
       run: |
-        docker-compose build ocs-fake-data-agent ocs-aggregator-agent ocs-crossbar
+        docker-compose build ocs ocs-fake-data-agent ocs-aggregator-agent ocs-crossbar
 
     - name: Run integration tests
       working-directory: ./tests

.github/workflows/pytest.yml

@@ -33,26 +33,18 @@ jobs:
     - name: Install ocs
       run: |
         pip3 install -r requirements.txt
-        pip3 install -e .
+        pip3 install -e .[so3g]
 
     # Unit Tests
     - name: Run unit tests
       working-directory: ./tests
       run: |
-        COVERAGE_FILE=.coverage.unit python3 -m pytest --cov -m 'not (integtest or spt3g)'
-
-    - name: Build docker image for containerized tests
-      run: |
-        docker-compose build ocs
-
-    - name: Test with pytest within a docker container
-      run: |
-        docker run -v $PWD:/coverage --rm -w="/app/ocs/tests/" ocs sh -c "COVERAGE_FILE=/coverage/.coverage.docker python3 -m pytest --cov -m 'spt3g'"
+        COVERAGE_FILE=.coverage.unit python3 -m pytest --cov -m 'not integtest'
 
     # Integration Tests
     - name: Build images for integration tests
       run: |
-        docker-compose build ocs-fake-data-agent ocs-aggregator-agent ocs-crossbar
+        docker-compose build ocs ocs-fake-data-agent ocs-aggregator-agent ocs-crossbar
 
     - name: Run integration tests
       working-directory: ./tests

README.rst

@@ -65,6 +65,10 @@ Install and update with pip::
 
     $ pip3 install -U ocs
 
+If you need to install the optional so3g module you can do so via::
+
+    $ pip3 install -U ocs[so3g]
+
 Installing from Source
 ``````````````````````
 

ocs/agent/aggregator.py

@@ -10,8 +10,8 @@
 
 from ocs.ocs_feed import Block, Feed
 
-from spt3g import core
 import so3g
+from spt3g import core
 
 
 HKAGG_VERSION = 2

requirements/testing.txt

@@ -2,5 +2,5 @@ pytest
 pytest-cov
 docker
 pytest-docker-compose
-pytest-dependency
 pytest-twisted
+so3g

setup.py

@@ -42,4 +42,7 @@
            'influxdb',
            'numpy',
        ],
+       extras_require={
+           "so3g": ["so3g"],
+       },
 )

tests/agents/test_aggregator_agent.py

@@ -10,23 +10,18 @@
     generate_data_for_queue
 )
 
-try:
-    # depends on spt3g
-    from aggregator_agent import AggregatorAgent
+# depends on spt3g
+from aggregator_agent import AggregatorAgent
 
-    args = mock.MagicMock()
-    args.time_per_file = 3
-    args.data_dir = '/tmp/data'
-    # start idle so we can use a tmpdir for data_dir
-    args.initial_state = 'idle'
+args = mock.MagicMock()
+args.time_per_file = 3
+args.data_dir = '/tmp/data'
+# start idle so we can use a tmpdir for data_dir
+args.initial_state = 'idle'
 
-    agent = create_agent_fixture(AggregatorAgent, {'args': args})
-except ModuleNotFoundError as e:
-    print(f"Unable to import: {e}")
+agent = create_agent_fixture(AggregatorAgent, {'args': args})
 
 
-@pytest.mark.spt3g
-@pytest.mark.dependency(depends=['so3g'], scope='session')
 class TestRecord:
     def test_aggregator_agent_record_no_data(self, agent, tmpdir):
         # repoint data_dir to tmpdir fixture
@@ -62,8 +57,6 @@ def test_aggregator_agent_record_data(self, agent, tmpdir):
         assert session.data['providers']['observatory.test-agent1.feeds.test_feed']['last_block_received'] == 'temps'
 
 
-@pytest.mark.spt3g
-@pytest.mark.dependency(depends=['so3g'], scope='session')
 def test_aggregator_agent_enqueue_data_no_aggregate(agent):
     agent.aggregate = False
 
@@ -73,8 +66,6 @@ def test_aggregator_agent_enqueue_data_no_aggregate(agent):
     assert agent.incoming_data.empty()
 
 
-@pytest.mark.spt3g
-@pytest.mark.dependency(depends=['so3g'], scope='session')
 class TestStopRecord:
     def test_aggregator_agent_stop_record_while_running(self, agent):
         session = create_session('record')

tests/integration/test_aggregator_agent_integration.py

@@ -21,7 +21,6 @@
 client = create_client_fixture('aggregator-local')
 
 
-@pytest.mark.dependency(depends=["so3g"], scope='session')
 @pytest.mark.integtest
 def test_aggregator_agent_record(wait_for_crossbar, run_agent, client):
     resp = client.record.start(test_mode=True)

tests/integration/test_crossbar_integration.py

@@ -5,14 +5,10 @@
 import docker
 
 from ocs.ocs_client import OCSClient
+from so3g import hk
 
 from integration.util import create_crossbar_fixture, restart_crossbar
 
-try:
-    from so3g import hk
-except ModuleNotFoundError as e:
-    print(f"Unable to import so3g: {e}")
-
 pytest_plugins = ("docker_compose",)
 
 
@@ -22,19 +18,6 @@
 CROSSBAR_SLEEP = 5  # time to wait before trying to make first connection
 
 
-@pytest.mark.spt3g
-@pytest.mark.dependency(name="so3g")
-def test_so3g_spt3g_import():
-    """Test that we can import so3g. Used to skip tests dependent on
-    this import.
-
-    """
-    import so3g
-
-    # Just to prevent flake8 from complaining
-    print(so3g.__file__)
-
-
 # @pytest.mark.integtest
 # def test_testing(wait_for_crossbar):
 #     "Just testing if the docker-compose/crossbar wait fixture is working."
@@ -76,7 +59,6 @@ def test_fake_data_after_crossbar_restart(wait_for_crossbar):
 #     """
 #     pass
 
-@pytest.mark.dependency(depends=["so3g"])
 @pytest.mark.integtest
 def test_aggregator_after_crossbar_restart(wait_for_crossbar):
     """Test that the aggregator reconnects after a crossbar restart and that

tests/pytest.ini

@@ -1,4 +1,3 @@
 [pytest]
 markers =
     integtest: marks tests as integration test (deselect with '-m "not integtest"')
-    spt3g: marks tests that depend on spt3g (deselect with '-m "not spt3g"')