switch to rabbitmq

.github/workflows/tests.yml

@@ -16,7 +16,8 @@ env:
   REALTIME_EVENTS_DIR: tests/data/realtime_events
   SKYSCAN_CACHE_DIR: cache-dir
   SKYSCAN_OUTPUT_DIR: output-dir
-  SKYSCAN_BROKER_ADDRESS: localhost
+  SKYSCAN_BROKER_CLIENT: rabbitmq
+  SKYSCAN_BROKER_ADDRESS: amqp://admin:admin@localhost/test?heartbeat=600
   SKYSCAN_DEBUG_DIR: debug-pkl-dir
   SKYSCAN_MQ_TIMEOUT_TO_CLIENTS: 60
   # SKYSCAN_MQ_TIMEOUT_FROM_CLIENTS: 60  # use default
@@ -40,13 +41,39 @@ jobs:
         with:
           context: .
           cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-to: type=gha,mode=min
           file: Dockerfile
           tags: icecube/skymap_scanner:local
 
+  test-build-docker-pulsar:
+    runs-on: ubuntu-latest
+    needs: test-build-docker
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          cache-from: type=gha
+          file: Dockerfile_pulsar
+          tags: icecube/skymap_scanner:local-pulsar
+
   test-run-singularity-dummy-reco:
     runs-on: ubuntu-latest
     needs: test-build-docker
+    services:
+      rabbitmq:
+        image: bitnami/rabbitmq:3.8
+        env:
+          RABBITMQ_USERNAME: admin
+          RABBITMQ_PASSWORD: admin
+          RABBITMQ_VHOST: test
+        options: --health-cmd "rabbitmqctl node_health_check" --health-interval 5s --health-timeout 5s --health-retries 10
+        ports:
+        - 5672:5672
+        - 15672:15672
     steps:
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@main
@@ -61,7 +88,7 @@ jobs:
         with:
           context: .
           cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-to: type=gha,mode=min
           file: Dockerfile
           tags: icecube/skymap_scanner:local
           load: true
@@ -72,15 +99,6 @@ jobs:
         run: |
           sudo singularity build skymap_scanner.sif docker-daemon://icecube/skymap_scanner:local
           ls -lh skymap_scanner.sif
-      - name: Checkout MQClient repo (for broker startup scripts)
-        uses: actions/checkout@v3
-        with:
-          repository: WIPACrepo/MQClient
-          path: MQClient
-      - name: Start Broker
-        run: |
-          set -x
-          ./MQClient/resources/docker-pulsar.sh
       - name: Run Singularity Container
         run: |
           mkdir $SKYSCAN_CACHE_DIR
@@ -135,27 +153,29 @@ jobs:
           run00136766-evt000007637140-GOLD.pkl,
           run00136662-evt000035405932-BRONZE.pkl
         ]
+    services:
+      rabbitmq:
+        image: bitnami/rabbitmq:3.8
+        env:
+          RABBITMQ_USERNAME: admin
+          RABBITMQ_PASSWORD: admin
+          RABBITMQ_VHOST: test
+        options: --health-cmd "rabbitmqctl node_health_check" --health-interval 5s --health-timeout 5s --health-retries 10
+        ports:
+        - 5672:5672
+        - 15672:15672
     env:
       CLIENTS_PER_CPU: 1  # there isn't any improvement when >1
     steps:
       - uses: actions/checkout@v3
-      - name: Checkout MQClient repo (for broker startup scripts)
-        uses: actions/checkout@v3
-        with:
-          repository: WIPACrepo/MQClient
-          path: MQClient
-      - name: Start Broker
-        run: |
-          set -x
-          ./MQClient/resources/docker-pulsar.sh
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Build and push
         uses: docker/build-push-action@v3
         with:
           context: .
           cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-to: type=gha,mode=min
           file: Dockerfile
           tags: icecube/skymap_scanner:local
           load: true
@@ -220,6 +240,17 @@ jobs:
           "GOLD",
           "JSON",
         ]
+    services:
+      rabbitmq:
+        image: bitnami/rabbitmq:3.8
+        env:
+          RABBITMQ_USERNAME: admin
+          RABBITMQ_PASSWORD: admin
+          RABBITMQ_VHOST: test
+        options: --health-cmd "rabbitmqctl node_health_check" --health-interval 5s --health-timeout 5s --health-retries 10
+        ports:
+        - 5672:5672
+        - 15672:15672
     steps:
       - uses: actions/checkout@v3
       - name: Set up Docker Buildx
@@ -229,7 +260,7 @@ jobs:
         with:
           context: .
           cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-to: type=gha,mode=min
           file: Dockerfile
           tags: icecube/skymap_scanner:local
           load: true
@@ -270,7 +301,6 @@ jobs:
             --baseline-gcd-file $(jq -r '.baseline_GCD_file' tests/data/reco_pixel_pkls/${{ matrix.dir }}/startup.json) \
             --out-pkl /local/pkls/out.pkl
 
-
       - name: Test Output Against Known Result (.pkl)
         run: |
           ls tests/data/reco_pixel_pkls/${{ matrix.dir }}

Dockerfile

@@ -3,53 +3,9 @@
 #
 ARG ICETRAY_VERSION=v1.5.0-ubuntu22.04
 
-FROM icecube/icetray:icetray-devel-$ICETRAY_VERSION as build
-
-#
-# Add Packages
-#
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y zstd libzstd-dev protobuf-compiler python3-pybind11 python3-pip && \
-    apt-get clean
-
-#
-# Manually compile Pulsar
-#
-WORKDIR /local
-RUN git clone --depth 1 --branch v3.1.0 https://github.com/apache/pulsar-client-cpp
-RUN cd pulsar-client-cpp && cmake -DBUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release . && make -j2 && make install
-RUN git clone --depth 1 --branch v3.0.0 https://github.com/apache/pulsar-client-python
-RUN cd pulsar-client-python && git submodule update --init \
-    && cmake -B build \
-    && cmake --build build && cmake --install build \
-    && python3 ./setup.py bdist_wheel
-
-
-#
-# Now make the prod image
-#
 FROM icecube/icetray:icetray-prod-$ICETRAY_VERSION as prod
 
 
-#
-# Add Packages
-#
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y libprotobuf23 python3-pybind11 python3-pip && \
-    apt-get clean
-
-
-#
-# Copy pulsar
-#
-WORKDIR /local
-COPY --from=build /usr/local/include/pulsar/ /usr/local/include/pulsar/
-COPY --from=build /usr/local/lib/libpulsar* /usr/local/lib/
-COPY --from=build /local/pulsar-client-python/dist/ pulsar-client-python/
-RUN python3 -m pip install pulsar-client-python/pulsar_client-*.whl
-
-
-
 # we need more spline tables (since we need to potentially re-do onlineL2)
 #RUN wget -nv -t 5 -O /opt/i3-data/photon-tables/splines/InfBareMu_mie_abs_z20a10_V2.fits \
 #        http://prod-exe.icecube.wisc.edu/spline-tables/InfBareMu_mie_abs_z20a10_V2.fits
@@ -86,7 +42,7 @@ RUN python3 -m pip install pulsar-client-python/pulsar_client-*.whl
 #
 WORKDIR /local
 COPY . .
-RUN pip install .
+RUN pip install .[rabbitmq]
 
 
 # set the entry point so that module is called with any parameters given to the `docker run` command

Dockerfile_pulsar

@@ -0,0 +1,95 @@
+#
+# Define the base image icetray version
+#
+ARG ICETRAY_VERSION=v1.5.0-ubuntu22.04
+
+FROM icecube/icetray:icetray-devel-$ICETRAY_VERSION as build
+
+#
+# Add Packages
+#
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y zstd libzstd-dev protobuf-compiler python3-pybind11 python3-pip && \
+    apt-get clean
+
+#
+# Manually compile Pulsar
+#
+WORKDIR /local
+RUN git clone --depth 1 --branch v3.1.0 https://github.com/apache/pulsar-client-cpp
+RUN cd pulsar-client-cpp && cmake -DBUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release . && make -j2 && make install
+RUN git clone --depth 1 --branch v3.0.0 https://github.com/apache/pulsar-client-python
+RUN cd pulsar-client-python && git submodule update --init \
+    && cmake -B build \
+    && cmake --build build && cmake --install build \
+    && python3 ./setup.py bdist_wheel
+
+
+#
+# Now make the prod image
+#
+FROM icecube/icetray:icetray-prod-$ICETRAY_VERSION as prod
+
+
+#
+# Add Packages
+#
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y libprotobuf23 python3-pybind11 python3-pip && \
+    apt-get clean
+
+
+#
+# Copy pulsar
+#
+WORKDIR /local
+COPY --from=build /usr/local/include/pulsar/ /usr/local/include/pulsar/
+COPY --from=build /usr/local/lib/libpulsar* /usr/local/lib/
+COPY --from=build /local/pulsar-client-python/dist/ pulsar-client-python/
+RUN python3 -m pip install pulsar-client-python/pulsar_client-*.whl
+
+
+
+# we need more spline tables (since we need to potentially re-do onlineL2)
+#RUN wget -nv -t 5 -O /opt/i3-data/photon-tables/splines/InfBareMu_mie_abs_z20a10_V2.fits \
+#        http://prod-exe.icecube.wisc.edu/spline-tables/InfBareMu_mie_abs_z20a10_V2.fits
+#RUN wget -nv -t 5 -O /opt/i3-data/photon-tables/splines/InfBareMu_mie_prob_z20a10_V2.fits \
+#        http://prod-exe.icecube.wisc.edu/spline-tables/InfBareMu_mie_prob_z20a10_V2.fits
+#RUN wget -nv -t 5 -O /opt/i3-data/photon-tables/splines/cascade_single_spice_bfr-v2_flat_z20_a5.abs.fits \
+#        http://prod-exe.icecube.wisc.edu/spline-tables/cascade_single_spice_bfr-v2_flat_z20_a5.abs.fits
+#RUN wget -nv -t 5 -O /opt/i3-data/photon-tables/splines/cascade_single_spice_bfr-v2_flat_z20_a5.prob.fits \
+#        http://prod-exe.icecube.wisc.edu/spline-tables/cascade_single_spice_bfr-v2_flat_z20_a5.prob.fits
+#RUN wget -nv -t 5 -O /opt/i3-data/photon-tables/splines/cascade_effectivedistance_spice_bfr-v2_z20.eff.fits \
+#        http://prod-exe.icecube.wisc.edu/spline-tables/cascade_effectivedistance_spice_bfr-v2_z20.eff.fits
+
+
+#
+# Get IceTray Setup
+#
+
+# add realtime_gfu python checkout from V21-06-00
+#RUN svn co http://code.icecube.wisc.edu/svn/meta-projects/realtime/releases/V21-06-00/realtime_gfu \
+#        /usr/local/icetray/realtime_gfu --username=icecube --password=skua --no-auth-cache && \
+#    ln -sf /usr/local/icetray/realtime_gfu/python /usr/local/icetray/lib/icecube/realtime_gfu
+# add realtime_hese
+#RUN svn co http://code.icecube.wisc.edu/svn/meta-projects/realtime/releases/V21-06-00/realtime_hese \
+#        /usr/local/icetray/realtime_hese --username=icecube --password=skua --no-auth-cache && \
+#    ln -sf /usr/local/icetray/realtime_hese/python /usr/local/icetray/lib/icecube/realtime_hese
+# add realtime_tools
+#RUN svn co http://code.icecube.wisc.edu/svn/meta-projects/realtime/releases/V21-06-00/realtime_tools \
+#        /usr/local/icetray/realtime_tools --username=icecube --password=skua --no-auth-cache && \
+#    ln -sf /usr/local/icetray/realtime_tools/python /usr/local/icetray/lib/icecube/realtime_tools
+
+
+#
+# Add Python Packages
+#
+WORKDIR /local
+COPY . .
+RUN pip install .[pulsar]
+
+
+# set the entry point so that module is called with any parameters given to the `docker run` command
+ENTRYPOINT ["/bin/bash", "/usr/local/icetray/env-shell.sh"]
+
+CMD []

README.md

@@ -9,6 +9,43 @@ Skymap Scanner is the computational core of the [SkyDriver orchestration service
 
 `skymap_scanner` is a python package containing two distinct applications meant to be deployed within containers (1 `skymap_scanner.server`, n `skymap_scanner.client`s), along with `skymap_scanner.utils` (utility functions) and `skymap_scanner.recos` (`icetray` reco-specific logic). Additional, package-independent, utility scripts are in `resources/utils/`.
 
+## Queue Types
+
+The default queue type in the container is RabbitMQ, since v3.1.0.
+Build `Dockerfile_pulsar` for a pulsar container.
+
+### RabbitMQ
+Env variables
+
+```
+export SKYSCAN_BROKER_CLIENT=rabbitmq
+export SKYSCAN_BROKER_ADDRESS=amqp://:<token>@<hostname>/<vhost>?heartbeat=600
+```
+
+Currently RabbitMQ uses URL parameters for settings.  This may change in future updates.
+
+Python install:
+```
+pip install .[rabbitmq]
+```
+
+### Pulsar
+Env variables
+
+```
+export SKYSCAN_BROKER_CLIENT=pulsar
+export SKYSCAN_BROKER_ADDRESS=<ip address>
+export SKYSCAN_BROKER_AUTH=<token>
+export PULSAR_UNACKED_MESSAGES_TIMEOUT_SEC=600
+```
+
+Python install:
+```
+pip install .[pulsar]
+```
+
+## Pulsar Example
+
 ### Example Startup
 You will need to get a pulsar broker address and authentication token to pass to both the server and client. Send a poke on slack #skymap-scanner to get those!
 
@@ -19,6 +56,7 @@ The server can be launched from anywhere with a stable network connection. You c
 ###### Environment Variables
 ```
 export SKYSCAN_BROKER_ADDRESS=BROKER_ADDRESS
+export SKYSCAN_BROKER_CLIENT=pulsar
 export SKYSCAN_BROKER_AUTH=$(cat ~/skyscan-broker.token)  # obfuscated for security
 ```
 ###### Command-Line Arguments
@@ -59,6 +97,7 @@ The client jobs can submitted via HTCondor from sub-2. Running the script below
 ###### Environment Variables
 ```
 export SKYSCAN_BROKER_ADDRESS=BROKER_ADDRESS
+export SKYSCAN_BROKER_CLIENT=pulsar
 export SKYSCAN_BROKER_AUTH=$(cat ~/skyscan-broker.token)  # obfuscated for security
 ```
 ###### Command-Line Arguments
@@ -110,6 +149,7 @@ For now, it's easy to scale up using the command line. Multiple server instances
 
 ```
 export SKYSCAN_BROKER_ADDRESS=BROKER_ADDRESS
+export SKYSCAN_BROKER_CLIENT=pulsar
 export SKYSCAN_BROKER_AUTH=$(cat ~/skyscan-broker.token)  # obfuscated for security
 ls *.json | xargs -n1 -PN -I{} bash -c 'mkdir /path/to/json/{} && python -m skymap_scanner.server --startup-json-dir /path/to/json/{} --cache-dir /path/to/cache --output-dir /path/to/out --reco-algo RECO_ALGO --event-file /path/to/data/{}'
 ```