DM-32979: initial checkin of hermesk daemon mods

.github/workflows/build.yaml

@@ -0,0 +1,38 @@
+name: Build Rucio/Butler ingestd container
+on:
+  push:
+    tags:
+    - v*
+  pull_request:
+
+env:
+  HERMESK_NAME: rucio-daemons-hermesk
+
+jobs:
+  push:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Build ingest image
+      working-directory: docker
+      run: |
+        docker-compose -f "docker-compose.yml" --env-file versions.env build $HERMESK_NAME
+
+    - name: Log in to GitHub Container Registry
+      run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
+
+    - name: Push image
+      run: |
+        HERMESK_ID=ghcr.io/${{ github.repository_owner }}/$HERMESK_NAME
+
+        # Strip git ref prefix from version
+        VERSION=$(cat docker/versions.env | grep RUCIO_DAEMONS_HERMESK_VERSION | sed 's/=/ /g'|cut -d " " -f2)
+        echo HERMESK_ID=$HERMESK_ID
+        echo VERSION=$VERSION
+        docker tag $HERMESK_NAME $HERMESK_ID:$VERSION
+        docker push $HERMESK_ID:$VERSION

SConstruct

@@ -0,0 +1,4 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+# Python-only package
+scripts.BasicSConstruct("ctrl_rucio_ingest", disableCc=True)

bin/rucio-hermesk

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# srp
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Hermesk is a daemon that gets the messages and sends them to external services (influxDB, ES, ActiveMQ, kafka).
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.hermes.kafka.hermesk import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="Hermesk is a daemon that get the messages and sends them to external services (influxDB, ES, ActiveMQ).", formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''
+Run the daemon::
+
+  $ rucio-hermesk --run-once
+    ''')
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: number of threads')
+    parser.add_argument("--bulk", action="store", default=1000, type=int, help='Bulk control: number of requests per cycle')
+    parser.add_argument("--sleep-time", action="store", default=10, type=int, help='Delay control: second control per cycle')
+    return parser
+
+
+if __name__ == "__main__":
+
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once,
+            threads=args.threads,
+            bulk=args.bulk,
+            sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

docker/docker-compose.yml

@@ -0,0 +1,13 @@
+version: "3.7"
+
+services:
+  rucio-daemons-hermesk:
+    container_name: rucio-daemons-hermesk
+    platform: linux/x86_64
+    image: rucio-daemons-hermesk
+    build:
+      dockerfile: ./docker/rucio-daemons-hermesk/Dockerfile
+      context: ../
+      args:
+        - RUCIO_DAEMONS_VERSION
+        - CONFLUENT_KAFKA_VERSION

docker/rucio-daemons-hermesk/Dockerfile

@@ -0,0 +1,14 @@
+ARG RUCIO_DAEMONS_VERSION
+FROM rucio/rucio-daemons:${RUCIO_DAEMONS_VERSION}
+
+ARG CONFLUENT_KAFKA_VERSION
+RUN pip install confluent_kafka==${CONFLUENT_KAFKA_VERSION}
+RUN pip install retrying
+
+RUN mkdir -p /etc/grid-security/certificates
+RUN mkdir /usr/local/lib/python3.9/site-packages/rucio/daemons/hermes/kafka
+COPY ./lib/rucio/daemons/hermes/kafka /usr/local/lib/python3.9/site-packages/rucio/daemons/hermes/kafka/
+COPY lib/rucio/common/constants.py /usr/local/lib/python3.9/site-packages/rucio/common/constants.py
+COPY bin/rucio-hermesk /usr/local/bin/rucio-hermesk
+
+# ENTRYPOINT ["/start-daemon.sh"]

docker/versions.env

@@ -0,0 +1,3 @@
+RUCIO_DAEMONS_HERMESK_VERSION=34.0.0.2
+RUCIO_DAEMONS_VERSION=release-34.0.0
+CONFLUENT_KAFKA_VERSION=1.9.2