Added files for 3.1 accumulo

apache · Feb 29, 2024 · 7e85355 · 7e85355
1 parent d503138
commit 7e85355
Show file tree

Hide file tree

Showing 13 changed files with 701 additions and 0 deletions.
diff --git a/helm/k8s-files/accumulo-compactor.yaml b/helm/k8s-files/accumulo-compactor.yaml
@@ -0,0 +1,62 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: accumulo-compactor
+  namespace: accumulo
+  labels:
+    app: accumulo-compactor
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: accumulo-compactor
+  template:
+    metadata:
+      labels:
+        app: accumulo-compactor
+    spec:
+      containers:
+      - name: accumulo-compactor
+        image: accumulo-s3-fs:2.1.0
+        imagePullPolicy: Never
+        command: ["/bin/bash", "-c"]
+        args: ["accumulo compactor -o compactor.queue=DCQ1 -o general.process.bind.addr=$(hostname -i)" ]
+        ports:
+        - containerPort: 9133
+        resources:
+          requests:
+            cpu: 100m
+            memory: "2048Mi"
+          limits:
+            cpu: 200m
+            memory: "4096Mi"
+        env:
+        - name: ACCUMULO_JAVA_OPTS
+          value: "-Xmx2g"
+        volumeMounts:
+        - name: "client-config"
+          mountPath: "/opt/accumulo/conf/accumulo-client.properties"
+          subPath: "accumulo-client.properties"
+        - name: "config"
+          mountPath: "/opt/accumulo/conf/accumulo.properties"
+          subPath: "accumulo.properties"
+        - name: "logging"
+          mountPath: "/opt/accumulo/conf/log4j2-service.properties"
+          subPath: "log4j2-service.properties"
+        - name: "core-site"
+          mountPath: "/opt/accumulo/conf/core-site.xml"
+          subPath: "core-site.xml"
+      terminationGracePeriodSeconds: 300  # Time to wait before moving from a TERM signal to the pod's main process to a KILL signal.
+      volumes:
+      - name: "client-config"
+        configMap:
+          name: "accumulo-client-properties"
+      - name: "config"
+        configMap:
+          name: "accumulo-properties"
+      - name: "logging"
+        configMap:
+          name: "accumulo-logging"
+      - name: "core-site"
+        configMap:
+          name: "core-site"
diff --git a/helm/k8s-files/accumulo-config.yaml b/helm/k8s-files/accumulo-config.yaml
@@ -0,0 +1,121 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: accumulo-client-properties
+  namespace: accumulo
+data:
+  accumulo-client.properties: |
+    auth.type=password
+    auth.principal=root
+    instance.name=s3test
+    instance.zookeepers=bitnami-zookeeper.zookeeper.svc.cluster.local:2181
+    rpc.transport.idle.timeout=60s
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: accumulo-properties
+  namespace: accumulo
+data:
+  accumulo.properties: |
+    general.rpc.timeout=240s
+    instance.secret=s3test
+    instance.zookeeper.host=bitnami-zookeeper.zookeeper.svc.cluster.local:2181
+    manager.wal.closer.implementation=org.apache.accumulo.server.manager.recovery.NoOpLogCloser
+    table.durability=sync
+    tserver.memory.maps.native.enabled=false
+    tserver.walog.max.size=512M
+    tserver.port.search=true
+    ## For S3 you must define volumes for accumulo and its write ahead logs. Replace the defaule instance volumes at the top
+    ## with the example below. Make sure the volumes match the general.custom.volume.preferred.default, and
+    ## general.custom.volume.preferred.logger property values below
+    instance.volumes=s3a://accumulo/database,s3a://accumulo/wal
+    ## The default deployment will use a RandomVolumeChooser and HadoopLogCloser and neither of which work when deploying to
+    ## S3. Remove the comments below to use the volume chooser and log closer that are compatible with S3
+    general.volume.chooser=org.apache.accumulo.core.spi.fs.PreferredVolumeChooser
+    ## Define the S3 objects to use for recording accumulo and write ahead log data
+    general.custom.volume.preferred.default=s3a://accumulo/database
+    general.custom.volume.preferred.logger=s3a://accumulo/wal
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: accumulo-logging
+  namespace: accumulo
+data:
+  log4j2-service.properties: |
+    status = info
+    dest = err
+    name = AccumuloCompactorLoggingProperties
+    appender.console.type = Console
+    appender.console.name = STDOUT
+    appender.console.target = SYSTEM_OUT
+    appender.console.layout.type = PatternLayout
+    appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
+    appender.console.filter.threshold.type = ThresholdFilter
+    appender.console.filter.threshold.level = debug
+    logger.hadoop.name = org.apache.hadoop
+    logger.zookeeper.level = warn
+    logger.zookeeper.name = org.apache.zookeeper
+    logger.zookeeper.level = error
+    logger.accumulo.name = org.apache.accumulo
+    logger.accumulo.level = debug
+    rootLogger.level = debug
+    rootLogger.appenderRef.console.ref = STDOUT
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+  name: core-site
+  namespace: accumulo
+data:
+  core-site.xml: |
+    <configuration>
+      <property>
+        <name>fs.defaultFS</name>
+        <value>s3a://accumulo/database</value>
+      </property>
+      <property>
+        <name>fs.s3a.impl</name>
+        <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
+        <description>The implementation class of the S3A Filesystem</description>
+      </property>
+      <property>
+        <name>fs.s3a.path.style.access</name>
+        <value>true</value>
+      </property>
+      <property>
+        <name>fs.s3a.endpoint</name>
+        <value>http://minio.minio-dev.svc.cluster.local:9000</value>
+      </property>
+      <property>
+        <name>fs.s3a.access.key</name>
+        <value>accumulo</value>
+      </property>
+      <property>
+        <name>fs.s3a.secret.key</name>
+        <value>changeme</value>
+      </property>
+      <property>
+        <name>fs.s3a.connection.ssl.enabled</name>
+        <value>false</value>
+      </property>
+          <!--
+          <property>
+            <name>fs.AbstractFileSystem.s3a.impl</name>
+            <value>org.apache.hadoop.fs.s3a.S3A</value>
+            <description>The implementation class of the S3A AbstractFileSystem.</description>
+          </property>
+          <property>
+            <name>fs.s3a.aws.credentials.provider</name>
+            <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
+          </property>
+          // OPTIONAL REGION PROPERTY
+          <property>
+              <name>fs.s3.region</name>
+              <value>{{ YOUR_S3_REGION }}</value>
+          </property>
+          -->
+    </configuration>
+---
diff --git a/helm/k8s-files/accumulo-coordinator.yaml b/helm/k8s-files/accumulo-coordinator.yaml
@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: accumulo-coordinator
+  namespace: accumulo
+  labels:
+    app: accumulo-coordinator
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: accumulo-coordinator
+  template:
+    metadata:
+      labels:
+        app: accumulo-coordinator
+    spec:
+      containers:
+      - name: accumulo-coordinator
+        image: accumulo-s3-fs:2.1.0
+        imagePullPolicy: Never
+        command: ["/bin/bash", "-c"]
+        args: ["accumulo compaction-coordinator -o general.process.bind.addr=$(hostname -i)"]
+        ports:
+        - containerPort: 9132
+          hostPort: 9132
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 1
+            memory: "2048Mi"
+          limits:
+            cpu: 2
+            memory: "4096Mi"
+        env:
+        - name: ACCUMULO_JAVA_OPTS
+          value: "-Xmx2g"
+        volumeMounts:
+        - name: "client-config"
+          mountPath: "/opt/accumulo/conf/accumulo-client.properties"
+          subPath: "accumulo-client.properties"
+        - name: "config"
+          mountPath: "/opt/accumulo/conf/accumulo.properties"
+          subPath: "accumulo.properties"
+        - name: "logging"
+          mountPath: "/opt/accumulo/conf/log4j2-service.properties"
+          subPath: "log4j2-service.properties"
+        - name: "core-site"
+          mountPath: "/opt/accumulo/conf/core-site.xml"
+          subPath: "core-site.xml"
+      terminationGracePeriodSeconds: 300  # Time to wait before moving from a TERM signal to the pod's main process to a KILL signal.
+      volumes:
+      - name: "client-config"
+        configMap:
+          name: "accumulo-client-properties"
+      - name: "config"
+        configMap:
+          name: "accumulo-properties"
+      - name: "logging"
+        configMap:
+          name: "accumulo-logging"
+      - name: "core-site"
+        configMap:
+          name: "core-site"
diff --git a/helm/k8s-files/accumulo-gc.yaml b/helm/k8s-files/accumulo-gc.yaml
@@ -0,0 +1,63 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: accumulo-gc
+  namespace: accumulo
+  labels:
+    app: accumulo-gc
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: accumulo-gc
+  template:
+    metadata:
+      labels:
+        app: accumulo-gc
+    spec:
+      containers:
+      - name: accumulo-gc
+        image: accumulo-s3-fs:2.1.0
+        imagePullPolicy: Never
+        command: ["/bin/bash", "-c"]
+        args: ["accumulo gc -o general.process.bind.addr=$(hostname -i)" ]
+        ports:
+        - containerPort: 9998
+        resources:
+          requests:
+            cpu: 100m
+            memory: "2048Mi"
+          limits:
+            cpu: 200m
+            memory: "4096Mi"
+        env:
+        - name: ACCUMULO_JAVA_OPTS
+          value: "-Xmx2g"
+        volumeMounts:
+        - name: "client-config"
+          mountPath: "/opt/accumulo/conf/accumulo-client.properties"
+          subPath: "accumulo-client.properties"
+        - name: "config"
+          mountPath: "/opt/accumulo/conf/accumulo.properties"
+          subPath: "accumulo.properties"
+        - name: "logging"
+          mountPath: "/opt/accumulo/conf/log4j2-service.properties"
+          subPath: "log4j2-service.properties"
+        - name: "core-site"
+          mountPath: "/opt/accumulo/conf/core-site.xml"
+          subPath: "core-site.xml"
+      terminationGracePeriodSeconds: 300  # Time to wait before moving from a TERM signal to the pod's main process to a KILL signal.
+      volumes:
+      - name: "client-config"
+        configMap:
+          name: "accumulo-client-properties"
+      - name: "config"
+        configMap:
+          name: "accumulo-properties"
+      - name: "logging"
+        configMap:
+          name: "accumulo-logging"
+      - name: "core-site"
+        configMap:
+          name: "core-site"
+
diff --git a/helm/k8s-files/accumulo-init.yaml b/helm/k8s-files/accumulo-init.yaml
@@ -0,0 +1,61 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: job
+  namespace: accumulo
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: accumulo-init
+        image: accumulo-s3-fs:2.1.0
+        imagePullPolicy: Never
+        command: ["/bin/bash", "-c"]
+        args: ["accumulo init --clear-instance-name --instance-name ${ACCUMULO_INSTANCE} --user ${ACCUMULO_USERNAME} --password ${ACCUMULO_PASSWORD}" ]
+        env:
+          - name: ACCUMULO_USERNAME
+            valueFrom:
+              secretKeyRef:
+                name: accumulo-secrets
+                key: accumulo-username
+                optional: false
+          - name: ACCUMULO_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: accumulo-secrets
+                key: accumulo-password
+                optional: false
+          - name: ACCUMULO_INSTANCE
+            valueFrom:
+              secretKeyRef:
+                name: accumulo-secrets
+                key: accumulo-instance-name
+                optional: false
+        volumeMounts:
+        - name: "client-config"
+          mountPath: "/opt/accumulo/conf/accumulo-client.properties"
+          subPath: "accumulo-client.properties"
+        - name: "config"
+          mountPath: "/opt/accumulo/conf/accumulo.properties"
+          subPath: "accumulo.properties"
+        - name: "logging"
+          mountPath: "/opt/accumulo/conf/log4j2-service.properties"
+          subPath: "log4j2-service.properties"
+        - name: "core-site"
+          mountPath: "/opt/accumulo/conf/core-site.xml"
+          subPath: "core-site.xml"
+      volumes:
+      - name: "client-config"
+        configMap:
+          name: "accumulo-client-properties"
+      - name: "config"
+        configMap:
+          name: "accumulo-properties"
+      - name: "logging"
+        configMap:
+          name: "accumulo-logging"
+      - name: "core-site"
+        configMap:
+          name: "core-site"
+