Migrate off deprecated K8s dependencies

azure-pipelines.yml

@@ -270,15 +270,7 @@ stages:
       # Install Helm on an agent machine
       - task: HelmInstaller@1
         inputs:
-          helmVersionToInstall: '2.16.3'
-
-      - task: HelmDeploy@0
-        displayName: Helm init
-        inputs:
-          connectionType: 'Kubernetes Service Connection'
-          kubernetesServiceConnection: $(CLUSTER)
-          command: init
-          arguments: --client-only
+          helmVersionToInstall: 'latest'
 
       - task: HelmDeploy@0
         displayName: Helm package
@@ -308,8 +300,8 @@ stages:
             api.dbinit.image.repository:$(backendImageName)
             workers.image.repository:$(backendImageName)
             workers.image.pullPolicy:Always
-            anonlink.objectstore.secure:false
             anonlink.objectstore.uploadEnabled:true
+            anonlink.objectstore.uploadSecure:false
             anonlink.objectstore.uploadAccessKey:testUploadAccessKey
             anonlink.objectstore.uploadSecretKey:testUploadSecretKey
             minio.accessKey:testMinioAccessKey

base/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.12.0
+FROM alpine:3.12.1
 
 ENV DOCKERIZE_VERSION v0.6.1
 RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
@@ -21,13 +21,13 @@ RUN apk add --no-cache \
     libstdc++=9.3.0-r2 \
     mpc1-dev=1.1.0-r1 \
     yajl=2.1.0-r1 \
-    libpq=12.3-r2 && \
+    libpq=12.5-r0 && \
     ln -s /usr/bin/python3 /usr/bin/python && \
     apk add --no-cache --virtual .build-deps \
     g++=9.3.0-r2 \
     python3-dev=3.8.5-r0 \
     yajl-dev=2.1.0-r1 \
-    postgresql-dev=12.3-r2 \
+    postgresql-dev=12.5-r0 \
     libffi-dev=3.3-r2 \
     gmp-dev=6.2.0-r0 \
     mpfr-dev=4.0.2-r4 \

base/requirements.txt

@@ -1,5 +1,6 @@
 anonlink==0.13.1
 anonlink-client==0.1.4
+bitarray-hardbyte==1.2.2
 bitmath==1.3.1.2
 celery==4.4.7
 clkhash==0.16.0
@@ -17,7 +18,7 @@ minio==5.0.10
 opentracing==2.3.0
 opentracing_instrumentation==3.2.1
 psycopg2==2.8.4
-pytest==5.4.3
+pytest==6.1.2
 pytest-xdist==1.29.0
 PyYAML==5.3
 redis==3.4.1

deployment/entity-service/requirements.yaml

@@ -1,11 +1,11 @@
 dependencies:
   - name: redis-ha
-    version: 4.4.1
-    repository: https://kubernetes-charts.storage.googleapis.com
+    version: 4.10.5
+    repository: https://dandydeveloper.github.io/charts
     condition: provision.redis
   - name: minio
-    version: 5.0.27
-    repository: https://kubernetes-charts.storage.googleapis.com
+    version: 8.0.6
+    repository: https://helm.min.io/
     condition: provision.minio
   - name: postgresql
     version: 8.9.1

deployment/entity-service/templates/configmap.yaml

@@ -31,11 +31,15 @@ data:
 
   UPLOAD_OBJECT_STORE_ENABLED: {{ .Values.anonlink.objectstore.uploadEnabled | quote }}
 
-  {{ if .Values.minio.ingress.enabled }}
-  UPLOAD_OBJECT_STORE_SERVER: {{ index .Values.minio.ingress.hosts 0 }}
+  {{ if .Values.anonlink.objectstore.uploadServer }}
+  UPLOAD_OBJECT_STORE_SERVER: {{ .Values.anonlink.objectstore.uploadServer | quote }}
+  {{ else if .Values.minio.ingress.enabled }}
+  UPLOAD_OBJECT_STORE_SERVER: {{ index .Values.minio.ingress.hosts 0 | quote }}
+  {{ else }}
+  UPLOAD_OBJECT_STORE_SERVER: "{{ .Release.Name }}-{{ .Values.minio.nameOverride }}:{{ .Values.minio.service.port }}"
   {{ end }}
 
-  UPLOAD_OBJECT_STORE_SECURE: {{ .Values.anonlink.objectstore.secure | quote }}
+  UPLOAD_OBJECT_STORE_SECURE: {{ .Values.anonlink.objectstore.uploadSecure | quote }}
 
   UPLOAD_OBJECT_STORE_BUCKET: {{ required "anonlink.objectstore.uploadBucket.name is required." .Values.anonlink.objectstore.uploadBucket.name | quote }}
 

deployment/entity-service/values.yaml

@@ -14,6 +14,13 @@ anonlink:
     ## via the object store. See section `minio.ingress` to create an ingress for minio.
     uploadEnabled: true
 
+    ## Server used as the external object store URL - provided to clients so should be externally
+    ## accessible. If not provided, the minio.ingress is used (if enabled).
+    #uploadServer: "s3.amazonaws.com"
+
+    ## Tell clients to make secure connections to the upload object store.
+    uploadSecure: true
+
     ## Object store credentials used to grant temporary upload access to clients
     ## Will be created with an "upload only" policy for a upload bucket if using the default
     ## MINIO provisioning.
@@ -24,9 +31,6 @@ anonlink:
     uploadBucket:
       name: "uploads"
 
-    ## Allow only secure connections to the object store.
-    secure: true
-
 
 api:
 
@@ -309,11 +313,11 @@ redis:
 
 redis-ha:
   ## Settings for configuration of a provisioned redis ha cluster.
-  ## https://github.com/helm/charts/tree/master/stable/redis-ha
+  ## https://github.com/DandyDeveloper/charts/tree/master/charts/redis-ha#configuration
   ## Provisioning is controlled in the `provision` section
   auth: true
   redisPassword: "exampleRedisPassword"
-  replicas: 3
+  #replicas: 3
   redis:
     resources:
       requests:
@@ -345,6 +349,7 @@ redis-ha:
       - |-
         sysctl -w net.core.somaxconn=10000
         echo never > /host-sys/kernel/mm/transparent_hugepage/enabled
+  # Enable prometheus exporter sidecar
   exporter:
     enabled: true
 
@@ -358,10 +363,6 @@ minio:
   #accessKey: "exampleMinioAccessKey"
   #secretKey: "exampleMinioSecretKet"
 
-  ## Settings required for connecting to another object store, the server is ignored
-  ## if provisioning minio during deployment.
-  server: "s3.amazonaws.com"
-
   defaultBucket:
     enabled: true
     name: "anonlink"

tools/docker-compose.yml

@@ -16,7 +16,7 @@ services:
       retries: 5
 
   minio:
-    image: minio/minio:RELEASE.2020-04-15T19-42-18Z
+    image: minio/minio:RELEASE.2020-12-03T05-49-24Z
     command: server /export
     environment:
       - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE