Migrate off deprecated K8s dependencies (#596)

* Update helm minio chart by several major versions
* Migrate off deprecated redis-ha repository
* Provide a fallback UPLOAD_OBJECT_STORE_SERVER option as an ingress isn't required for minio to work.
* Documents upload object store configuration.
* Update azure pipelines
* Update base image deps
* Pin an older version of bitarray
* Update minio image used with docker compose
* Bump the chart version

azure-pipelines.yml

@@ -270,15 +270,7 @@ stages:
       # Install Helm on an agent machine
       - task: HelmInstaller@1
         inputs:
-          helmVersionToInstall: '2.16.3'
-
-      - task: HelmDeploy@0
-        displayName: Helm init
-        inputs:
-          connectionType: 'Kubernetes Service Connection'
-          kubernetesServiceConnection: $(CLUSTER)
-          command: init
-          arguments: --client-only
+          helmVersionToInstall: 'latest'
 
       - task: HelmDeploy@0
         displayName: Helm package
@@ -308,8 +300,8 @@ stages:
             api.dbinit.image.repository:$(backendImageName)
             workers.image.repository:$(backendImageName)
             workers.image.pullPolicy:Always
-            anonlink.objectstore.secure:false
             anonlink.objectstore.uploadEnabled:true
+            anonlink.objectstore.uploadSecure:false
             anonlink.objectstore.uploadAccessKey:testUploadAccessKey
             anonlink.objectstore.uploadSecretKey:testUploadSecretKey
             minio.accessKey:testMinioAccessKey

base/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.12.0
+FROM alpine:3.12.1
 
 ENV DOCKERIZE_VERSION v0.6.1
 RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
@@ -21,13 +21,13 @@ RUN apk add --no-cache \
     libstdc++=9.3.0-r2 \
     mpc1-dev=1.1.0-r1 \
     yajl=2.1.0-r1 \
-    libpq=12.3-r2 && \
+    libpq=12.5-r0 && \
     ln -s /usr/bin/python3 /usr/bin/python && \
     apk add --no-cache --virtual .build-deps \
     g++=9.3.0-r2 \
     python3-dev=3.8.5-r0 \
     yajl-dev=2.1.0-r1 \
-    postgresql-dev=12.3-r2 \
+    postgresql-dev=12.5-r0 \
     libffi-dev=3.3-r2 \
     gmp-dev=6.2.0-r0 \
     mpfr-dev=4.0.2-r4 \

base/requirements.txt

@@ -1,5 +1,6 @@
 anonlink==0.13.1
 anonlink-client==0.1.4
+bitarray-hardbyte==1.2.2
 bitmath==1.3.1.2
 celery==4.4.7
 clkhash==0.16.0
@@ -17,7 +18,7 @@ minio==5.0.10
 opentracing==2.3.0
 opentracing_instrumentation==3.2.1
 psycopg2==2.8.4
-pytest==5.4.3
+pytest==6.1.2
 pytest-xdist==1.29.0
 PyYAML==5.3
 redis==3.4.1

deployment/entity-service/Chart.yaml

@@ -1,6 +1,6 @@
 name: entity-service
 appVersion: 1.13.0
-version: 1.13.3
+version: 1.13.4
 description: Privacy preserving record linkage service
 sources:
   - https://github.com/data61/anonlink-entity-service

deployment/entity-service/requirements.yaml

@@ -1,11 +1,11 @@
 dependencies:
   - name: redis-ha
-    version: 4.4.1
-    repository: https://kubernetes-charts.storage.googleapis.com
+    version: 4.10.5
+    repository: https://dandydeveloper.github.io/charts
     condition: provision.redis
   - name: minio
-    version: 5.0.27
-    repository: https://kubernetes-charts.storage.googleapis.com
+    version: 8.0.6
+    repository: https://helm.min.io/
     condition: provision.minio
   - name: postgresql
     version: 8.9.1

deployment/entity-service/templates/configmap.yaml

@@ -31,11 +31,15 @@ data:
 
   UPLOAD_OBJECT_STORE_ENABLED: {{ .Values.anonlink.objectstore.uploadEnabled | quote }}
 
-  {{ if .Values.minio.ingress.enabled }}
-  UPLOAD_OBJECT_STORE_SERVER: {{ index .Values.minio.ingress.hosts 0 }}
+  {{ if .Values.anonlink.objectstore.uploadServer }}
+  UPLOAD_OBJECT_STORE_SERVER: {{ .Values.anonlink.objectstore.uploadServer | quote }}
+  {{ else if .Values.minio.ingress.enabled }}
+  UPLOAD_OBJECT_STORE_SERVER: {{ index .Values.minio.ingress.hosts 0 | quote }}
+  {{ else }}
+  UPLOAD_OBJECT_STORE_SERVER: "{{ .Release.Name }}-{{ .Values.minio.nameOverride }}:{{ .Values.minio.service.port }}"
   {{ end }}
 
-  UPLOAD_OBJECT_STORE_SECURE: {{ .Values.anonlink.objectstore.secure | quote }}
+  UPLOAD_OBJECT_STORE_SECURE: {{ .Values.anonlink.objectstore.uploadSecure | quote }}
 
   UPLOAD_OBJECT_STORE_BUCKET: {{ required "anonlink.objectstore.uploadBucket.name is required." .Values.anonlink.objectstore.uploadBucket.name | quote }}
 

deployment/entity-service/values.yaml

@@ -14,6 +14,13 @@ anonlink:
     ## via the object store. See section `minio.ingress` to create an ingress for minio.
     uploadEnabled: true
 
+    ## Server used as the external object store URL - provided to clients so should be externally
+    ## accessible. If not provided, the minio.ingress is used (if enabled).
+    #uploadServer: "s3.amazonaws.com"
+
+    ## Tell clients to make secure connections to the upload object store.
+    uploadSecure: true
+
     ## Object store credentials used to grant temporary upload access to clients
     ## Will be created with an "upload only" policy for a upload bucket if using the default
     ## MINIO provisioning.
@@ -24,9 +31,6 @@ anonlink:
     uploadBucket:
       name: "uploads"
 
-    ## Allow only secure connections to the object store.
-    secure: true
-
 
 api:
 
@@ -309,11 +313,11 @@ redis:
 
 redis-ha:
   ## Settings for configuration of a provisioned redis ha cluster.
-  ## https://github.com/helm/charts/tree/master/stable/redis-ha
+  ## https://github.com/DandyDeveloper/charts/tree/master/charts/redis-ha#configuration
   ## Provisioning is controlled in the `provision` section
   auth: true
   redisPassword: "exampleRedisPassword"
-  replicas: 3
+  #replicas: 3
   redis:
     resources:
       requests:
@@ -345,6 +349,7 @@ redis-ha:
       - |-
         sysctl -w net.core.somaxconn=10000
         echo never > /host-sys/kernel/mm/transparent_hugepage/enabled
+  # Enable prometheus exporter sidecar
   exporter:
     enabled: true
 
@@ -358,10 +363,6 @@ minio:
   #accessKey: "exampleMinioAccessKey"
   #secretKey: "exampleMinioSecretKet"
 
-  ## Settings required for connecting to another object store, the server is ignored
-  ## if provisioning minio during deployment.
-  server: "s3.amazonaws.com"
-
   defaultBucket:
     enabled: true
     name: "anonlink"

docs/changelog.rst

@@ -7,6 +7,8 @@ Changelog
 Next Version
 ------------
 
+Updates Redis and MinIO dependencies.
+
 Version 1.13.0
 --------------
 

docs/development.rst

@@ -105,11 +105,12 @@ The run info ``HASH`` stores:
 Object Store
 ------------
 
-MinIO is an S3 compatible object store which is used to handle uploads and store
-files for the Anonlink Entity Service.
-
-
+Write access to an AWS S3 compatible object store is required to store intermediate files for the
+Anonlink Entity Service. The optional feature for data upload via object store also requires access
+to an AWS S3 compatible object store - along with authorization to create temporary credentials.
 
+`MinIO <https://min.io/>`__ is an open source object store implementation which can be used with
+both Docker Compose and Kubernetes deployments instead of AWS S3.
 
 
 Deployment Testing
@@ -139,4 +140,3 @@ This job includes a ``1GiB`` `persistent volume claim <https://kubernetes.io/doc
 to which the results are written (as ``results.xml``). During the testing the pytest output will be rendered,
 and then the Job's pod terminates. We create a temporary pod which mounts the same results volume and then we copy
 across the produced test result artifact.
-

docs/future.rst

@@ -4,7 +4,6 @@ Road map for the entity service
 ===============================
 
 - baseline benchmarking vs known datasets (accuracy and speed) e.g ``recordspeed`` datasets
-- blocking
 - Schema specification and tooling
 - Algorithmic improvements. e.g., implementing canopy clustering solver
 - A web front end including authentication and access control
@@ -13,8 +12,6 @@ Road map for the entity service
 - Semi supervised machine learning methods to learn thresholds
 - Handle 1 to many relationships. E.g. familial groups
 - Larger scale graph solving methods
-- Remove bottleneck of sparse links having to fit in redis.
-- improve uploads by allowing direct binary file transfer into object store
 - optimise anonlink memory management and C++ code
 
 Bigger Projects

docs/production-deployment.rst

@@ -235,9 +235,9 @@ to ``false``, set the database server in ``postgresql.nameOverride``, and add cr
 Object Store Deployment Options
 -------------------------------
 
-At deployment time you can decide to deploy MINIO or instead use an existing service such as AWS S3.
+At deployment time you can decide to deploy MinIO or instead use an existing object store service compatible with AWS S3.
 
-Note that there is a trade off between using a local deployment of minio vs S3. In our AWS based experimentation
+Note that there is a trade off between using a local deployment of MinIO vs AWS S3. In our AWS based experimentation
 Minio is noticeably faster, but more expensive and less reliable than AWS S3, your own mileage may vary.
 
 To configure a deployment to use an external object store, set ``provision.minio`` to ``false`` and add
@@ -247,12 +247,40 @@ credentials (and disable provisioning minio)::
     helm install entity-service --name="es-s3" --set provision.minio=false --set minio.accessKey=XXX --set minio.secretKey=YYY --set minio.bucket=<bucket>
 
 
+Object Store for client uploads
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Optionally client's can upload data via an object store instead of via the REST API. This requires external access
+to an object store, and the service must have authorization to create temporary credentials.
+
+The following settings control this optional feature:
+
+==================================  ==========================================
+     Environment Variable              Helm Config
+==================================  ==========================================
+``UPLOAD_OBJECT_STORE_ENABLED``      ``anonlink.objectstore.uploadEnabled``
+``UPLOAD_OBJECT_STORE_SERVER``       ``anonlink.objectstore.uploadServer``
+``UPLOAD_OBJECT_STORE_SECURE``       ``anonlink.objectstore.uploadSecure``
+``UPLOAD_OBJECT_STORE_BUCKET``       ``anonlink.objectstore.uploadBucket.name``
+``UPLOAD_OBJECT_STORE_ACCESS_KEY``   ``anonlink.objectstore.uploadAccessKey``
+``UPLOAD_OBJECT_STORE_SECRET_KEY``   ``anonlink.objectstore.uploadSecretKey``
+===================================  ==========================================
+
+
+.. note::
+
+   If the ``uploadServer`` config isn't provided, the deployment will assume that MinIO has been
+   deployed along with the service and fallback to using the MinIO ingress host (if present),
+   otherwise the cluster internal address of the deployed MinIO service. This last fallback is
+   in place simply to make e2e testing easier.
+
 
 Redis Deployment Options
 ------------------------
 
 At deployment time you can decide to provision redis using our chart, or instead use an existing redis installation or
 managed service. The provisioned redis is a highly available 3 node redis cluster using the `redis-ha` helm chart.
+
 Directly connecting to redis, and discovery via the sentinel protocol are supported. When using sentinel protocol
 for redis discovery read only requests are dispatched to redis replicas.
 
@@ -266,6 +294,8 @@ To use a separate install of redis using the server ``shared-redis-ha-redis-ha.d
          --set redis.use_sentinel=true
 
 
+Note these settings can also be provided via a ``values.yaml`` deployment configuration file.
+
 Uninstalling
 ------------
 

frontend/Dockerfile

@@ -1,19 +1,15 @@
 # To run this Dockerfile, it is required to be in the source `anonlink-entity-service` folder and run `docker build -f frontend/Dockerfile .`
 # Otherwise, it cannot access the `docs` folder to build the documentation before serving it.
 
-FROM data61/anonlink-docs-builder:20190625 AS docsbuilder
+FROM data61/anonlink-docs-builder:20201221 AS docsbuilder
 COPY ./docs /src/docs
 COPY ./backend/entityservice/api_def/openapi.yaml /src/docs/_static/openapi.yaml
 WORKDIR /src
 RUN python -m sphinx /src/docs /build
 
 
-
-
 FROM nginx:1.15.12-alpine
 
-MAINTAINER "Brian Thorne <brian.thorne@data61.csiro.au>"
-
 RUN apk add --no-cache curl
 
 # Copy the static site assets:

tools/docker-compose.yml

@@ -16,7 +16,7 @@ services:
       retries: 5
 
   minio:
-    image: minio/minio:RELEASE.2020-04-15T19-42-18Z
+    image: minio/minio:RELEASE.2020-12-03T05-49-24Z
     command: server /export
     environment:
       - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE