Merge pull request #13 from magda-io/issue/8

Issue/8,11,12
magda-io · Jul 4, 2021 · 06ceb8d · 06ceb8d
2 parents 848d8de + 4e3b33d
commit 06ceb8d
Show file tree

Hide file tree

Showing 22 changed files with 1,075 additions and 2,818 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -11,37 +11,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Use Node.js 10
+      - name: Use Node.js 12
         uses: actions/setup-node@v1
         with:
-          node-version: 10
+          node-version: 12
       - run: yarn install
       - run: yarn build
       - run: yarn test
 
-      - name: Setup Helm
-        uses: azure/setup-helm@v1
-        with:
-          version: v3.2.0
+      - name: helm-build-dependencies
+        run: helm dep up deploy/${REPO_NAME}
 
       - run: yarn helm-lint
 
       - name: Login to GitHub Package Repository
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_ACCESS_TOKEN }}
+          GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
         run: docker login docker.pkg.github.com -u magdabot -p ${GH_TOKEN}
       - name: Build Docker Image & Push
         run: yarn docker-build-prod --repository=docker.pkg.github.com/magda-io/${REPO_NAME} --name=${REPO_NAME} --version=${GITHUB_SHA}
-
-  build-test-node-12:
-    name: Build, Test Only with Node 12
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Use Node.js 12
-        uses: actions/setup-node@v1
-        with:
-          node-version: 12
-      - run: yarn install
-      - run: yarn build
-      - run: yarn test
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -5,7 +5,17 @@ on:
     types: [published]
 
 env:
+  # [avoid issue when Github action upgraded from 18.04 default to 20.04](https://github.com/aws/aws-cli/issues/5262)
+  AWS_EC2_METADATA_DISABLED: true
   REPO_NAME: magda-minion-format
+  # Github account username (used for access github registry)
+  GH_USERNAME: magdabot
+  # Github Orgnisation name or user name for this repo
+  GH_ORGNAME: magda-io
+  #Docker Hub username
+  DH_USERNAME: magdabot
+  #S3 bucket name: this s3 bucket will be used to store published helm chart and index
+  S3_BUCKET: magda-charts
 
 jobs:
   release-helm-chart:
@@ -22,49 +32,47 @@ jobs:
       - run: yarn build
       - run: yarn test
 
-      - name: Setup Helm
-        uses: azure/setup-helm@v1
-        with:
-          version: v3.2.0
+      - name: helm-build-dependencies
+        run: helm dep up deploy/${REPO_NAME}
+
+      - name: helm-check
+        run: yarn helm-lint
 
-      - run: yarn helm-lint
+      - name: helm-chart-version-check
+        run: yarn check-helm-chart-version deploy/${REPO_NAME}/Chart.yaml
 
       - name: Login to GitHub Package Repository
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_ACCESS_TOKEN }}
-        run: docker login docker.pkg.github.com -u magdabot -p ${GH_TOKEN}
+          GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
+        run: docker login docker.pkg.github.com -u ${GH_USERNAME} -p ${GH_TOKEN}
 
       - name: Build Docker Image & Push to Github Registry
-        run: yarn docker-build-prod --repository=docker.pkg.github.com/magda-io/${REPO_NAME} --name=${REPO_NAME}
+        run: yarn docker-build-prod --repository=docker.pkg.github.com/${GH_ORGNAME}/${REPO_NAME} --name=${REPO_NAME}
 
       - name: Login to Docker Hub
         env:
           DH_TOKEN: ${{ secrets.DOCKER_HUB_PASSWORD }}
-        run: docker login -u magdabot -p ${DH_TOKEN}
+        run: docker login -u ${DH_USERNAME} -p ${DH_TOKEN}
 
       - name: Re-tag & Push Docker Image to Docker Hub
-        run: yarn retag-and-push --fromPrefix=docker.pkg.github.com/magda-io/${REPO_NAME}/ --fromName=${REPO_NAME}
+        run: yarn retag-and-push --fromPrefix=docker.pkg.github.com/${GH_ORGNAME}/${REPO_NAME}/ --fromName=${REPO_NAME}
 
       - name: Configure Git
         run: |
           git config user.name "$GITHUB_ACTOR"
           git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
-
-      - name: Setup AWS CLi Tools
-        uses: chrislennon/action-aws-cli@v1.1
-
       - name: Release Helm Chart
         env:
-          CR_TOKEN: "${{ secrets.GITHUB_ACCESS_TOKEN }}"
+          CR_TOKEN: "${{ secrets.GH_ACCESS_TOKEN }}"
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
           cd deploy
           mkdir -p sync_dir
           mkdir -p index_dir
-          if ! aws s3 cp s3://magda-charts/index.yaml index_dir/index.yaml; then echo "failed to copy current repo index" && exit 1; fi
+          if ! aws s3 cp s3://${S3_BUCKET}/index.yaml index_dir/index.yaml; then echo "failed to copy current repo index" && exit 1; fi
           helm package -d sync_dir ${REPO_NAME}
           helm repo index --merge "index_dir/index.yaml" sync_dir
           mv -f sync_dir/index.yaml index_dir/index.yaml
-          aws s3 sync sync_dir s3://magda-charts/ --acl public-read
-          aws s3 cp index_dir/index.yaml s3://magda-charts/index.yaml --acl public-read
+          aws s3 sync sync_dir s3://${S3_BUCKET}/
+          aws s3 cp index_dir/index.yaml s3://${S3_BUCKET}/index.yaml
diff --git a/CHANGES.md b/CHANGES.md
@@ -0,0 +1,11 @@
+> The repo was part of [magda main repo](https://github.com/magda-io/magda). For history before v1.0.0, please check [CHANGES.md of main repo](https://github.com/magda-io/magda/blob/master/CHANGES.md).
+
+# 1.0.0
+
+-   #8 Better GeoSpatial API format categorising
+-   #11 Use format info in original metadata if it is "CSV-GEO-AU"
+-   #12 Recognise zipped GeoTiff file as GeoTiff format
+-   Increase default memory to 50mb
+-   Use Chart.version as default image version
+-   Upgrade CI scripts
+-   Make ArcGIS FeatureServer format as FEATURESERVER
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:10
+FROM node:12
 
 RUN mkdir -p /usr/src/app
 COPY . /usr/src/app

diff --git a/README.md b/README.md
@@ -8,6 +8,14 @@ Other aspects exist that are written to by many minions - for instance, we have
 
 This magda minion can generate more accurate distribution format information.
 
+### Docker Image
+
+Docker image releases can be found from Docker Hub:
+
+https://hub.docker.com/r/data61/magda-minion-format/
+
+Development releases (per commit) are also available from [GitHub Registry](https://github.com/magda-io/magda-minion-format/packages) and accessible with access token.
+
 ### Helm Chart
 
 It's recommanded to deploy minions with as [dependencies](https://helm.sh/docs/topics/chart_best_practices/dependencies/) of a Magda helm deployment. Example can be found from [here](https://github.com/magda-io/magda-config).
@@ -22,10 +30,22 @@ The [helm chart](https://helm.sh/docs/topics/charts/) for this minion is auto re
 helm repo add magda-io https://charts.magda.io
 ```
 
-### Docker Image
-
-Docker image releases can be found from Docker Hub:
-
-https://hub.docker.com/r/data61/magda-minion-format/
-
-Development releases (per commit) are also available from [GitHub Registry](https://github.com/magda-io/magda-minion-format/packages) and accessible with access token.
+## Requirements
+
+Kubernetes: `>= 1.14.0-0`
+
+## Values
+
+| Key                          | Type   | Default                                  | Description |
+| ---------------------------- | ------ | ---------------------------------------- | ----------- |
+| defaultAdminUserId           | string | `"00000000-0000-4000-8000-000000000000"` |             |
+| defaultImage.imagePullSecret | bool   | `false`                                  |             |
+| defaultImage.pullPolicy      | string | `"IfNotPresent"`                         |             |
+| defaultImage.repository      | string | `"docker.io/data61"`                     |             |
+| global.image                 | object | `{}`                                     |             |
+| global.minions.image         | object | `{}`                                     |             |
+| global.rollingUpdate         | object | `{}`                                     |             |
+| image.name                   | string | `"magda-minion-format"`                  |             |
+| resources.limits.cpu         | string | `"100m"`                                 |             |
+| resources.requests.cpu       | string | `"10m"`                                  |             |
+| resources.requests.memory    | string | `"50Mi"`                                 |             |
diff --git a/README.md.gotmpl b/README.md.gotmpl
@@ -0,0 +1,40 @@
+## Magda Format Minion
+
+![CI Workflow](https://github.com/magda-io/magda-minion-format/workflows/Main%20CI%20Workflow/badge.svg?branch=master) [![Release](https://img.shields.io/github/release/magda-io/magda-minion-format.svg)](https://github.com/magda-io/magda-minion-format/releases)
+
+A [Magda](https://github.com/magda-io/magda) minion is a service that listens for new records or changes to existing records, performs some kind of operation and then writes the result back to the registry. For instance, we have a broken link minion that listens for changes to distributions, retrieves the URLs described, records whether they were able to be accessed successfully and then writes that back to the registry in its own aspect.
+
+Other aspects exist that are written to by many minions - for instance, we have a "quality" aspect that contains a number of different quality ratings from different sources, which are averaged out and used by search.
+
+This magda minion can generate more accurate distribution format information.
+
+
+### Docker Image
+
+Docker image releases can be found from Docker Hub:
+
+https://hub.docker.com/r/data61/magda-minion-format/
+
+Development releases (per commit) are also available from [GitHub Registry](https://github.com/magda-io/magda-minion-format/packages) and accessible with access token.
+
+### Helm Chart
+
+It's recommanded to deploy minions with as [dependencies](https://helm.sh/docs/topics/chart_best_practices/dependencies/) of a Magda helm deployment. Example can be found from [here](https://github.com/magda-io/magda-config).
+
+-   Magda Helm Charts Repository Url: https://charts.magda.io
+
+The [helm chart](https://helm.sh/docs/topics/charts/) for this minion is auto released when a [Github Release](https://help.github.com/en/github/administering-a-repository/creating-releases) is created for this repo.
+
+-   Add repository to helm:
+
+```bash
+helm repo add magda-io https://charts.magda.io
+```
+
+{{ template "chart.maintainersSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+{{ template "chart.valuesHeader" . }}
+
+{{ template "chart.valuesTable" . }}
diff --git a/deploy/magda-minion-format/Chart.yaml b/deploy/magda-minion-format/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: magda-minion-format
 description: A Helm chart for Magda Format Minion
-version: "0.0.57-0"
+version: "1.0.0"
 kubeVersion: ">= 1.14.0-0"
 home: "https://github.com/magda-io/magda-minion-format"
 sources: ["https://github.com/magda-io/magda-minion-format"]

diff --git a/deploy/magda-minion-format/templates/deployment.yaml b/deploy/magda-minion-format/templates/deployment.yaml
@@ -25,7 +25,7 @@ See chart value file for details of the logic used to generate this setting valu
       {{- end }}
       containers:
       - name: minion-format
-        image: "{{ .Values.image.repository | default .Values.global.minions.image.repository | default .Values.global.image.repository | default .Values.defaultImage.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Values.global.minions.image.tag | default .Values.global.image.tag | default .Values.defaultImage.tag }}"
+        image: "{{ .Values.image.repository | default .Values.global.minions.image.repository | default .Values.global.image.repository | default .Values.defaultImage.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Values.global.minions.image.tag | default .Values.global.image.tag | default .Chart.Version }}"
         imagePullPolicy: {{ .Values.image.pullPolicy | default .Values.global.minions.image.pullPolicy | default .Values.global.image.pullPolicy | default .Values.defaultImage.pullPolicy }}
 {{- if .Values.global.enableLivenessProbes }}
         livenessProbe:

diff --git a/deploy/magda-minion-format/values.yaml b/deploy/magda-minion-format/values.yaml
@@ -20,7 +20,6 @@ image:
 
 defaultImage:
   repository: docker.io/data61
-  tag: 0.0.57-0
   pullPolicy: IfNotPresent
   imagePullSecret: false
 
@@ -29,6 +28,6 @@ defaultAdminUserId: "00000000-0000-4000-8000-000000000000"
 resources:
   requests:
     cpu: 10m
-    memory: 30Mi
+    memory: 50Mi
   limits:
     cpu: 100m
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@magda/minion-format",
   "description": "MAGDA Layering Minion",
-  "version": "0.0.57-0",
+  "version": "1.0.0",
   "scripts": {
     "prebuild": "rimraf dist tsconfig.tsbuildinfo",
     "build": "yarn run compile",
@@ -13,46 +13,48 @@
     "docker-build-prod": "create-docker-context-for-node-component --build --push --tag auto",
     "test": "mocha --require ts-node/register --require tsconfig-paths/register \"src/test/**/*.spec.ts\"",
     "helm-lint": "helm lint deploy/magda-minion-format -f deploy/test-deploy.yaml",
+    "helm-docs": "helm-docs -t ./README.md.gotmpl -o ../../README.md",
+    "update-all-charts": "helm dep up ./deploy/magda-minion-format",
+    "add-all-chart-version-changes": "git ls-files -m | grep Chart.yaml | xargs git add && git ls-files -m | grep Chart.lock | xargs git add",
+    "add-all-helm-docs-changes": "yarn helm-docs && git ls-files -m | grep -i readme.md | xargs git add",
+    "version": "yarn update-helm-chart-version && yarn update-all-charts && yarn add-all-chart-version-changes && yarn add-all-helm-docs-changes",
     "retag-and-push": "retag-and-push"
   },
   "author": "",
   "license": "Apache-2.0",
   "devDependencies": {
-    "@magda/scripts": "^0.0.57-0",
+    "@magda/ci-utils": "^1.0.2",
+    "@magda/docker-utils": "^0.0.60-alpha.15",
     "@types/chai": "^4.2.8",
-    "@types/ftp": "^0.3.29",
+    "@types/express": "4.17.9",
     "@types/lodash": "^4.14.88",
-    "@types/lru-cache": "4.0.0",
     "@types/mime-types": "^2.1.0",
     "@types/mocha": "^7.0.1",
-    "@types/request": "^2.48.1",
     "@types/sinon": "^7.5.1",
-    "@types/urijs": "^1.15.34",
+    "chai": "^4.2.0",
     "husky": "^3.1.0",
+    "mocha": "^7.0.1",
+    "nock": "^11.7.2",
     "prettier": "^1.19.1",
     "pretty-quick": "^2.0.1",
     "rimraf": "^3.0.0",
-    "chai": "^4.2.0",
-    "mocha": "^7.0.1",
-    "nock": "^11.7.2",
-    "jsverify": "^0.8.2",
     "sinon": "^8.1.1",
-    "typescript": "^3.7.2"
+    "ts-node": "^8.10.2",
+    "tsconfig-paths": "^3.9.0",
+    "typescript": "^3.9.5"
   },
   "dependencies": {
-    "@magda/minion-sdk": "^0.0.57-0",
-    "@magda/registry-aspects": "^0.0.57-0",
-    "@magda/utils": "^0.0.57-0",
+    "@magda/minion-sdk": "^0.0.60-alpha.15",
+    "@magda/registry-aspects": "^0.0.60-alpha.15",
+    "@magda/utils": "^0.0.60-alpha.15",
     "@types/escape-string-regexp": "^0.0.32",
     "@types/lodash": "^4.14.88",
     "escape-string-regexp": "^1.0.5",
-    "ftp": "^0.3.10",
-    "load-json-file": "^4.0.0",
     "lodash": "^4.17.4",
-    "lru-cache": "4.0.2",
-    "mime-types": "^1.0.0",
-    "request": "^2.88.0",
-    "urijs": "^1.18.12"
+    "mime-types": "^2.1.31"
+  },
+  "resolutions": {
+    "@types/express": "4.17.9"
   },
   "config": {
     "registryUrl": "http://localhost:6101/v0",

diff --git a/src/format-engine/measureEvaluatorByHierarchy.ts b/src/format-engine/measureEvaluatorByHierarchy.ts
@@ -43,6 +43,19 @@ export default function getBestMeasureResult(
             ) {
                 finalCandidate = dcatSet;
             }
+
+            // if format get from original metadata is "WMS" or "WFS", we should trust it
+            if (
+                ["WMS", "WFS"].indexOf(dcatFormat) !== -1 &&
+                dcatFormat !== sortedFormat
+            ) {
+                finalCandidate = dcatSet;
+            }
+
+            // if format get from original metadata is "CSV-GEO-AU", we should trust it
+            if (dcatFormat === "CSV-GEO-AU" && dcatFormat !== sortedFormat) {
+                finalCandidate = dcatSet;
+            }
         }
         return {
             format: finalCandidate.measureResult.formats[0],