Merge pull request #14 from magda-io/use-magda-common

Use magda common
magda-io · Oct 11, 2021 · 15b71d5 · 15b71d5
2 parents c932e59 + 9873d86
commit 15b71d5
Show file tree

Hide file tree

Showing 16 changed files with 1,239 additions and 2,215 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -11,38 +11,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Use Node.js 10
+      - name: Use Node.js 12
         uses: actions/setup-node@v1
         with:
-          node-version: 10
+          node-version: 12
       - run: yarn install
       - run: yarn build
       - run: yarn test
-
-      - name: Setup Helm
-        uses: azure/setup-helm@v1
-        with:
-          version: v3.2.0
 
-      - name: helm-check
-        run: yarn helm-lint
+      - name: helm-build-dependencies
+        run: helm dep up deploy/${REPO_NAME}
+
+      - run: yarn helm-lint
 
       - name: Login to GitHub Package Repository
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_ACCESS_TOKEN }}
+          GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
         run: docker login docker.pkg.github.com -u magdabot -p ${GH_TOKEN}
       - name: Build Docker Image & Push
         run: yarn docker-build-prod --repository=docker.pkg.github.com/magda-io/${REPO_NAME} --name=${REPO_NAME} --version=${GITHUB_SHA}
-
-  build-test-node-8:
-    name: Build, Test Only with Node 12
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Use Node.js 12
-        uses: actions/setup-node@v1
-        with:
-          node-version: 12
-      - run: yarn install
-      - run: yarn build
-      - run: yarn test
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -5,7 +5,17 @@ on:
     types: [published]
 
 env:
+  # [avoid issue when Github action upgraded from 18.04 default to 20.04](https://github.com/aws/aws-cli/issues/5262)
+  AWS_EC2_METADATA_DISABLED: true
   REPO_NAME: magda-dap-connector
+  # Github account username (used for access github registry)
+  GH_USERNAME: magdabot
+  # Github Orgnisation name or user name for this repo
+  GH_ORGNAME: magda-io
+  #Docker Hub username
+  DH_USERNAME: magdabot
+  #S3 bucket name: this s3 bucket will be used to store published helm chart and index
+  S3_BUCKET: magda-charts
 
 jobs:
   release-helm-chart:
@@ -21,51 +31,49 @@ jobs:
       - run: yarn install
       - run: yarn build
       - run: yarn test
-
-      - name: Setup Helm
-        uses: azure/setup-helm@v1
-        with:
-          version: v3.2.0
+
+      - name: helm-build-dependencies
+        run: helm dep up deploy/${REPO_NAME}
 
       - name: helm-check
         run: yarn helm-lint
 
+      - name: helm-chart-version-check
+        run: yarn check-helm-chart-version deploy/${REPO_NAME}/Chart.yaml
+
       - name: Login to GitHub Package Repository
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_ACCESS_TOKEN }}
-        run: docker login docker.pkg.github.com -u magdabot -p ${GH_TOKEN}
+          GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
+        run: docker login docker.pkg.github.com -u ${GH_USERNAME} -p ${GH_TOKEN}
 
       - name: Build Docker Image & Push to Github Registry
-        run: yarn docker-build-prod --repository=docker.pkg.github.com/magda-io/${REPO_NAME} --name=${REPO_NAME}
+        run: yarn docker-build-prod --repository=docker.pkg.github.com/${GH_ORGNAME}/${REPO_NAME} --name=${REPO_NAME}
 
       - name: Login to Docker Hub
         env:
           DH_TOKEN: ${{ secrets.DOCKER_HUB_PASSWORD }}
-        run: docker login -u magdabot -p ${DH_TOKEN}
+        run: docker login -u ${DH_USERNAME} -p ${DH_TOKEN}
 
       - name: Re-tag & Push Docker Image to Docker Hub
-        run: yarn retag-and-push --fromPrefix=docker.pkg.github.com/magda-io/${REPO_NAME}/ --fromName=${REPO_NAME}
+        run: yarn retag-and-push --fromPrefix=docker.pkg.github.com/${GH_ORGNAME}/${REPO_NAME}/ --fromName=${REPO_NAME}
 
       - name: Configure Git
         run: |
           git config user.name "$GITHUB_ACTOR"
           git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
 
-      - name: Setup AWS CLi Tools
-        uses: chrislennon/action-aws-cli@v1.1
-
       - name: Release Helm Chart
         env:
-          CR_TOKEN: "${{ secrets.GITHUB_ACCESS_TOKEN }}"
+          CR_TOKEN: "${{ secrets.GH_ACCESS_TOKEN }}"
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
           cd deploy
           mkdir -p sync_dir
           mkdir -p index_dir
-          if ! aws s3 cp s3://magda-charts/index.yaml index_dir/index.yaml; then echo "failed to copy current repo index" && exit 1; fi
+          if ! aws s3 cp s3://${S3_BUCKET}/index.yaml index_dir/index.yaml; then echo "failed to copy current repo index" && exit 1; fi
           helm package -d sync_dir ${REPO_NAME}
           helm repo index --merge "index_dir/index.yaml" sync_dir
           mv -f sync_dir/index.yaml index_dir/index.yaml
-          aws s3 sync sync_dir s3://magda-charts/ --acl public-read
-          aws s3 cp index_dir/index.yaml s3://magda-charts/index.yaml --acl public-read
+          aws s3 sync sync_dir s3://${S3_BUCKET}/
+          aws s3 cp index_dir/index.yaml s3://${S3_BUCKET}/index.yaml
diff --git a/.gitignore b/.gitignore
@@ -50,3 +50,5 @@ magda-tenant-api/lib/
 
 #typescript
 tsconfig.tsbuildinfo
+
+deploy/*/charts
diff --git a/.helmdocsignore b/.helmdocsignore
@@ -0,0 +1 @@
+deploy/*/charts
diff --git a/CHANGES.md b/CHANGES.md
@@ -0,0 +1,5 @@
+# 1.0.0
+
+-   Upgrade dependencies
+-   Upgrade CI scripts
+-   Related to https://github.com/magda-io/magda/issues/3229, Use magda-common for docker image related logic
diff --git a/README.md b/README.md
@@ -26,4 +26,33 @@ Docker image releases can be found from Docker Hub:
 
 https://hub.docker.com/r/data61/magda-dap-connector/
 
-Development releases (per commit) are also available from [GitHub Registry](https://github.com/magda-io/magda-dap-connector/packages) and accessible with access token.
+Development releases (per commit) are also available from [GitHub Registry](https://github.com/magda-io/magda-dap-connector/packages) and accessible with access token.
+
+## Requirements
+
+Kubernetes: `>= 1.14.0-0`
+
+| Repository              | Name         | Version       |
+| ----------------------- | ------------ | ------------- |
+| https://charts.magda.io | magda-common | 1.0.0-alpha.4 |
+
+## Values
+
+| Key                                | Type   | Default                              | Description |
+| ---------------------------------- | ------ | ------------------------------------ | ----------- |
+| config.id                          | string | `"dap"`                              |             |
+| config.name                        | string | `"CSIRO"`                            |             |
+| config.pageSize                    | int    | `100`                                |             |
+| config.sourceUrl                   | string | `"https://data.csiro.au/dap/ws/v2/"` |             |
+| defaultImage.imagePullSecret       | bool   | `false`                              |             |
+| defaultImage.pullPolicy            | string | `"IfNotPresent"`                     |             |
+| defaultImage.repository            | string | `"docker.io/data61"`                 |             |
+| defaultSettings.includeCronJobs    | bool   | `true`                               |             |
+| defaultSettings.includeInitialJobs | bool   | `false`                              |             |
+| defaultTenantId                    | int    | `0`                                  |             |
+| global.connectors.image            | object | `{}`                                 |             |
+| global.image                       | object | `{}`                                 |             |
+| image.name                         | string | `"magda-dap-connector"`              |             |
+| resources.limits.cpu               | string | `"100m"`                             |             |
+| resources.requests.cpu             | string | `"50m"`                              |             |
+| resources.requests.memory          | string | `"30Mi"`                             |             |
diff --git a/README.md.gotmpl b/README.md.gotmpl
@@ -0,0 +1,37 @@
+## Magda DAP Connector
+
+![CI Workflow](https://github.com/magda-io/magda-dap-connector/workflows/Main%20CI%20Workflow/badge.svg?branch=master) [![Release](https://img.shields.io/github/release/magda-io/magda-dap-connector.svg)](https://github.com/magda-io/magda-dap-connector/releases)
+
+[Magda](https://github.com/magda-io/magda) connectors go out to external datasources and copy their metadata into the Registry, so that they can be searched and have other aspects attached to them. A connector is simply a docker-based microservice that is invoked as a job. It scans the target datasource (usually an open-data portal), then completes and shuts down.
+
+Magda dap Connector is created for crawling data from CSIRO Data Access Portal (DAP).
+
+### Helm Chart
+
+It's recommanded to deploy connectors with as [dependencies](https://helm.sh/docs/topics/chart_best_practices/dependencies/) of a Magda helm deployment. Example can be found from [here](https://github.com/magda-io/magda-config).
+
+-   Magda Helm Charts Repository Url: https://charts.magda.io
+
+The [helm chart](https://helm.sh/docs/topics/charts/) for this connector is auto released when a [Github Release](https://help.github.com/en/github/administering-a-repository/creating-releases) is created for this repo.
+
+-   Add repository to helm:
+
+```bash
+helm repo add magda-io https://charts.magda.io
+```
+
+### Docker Image
+
+Docker image releases can be found from Docker Hub:
+
+https://hub.docker.com/r/data61/magda-dap-connector/
+
+Development releases (per commit) are also available from [GitHub Registry](https://github.com/magda-io/magda-dap-connector/packages) and accessible with access token.
+
+{{ template "chart.maintainersSection" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+{{ template "chart.valuesHeader" . }}
+
+{{ template "chart.valuesTable" . }}
diff --git a/deploy/magda-dap-connector/Chart.lock b/deploy/magda-dap-connector/Chart.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: magda-common
+  repository: https://charts.magda.io
+  version: 1.0.0-alpha.4
+digest: sha256:a5dcf2df16ca5a3972f92d91434c2e78be0ce411994d2a3cb89c674a711ddc24
+generated: "2021-10-11T17:26:37.278002+11:00"
diff --git a/deploy/magda-dap-connector/Chart.yaml b/deploy/magda-dap-connector/Chart.yaml
@@ -1,7 +1,13 @@
 apiVersion: v2
 name: magda-dap-connector
 description: A Helm chart for Magda DAP Connector
-version: "0.0.57-0"
+version: "1.0.0"
 kubeVersion: ">= 1.14.0-0"
 home: "https://github.com/magda-io/magda-dap-connector"
-sources: ["https://github.com/magda-io/magda-dap-connector"]
+sources: [ "https://github.com/magda-io/magda-dap-connector" ]
+annotations:
+  magdaModuleType: "connector"
+dependencies:
+  - name: magda-common
+    version: "1.0.0-alpha.4"
+    repository: "https://charts.magda.io"
diff --git a/deploy/magda-dap-connector/templates/cronjobs.yaml b/deploy/magda-dap-connector/templates/cronjobs.yaml
@@ -17,18 +17,11 @@ spec:
         metadata:
           name: connector-{{ .Values.config.id }}
         spec:
-{{- /*
-See chart value file for details of the logic used to generate this setting value below.!!!
-*/}}
-{{- $imagePullSecret := (ne (.Values.image.imagePullSecret | typeOf) "<nil>") | ternary .Values.image.imagePullSecret ( (ne (.Values.global.connectors.image.imagePullSecret | typeOf) "<nil>") | ternary .Values.global.connectors.image.imagePullSecret (  (ne (.Values.global.image.imagePullSecret | typeOf) "<nil>") | ternary .Values.global.image.imagePullSecret .Values.defaultImage.imagePullSecret )  ) -}}
-          {{- if ne ($imagePullSecret | toString) "false" }}
-          imagePullSecrets:
-            - name: {{ $imagePullSecret }}
-          {{- end }}
+          {{- include "magda.imagePullSecrets" . | indent 10 }}
           containers:
             - name: connector-{{ .Values.config.id }}
-              image: "{{ .Values.image.repository | default .Values.global.connectors.image.repository | default .Values.global.image.repository | default .Values.defaultImage.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Values.global.connectors.image.tag | default .Values.global.image.tag | default .Values.defaultImage.tag }}"
-              imagePullPolicy: {{ .Values.image.pullPolicy | default .Values.global.connectors.image.pullPolicy | default .Values.global.image.pullPolicy | default .Values.defaultImage.pullPolicy }}
+              image: {{ include "magda.image" . | quote }}
+              imagePullPolicy: {{ include "magda.imagePullPolicy" . | quote }}
               command:
                 - "node"
                 - "/usr/src/app/component/dist/index.js"

diff --git a/deploy/magda-dap-connector/templates/jobs.yaml b/deploy/magda-dap-connector/templates/jobs.yaml
@@ -13,18 +13,11 @@ spec:
     metadata:
       name: connector-{{ .Values.config.id }}
     spec:
-{{- /*
-See chart value file for details of the logic used to generate this setting value below.!!!
-*/}}
-{{- $imagePullSecret := (ne (.Values.image.imagePullSecret | typeOf) "<nil>") | ternary .Values.image.imagePullSecret ( (ne (.Values.global.connectors.image.imagePullSecret | typeOf) "<nil>") | ternary .Values.global.connectors.image.imagePullSecret (  (ne (.Values.global.image.imagePullSecret | typeOf) "<nil>") | ternary .Values.global.image.imagePullSecret .Values.defaultImage.imagePullSecret )  ) -}}
-      {{- if ne ($imagePullSecret | toString) "false" }}
-      imagePullSecrets:
-        - name: {{ $imagePullSecret }}
-      {{- end }}
+      {{- include "magda.imagePullSecrets" . | indent 6 }}
       containers:
         - name: connector-{{ .Values.config.id }}
-          image: "{{ .Values.image.repository | default .Values.global.connectors.image.repository | default .Values.global.image.repository | default .Values.defaultImage.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Values.global.connectors.image.tag | default .Values.global.image.tag | default .Values.defaultImage.tag }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy | default .Values.global.connectors.image.pullPolicy | default .Values.global.image.pullPolicy | default .Values.defaultImage.pullPolicy }}
+          image: {{ include "magda.image" . | quote }}
+          imagePullPolicy: {{ include "magda.imagePullPolicy" . | quote }}
           command:
             - "node"
             - "/usr/src/app/component/dist/index.js"

diff --git a/deploy/magda-dap-connector/values.yaml b/deploy/magda-dap-connector/values.yaml
@@ -6,7 +6,6 @@ global:
 # image setting loadding order: (from higher priority to lower priority)
 # - Values.image.x
 # - Values.global.connectors.image.x
-# - Values.global.image.x
 # - Values.defaultImage.x
 # You can set `imagePullSecret` value to `false` (bool) to reset the value (see example below)
 
@@ -20,7 +19,6 @@ image:
 
 defaultImage:
   repository: docker.io/data61
-  tag: 0.0.57-0
   pullPolicy: IfNotPresent
   imagePullSecret: false
 

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@magda/dap-connector",
   "description": "MAGDA DAP Connector",
-  "version": "0.0.57-0",
+  "version": "1.0.0",
   "scripts": {
     "prebuild": "rimraf dist tsconfig.tsbuildinfo",
     "build": "yarn run compile",
@@ -15,40 +15,45 @@
     "docker-build-prod": "create-docker-context-for-node-component --build --push --tag auto",
     "retag-and-push": "retag-and-push",
     "helm-lint": "helm lint deploy/magda-dap-connector -f deploy/test-deploy.yaml",
-    "test": "echo \"Test cases to be created for DAP connector\""
+    "test": "echo \"Test cases to be created for DAP connector\"",
+    "helm-docs": "helm-docs -t ./README.md.gotmpl -o ../../README.md",
+    "update-all-charts": "helm dep up ./deploy/magda-dap-connector",
+    "add-all-chart-version-changes": "git ls-files -m | grep Chart.yaml | xargs git add && git ls-files -m | grep Chart.lock | xargs git add",
+    "add-all-helm-docs-changes": "yarn helm-docs && git ls-files -m | grep -i readme.md | xargs git add",
+    "version": "yarn update-helm-chart-version && yarn update-all-charts && yarn add-all-chart-version-changes && yarn add-all-helm-docs-changes"
   },
   "author": "",
   "license": "Apache-2.0",
   "devDependencies": {
-    "@magda/scripts": "^0.0.57-0",
+    "@magda/ci-utils": "^1.0.2",
+    "@magda/connector-test-utils": "^0.0.60",
+    "@magda/docker-utils": "^0.0.60",
+    "@types/chai": "^4.2.8",
     "@types/express": "^4.0.37",
+    "@types/mocha": "^7.0.1",
     "@types/read-pkg-up": "^3.0.1",
     "@types/request": "^2.48.1",
-    "@types/urijs": "^1.15.34",
     "@types/yargs": "^12.0.8",
-    "@types/chai": "^4.2.8",
-    "@types/mocha": "^7.0.1",
-    "husky": "^3.1.0",
+    "babel-core": "^6.26.0",
+    "babel-loader": "^7.1.2",
+    "babel-preset-env": "^1.6.0",
     "chai": "^4.2.0",
+    "husky": "^3.1.0",
     "mocha": "^7.0.1",
     "nock": "^11.7.2",
     "prettier": "^1.19.1",
     "pretty-quick": "^2.0.1",
-    "babel-core": "^6.26.0",
-    "babel-loader": "^7.1.2",
-    "babel-preset-env": "^1.6.0",
     "ts-loader": "^6.2.1",
     "ts-node": "^8.5.2",
     "tsconfig-paths-webpack-plugin": "^3.2.0",
     "typescript": "^3.7.2",
     "webpack": "^4.28.3",
-    "webpack-cli": "^3.3.10",
-    "@magda/connector-test-utils": "^0.0.57-0"
+    "webpack-cli": "^3.3.10"
   },
   "dependencies": {
-    "@magda/connector-sdk": "^0.0.57-0",
-    "@magda/registry-aspects": "^0.0.57-0",
-    "@magda/utils": "^0.0.57-0",
+    "@magda/connector-sdk": "^0.0.60",
+    "@magda/registry-aspects": "^0.0.60",
+    "@magda/utils": "^0.0.60",
     "body-parser": "^1.18.1",
     "express": "^4.15.4",
     "isomorphic-fetch": "^2.2.1",

diff --git a/src/Dap.ts b/src/Dap.ts
@@ -230,7 +230,7 @@ export default class Dap implements ConnectorSource {
 
     // Custom this function following the DAP API specification: https://confluence.csiro.au/display/daphelp/Web+Services+Interface
     private requestPackageSearchPage(
-        url: uri.URI,
+        url: URI,
         fqComponent: string,
         startIndex: number,
         maxResults: number

diff --git a/src/DapUrlBuilder.ts b/src/DapUrlBuilder.ts
@@ -10,8 +10,8 @@ export interface DapUrlBuilderOptions {
 export default class DapUrlBuilder {
     public readonly id: string;
     public readonly name: string;
-    public readonly baseUrl: uri.URI;
-    public readonly apiBaseUrl: uri.URI;
+    public readonly baseUrl: URI;
+    public readonly apiBaseUrl: URI;
 
     constructor(options: DapUrlBuilderOptions) {
         this.id = options.id;