Merge remote-tracking branch 'origin/master' into issue/1923-wrong-po…

…stgres-password-secret-key

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,6 +2,8 @@
 
 Fixes #<insert issue number here if relevant>
 
+This is a description of what I've done in this PR, especially explaining choices made to make the reviewer's job easier. If I don't replace this paragraph with an actual description, my PR will probably be ignored.
+
 ### Checklist
 
 -   [ ] There are unit tests to verify my changes are correct or unit tests aren't applicable

.gitignore

@@ -35,3 +35,4 @@ dist/
 pancake
 package-lock.json
 */package-lock.json
+.nyc_*

CHANGES.md

@@ -1,6 +1,22 @@
 ## 0.0.51
 
 -   Changed the migrators to look for "password" instead of "postgres-password" in the cloudsql-db-credentials secret.
+-   Add readiness and liveness probes to all services
+-   Got screen reader to say "Open Data Quality: 3/5 stars" instead of repeating star rating text
+-   Redirect HTTP requests to HTTPS URLs
+-   Fixed chart won't displayed correctly under IE11
+-   Added boosting to results which have location information that overlaps a known region whose name/shortname appears in query text.
+-   Fixed stream processing issue when re-indexing, mitigating out-of-memory risk too
+-   Fixed index trimming failure issue when re-indexing
+-   Fixed form error message not being heard in order by screen readers and also when field is in focus by putting them in the same label element.
+-   Fixed: indexer throws an error when processes spatial data number with more than one decimal places
+-   Made the image and the text wrapped in a single link and the set image alt value to null.
+-   Fixed the image and the text should be wrapped in a single link and the image alt text should have a null value.chart won't displayed correctly under IE11
+-   Fixed stream processing issue when re-indexing, mitigating out-of-memory risk too
+-   Fixed index trimming failure issue when re-indexing
+-   Fixed: indexer throws an error when processes spatial data number with more than one decimal places
+-   Fixed: Error messages are not associated with their form fields on suggest dataset form
+-   Added shortcut to build create secrets script for magda-config repo
 
 ## 0.0.50
 
@@ -55,7 +71,7 @@
 -   `create-secrets` will load ENV vars according to question data types
 -   Include Magda user agent in external HTTP resource accesses
 -   Made admin UI create CSV connector with internal URL
--   Fixed magda-apidocs-server incorrectly builds into $PWD directory on windows
+-   Fixed magda-apidocs-server incorrectly builds into \$PWD directory on windows
 -   Fixed an issue that DAP connector not handle access error correctly
 -   Stopped caching anything requested through the admin ui.
 -   Removed old admin ui code

deploy/helm/magda-dev.yml

@@ -25,6 +25,7 @@ gateway:
   ckanRedirectionDomain: "ckan.data.gov.au"
   ckanRedirectionPath: ""
   enableAuthEndpoint: true
+  enableHttpsRedirection: true
   auth:
     facebookClientId: "173073926555600"
     googleClientId: "275237095477-f7ej2gsvbl2alb8bcqcn7r5jk0ur719p.apps.googleusercontent.com"

deploy/helm/magda/charts/admin-api/templates/deployment.yaml

@@ -31,10 +31,17 @@ spec:
 {{- if .Values.global.enableLivenessProbes }}
         livenessProbe:
           httpGet:
-            path: /v0/healthz
+            path: /v0/status/live
             port: 80
-          initialDelaySeconds: 30
-          periodSeconds: 60
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /v0/status/ready
+            port: 80
+          initialDelaySeconds: 10
+          periodSeconds: 10
           timeoutSeconds: 10
 {{- end }}
         resources:

deploy/helm/magda/charts/authorization-api/templates/deployment.yaml

@@ -26,7 +26,14 @@ spec:
 {{- if .Values.global.enableLivenessProbes }}
         livenessProbe:
           httpGet:
-            path: /v0/public/users/{{ .Values.global.defaultAdminUserId }}
+            path: /v0/status/live
+            port: 80
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /v0/status/ready
             port: 80
           initialDelaySeconds: 10
           periodSeconds: 10

deploy/helm/magda/charts/content-api/templates/deployment.yaml

@@ -5,6 +5,7 @@ metadata:
 spec:
   replicas: {{ .Values.replicas | default 1 }}
   strategy:
+    type: RollingUpdate
     rollingUpdate:
       maxUnavailable: {{ .Values.global.rollingUpdate.maxUnavailable | default 0 }}
   template:
@@ -27,7 +28,14 @@ spec:
 {{- if .Values.global.enableLivenessProbes }}
         livenessProbe:
           httpGet:
-            path: /v0/healthz
+            path: /v0/status/live
+            port: 80
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /v0/status/ready
             port: 80
           initialDelaySeconds: 10
           periodSeconds: 10

deploy/helm/magda/charts/correspondence-api/templates/deployment.yaml

@@ -31,9 +31,16 @@ spec:
         ]
         livenessProbe:
           httpGet:
-            path: /v0/healthz
+            path: /v0/status/live
             port: 80
-          initialDelaySeconds: 60
+          initialDelaySeconds: 10
+          periodSeconds: 60
+          timeoutSeconds: 30
+        readinessProbe:
+          httpGet:
+            path: /v0/status/ready
+            port: 80
+          initialDelaySeconds: 10
           periodSeconds: 60
           timeoutSeconds: 30
         resources:
@@ -57,4 +64,4 @@ spec:
           valueFrom:
             secretKeyRef:
               name: auth-secrets
-              key: jwt-secret
+              key: jwt-secret

deploy/helm/magda/charts/gateway/templates/deployment.yaml

@@ -41,6 +41,9 @@ spec:
 {{- end }}
 {{- if .Values.enableWebAccessControl }}
             "--enableWebAccessControl", {{ .Values.enableWebAccessControl | quote }},
+{{- end }}
+{{- if .Values.enableHttpsRedirection }}
+            "--enableHttpsRedirection", {{ .Values.enableHttpsRedirection | quote }},
 {{- end }}
             "--proxyRoutesJson", "/etc/config/routes.json",
             "--helmetJson", "/etc/config/helmet.json",
@@ -54,17 +57,17 @@ spec:
         volumeMounts:
           - name: config
             mountPath: /etc/config
-        readinessProbe:
+{{- if .Values.global.enableLivenessProbes }}
+        livenessProbe:
           httpGet:
-            path: /v0/healthz
+            path: /status/live
             port: 80
           initialDelaySeconds: 10
           periodSeconds: 10
           timeoutSeconds: 10
-{{- if .Values.global.enableLivenessProbes }}
-        livenessProbe:
+        readinessProbe:
           httpGet:
-            path: /v0/healthz
+            path: /status/live
             port: 80
           initialDelaySeconds: 10
           periodSeconds: 10

deploy/helm/magda/charts/registry-api/templates/_helpers.tpl

@@ -51,14 +51,21 @@ spec:
 {{- if .root.Values.global.enableLivenessProbes }}
         livenessProbe:
           httpGet:
-            path: "/v0/ping"
+            path: /v0/status/live
             port: 80
           initialDelaySeconds: 60
           periodSeconds: 10
           timeoutSeconds: {{ .root.Values.livenessProbe.timeoutSeconds | default 10 }}
+        readinessProbe:
+          httpGet:
+            path: /v0/status/ready
+            port: 80
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 10
 {{- end }}
         ports:
         - containerPort: 80
         resources:
 {{ .deploymentConfig.resources | default .root.Values.resources | toYaml | indent 10 }}
-{{- end -}}
+{{- end -}}

deploy/helm/magda/charts/search-api/templates/deployment.yaml

@@ -27,19 +27,19 @@ spec:
 {{- if .Values.regionsIndexVersion }}
           - "-DelasticSearch.indices.regions.version={{ .Values.regionsIndexVersion }}"
 {{- end }}
-        readinessProbe:
+{{- if .Values.global.enableLivenessProbes }}
+        livenessProbe:
           httpGet:
-            path: "/v0/datasets"
+            path: /v0/status/live
             port: 80
-          initialDelaySeconds: 10
+          initialDelaySeconds: 60
           periodSeconds: 10
           timeoutSeconds: 10
-{{- if .Values.global.enableLivenessProbes }}
-        livenessProbe:
+        readinessProbe:
           httpGet:
-            path: "/v0/datasets"
+            path: /v0/status/ready
             port: 80
-          initialDelaySeconds: 60
+          initialDelaySeconds: 10
           periodSeconds: 10
           timeoutSeconds: 10
 {{- end }}

deploy/helm/preview.yml

@@ -21,6 +21,7 @@ gateway:
   autoscaler:
     enabled: false
   enableAuthEndpoint: true
+  enableHttpsRedirection: true
   ckanRedirectionDomain: "ckan.data.gov.au"
   ckanRedirectionPath: ""
   helmet:

magda-admin-api/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:6
+FROM node:10
 
 RUN mkdir -p /usr/src/app
 COPY . /usr/src/app

magda-admin-api/package.json

@@ -9,7 +9,7 @@
     "dev": "run-typescript-in-nodemon src/index.ts",
     "docker-build-local": "create-docker-context-for-node-component --build --push --tag auto --local",
     "docker-build-prod": "create-docker-context-for-node-component --build --push --tag auto",
-    "test": "mocha --compilers ts:ts-node/register,tsx:ts-node/register --require tsconfig-paths/register \"src/test/**/*.spec.ts\"",
+    "test": "nyc mocha --compilers ts:ts-node/register,tsx:ts-node/register --require tsconfig-paths/register \"src/test/**/*.spec.ts\"",
     "retag-and-push": "retag-and-push"
   },
   "dependencies": {
@@ -21,7 +21,8 @@
     "kubernetes-client": "3.17.2",
     "lodash": "^4.17.4",
     "util.promisify": "^1.0.0",
-    "yargs": "^8.0.2"
+    "yargs": "^8.0.2",
+    "nyc": "^13.1.0"
   },
   "devDependencies": {
     "@magda/scripts": "^0.0.51-0",
@@ -58,5 +59,13 @@
     "categories": {
       "api": true
     }
+  },
+  "nyc": {
+    "extension": [
+      ".ts"
+    ],
+    "exclude": [
+      "src/test/**"
+    ]
   }
 }

magda-admin-api/src/buildApiRouter.ts

@@ -5,6 +5,11 @@ import * as _ from "lodash";
 import K8SApi, { K8SApiType } from "./k8sApi";
 import { mustBeAdmin } from "@magda/typescript-common/dist/authorization-api/authMiddleware";
 
+import {
+    installStatusRouter,
+    createServiceProbe
+} from "@magda/typescript-common/dist/express/status";
+
 export interface Options {
     dockerRepo: string;
     authApiUrl: string;
@@ -23,14 +28,18 @@ export default function buildApiRouter(options: Options) {
 
     const k8sApi = new K8SApi(options.kubernetesApiType, options.namespace);
 
-    router.get("/healthz", (req, res) => {
-        k8sApi
-            .getJobs()
-            .then(() => {
-                res.status(200).send("OK");
-            })
-            .catch(() => res.status(500).send("Error"));
-    });
+    const status = {
+        probes: {
+            k8s: async () => {
+                await k8sApi.getJobs();
+                return {
+                    ready: true
+                };
+            },
+            auth: createServiceProbe(options.authApiUrl)
+        }
+    };
+    installStatusRouter(router, status);
 
     router.use(mustBeAdmin(options.authApiUrl, options.jwtSecret));
 

magda-admin-api/tsconfig-build.json

@@ -1,6 +1,7 @@
 {
     "extends": "../tsconfig-global.json",
     "compilerOptions": {
+        "lib": ["es2017", "dom"],
         "declaration": true,
         "outDir": "dist",
         "baseUrl": "."

magda-apidocs-server/default.conf

@@ -3,6 +3,24 @@ server {
     server_name  localhost;
     add_header Content-Security-Policy "default-src * 'unsafe-inline' 'unsafe-eval'; script-src * 'unsafe-inline' 'unsafe-eval'; connect-src * 'unsafe-inline'; img-src * data: blob: 'unsafe-inline'; frame-src *; style-src * 'unsafe-inline';";
 
+    location /healthz {
+      access_log off;
+      default_type text/plain;
+      return 200 "OK\n";
+    }
+
+    location /status/live {
+      access_log off;
+      default_type text/plain;
+      return 200 "OK\n";
+    }
+
+    location /status/ready {
+      access_log off;
+      default_type application/json;
+      return 200 "{\"ready\": true}";
+    }
+
     location / {
         root   /usr/share/nginx/html;
         index  index.html index.htm;

magda-authorization-api/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:latest
+FROM node:10
 
 RUN mkdir -p /usr/src/app
 COPY . /usr/src/app

magda-authorization-api/package.json

@@ -7,7 +7,7 @@
     "watch": "tsc -p tsconfig-build.json --watch",
     "start": "node dist/index.js",
     "dev": "run-typescript-in-nodemon src/index.ts",
-    "test": "mocha --compilers ts:ts-node/register,tsx:ts-node/register --require tsconfig-paths/register \"src/test/**/*.spec.ts\"",
+    "test": "nyc mocha --compilers ts:ts-node/register,tsx:ts-node/register --require tsconfig-paths/register \"src/test/**/*.spec.ts\"",
     "docker-build-local": "create-docker-context-for-node-component --build --push --tag auto --local",
     "docker-build-prod": "create-docker-context-for-node-component --build --push --tag auto",
     "retag-and-push": "retag-and-push"
@@ -37,6 +37,7 @@
     "lodash": "^4.17.4",
     "mocha": "^3.5.3",
     "nock": "^9.1.6",
+    "nyc": "^13.1.0",
     "sinon": "^4.2.1",
     "supertest": "^3.0.0",
     "typescript": "~2.5.0"
@@ -54,5 +55,13 @@
     "categories": {
       "api": true
     }
+  },
+  "nyc": {
+    "extension": [
+      ".ts"
+    ],
+    "exclude": [
+      "src/test/**"
+    ]
   }
 }

magda-authorization-api/src/Database.ts

@@ -67,4 +67,11 @@ export default class Database {
             )
             .then(result => result.rows[0]);
     }
+
+    async check(): Promise<any> {
+        await this.pool.query("SELECT id FROM users LIMIT 1");
+        return {
+            ready: true
+        };
+    }
 }