Merge pull request #17 from magda-io/issue/14

Issue/14 & 16
magda-io · Jul 27, 2021 · 5f50526 · 5f50526
2 parents c2999d7 + 7d748ca
commit 5f50526
Show file tree

Hide file tree

Showing 9 changed files with 153 additions and 11 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,10 @@
 > The repo was part of [magda main repo](https://github.com/magda-io/magda). For history before v1.0.0, please check [CHANGES.md of main repo](https://github.com/magda-io/magda/blob/master/CHANGES.md).
 
+# 1.0.1
+
+-   #14 Fixed: TypeError: Cannot read property 'trim' of null when processing invalid metadata
+-   #16 Should recognise ZIP or other format file stored on ESRI portal correctly
+
 # 1.0.0
 
 -   #8 Better GeoSpatial API format categorising

diff --git a/deploy/magda-minion-format/Chart.yaml b/deploy/magda-minion-format/Chart.yaml
@@ -1,8 +1,7 @@
 apiVersion: v2
 name: magda-minion-format
 description: A Helm chart for Magda Format Minion
-version: "1.0.0"
+version: "1.0.1-alpha.0"
 kubeVersion: ">= 1.14.0-0"
 home: "https://github.com/magda-io/magda-minion-format"
-sources: ["https://github.com/magda-io/magda-minion-format"]
-
+sources: [ "https://github.com/magda-io/magda-minion-format" ]
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@magda/minion-format",
   "description": "MAGDA Layering Minion",
-  "version": "1.0.0",
+  "version": "1.0.1-alpha.0",
   "scripts": {
     "prebuild": "rimraf dist tsconfig.tsbuildinfo",
     "build": "yarn run compile",

diff --git a/src/format-engine/measureEvaluatorByHierarchy.ts b/src/format-engine/measureEvaluatorByHierarchy.ts
@@ -30,16 +30,16 @@ export default function getBestMeasureResult(
             const dcatFormat = ("" + dcatSet.measureResult.formats[0].format)
                 .trim()
                 .toUpperCase();
+
             if (
                 /**
                  * if sortedFormat is `ZIP` & is different from dcatFormat, we should trust dcatFormat
+                 * However, if dcatFormat is "esri", likely a zip resource hosted on esri portal.
+                 * Thus, should keep as ZIP
                  */
-                (sortedFormat === "ZIP" && dcatFormat !== sortedFormat) ||
-                /**
-                 * if sortedFormat is `ESRI REST` & is different from dcatFormat, we should trust dcatFormat
-                 * The Regex for testing `ESRI REST` URL cannot be very specific. Thus, should only be used when DcatFormat not present
-                 */
-                (sortedFormat === "ESRI REST" && dcatFormat !== sortedFormat)
+                sortedFormat === "ZIP" &&
+                dcatFormat !== sortedFormat &&
+                dcatFormat.indexOf("ESRI") === -1
             ) {
                 finalCandidate = dcatSet;
             }

diff --git a/src/format-engine/measures/dcatFormatMeasure.ts b/src/format-engine/measures/dcatFormatMeasure.ts
@@ -187,7 +187,9 @@ export default function getMeasureResult(
                 currentTransformer(accumulation),
             processedFormats
         )
-        .filter(format => format.trim().length > 0);
+        .filter(
+            format => typeof format === "string" && format.trim().length > 0
+        );
 
     if (processedFormats.length < 1) {
         return null;

diff --git a/src/test/onRecordFound.spec.ts b/src/test/onRecordFound.spec.ts
@@ -132,6 +132,15 @@ describe("onRecordFound", async function(this) {
         });
     });
 
+    testDistFileReturnsFormat("./sampleDataFiles/esri-zip.json", "ZIP");
+
+    testDistFileReturnsFormat("./sampleDataFiles/esri-pdf.json", "PDF");
+
+    testDistFileReturnsFormat(
+        "./sampleDataFiles/esri-rest-map-server.json",
+        "ESRI MAPSERVER"
+    );
+
     testDistFileReturnsFormat(
         "./sampleDataFiles/esri-featureserver.json",
         "ESRI FEATURESERVER"

diff --git a/src/test/sampleDataFiles/esri-pdf.json b/src/test/sampleDataFiles/esri-pdf.json
@@ -0,0 +1,50 @@
+{
+    "aspects": {
+        "source": {
+            "id": "dga",
+            "name": "data.gov.au",
+            "type": "ckan-resource",
+            "url": "https://data.gov.au/data/api/3/action/resource_show?id=9ba09c88-c7ea-4b6a-8a48-172bae0ad8ef"
+        },
+        "ckan-resource": {
+            "cache_last_updated": null,
+            "cache_url": null,
+            "created": "2017-06-13T15:01:46.421325",
+            "datastore_active": false,
+            "description": "test PDF file on esri portal",
+            "format": "esri pdf files",
+            "hash": "",
+            "id": "9ba09c88-c7ea-4b6a-8a48-172bae0ad8ef",
+            "last_modified": "2017-06-13T05:01:46.220210",
+            "mimetype": null,
+            "mimetype_inner": null,
+            "name": "test PDF file on esri portal",
+            "package_id": "5fdef41b-bbd1-42ed-bbea-7f6db1a4ab26",
+            "position": 2,
+            "resource_type": null,
+            "revision_id": "af166725-3f6c-47a0-b626-fd8eb85dd79a",
+            "size": null,
+            "state": "active",
+            "url": "https://data.gov.au/dataset/5fdef41b-bbd1-42ed-bbea-7f6db1a4ab26/resource/9ba09c88-c7ea-4b6a-8a48-172bae0ad8ef/download/soe2016bio10bio26.zip",
+            "url_type": "upload",
+            "wms_layer": ""
+        },
+        "dcat-distribution-strings": {
+            "description": "test PDF file on esri portal",
+            "downloadURL": "https://data.gov.au/dataset/5fdef41b-bbd1-42ed-bbea-7f6db1a4ab26/resource/9ba09c88-c7ea-4b6a-8a48-172bae0ad8ef/download/soe2016bio10bio26.pdf",
+            "format": "esri pdf files",
+            "issued": "2017-06-13T15:01:46Z",
+            "license": "Creative Commons Attribution 4.0 International",
+            "modified": "2017-06-13T05:01:46Z",
+            "title": "test PDF file on esri portal"
+        },
+        "source-link-status": {
+            "httpStatusCode": 200,
+            "status": "active"
+        }
+    },
+    "id": "dist-dga-9ba09c88-c7ea-4b6a-8a48-172bae0ad8ef",
+    "name": "test PDF file on esri portal",
+    "sourceTag": "a9bb685b-ba6f-470b-8138-4a7b9defb993",
+    "tenantId": 0
+}
diff --git a/src/test/sampleDataFiles/esri-rest-map-server.json b/src/test/sampleDataFiles/esri-rest-map-server.json
@@ -0,0 +1,27 @@
+{
+    "aspects": {
+        "source": {
+            "id": "dga",
+            "name": "data.gov.au",
+            "type": "ckan-resource",
+            "url": "https://data.gov.au/data/api/3/action/resource_show?id=0ca78178-7482-486f-ae85-9cd97c7c97c9"
+        },
+        "dcat-distribution-strings": {
+            "description": "Map prepared by the Department of Environment and Energy\r\n\r\nSource: Environmental Resources Information Network, Australian Government Department of the Environment and Energy.\r\n",
+            "downloadURL": "http://www.environment.gov.au/mapping/rest/services/digital_soe/soe2016_bio1/MapServer/0",
+            "format": "Esri REST",
+            "issued": "2017-01-24T18:21:45Z",
+            "license": "Creative Commons Attribution 4.0 International",
+            "modified": "2017-01-24T18:20:26Z",
+            "title": "BIO26 Numbers of fish species listed under the EPBC Act 1999 in each IBRA region"
+        },
+        "source-link-status": {
+            "httpStatusCode": 200,
+            "status": "active"
+        }
+    },
+    "id": "dist-dga-0ca78178-7482-486f-ae85-9cd97c7c97c9",
+    "name": "BIO26 Numbers of fish species listed under the EPBC Act 1999 in each IBRA region",
+    "sourceTag": "2184d3f1-3cc4-4aee-8629-d4810172b51e",
+    "tenantId": 0
+}
diff --git a/src/test/sampleDataFiles/esri-zip.json b/src/test/sampleDataFiles/esri-zip.json
@@ -0,0 +1,50 @@
+{
+    "aspects": {
+        "source": {
+            "id": "dga",
+            "name": "data.gov.au",
+            "type": "ckan-resource",
+            "url": "https://data.gov.au/data/api/3/action/resource_show?id=9ba09c88-c7ea-4b6a-8a48-172bae0ad8ef"
+        },
+        "ckan-resource": {
+            "cache_last_updated": null,
+            "cache_url": null,
+            "created": "2017-06-13T15:01:46.421325",
+            "datastore_active": false,
+            "description": "Downloadable shape and layer files depicting the total number of EPBC Act listed fish species in each IBRA region across Australia. Data supplied by the Environmental Resources Information Network, Australian Government Department of the Environment and Energy. For further information see: http://www.environment.gov.au/about-us/environmental-information-data/open-data\r\n\r\nMap showing the number of fish species listed as threatened under the EPBC Act in each IBRA region across Australia. Data has been sourced from the Species of Environmental Significance database and IBRA version 7 (Australian Government Department of the Environment and Energy).\r\n\r\nMap prepared by the Department of Environment and Energy in order to produce Figure BIO26 in the Biodiversity theme of the 2016 State of the Environment Report, available at http://www.soe.environment.gov.au",
+            "format": "esri shape and layer files",
+            "hash": "",
+            "id": "9ba09c88-c7ea-4b6a-8a48-172bae0ad8ef",
+            "last_modified": "2017-06-13T05:01:46.220210",
+            "mimetype": null,
+            "mimetype_inner": null,
+            "name": "BIO26 Number of EPBC fish species in each IBRA region",
+            "package_id": "5fdef41b-bbd1-42ed-bbea-7f6db1a4ab26",
+            "position": 2,
+            "resource_type": null,
+            "revision_id": "af166725-3f6c-47a0-b626-fd8eb85dd79a",
+            "size": null,
+            "state": "active",
+            "url": "https://data.gov.au/dataset/5fdef41b-bbd1-42ed-bbea-7f6db1a4ab26/resource/9ba09c88-c7ea-4b6a-8a48-172bae0ad8ef/download/soe2016bio10bio26.zip",
+            "url_type": "upload",
+            "wms_layer": ""
+        },
+        "dcat-distribution-strings": {
+            "description": "Downloadable shape and layer files depicting the total number of EPBC Act listed fish species in each IBRA region across Australia. Data supplied by the Environmental Resources Information Network, Australian Government Department of the Environment and Energy. For further information see: http://www.environment.gov.au/about-us/environmental-information-data/open-data\r\n\r\nMap showing the number of fish species listed as threatened under the EPBC Act in each IBRA region across Australia. Data has been sourced from the Species of Environmental Significance database and IBRA version 7 (Australian Government Department of the Environment and Energy).\r\n\r\nMap prepared by the Department of Environment and Energy in order to produce Figure BIO26 in the Biodiversity theme of the 2016 State of the Environment Report, available at http://www.soe.environment.gov.au",
+            "downloadURL": "https://data.gov.au/dataset/5fdef41b-bbd1-42ed-bbea-7f6db1a4ab26/resource/9ba09c88-c7ea-4b6a-8a48-172bae0ad8ef/download/soe2016bio10bio26.zip",
+            "format": "esri shape and layer files",
+            "issued": "2017-06-13T15:01:46Z",
+            "license": "Creative Commons Attribution 4.0 International",
+            "modified": "2017-06-13T05:01:46Z",
+            "title": "BIO26 Number of EPBC fish species in each IBRA region"
+        },
+        "source-link-status": {
+            "httpStatusCode": 200,
+            "status": "active"
+        }
+    },
+    "id": "dist-dga-9ba09c88-c7ea-4b6a-8a48-172bae0ad8ef",
+    "name": "BIO26 Number of EPBC fish species in each IBRA region",
+    "sourceTag": "a9bb685b-ba6f-470b-8138-4a7b9defb993",
+    "tenantId": 0
+}