Fix BCDC Datasets out of sync (#837)

---------

Co-authored-by: James Elson <james.elson@gov.bc.ca>
Co-authored-by: Elson9 <Elson9@users.noreply.github.com>

feeds/README.md

@@ -39,11 +39,14 @@
 ## Configuration
 
 ```
+export LOG_FEEDS=ON
 export DESTINATION_URL=https://aps-portal-feature-sprint13-poc.apps.silver.devops.gov.bc.ca
 export WORKING_PATH=`pwd`/_tmp
 export KONG_ADMIN_URL=https://adminapi-264e6f-dev.apps.silver.devops.gov.bc.ca
 export CKAN_URL=https://catalog.data.gov.bc.ca
 export PROM_URL=https://prom-264e6f-dev.apps.silver.devops.gov.bc.ca
+
+npm run start
 ```
 
 #### GatewayService

feeds/ckan/index.js

@@ -83,23 +83,15 @@ async function sync({ url, workingPath, destinationUrl }) {
   const xfer = transfers(workingPath, url, exceptions);
 
   await xfer.copy('/api/action/group_list?limit=100&offset=0', 'group-keys');
-  await xfer.copy(
-    '/api/action/package_list?limit=100&offset=0',
-    'package-keys'
+  await xfer.copySearch(
+    '/api/action/package_search?fq=groups:(bc-government-api-registry)',
+    'packages/'
   );
 
   await xfer.concurrentWork(
     getCkanDataProducer(xfer, 'group-keys', '/api/action/group_show', 'groups/')
   );
-  await xfer.concurrentWork(
-    getCkanDataProducer(
-      xfer,
-      'package-keys',
-      '/api/action/package_show',
-      'packages/'
-    ),
-    10
-  );
+
   console.log('Exceptions? ' + (exceptions.length == 0 ? 'NO' : 'YES!'));
   console.log(JSON.stringify(exceptions, null, 4));
 

feeds/index.js

@@ -171,6 +171,8 @@ if (process.env.SCHEDULE == 'true') {
   runTimedJob('prometheus', 120, { numDays: 1 });
   runTimedJob('prometheus', 24 * 60 + 5, { numDays: 5 });
   runTimedJob('kong', 1 * 60, {});
+  runTimedJob('ckan', (24 * 60) + 30, {});
+  runTimedJob('ckan_org', 24 * 60, {});
 }
 
 const server = app.listen(port, () => {

feeds/utils/transfers.js

@@ -71,6 +71,32 @@ function transfers(workingPath, baseUrl, exceptions) {
         });
     },
 
+    copySearch: async function (_url, filename, index = 0) {
+      log.debug('[copySearch] %s%s', baseUrl, _url);
+      return fetch(baseUrl + _url)
+        .then(checkStatus)
+        .then((data) => data.json())
+        .then((json) => {
+          for (const item of json.result.results) {
+            const out =
+              workingPath + '/' + filename + item.name + '-' + index + '.json';
+            fs.writeFileSync(
+              out,
+              JSON.stringify({ result: item }, null, 4),
+              null
+            );
+          }
+        })
+        .catch((err) => {
+          log.error('[copySearch] %s, %s', filename, err);
+          exceptions.push({
+            relativeUrl: url,
+            filename: filename,
+            error: '' + err,
+          });
+        });
+    },
+
     read: function (filename) {
       const infile = workingPath + '/' + filename + '.json';
       return JSON.parse(fs.readFileSync(infile));