Enable additional sources for federated API

.github/workflows/changelog.yml

@@ -15,4 +15,4 @@ on:
 
 jobs:
   check_changelog:
-    uses: OpenTermsArchive/engine/.github/workflows/changelog.yml@main
+    uses: OpenTermsArchive/engine/.github/workflows/changelog.yml@v0.37.1

CHANGELOG.md

@@ -2,7 +2,15 @@
 
 All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the [engine CONTRIBUTING file](https://github.com/OpenTermsArchive/engine/blob/main/CONTRIBUTING.md#changelog). This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## Unreleased
+## Unreleased [minor]
+
+_Full changeset and discussions: [#4](https://github.com/OpenTermsArchive/engine/pull/4)._
+
+> Development of this release was [supported](https://nlnet.nl/project/TOSDR-OTA/) by the [NGI0 Entrust Fund](https://nlnet.nl/entrust), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://www.ngi.eu) programme, under the aegis of DG CNECT under grant agreement N°101069594.
+
+### Changed
+
+- **Breaking:** Replaced `collectionsUrl` config entry with `collections` key, now supporting both URLs and directly specified collections. Refer to the [configuration documentation](https://docs.opentermsarchive.org/api/federated/#configuring) for details.
 
 ## 0.1.1 - 2024-02-15
 

config/default.json

@@ -1,11 +1,13 @@
 {
-  "port": 3333,
-  "collectionsUrl": "https://opentermsarchive.org/collections.json",
   "logger": {
     "smtp": {
       "host": "smtp-relay.sendinblue.com",
       "username": "admin@opentermsarchive.org"
     },
     "sendMailOnError": false
-  }
+  },
+  "port": 3333,
+  "collections": [
+    "https://opentermsarchive.org/collections.json"
+  ]
 }

config/production.json

@@ -1,9 +1,9 @@
 {
-  "port": 3333,
   "logger": {
     "sendMailOnError": {
       "from": "noreply@opentermsarchive.org",
       "to": "admin@opentermsarchive.org"
     }
-  }
+  },
+  "port": 3333
 }

src/controllers/collections.js

@@ -1,5 +1,7 @@
+import config from 'config';
+
 import { fetchCollections } from '../services/collections.js';
 
 export const getCollections = async (req, res) => {
-  res.json(await fetchCollections());
+  res.json(await fetchCollections(config.get('collections')));
 };

src/controllers/services.js

@@ -1,10 +1,12 @@
+import config from 'config';
+
 import { fetchCollections } from '../services/collections.js';
 import { fetchServices, isServiceIDValid } from '../services/services.js';
 
 export const getServices = async (req, res) => {
   const { name: requestedName, termsType: requestedTermsType } = req.query;
 
-  const collections = await fetchCollections();
+  const collections = await fetchCollections(config.get('collections'));
 
   const results = [];
   const failures = [];
@@ -51,7 +53,7 @@ export const getService = async (req, res) => {
     return res.status(400).json();
   }
 
-  const collections = await fetchCollections();
+  const collections = await fetchCollections(config.get('collections'));
 
   const results = [];
   const failures = [];

src/routes/collections.test.js

@@ -1,24 +1,25 @@
 import { expect } from 'chai';
-import config from 'config';
 import nock from 'nock';
 import request from 'supertest';
 
 import app, { BASE_PATH } from '../index.js';
 
-const COLLECTIONS_RESULT = {
-  'Collection 1': {
+const COLLECTIONS_RESULT = [
+  {
+    name: 'Collection 1',
     id: 'collection-1',
     endpoint: 'http://collection-1.example/api/v1',
   },
-  'Collection 2': {
+  {
+    name: 'Collection 2',
     id: 'collection-2',
     endpoint: 'https://2.collection.example/api/v1',
   },
-};
+];
 
-describe('Collections routes', () => {
+describe('Routes: Collections', () => {
   before(() => {
-    nock(config.get('collectionsUrl')).persist().get('').reply(200, COLLECTIONS_RESULT);
+    nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
   });
 
   after(() => {

src/routes/docs.test.js

@@ -3,7 +3,7 @@ import request from 'supertest';
 
 import app, { BASE_PATH } from '../index.js';
 
-describe('Docs API', () => {
+describe('Routes: Docs', () => {
   describe('GET /docs', () => {
     let response;
 

src/routes/services.test.js

@@ -1,20 +1,21 @@
 import { expect } from 'chai';
-import config from 'config';
 import nock from 'nock';
 import request from 'supertest';
 
 import app, { BASE_PATH } from '../index.js';
 
-export const COLLECTIONS_RESULT = {
-  'Collection 1': {
+export const COLLECTIONS_RESULT = [
+  {
+    name: 'Collection 1',
     id: 'collection-1',
     endpoint: 'http://collection-1.example/api/v1',
   },
-  'Collection 2': {
+  {
+    name: 'Collection 2',
     id: 'collection-2',
     endpoint: 'https://2.collection.example/api/v1',
   },
-};
+];
 
 const COLLECTION_1_SERVICES_RESULT = [
   {
@@ -66,11 +67,11 @@ const COLLECTION_2_SERVICES_RESULT = [
   },
 ];
 
-describe('Services routes', () => {
+describe('Routes: Services', () => {
   const serviceWithUrlEncodedChineseCharactersName = '%E6%8A%96%E9%9F%B3%E7%9F%AD%E8%A7%86%E9%A2%91';
 
   before(() => {
-    nock(config.get('collectionsUrl')).persist().get('').reply(200, COLLECTIONS_RESULT);
+    nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
     nock('http://collection-1.example').persist().get('/api/v1/services').reply(200, COLLECTION_1_SERVICES_RESULT);
     nock('https://2.collection.example').persist().get('/api/v1/services').reply(200, COLLECTION_2_SERVICES_RESULT);
   });
@@ -267,7 +268,7 @@ describe('Services routes', () => {
     context('when an error occurs in one of the underlying collections', () => {
       before(async () => {
         nock.cleanAll();
-        nock(config.get('collectionsUrl')).persist().get('').reply(200, COLLECTIONS_RESULT);
+        nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
         nock('http://collection-1.example').persist().get('/api/v1/services').reply(200, COLLECTION_1_SERVICES_RESULT);
         nock('https://2.collection.example').get('/api/v1/services').replyWithError({
           message: 'something went wrong',

src/services/collections.js

@@ -1,19 +1,54 @@
-import config from 'config';
+import { URL } from 'url';
 
 import fetch from '../utils/fetch.js';
+import logger from '../utils/logger.js';
 
-export const fetchCollections = async () => {
-  const collections = await fetch(config.get('collectionsUrl'));
+export const fetchCollections = async collectionsConfig => {
+  const unqiueCollections = new Map();
 
-  return Object.keys(collections).reduce((result, collectionName) => {
-    if (collections[collectionName].endpoint) {
-      result.push({
-        name: collectionName,
-        id: collections[collectionName].id,
-        endpoint: collections[collectionName].endpoint,
-      });
+  const result = await Promise.allSettled(collectionsConfig.map(async item => {
+    let collections = [];
+
+    if (typeof item === 'string') {
+      collections = await fetch(item);
+    }
+
+    if (typeof item === 'object') {
+      collections = [item];
     }
 
-    return result;
-  }, []);
+    return filterInvalidCollections(collections);
+  }));
+
+  result.forEach(({ value }) => value.forEach(collection => unqiueCollections.set(collection.id, collection)));
+
+  return Array.from(unqiueCollections.values());
 };
+
+export function filterInvalidCollections(collections) {
+  return collections.filter(collection => {
+    const haveMandatoryFields = collection.id && collection.name && collection.endpoint;
+
+    if (!haveMandatoryFields) {
+      logger.warn(`Ignore the following collection lacking mandatory fields 'id', 'name', or 'endpoint': \n${JSON.stringify(collection, null, 4)}`);
+    }
+
+    const isEndpointValidUrl = isURL(collection.endpoint);
+
+    if (!isEndpointValidUrl) {
+      logger.warn(`Ignore the following collection as 'endpoint' is not valid URL: \n${JSON.stringify(collection, null, 4)}`);
+    }
+
+    return haveMandatoryFields && isEndpointValidUrl;
+  });
+}
+
+function isURL(string) {
+  try {
+    new URL(string); // eslint-disable-line no-new
+
+    return true;
+  } catch (error) {
+    return false;
+  }
+}

src/services/collections.test.js

@@ -0,0 +1,106 @@
+import { expect } from 'chai';
+import nock from 'nock';
+
+import { fetchCollections } from './collections.js';
+
+describe('Services: Collections', () => {
+  const COLLECTION_1 = { id: 'collection1', name: 'Collection 1', endpoint: 'http://domain1.example/endpoint' };
+  const COLLECTION_2 = { id: 'collection2', name: 'Collection 2', endpoint: 'http://domain1.example/endpoint' };
+  const COLLECTION_3 = { id: 'collection3', name: 'Collection 3', endpoint: 'http://domain1.example/endpoint' };
+  const COLLECTION_1_OVERRIDEN = { id: COLLECTION_1.id, name: 'Override Collection 1', endpoint: 'http://domain2.example/endpoint' };
+
+  describe('fetchCollections', () => {
+    it('fetches collections from URLs and includes directly given collections', async () => {
+      nock('http://domain1.example')
+        .get('/collections.json')
+        .reply(200, [ COLLECTION_1, COLLECTION_2 ]);
+
+      const config = [
+        'http://domain1.example/collections.json',
+        COLLECTION_3,
+      ];
+
+      expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1, COLLECTION_2, COLLECTION_3 ]);
+    });
+
+    context('when there are duplicates', () => {
+      it('removes duplicates', async () => {
+        nock('http://domain1.example')
+          .get('/collections.json')
+          .reply(200, [ COLLECTION_1, COLLECTION_2 ]);
+
+        const config = [
+          'http://domain1.example/collections.json',
+          COLLECTION_1_OVERRIDEN,
+        ];
+
+        expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1_OVERRIDEN, COLLECTION_2 ]);
+      });
+    });
+
+    context('with invalid collections', () => {
+      context('when mandatory fields are missing', () => {
+        context('when "id" is missing', () => {
+          it('removes invalid collection', async () => {
+            const config = [
+              COLLECTION_1,
+              {
+                name: 'Invalid collection',
+                endpoint: 'http://domain1.example/endpoint',
+              },
+              COLLECTION_3,
+            ];
+
+            expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1, COLLECTION_3 ]);
+          });
+        });
+
+        context('when "name" is missing', () => {
+          it('removes invalid collection', async () => {
+            const config = [
+              COLLECTION_1,
+              {
+                id: 'invalid-collection',
+                endpoint: 'http://domain1.example/endpoint',
+              },
+              COLLECTION_3,
+            ];
+
+            expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1, COLLECTION_3 ]);
+          });
+        });
+
+        context('when "endpoint" is missing', () => {
+          it('removes invalid collection', async () => {
+            const config = [
+              COLLECTION_1,
+              {
+                id: 'invalid-collection',
+                name: 'Invalid collection',
+              },
+              COLLECTION_3,
+            ];
+
+            expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1, COLLECTION_3 ]);
+          });
+        });
+      });
+
+      context('when endpoint is not a valid URL ', () => {
+        it('removes invalid collection', async () => {
+          const config = [
+            COLLECTION_1,
+            {
+              id: 'invalid-endpoint',
+              name: 'Invalid collection endpoint',
+              endpoint: 'no url endoint',
+            },
+            COLLECTION_3,
+          ];
+
+          expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1, COLLECTION_3 ]);
+        });
+      });
+    });
+  });
+});