Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable additional sources for federated API #4

Closed
wants to merge 13 commits into from
2 changes: 1 addition & 1 deletion .github/workflows/changelog.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@ on:

jobs:
check_changelog:
uses: OpenTermsArchive/engine/.github/workflows/changelog.yml@main
uses: OpenTermsArchive/engine/.github/workflows/changelog.yml@v0.37.1
Ndpnt marked this conversation as resolved.
Show resolved Hide resolved
10 changes: 9 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,15 @@

All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the [engine CONTRIBUTING file](https://github.com/OpenTermsArchive/engine/blob/main/CONTRIBUTING.md#changelog). This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## Unreleased
## Unreleased [minor]

_Full changeset and discussions: [#4](https://github.com/OpenTermsArchive/engine/pull/4)._

> Development of this release was [supported](https://nlnet.nl/project/TOSDR-OTA/) by the [NGI0 Entrust Fund](https://nlnet.nl/entrust), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://www.ngi.eu) programme, under the aegis of DG CNECT under grant agreement N°101069594.

### Changed

- **Breaking:** Replaced `collectionsUrl` config entry with `collections` key, now supporting both URLs and directly specified collections. Refer to the [configuration documentation](https://docs.opentermsarchive.org/api/federated/#configuring) for details.

## 0.1.1 - 2024-02-15

Expand Down
8 changes: 5 additions & 3 deletions config/default.json
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
{
"port": 3333,
"collectionsUrl": "https://opentermsarchive.org/collections.json",
"logger": {
"smtp": {
"host": "smtp-relay.sendinblue.com",
"username": "admin@opentermsarchive.org"
},
"sendMailOnError": false
}
},
"port": 3333,
"collections": [
"https://opentermsarchive.org/collections.json"
]
}
4 changes: 2 additions & 2 deletions config/production.json
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"port": 3333,
"logger": {
"sendMailOnError": {
"from": "noreply@opentermsarchive.org",
"to": "admin@opentermsarchive.org"
}
}
},
"port": 3333
}
4 changes: 3 additions & 1 deletion src/controllers/collections.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import config from 'config';

import { fetchCollections } from '../services/collections.js';

export const getCollections = async (req, res) => {
res.json(await fetchCollections());
res.json(await fetchCollections(config.get('collections')));
};
6 changes: 4 additions & 2 deletions src/controllers/services.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import config from 'config';

import { fetchCollections } from '../services/collections.js';
import { fetchServices, isServiceIDValid } from '../services/services.js';

export const getServices = async (req, res) => {
const { name: requestedName, termsType: requestedTermsType } = req.query;

const collections = await fetchCollections();
const collections = await fetchCollections(config.get('collections'));

const results = [];
const failures = [];
Expand Down Expand Up @@ -51,7 +53,7 @@ export const getService = async (req, res) => {
return res.status(400).json();
}

const collections = await fetchCollections();
const collections = await fetchCollections(config.get('collections'));

const results = [];
const failures = [];
Expand Down
15 changes: 8 additions & 7 deletions src/routes/collections.test.js
Original file line number Diff line number Diff line change
@@ -1,24 +1,25 @@
import { expect } from 'chai';
import config from 'config';
import nock from 'nock';
import request from 'supertest';

import app, { BASE_PATH } from '../index.js';

const COLLECTIONS_RESULT = {
'Collection 1': {
const COLLECTIONS_RESULT = [
{
name: 'Collection 1',
id: 'collection-1',
endpoint: 'http://collection-1.example/api/v1',
},
'Collection 2': {
{
name: 'Collection 2',
id: 'collection-2',
endpoint: 'https://2.collection.example/api/v1',
},
};
];

describe('Collections routes', () => {
describe('Routes: Collections', () => {
before(() => {
nock(config.get('collectionsUrl')).persist().get('').reply(200, COLLECTIONS_RESULT);
nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
});

after(() => {
Expand Down
2 changes: 1 addition & 1 deletion src/routes/docs.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import request from 'supertest';

import app, { BASE_PATH } from '../index.js';

describe('Docs API', () => {
describe('Routes: Docs', () => {
describe('GET /docs', () => {
let response;

Expand Down
17 changes: 9 additions & 8 deletions src/routes/services.test.js
Original file line number Diff line number Diff line change
@@ -1,20 +1,21 @@
import { expect } from 'chai';
import config from 'config';
import nock from 'nock';
import request from 'supertest';

import app, { BASE_PATH } from '../index.js';

export const COLLECTIONS_RESULT = {
'Collection 1': {
export const COLLECTIONS_RESULT = [
{
name: 'Collection 1',
id: 'collection-1',
endpoint: 'http://collection-1.example/api/v1',
},
'Collection 2': {
{
name: 'Collection 2',
id: 'collection-2',
endpoint: 'https://2.collection.example/api/v1',
},
};
];

const COLLECTION_1_SERVICES_RESULT = [
{
Expand Down Expand Up @@ -66,11 +67,11 @@ const COLLECTION_2_SERVICES_RESULT = [
},
];

describe('Services routes', () => {
describe('Routes: Services', () => {
const serviceWithUrlEncodedChineseCharactersName = '%E6%8A%96%E9%9F%B3%E7%9F%AD%E8%A7%86%E9%A2%91';

before(() => {
nock(config.get('collectionsUrl')).persist().get('').reply(200, COLLECTIONS_RESULT);
nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
nock('http://collection-1.example').persist().get('/api/v1/services').reply(200, COLLECTION_1_SERVICES_RESULT);
nock('https://2.collection.example').persist().get('/api/v1/services').reply(200, COLLECTION_2_SERVICES_RESULT);
});
Expand Down Expand Up @@ -267,7 +268,7 @@ describe('Services routes', () => {
context('when an error occurs in one of the underlying collections', () => {
before(async () => {
nock.cleanAll();
nock(config.get('collectionsUrl')).persist().get('').reply(200, COLLECTIONS_RESULT);
nock('https://opentermsarchive.org/collections.json').persist().get('').reply(200, COLLECTIONS_RESULT);
nock('http://collection-1.example').persist().get('/api/v1/services').reply(200, COLLECTION_1_SERVICES_RESULT);
nock('https://2.collection.example').get('/api/v1/services').replyWithError({
message: 'something went wrong',
Expand Down
61 changes: 49 additions & 12 deletions src/services/collections.js
Original file line number Diff line number Diff line change
@@ -1,19 +1,56 @@
import config from 'config';
import { URL } from 'url';

import fetch from '../utils/fetch.js';
import logger from '../utils/logger.js';

export const fetchCollections = async () => {
const collections = await fetch(config.get('collectionsUrl'));
export const fetchCollections = async collectionsConfig => {
const uniqueCollections = new Map();

return Object.keys(collections).reduce((result, collectionName) => {
if (collections[collectionName].endpoint) {
result.push({
name: collectionName,
id: collections[collectionName].id,
endpoint: collections[collectionName].endpoint,
});
(await Promise.allSettled(collectionsConfig.map(async item => {
let collections = [];

if (typeof item === 'string') {
collections = await fetch(item);
}
MattiSG marked this conversation as resolved.
Show resolved Hide resolved

if (typeof item === 'object') {
collections = [item];
}

return result;
}, []);
return filterInvalidCollections(collections);
}))).forEach(({ value }) => {
value.forEach(collection => {
uniqueCollections.set(collection.id, collection);
});
});

return Array.from(uniqueCollections.values());
};

export function filterInvalidCollections(collections) {
return collections.filter(collection => {
const hasMandatoryFields = collection.id && collection.name && collection.endpoint;

if (!hasMandatoryFields) {
logger.warn(`Ignore the following collection lacking mandatory fields 'id', 'name', or 'endpoint': \n${JSON.stringify(collection, null, 4)}`);
}

const isEndpointValidUrl = isURL(collection.endpoint);

if (!isEndpointValidUrl) {
logger.warn(`Ignore the following collection as 'endpoint' is not a valid URL: \n${JSON.stringify(collection, null, 4)}`);
}

return hasMandatoryFields && isEndpointValidUrl;
});
}

function isURL(string) {
try {
new URL(string); // eslint-disable-line no-new

return true;
} catch (error) {
return false;
}
}
106 changes: 106 additions & 0 deletions src/services/collections.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
import { expect } from 'chai';
import nock from 'nock';

import { fetchCollections } from './collections.js';

describe('Services: Collections', () => {
const COLLECTION_1 = { id: 'collection1', name: 'Collection 1', endpoint: 'http://domain1.example/endpoint' };
const COLLECTION_2 = { id: 'collection2', name: 'Collection 2', endpoint: 'http://domain1.example/endpoint' };
const COLLECTION_3 = { id: 'collection3', name: 'Collection 3', endpoint: 'http://domain1.example/endpoint' };
const COLLECTION_1_OVERRIDEN = { id: COLLECTION_1.id, name: 'Override Collection 1', endpoint: 'http://domain2.example/endpoint' };

describe('fetchCollections', () => {
it('fetches collections from URLs and includes directly given collections', async () => {
nock('http://domain1.example')
.get('/collections.json')
.reply(200, [ COLLECTION_1, COLLECTION_2 ]);

const config = [
'http://domain1.example/collections.json',
COLLECTION_3,
];

expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1, COLLECTION_2, COLLECTION_3 ]);
});

context('when there are duplicates', () => {
it('removes duplicates', async () => {
nock('http://domain1.example')
.get('/collections.json')
.reply(200, [ COLLECTION_1, COLLECTION_2 ]);

const config = [
'http://domain1.example/collections.json',
COLLECTION_1_OVERRIDEN,
];

expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1_OVERRIDEN, COLLECTION_2 ]);
});
});

context('with invalid collections', () => {
context('when mandatory fields are missing', () => {
context('when "id" is missing', () => {
it('removes invalid collection', async () => {
const config = [
COLLECTION_1,
{
name: 'Invalid collection',
endpoint: 'http://domain1.example/endpoint',
},
COLLECTION_3,
];

expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1, COLLECTION_3 ]);
});
});

context('when "name" is missing', () => {
it('removes invalid collection', async () => {
const config = [
COLLECTION_1,
{
id: 'invalid-collection',
endpoint: 'http://domain1.example/endpoint',
},
COLLECTION_3,
];

expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1, COLLECTION_3 ]);
});
});

context('when "endpoint" is missing', () => {
it('removes invalid collection', async () => {
const config = [
COLLECTION_1,
{
id: 'invalid-collection',
name: 'Invalid collection',
},
COLLECTION_3,
];

expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1, COLLECTION_3 ]);
});
});
});

context('when endpoint is not a valid URL ', () => {
it('removes invalid collection', async () => {
const config = [
COLLECTION_1,
{
id: 'invalid-endpoint',
name: 'Invalid collection endpoint',
endpoint: 'no url endoint',
},
COLLECTION_3,
];

expect(await fetchCollections(config)).to.deep.equal([ COLLECTION_1, COLLECTION_3 ]);
});
});
});
});
});
Loading
Loading