diff --git a/.prettierrc b/.prettierrc index 5149b1b..96f977f 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,5 +1,7 @@ { "trailingComma": "es5", + "tabWidth": 2, "semi": false, - "singleQuote": true + "singleQuote": true, + "printWidth": 79 } diff --git a/lib/index.js b/lib/index.js index 1b5f756..b0a9ee4 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,16 +1,17 @@ -const mapper = require('frictionless-ckan-mapper-js') +const frictionlessCkanMapper = require('frictionless-ckan-mapper-js') const axios = require('axios') const CkanAuthApi = require('./util/ckan-auth-api') const CkanUploadAPI = require('./util/ckan-upload-api') const ActionApi = require('./util/action-api') const Open = require('./file') + const { camelToSnakeCase } = require('./util/general') /** * @class Client. * @param {string} apiKey - * @param {number} organizationId + * @param {string} organizationId * @param {string} datasetId * @param {string} api */ @@ -70,8 +71,36 @@ class Client { return response.data } - put(actionType, data) { - return ActionApi.action(this.api, this.apiKey, actionType, data) + /** + * @async + * @param {(string|Object)} datasetNameOrMetadata + * @return {Promise} The frictionless dataset + */ + async create(datasetNameOrMetadata) { + const datasetMetadata = + typeof datasetNameOrMetadata === 'string' + ? { name: datasetNameOrMetadata } + : datasetNameOrMetadata + + const ckanMetadata = frictionlessCkanMapper.packageFrictionlessToCkan( + datasetMetadata + ) + + const response = await this.action('package_create', ckanMetadata) + return frictionlessCkanMapper.packageCkanToFrictionless(response.result) + } + + /** + * @async + * @param {Object} datasetMetadata + * @return {Promise} The frictionless dataset + */ + async push(datasetMetadata) { + const ckanMetadata = frictionlessCkanMapper.packageFrictionlessToCkan( + datasetMetadata + ) + const response = await this.action('package_update', ckanMetadata) + return frictionlessCkanMapper.packageCkanToFrictionless(response.result) } /** @@ -90,7 +119,7 @@ class Client { true ) - return mapper.packageCkanToFrictionless(response.result) + return frictionlessCkanMapper.packageCkanToFrictionless(response.result) } /** diff --git a/package.json b/package.json index 0c2eb1b..521bcb1 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "test": "test" }, "scripts": { - "test": "ava test/upload.test.js --verbose", + "test": "ava test/*.test.js --verbose", "test:watch": "ava test/*.test.js --verbose --watch", "build": "webpack", "build:watch": "webpack --watch" diff --git a/test/get.test.js b/test/get.test.js index 78cf639..e7f33d9 100644 --- a/test/get.test.js +++ b/test/get.test.js @@ -25,7 +25,7 @@ test('get package', async (t) => { ) const packageResponseConvertedToFrictionless = JSON.parse( await fs.readFileSync( - __dirname + '/mocks/getPackageFrictionlessResponse.json' + __dirname + '/mocks/frictionless-dataset-converted-from-ckan.json' ) ) // Testing dataname/id diff --git a/test/metadata.test.js b/test/metadata.test.js deleted file mode 100644 index 15f78a7..0000000 --- a/test/metadata.test.js +++ /dev/null @@ -1,234 +0,0 @@ -const test = require('ava') -const nock = require('nock') -const f11s = require('data.js') -const { Client } = require('../lib/index') - -/** - * Push stuff - */ -const config = { - api: 'http://localhost:5000', - authToken: 'be270cae-1c77-4853-b8c1-30b6cf5e9878', - organizationId: 'myorg', - datasetId: 'my_dataset', -} - -const client = new Client( - config.authToken, - config.organizationId, - config.datasetId, - config.api -) - -const metadataBody = { - path: 'test/fixtures/sample.csv', - pathType: 'local', - name: 'sample', - format: 'csv', - mediatype: 'text/csv', - encoding: 'UTF-8', - resources: [], -} - -const resourceBody = { - path: 'test/fixtures/sample.csv', - pathType: 'local', - name: 'sample', - format: 'csv', - mediatype: 'text/csv', - encoding: 'UTF-8', - package_id: 'my_dataset_name' -} - -/** - * Mock - */ -const packageCreateMock = nock(config.api) - .persist() - .post('/api/3/action/package_create', { - "name": config.datasetId, - "owner_org": config.organizationId, - }) - .reply(200, { - help: 'http://localhost:5000/api/3/action/help_show?name=package_create', - success: true, - result: { - license_title: null, - maintainer: null, - relationships_as_object: [], - private: false, - maintainer_email: null, - num_tags: 0, - id: '3d27173c-dcbe-4c07-b1a9-70f683623f33', - metadata_created: '2020-09-11T00:12:07.550325', - metadata_modified: '2020-09-11T00:12:07.550334', - author: null, - author_email: null, - state: 'active', - version: null, - creator_user_id: 'ceb0d8c2-352b-4a6c-b2c8-2a1a4190192b', - type: 'dataset', - resources: [], - num_resources: 0, - tags: [], - groups: [], - license_id: null, - relationships_as_subject: [], - organization: { - description: 'my org', - created: '2020-04-14T13:27:47.434967', - title: 'myorg', - name: 'myorg', - is_organization: true, - state: 'active', - image_url: '', - revision_id: 'bff67202-a21e-470b-9b67-14e65ad85fa2', - type: 'organization', - id: 'f3fce98a-4750-4d09-a5d1-d5e1b995a2e8', - approval_status: 'approved', - }, - name: 'my_dataset', - isopen: false, - url: null, - notes: null, - owner_org: 'f3fce98a-4750-4d09-a5d1-d5e1b995a2e8', - extras: [], - title: 'my_dataset', - revision_id: '3b10c815-d8aa-4726-ba06-bff4cb5874cb', - }, - }) - -const metadataMock = nock(config.api) - .persist() - .post('/api/3/action/package_create', metadataBody) - .reply(200, { - "help": "http://localhost:5000/api/3/action/help_show?name=package_create", - "success": true, - "result": { - "license_title": null, - "maintainer": null, - "relationships_as_object": [], - "private": false, - "maintainer_email": null, - "num_tags": 0, - "id": "9a25bbb9-a21f-430c-88c7-291a7bb37350", - "metadata_created": "2020-09-11T02:06:23.785055", - "metadata_modified": "2020-09-11T02:06:23.785059", - "author": null, - "author_email": null, - "state": "active", - "version": null, - "creator_user_id": "ceb0d8c2-352b-4a6c-b2c8-2a1a4190192b", - "type": "dataset", - "resources": [], - "num_resources": 0, - "tags": [], - "groups": [], - "license_id": null, - "relationships_as_subject": [], - "organization": { - "description": "my org", - "created": "2020-04-14T13:27:47.434967", - "title": "myorg", - "name": "myorg", - "is_organization": true, - "state": "active", - "image_url": "", - "revision_id": "bff67202-a21e-470b-9b67-14e65ad85fa2", - "type": "organization", - "id": "f3fce98a-4750-4d09-a5d1-d5e1b995a2e8", - "approval_status": "approved" - }, - "name": "sample", - "isopen": false, - "url": null, - "notes": null, - "owner_org": "f3fce98a-4750-4d09-a5d1-d5e1b995a2e8", - "extras": [], - "title": "sample", - "revision_id": "341b8af7-3586-450a-b38a-98fb96a7afc4" - } - }) - -const resourceMock = nock(config.api) - .persist() - .post('/api/3/action/resource_create', resourceBody) - .reply(200, { - "help": "http://localhost:5000/api/3/action/help_show?name=resource_create", - "success": true, - "result": { - "cache_last_updated": null, - "package_id": "3a1aa4af-9f00-490b-956e-2b063ed04961", - "datastore_active": false, - "id": "d5f496ce-2fe2-48d3-898f-3c0cc22f4015", - "size": null, - "encoding": "UTF-8", - "state": "active", - "hash": "", - "description": "", - "format": "CSV", - "mediatype": "text/csv", - "pathType": "local", - "mimetype_inner": null, - "url_type": null, - "path": "test/fixtures/sample.csv", - "mimetype": null, - "cache_url": null, - "name": "sample", - "created": "2020-09-11T02:51:20.697102", - "url": "", - "last_modified": null, - "position": 3, - "revision_id": "17aea680-6dc8-4ce4-ab8b-fec31996242c", - "resource_type": null - } - }) - -test('create a dataset', async (t) => { - const client = new Client( - config.authToken, - config.organizationId, - config.datasetId, - config.api - ) - - const response = await client.put('package_create', { "name": "my_dataset", "owner_org": "myorg"}) - - t.is(client.api, config.api) - t.is(packageCreateMock.isDone(), true) - t.is(response.success, true) - t.is(response.result.name, "my_dataset") - t.is(response.result.organization.name, "myorg") -}) - -test('put metadata', async (t) => { - const path = 'test/fixtures/sample.csv' - const file = await f11s.open(path) - const dataset = new f11s.Dataset({ - ...file.descriptor, - resources: [], - }) - - const response = await client.put('package_create', dataset.descriptor) - - t.is(packageCreateMock.isDone(), true) - t.is(response.success, true) - t.is(response.result.name, "sample") - t.deepEqual(response.result.resources, []) -}) - -test('put_resource create or update a resource', async (t) => { - const path = 'test/fixtures/sample.csv' - const file = await f11s.open(path) - - // Dataset must exist - file.descriptor.package_id = "my_dataset_name" - - const response = await client.put('resource_create', file.descriptor) - - t.is(resourceMock.isDone(), true) - t.is(response.success, true) - t.is(response.result.name, "sample") - t.is(response.result.format, "CSV") - t.is(response.result.mediatype, "text/csv") -}) diff --git a/test/methods.test.js b/test/methods.test.js new file mode 100644 index 0000000..eb63e83 --- /dev/null +++ b/test/methods.test.js @@ -0,0 +1,141 @@ +const test = require('ava') +const nock = require('nock') +const f11s = require('data.js') +const fs = require('fs') + +const { Client } = require('../lib/index') + +/** + * Push stuff + */ +const config = { + api: 'http://localhost:5000', + authToken: 'be270cae-1c77-4853-b8c1-30b6cf5e9878', + organizationId: 'myorg', + datasetId: 'my_dataset', +} + +const client = new Client( + config.authToken, + config.organizationId, + config.datasetId, + config.api +) + +test('push a metadata', async (t) => { + // Mock datasets + const frictionlessDataset = JSON.parse( + await fs.readFileSync( + __dirname + '/mocks/frictionless-dataset-converted-from-ckan.json' + ) + ) + const ckanDataset = JSON.parse( + await fs.readFileSync(__dirname + '/mocks/ckan-dataset.json') + ) + + const packagePushMock = nock(config.api) + .persist() + .post('/api/3/action/package_update', (body) => { + t.deepEqual(body, { + notes: 'GDPs list', + url: 'https://datopian.com', + extras: [ + { + key: 'schema', + value: + '{"fields":[{"name":"id","type":"integer"},{"name":"title","type":"string"}]}', + }, + ], + name: 'gdp', + resources: [ + { + name: 'data.csv', + url: 'http://someplace.com/data.csv', + size: 100, + }, + ], + }) + return true + }) + .reply(200, { + help: 'http://localhost:5000/api/3/action/help_show?name=package_create', + success: true, + result: ckanDataset, + }) + + const response = await client.push(frictionlessDataset) + + t.is(packagePushMock.isDone(), true) + t.deepEqual(response, frictionlessDataset) +}) + +test('create a metadata', async (t) => { + // Mock datasets + const frictionlessDataset = JSON.parse( + await fs.readFileSync( + __dirname + '/mocks/frictionless-dataset-converted-from-ckan.json' + ) + ) + const ckanDataset = JSON.parse( + await fs.readFileSync(__dirname + '/mocks/ckan-dataset.json') + ) + + // test with object + let packageCreateMock = nock(config.api) + .post('/api/3/action/package_create', (body) => { + t.deepEqual(body, { + notes: 'GDPs list', + url: 'https://datopian.com', + extras: [ + { + key: 'schema', + value: + '{"fields":[{"name":"id","type":"integer"},{"name":"title","type":"string"}]}', + }, + ], + name: 'gdp', + resources: [ + { + name: 'data.csv', + url: 'http://someplace.com/data.csv', + size: 100, + }, + ], + }) + return true + }) + .reply(200, { + help: 'http://localhost:5000/api/3/action/help_show?name=package_create', + success: true, + result: ckanDataset, + }) + + let response = await client.create(frictionlessDataset) + + t.is(packageCreateMock.isDone(), true) + t.deepEqual(response, frictionlessDataset) + + // test with string + packageCreateMock = nock(config.api) + .persist() + .post('/api/3/action/package_create', (body) => { + t.deepEqual(body, { + name: 'My Dataset Name', + }) + return true + }) + .reply(200, { + help: 'http://localhost:5000/api/3/action/help_show?name=package_create', + success: true, + result: { + name: 'My Dataset Name', + }, + }) + + response = await client.create('My Dataset Name') + + t.is(packageCreateMock.isDone(), true) + t.deepEqual(response, { + name: 'My Dataset Name', + }) +}) diff --git a/test/mocks/ckan-dataset.json b/test/mocks/ckan-dataset.json new file mode 100644 index 0000000..60158be --- /dev/null +++ b/test/mocks/ckan-dataset.json @@ -0,0 +1,22 @@ +{ + "name": "gdp", + "url": "https://datopian.com", + "resources": [ + { + "name": "data.csv", + "url": "http://someplace.com/data.csv", + "size": 100 + } + ], + "description": "GDPs list", + "schema": { + "fields": [ + { "name": "id", "type": "integer" }, + { "name": "title", "type": "string" } + ] + }, + "isopen": true, + "num_tags": 1, + "num_resources": 10, + "state": "active" +} diff --git a/test/mocks/getPackageFrictionlessResponse.json b/test/mocks/frictionless-dataset-converted-from-ckan.json similarity index 88% rename from test/mocks/getPackageFrictionlessResponse.json rename to test/mocks/frictionless-dataset-converted-from-ckan.json index 25639c3..c677f3a 100644 --- a/test/mocks/getPackageFrictionlessResponse.json +++ b/test/mocks/frictionless-dataset-converted-from-ckan.json @@ -1,5 +1,6 @@ { "description": "GDPs list", + "homepage": "https://datopian.com", "schema": { "fields": [ { "name": "id", "type": "integer" }, diff --git a/test/mocks/getPackageCkanResponse.json b/test/mocks/getPackageCkanResponse.json index 7d5bfe7..1f36c5a 100644 --- a/test/mocks/getPackageCkanResponse.json +++ b/test/mocks/getPackageCkanResponse.json @@ -2,6 +2,7 @@ "success": true, "result": { "name": "gdp", + "url": "https://datopian.com", "resources": [ { "name": "data.csv", diff --git a/test/upload.test.js b/test/upload.test.js index 1753580..7ceef1c 100644 --- a/test/upload.test.js +++ b/test/upload.test.js @@ -33,7 +33,7 @@ const accessGranterConfig = { }, headers: { Accept: 'application/vnd.git-lfs+json', - "Content-Type": 'application/vnd.git-lfs+json', + 'Content-Type': 'application/vnd.git-lfs+json', Authorization: 'Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzY29wZXMi===', }, @@ -60,21 +60,19 @@ const file = new Open.NodeFileSystemFile('./test/fixtures/sample.csv') /** * Mock */ -const ckanAuthzMock = nock("http://127.0.0.1") +const ckanAuthzMock = nock('http://127.0.0.1') .persist() .post('/api/3/action/authz_authorize', ckanAuthzConfig.body) .reply(200, { - "help": "http://localhost:5000/api/3/action/help_show?name=authz_authorize", - "success": true, - "result": { - "requested_scopes": [ - "obj:myorg/dataset-name/*:write" - ], - "granted_scopes": [], - "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzY29wZXMiOiIiLCJ== ", - "user_id": "admin", - "expires_at": "2020-04-22T20:08:41.102934+00:00" - } + help: 'http://localhost:5000/api/3/action/help_show?name=authz_authorize', + success: true, + result: { + requested_scopes: ['obj:myorg/dataset-name/*:write'], + granted_scopes: [], + token: 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzY29wZXMiOiIiLCJ== ', + user_id: 'admin', + expires_at: '2020-04-22T20:08:41.102934+00:00', + }, }) const mainAuthzMock_forCloudStorageAccessGranterServiceMock = nock(config.api) @@ -87,12 +85,14 @@ const mainAuthzMock_forCloudStorageAccessGranterServiceMock = nock(config.api) transfer: 'basic', objects: [ { - oid: "8857053d874453bbe8e7613b09874e2d8fc9ddffd2130a579ca918301c31b369", + oid: + '8857053d874453bbe8e7613b09874e2d8fc9ddffd2130a579ca918301c31b369', size: 701, authenticated: true, actions: { upload: { - href: 'https://myaccount.blob.core.windows.net/mycontainer/my-blob', + href: + 'https://myaccount.blob.core.windows.net/mycontainer/my-blob', header: accessGranterConfig.headers, expires_in: 86400, }, @@ -120,7 +120,7 @@ const verifyFileUploadMock = nock('https://some-verify-callback.com') .persist() .post('/') .reply(200, { - message: "Verify Uploaded Successfully", + message: 'Verify Uploaded Successfully', success: true, }) @@ -141,7 +141,7 @@ test('Can get JWT token', async (t) => { const token = await client.doBlobAuthz() t.is(ckanAuthzMock.isDone(), true) - t.is(token, "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzY29wZXMiOiIiLCJ== ") + t.is(token, 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzY29wZXMiOiIiLCJ== ') }) test('Push works with packaged dataset', async (t) => { @@ -159,5 +159,8 @@ test('Dataset not altered', async (t) => { const sha256 = await file.sha256() t.is(size, 701) - t.is(sha256, '7b28186dca74020a82ed969101ff551f97aed110d8737cea4763ce5be3a38b47') + t.is( + sha256, + '7b28186dca74020a82ed969101ff551f97aed110d8737cea4763ce5be3a38b47' + ) })