From fe1ecaf8f7820063bf818c8047ada655f9c2813f Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Sat, 26 Aug 2023 13:19:17 +0200 Subject: [PATCH] Add support for partial upload --- package-lock.json | 16 +- package.json | 3 +- src/api.js | 32 ++- src/request.js | 128 ++++++++---- src/response.js | 11 +- test/api.spec.js | 23 ++- test/request.spec.js | 458 +++++++++++++++++++++++++++++-------------- 7 files changed, 461 insertions(+), 210 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5cf03a3..a79ac19 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,8 @@ "@babel/runtime": "^7.22.10", "@babel/runtime-corejs3": "^7.22.10", "cross-fetch": "^4.0.0", - "spark-md5": "^3.0.2" + "spark-md5": "^3.0.2", + "xdelta3-wasm": "^1.0.0" }, "devDependencies": { "@babel/core": "^7.22.10", @@ -8314,6 +8315,14 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "dev": true }, + "node_modules/xdelta3-wasm": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/xdelta3-wasm/-/xdelta3-wasm-1.0.0.tgz", + "integrity": "sha512-vhS28BhVaE3S/PGG1KQIwjBVqJecuS5Sdh82UAZysbnYaU93KS6l8ZPtsqonNZMeuyFgrGW9D44hh2lwQCsidA==", + "engines": { + "node": ">=12" + } + }, "node_modules/xmlcreate": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.4.tgz", @@ -14513,6 +14522,11 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "dev": true }, + "xdelta3-wasm": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/xdelta3-wasm/-/xdelta3-wasm-1.0.0.tgz", + "integrity": "sha512-vhS28BhVaE3S/PGG1KQIwjBVqJecuS5Sdh82UAZysbnYaU93KS6l8ZPtsqonNZMeuyFgrGW9D44hh2lwQCsidA==" + }, "xmlcreate": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.4.tgz", diff --git a/package.json b/package.json index 5081369..e9d2755 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,8 @@ "@babel/runtime": "^7.22.10", "@babel/runtime-corejs3": "^7.22.10", "cross-fetch": "^4.0.0", - "spark-md5": "^3.0.2" + "spark-md5": "^3.0.2", + "xdelta3-wasm": "^1.0.0" }, "devDependencies": { "@babel/core": "^7.22.10", diff --git a/src/api.js b/src/api.js index df7e420..e1e5134 100644 --- a/src/api.js +++ b/src/api.js @@ -362,32 +362,44 @@ const api = function() { /** * Configure api to upload or download an attachment file - * Can be only used in conjuction with items() and post()/get() + * Can be only used in conjuction with items() and post()/get()/patch() + * Method patch() can only be used to upload a file patch, in this case algorithm must be provided * Use items() to select attachment item for which file is uploaded/downloaded * Will populate format on download as well as Content-Type, If-None-Match headers * in case of an upload - * @param {String} fileName - name of the file, should match values in attachment - * item entry - * @param {ArrayBuffer} file - file to be uploaded - * @param {Number} mtime - file's mtime, if not provided current time is used - * @param {Number} md5sum - existing file md5sum, if matches will override existing file. Leave empty to perform new upload. + * @param {String} fileName - name of the file, should match values in attachment item entry + * @param {ArrayBuffer} file - file to be uploaded + * @param {Number} md5OrNewFile - existing file or existing file's md5sum. Former will calculate + * diff and trigger partial update, latter will replace existing + * file with a full upload if md5sum matches previous file. + * Leave empty to perform new upload. * @return {Object} Partially configured api functions * @chainable */ - const attachment = function(fileName, file, mtime = null, md5sum = null) { + const attachment = function (fileName, file, oldFileOrMd5 = null) { let resource = { ...this.resource, file: null }; + + let bindParams = {}; + + if(typeof(oldFileOrMd5) === 'string' && oldFileOrMd5.length === 32) { + bindParams.ifMatch = oldFileOrMd5; + } else if(oldFileOrMd5) { + bindParams.oldFile = oldFileOrMd5; + } else { + bindParams.ifNoneMatch = '*'; + } + if(fileName && file) { return ef.bind(this)({ format: null, - [md5sum ? 'ifMatch' : 'ifNoneMatch']: md5sum || '*', contentType: 'application/x-www-form-urlencoded', fileName, - file, resource, - mtime + file, + ...bindParams }) } else { return ef.bind(this)({ format: null, resource }); diff --git a/src/request.js b/src/request.js index 8038ccb..3468ba0 100644 --- a/src/request.js +++ b/src/request.js @@ -1,10 +1,15 @@ import SparkMD5 from 'spark-md5'; + +//todo move to worker +import { init, xd3_encode_memory, xd3_smatch_cfg } from "xdelta3-wasm"; + + import { ApiResponse, DeleteResponse, ErrorResponse, FileDownloadResponse, FileUploadResponse, FileUrlResponse, MultiReadResponse, MultiWriteResponse, PretendResponse, RawApiResponse, SchemaResponse, SingleReadResponse, SingleWriteResponse, } from './response.js'; -const headerNames = { +const acceptedHeaderNames = { authorization: 'Authorization', contentType: 'Content-Type', ifMatch: 'If-Match', @@ -43,6 +48,11 @@ const queryParamNames = [ 'tag', ]; +const filePatchQueryParamNames = [ + 'algorithm', + 'upload', +]; + const fetchParamNames = [ 'body', 'cache', @@ -118,9 +128,9 @@ const makeUrlPath = resource => { return path.join('/'); }; -const makeUrlQuery = options => { +const makeUrlQuery = (options, paramNames) => { let params = []; - for(let name of queryParamNames) { + for(let name of paramNames) { if(options[name]) { if(queryParamsWithArraySupport.includes(name) && Array.isArray(options[name])) { params.push(...options[name].map(k => `${name}=${encodeURIComponent(k)}`)); @@ -132,6 +142,16 @@ const makeUrlQuery = options => { return params.length ? '?' + params.join('&') : ''; }; +const makeHeaders = (options, headerNames) => { + let headers = {}; + for (let header of Object.keys(headerNames)) { + if (header in options) { + headers[headerNames[header]] = options[header]; + } + } + return headers; +} + const hasDefinedKey = (object, key) => { return key in object && object[key] !== null && typeof(object[key]) !== 'undefined'; } @@ -152,6 +172,8 @@ const sleep = seconds => { }); }; + + /** * Executes request and returns a response. Not meant to be called directly, instead use {@link module:zotero-api-client~api}. @@ -224,20 +246,38 @@ const request = async config => { } const options = {...defaults, ...config}; - - if (['POST', 'PUT', 'PATCH'].includes(options.method.toUpperCase()) && !('contentType' in config)) { - options.contentType = 'application/json'; + + if (hasDefinedKey(options, 'body') && (hasDefinedKey(options, 'file') || hasDefinedKey(options, 'oldFile'))) { + throw new Error('Cannot use both "file" and "body" in a single request.'); } - const headers = {}; + if (hasDefinedKey(options, 'oldFile')) { + // for partial file upload uses patch(), however file authorisation still uses POST + options.method = 'POST'; + const fileView = new Uint8Array(options.file); + const oldFileView = new Uint8Array(options.oldFile); + + // clamp max size to 1K - 10M + const maxSize = Math.max(Math.min(fileView.byteLength, oldFileView.byteLength, 10 * 1024 * 1024), 1024); - for(let header of Object.keys(headerNames)) { - if(header in options) { - headers[headerNames[header]] = options[header]; + //todo move to worker + await init(); + const { str, output } = xd3_encode_memory(fileView, oldFileView, maxSize, xd3_smatch_cfg.DEFAULT); + if(str !== 'SUCCESS') { + throw new Error(`Failed to calculate diff: ${str}`); } + options.ifMatch = SparkMD5.ArrayBuffer.hash(options.oldFile); + options.filePatch = output; + options.algorithm = 'xdelta'; + } + + if (['POST', 'PUT', 'PATCH'].includes(options.method.toUpperCase()) && !('contentType' in config)) { + options.contentType = 'application/json'; } + + const headers = makeHeaders(options, acceptedHeaderNames); const path = makeUrlPath(options.resource); - const query = makeUrlQuery(options); + const query = makeUrlQuery(options, queryParamNames); const url = `https://${options.apiAuthorityPart}/${path}${query}`; const fetchConfig = {}; @@ -249,17 +289,12 @@ const request = async config => { } } - // build pre-upload (authorisation) request body based on the file provided - if(hasDefinedKey(options, 'body') && hasDefinedKey(options, 'file')) { - throw new Error('Cannot use both "file" and "body" in a single request.'); - } - // process the request for file upload authorisation request - if(hasDefinedKey(options, 'file') && hasDefinedKey(options, 'fileName')) { + if ((hasDefinedKey(options, 'filePatch') || hasDefinedKey(options, 'file')) && hasDefinedKey(options, 'fileName')) { let fileName = options.fileName; let md5sum = SparkMD5.ArrayBuffer.hash(options.file); + let mtime = Date.now(); let filesize = options.file.byteLength; - let mtime = options.mtime || Date.now(); fetchConfig['body'] = `md5=${md5sum}&filename=${fileName}&filesize=${filesize}&mtime=${mtime}`; } @@ -296,7 +331,7 @@ const request = async config => { } } - if((hasDefinedKey(options, 'file') && hasDefinedKey(options, 'fileName')) || options.uploadRegisterOnly === true) { + if (((hasDefinedKey(options, 'file') || hasDefinedKey(options, 'filePatch')) && hasDefinedKey(options, 'fileName')) || options.uploadRegisterOnly === true) { if(rawResponse.ok) { let authData = await rawResponse.json(); if('exists' in authData && authData.exists) { @@ -309,30 +344,47 @@ const request = async config => { rawResponse, options ); } - let prefix = new Uint8ClampedArray(authData.prefix.split('').map(e => e.charCodeAt(0))); - let suffix = new Uint8ClampedArray(authData.suffix.split('').map(e => e.charCodeAt(0))); - let body = new Uint8ClampedArray(prefix.byteLength + options.file.byteLength + suffix.byteLength); - body.set(prefix, 0); - body.set(new Uint8ClampedArray(options.file), prefix.byteLength); - body.set(suffix, prefix.byteLength + options.file.byteLength); - - // follow-up request - let uploadResponse = await fetch(authData.url, { - headers: { - [headerNames['contentType']]: authData.contentType, - }, - method: 'post', - body: body.buffer - }); - - if(uploadResponse.status === 201) { - let registerResponse = await fetch(url, { + let uploadResponse, isUploadSuccessful, registerResponse; + if(hasDefinedKey(options, 'filePatch')) { + const uploadQuery = makeUrlQuery({ ...options, upload: authData.uploadKey }, filePatchQueryParamNames); + const uploadUrl = `https://${options.apiAuthorityPart}/${path}${uploadQuery}`; + + // upload file patch request + uploadResponse = await fetch(uploadUrl, { + ...fetchConfig, + method: 'PATCH', + body: options.filePatch, + }); + isUploadSuccessful = uploadResponse.status === 204; + } else { + let prefix = new Uint8ClampedArray(authData.prefix.split('').map(e => e.charCodeAt(0))); + let suffix = new Uint8ClampedArray(authData.suffix.split('').map(e => e.charCodeAt(0))); + let body = new Uint8ClampedArray(prefix.byteLength + options.file.byteLength + suffix.byteLength); + body.set(prefix, 0); + body.set(new Uint8ClampedArray(options.file), prefix.byteLength); + body.set(suffix, prefix.byteLength + options.file.byteLength); + + // full file upload request + uploadResponse = await fetch(authData.url, { + headers: { + [acceptedHeaderNames['contentType']]: authData.contentType, + }, + method: 'POST', + body: body.buffer + }); + isUploadSuccessful = uploadResponse.status === 201; + + // register file request + registerResponse = await fetch(url, { ...fetchConfig, body: `upload=${authData.uploadKey}` }); - if(!registerResponse.ok) { + if (!registerResponse.ok) { return await throwErrorResponse(registerResponse, options, 'Upload stage 3: '); } + } + + if (isUploadSuccessful) { response = new FileUploadResponse({}, options, rawResponse, uploadResponse, registerResponse); } else { return await throwErrorResponse(uploadResponse, options, 'Upload stage 2: '); diff --git a/src/response.js b/src/response.js index 83ac7b7..73ed5e6 100644 --- a/src/response.js +++ b/src/response.js @@ -1,5 +1,5 @@ const parseIntHeaders = (headers, headerName) => { - const value = headers && headers.get(headerName); + const value = (headers && headers.get(headerName)) || null; return value === null ? null : parseInt(value, 10); } @@ -371,8 +371,13 @@ class FileUploadResponse extends ApiResponse { * @see {@link module:zotero-api-client~ApiResponse#getVersion} */ getVersion() { - return this.registerResponse ? - parseIntHeaders(this.registerResponse?.headers, 'Last-Modified-Version') : + // full upload will have latest version in the final register response, + // partial upload could have latest version in the upload response + // (because that goes through API), however currently that's not the case. + // If file existed before, and currently for partial uploads, latest version + // will be in obtained from the initial response + return parseIntHeaders(this.registerResponse?.headers, 'Last-Modified-Version') ?? + parseIntHeaders(this.uploadResponse?.headers, 'Last-Modified-Version') ?? parseIntHeaders(this.response?.headers, 'Last-Modified-Version'); } } diff --git a/test/api.spec.js b/test/api.spec.js index 43a72eb..21d3366 100644 --- a/test/api.spec.js +++ b/test/api.spec.js @@ -11,10 +11,10 @@ const SEARCH_KEY = 'SEARCH_KEY'; const SETTING_KEY = 'SETTING_KEY'; const URL_ENCODED_TAGS = 'URL_ENCODED_TAGS'; const FILE = Uint8ClampedArray.from('lorem ipsum'.split('').map(e => e.charCodeAt(0))).buffer; +const NEW_FILE = Uint8ClampedArray.from('lorem dolot ipsum'.split('').map(e => e.charCodeAt(0))).buffer; const FILE_NAME = 'test.txt'; const MD5 = '9edb2ca32f7b57662acbc112a80cc59d'; - describe('Zotero Api Client', () => { var lrc; // mock api so it never calls request(), instead @@ -449,7 +449,7 @@ describe('Zotero Api Client', () => { }); describe('Construct attachment requests', () => { - it('handles api.library.items(I).attachment(Fname, F).post()', () => { + it('handles api.library.items(I).attachment(Fname, F).post() to upload new file', () => { api(KEY).library(LIBRARY_KEY).items(ITEM_KEY).attachment(FILE_NAME, FILE).post(); assert.equal(lrc.method, 'post'); assert.equal(lrc.resource.library, LIBRARY_KEY); @@ -463,20 +463,33 @@ describe('Zotero Api Client', () => { assert.isUndefined(lrc.body); }); - it('handles api.library.items(I).attachment(Fname, F, Fmtime, F2md5sum).post()', () => { - api(KEY).library(LIBRARY_KEY).items(ITEM_KEY).attachment(FILE_NAME, FILE, 22, MD5).post(); + it('handles api.library.items(I).attachment(Fname, F, OLD_MD5).post() to update file by performing full upload', () => { + api(KEY).library(LIBRARY_KEY).items(ITEM_KEY).attachment(FILE_NAME, FILE, MD5).post(); assert.equal(lrc.method, 'post'); assert.equal(lrc.resource.library, LIBRARY_KEY); assert.equal(lrc.resource.items, ITEM_KEY); assert.isNull(lrc.resource.file); assert.equal(lrc.file.byteLength, FILE.byteLength); assert.equal(lrc.fileName, FILE_NAME); - assert.equal(lrc.mtime, 22); assert.equal(lrc.ifMatch, MD5); assert.equal(lrc.contentType, 'application/x-www-form-urlencoded'); assert.isNull(lrc.format); assert.isUndefined(lrc.body); }); + + it('handles api.library.items(I).attachment(Fname, F, OLD_FILE).patch() to update file by performing partial upload', () => { + api(KEY).library(LIBRARY_KEY).items(ITEM_KEY).attachment(FILE_NAME, NEW_FILE, FILE).patch(); + assert.equal(lrc.method, 'patch'); + assert.equal(lrc.resource.library, LIBRARY_KEY); + assert.equal(lrc.resource.items, ITEM_KEY); + assert.isNull(lrc.resource.file); + assert.equal(lrc.fileName, FILE_NAME); + assert.equal(lrc.oldFile.byteLength, FILE.byteLength); + assert.equal(lrc.file.byteLength, NEW_FILE.byteLength); + assert.equal(lrc.contentType, 'application/x-www-form-urlencoded'); + assert.isNull(lrc.format); + assert.isUndefined(lrc.body); + }); it('handles api.library.items(I).attachment().get()', () => { api(KEY).library(LIBRARY_KEY).items(ITEM_KEY).attachment().get(); diff --git a/test/request.spec.js b/test/request.spec.js index 4cabd2d..1f24b8b 100644 --- a/test/request.spec.js +++ b/test/request.spec.js @@ -1,11 +1,12 @@ /* eslint-env mocha */ -import URL from 'url'; import fetchMock from 'fetch-mock'; import { assert } from 'chai'; import _request from '../src/request.js'; -import { ApiResponse, DeleteResponse, ErrorResponse, FileDownloadResponse, FileUploadResponse, +import { + ApiResponse, DeleteResponse, ErrorResponse, FileDownloadResponse, FileUploadResponse, FileUrlResponse, MultiReadResponse, MultiWriteResponse, PretendResponse, RawApiResponse, - SchemaResponse, SingleReadResponse, SingleWriteResponse, } from '../src/response.js'; + SchemaResponse, SingleReadResponse, SingleWriteResponse, +} from '../src/response.js'; import singleGetResponseFixture from './fixtures/single-object-get-response.js'; import multiGetResponseFixture from './fixtures/multi-object-get-response.js'; @@ -21,7 +22,9 @@ import userGroupsFixture from './fixtures/user-groups-response.js'; const FILE = Uint8ClampedArray.from('lorem ipsum'.split('').map(e => e.charCodeAt(0))).buffer; const FILE_MD5 = '80a751fde577028640c419000e33eba6'; const FILE_NAME = 'test.txt'; -const FILE_SIZE = FILE.byteLength; +const NEW_FILE = Uint8ClampedArray.from('lorem dolot ipsum'.split('').map(e => e.charCodeAt(0))).buffer; +const NEW_FILE_MD5 = '0293973cad1be8dbda55d94c53069865'; +const API_KEY = 'test'; const request = async (opts) => { var config = await _request(opts); @@ -30,11 +33,11 @@ const request = async (opts) => { describe('ZoteroJS request', () => { beforeEach(() => { - fetchMock.config.overwriteRoutes = false; - fetchMock.catch(request => { - throw(new Error(`A request to ${request} was not expected`)); - }); + fetchMock.config.overwriteRoutes = false; + fetchMock.catch(request => { + throw (new Error(`A request to ${request} was not expected`)); }); + }); afterEach(() => fetchMock.restore()); @@ -98,19 +101,19 @@ describe('ZoteroJS request', () => { assert.strictEqual(Object.keys(response.getData()).length, 2); }); }); - + it('should get annotation template', () => { fetchMock.mock( url => { return url.startsWith('https://api.zotero.org/items/new') && - [ - ['itemType', 'annotation'], - ['annotationType', 'highlight'] - ].every(([q, v]) => url.match(new RegExp(`\\b${q}=${v}\\b`))); + [ + ['itemType', 'annotation'], + ['annotationType', 'highlight'] + ].every(([q, v]) => url.match(new RegExp(`\\b${q}=${v}\\b`))); }, { - itemType: 'annotation', - annotationType: 'highlight' - } + itemType: 'annotation', + annotationType: 'highlight' + } ); return request({ @@ -150,11 +153,11 @@ describe('ZoteroJS request', () => { }); it('should get /top items from a user library', () => { - fetchMock.mock( (url, opts) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/top')); - assert.strictEqual(opts.method, 'GET'); - return true; - }, { + fetchMock.mock((url, opts) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/top')); + assert.strictEqual(opts.method, 'GET'); + return true; + }, { headers: { 'Last-Modified-Version': 1337, 'Total-Results': 15, @@ -303,7 +306,7 @@ describe('ZoteroJS request', () => { resource: { library: 'u475425', tags: null - }, + }, tag: 'Fiction' }).then(response => { assert.instanceOf(response, ApiResponse); @@ -321,7 +324,7 @@ describe('ZoteroJS request', () => { resource: { library: 'u475425', searches: null - }, + }, searchKey: 'HHF7BB4C' }).then(response => { assert.instanceOf(response, ApiResponse); @@ -332,19 +335,19 @@ describe('ZoteroJS request', () => { it('should handle sorting and pagination', () => { fetchMock.mock( url => { - let parsedUrl = URL.parse(url); + let parsedUrl = new URL(url); parsedUrl = parsedUrl.search.slice(1); parsedUrl = parsedUrl.split('&'); - if(!parsedUrl.includes('sort=title')) { + if (!parsedUrl.includes('sort=title')) { return false; } - if(!parsedUrl.includes('direction=asc')) { + if (!parsedUrl.includes('direction=asc')) { return false; } - if(!parsedUrl.includes('limit=50')) { + if (!parsedUrl.includes('limit=50')) { return false; } - if(!parsedUrl.includes('start=25')) { + if (!parsedUrl.includes('start=25')) { return false; } return true; @@ -356,7 +359,7 @@ describe('ZoteroJS request', () => { resource: { library: 'u475425', items: null - }, + }, sort: 'title', direction: 'asc', limit: 50, @@ -370,10 +373,10 @@ describe('ZoteroJS request', () => { it('should handle searching by itemKey', () => { fetchMock.mock( url => { - let parsedUrl = URL.parse(url); + let parsedUrl = new URL(url); parsedUrl = parsedUrl.search.slice(1); parsedUrl = parsedUrl.split('&'); - if(!parsedUrl.includes('itemKey=N7W92H48')) { + if (!parsedUrl.includes('itemKey=N7W92H48')) { return false; } return true; @@ -385,7 +388,7 @@ describe('ZoteroJS request', () => { resource: { library: 'u475425', items: null - }, + }, itemKey: 'N7W92H48' }).then(response => { assert.instanceOf(response, MultiReadResponse); @@ -394,11 +397,11 @@ describe('ZoteroJS request', () => { }); it('should handle multiple response with missing headers', () => { - fetchMock.mock( (url, opts) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/top')); - assert.strictEqual(opts.method, 'GET'); - return true; - }, { + fetchMock.mock((url, opts) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/top')); + assert.strictEqual(opts.method, 'GET'); + return true; + }, { headers: {}, body: multiGetResponseFixture }); @@ -430,7 +433,7 @@ describe('ZoteroJS request', () => { resource: { library: 'u475425', searches: null - }, + }, searchKey: 'HHF7BB4C' }).then(response => { assert.instanceOf(response, MultiReadResponse); @@ -448,7 +451,7 @@ describe('ZoteroJS request', () => { resource: { library: 'u475425', searches: null - } + } }).then(response => { assert.instanceOf(response, ApiResponse); assert.strictEqual(response.getData().length, 1); @@ -556,7 +559,7 @@ describe('ZoteroJS request', () => { it('should get a set of tags for filtered by itemsQ and itemsTag', () => { fetchMock.mock( - url => { + url => { assert.isOk( url.startsWith('https://api.zotero.org/users/475425/items/tags') ); @@ -651,7 +654,7 @@ describe('ZoteroJS request', () => { /https:\/\/api.zotero.org\/users\/475425\/?.*?since=42/i, responseRaw ); - + return request({ resource: { library: 'u475425', @@ -697,7 +700,7 @@ describe('ZoteroJS request', () => { keysCurrentResponse ); - return request({ + return request({ resource: { verifyKeyAccess: null } }).then(response => { assert.instanceOf(response, ApiResponse); @@ -729,7 +732,7 @@ describe('ZoteroJS request', () => { 'authorization': 'a', 'zoteroWriteToken': 'b', 'ifModifiedSinceVersion': 1, - 'ifUnmodifiedSinceVersion': 1 , + 'ifUnmodifiedSinceVersion': 1, 'contentType': 'c' }); }); @@ -746,7 +749,7 @@ describe('ZoteroJS request', () => { return true; }, {} ); - + return request({ resource: { schema: null, @@ -756,7 +759,7 @@ describe('ZoteroJS request', () => { it('should include query params in the request', () => { fetchMock.mock( - url => { + url => { return [ 'collectionKey', 'content', 'direction', 'format', 'include', 'includeTrashed', 'itemKey', 'itemType', 'limit', 'q', 'qmode', 'searchKey', 'since', 'sort', @@ -795,7 +798,7 @@ describe('ZoteroJS request', () => { 'https://api.zotero.org/users/475425/items?format=json&tag=aaa&tag=bbb', multiGetResponseFixture ); - + return request({ resource: { library: 'u475425', @@ -941,19 +944,19 @@ describe('ZoteroJS request', () => { describe('Item write & delete requests', () => { it('should post a single item', () => { - fetchMock.mock( (url, opts) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items')); - assert.strictEqual(opts.method, 'POST'); - assert.propertyVal(opts.headers, 'Content-Type', 'application/json'); - return true; - }, { + fetchMock.mock((url, opts) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items')); + assert.strictEqual(opts.method, 'POST'); + assert.propertyVal(opts.headers, 'Content-Type', 'application/json'); + return true; + }, { headers: { 'Last-Modified-Version': 1337 }, body: multiSuccessWriteResponseFixture }); - const item ={ + const item = { 'key': 'AZBCAADA', 'version': 0, 'itemType': 'book', @@ -980,16 +983,16 @@ describe('ZoteroJS request', () => { }); it('should use new data from successful write response', () => { - const item ={ + const item = { 'version': 0, 'itemType': 'book', 'title': 'My Amazing Book' }; - fetchMock.post( (url) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items')); - return true; - }, { + fetchMock.post((url) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items')); + return true; + }, { headers: { 'Last-Modified-Version': 1337 }, @@ -1018,7 +1021,7 @@ describe('ZoteroJS request', () => { } } }); - + return request({ method: 'post', body: [item], @@ -1073,19 +1076,19 @@ describe('ZoteroJS request', () => { 'title': 'My super paper' }; - const serverSideData = { + const serverSideData = { dateAdded: "2018-07-05T09:24:36Z", dateModified: "2018-07-05T09:24:36Z", tags: [], relations: {} }; - fetchMock.mock( (url, opts) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items')); - assert.strictEqual(opts.method, 'POST'); - assert.propertyVal(opts.headers, 'Content-Type', 'application/json'); - return true; - }, { + fetchMock.mock((url, opts) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items')); + assert.strictEqual(opts.method, 'POST'); + assert.propertyVal(opts.headers, 'Content-Type', 'application/json'); + return true; + }, { headers: { 'Last-Modified-Version': 1337 }, @@ -1093,9 +1096,9 @@ describe('ZoteroJS request', () => { ...multiMixedWriteResponseFixture, "successful": { // add server side data to one of the responses - "0": { + "0": { data: { - ...book, ...serverSideData + ...book, ...serverSideData }, meta: { parsedDate: "1987" @@ -1146,12 +1149,12 @@ describe('ZoteroJS request', () => { }); it('should update put a single, complete item', () => { - fetchMock.mock( (url, opts) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/ABCD1111')); - assert.strictEqual(opts.method, 'PUT'); - assert.propertyVal(opts.headers, 'Content-Type', 'application/json'); - return true; - }, { + fetchMock.mock((url, opts) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/ABCD1111')); + assert.strictEqual(opts.method, 'PUT'); + assert.propertyVal(opts.headers, 'Content-Type', 'application/json'); + return true; + }, { status: 204, headers: { 'Last-Modified-Version': 42 @@ -1184,12 +1187,12 @@ describe('ZoteroJS request', () => { }); it('should patch a single item', () => { - fetchMock.mock( (url, opts) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/ABCD1111')); - assert.strictEqual(opts.method, 'PATCH'); - assert.propertyVal(opts.headers, 'Content-Type', 'application/json'); - return true; - }, { + fetchMock.mock((url, opts) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/ABCD1111')); + assert.strictEqual(opts.method, 'PATCH'); + assert.propertyVal(opts.headers, 'Content-Type', 'application/json'); + return true; + }, { status: 204, headers: { 'Last-Modified-Version': 42 @@ -1219,12 +1222,12 @@ describe('ZoteroJS request', () => { }); it('should delete a single item', () => { - fetchMock.mock( (url, opts) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/ABCD1111')); - assert.strictEqual(opts.method, 'DELETE'); - assert.notProperty(opts.headers, 'Content-Type'); - return true; - }, { + fetchMock.mock((url, opts) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/ABCD1111')); + assert.strictEqual(opts.method, 'DELETE'); + assert.notProperty(opts.headers, 'Content-Type'); + return true; + }, { status: 204, headers: { 'Last-Modified-Version': 43 @@ -1248,14 +1251,14 @@ describe('ZoteroJS request', () => { }); it('should delete multiple items', () => { - fetchMock.mock( (url, opts) => { - assert.strictEqual(opts.method, 'DELETE'); - let parsedUrl = URL.parse(url); - parsedUrl = parsedUrl.search.slice(1); - parsedUrl = parsedUrl.split('&'); - assert.isOk(parsedUrl.includes('itemKey=ABCD1111%2CABCD2222%2CABCD3333')); - return true; - }, { + fetchMock.mock((url, opts) => { + assert.strictEqual(opts.method, 'DELETE'); + let parsedUrl = new URL(url); + parsedUrl = parsedUrl.search.slice(1); + parsedUrl = parsedUrl.split('&'); + assert.isOk(parsedUrl.includes('itemKey=ABCD1111%2CABCD2222%2CABCD3333')); + return true; + }, { status: 204, headers: { 'Last-Modified-Version': 100 @@ -1278,22 +1281,26 @@ describe('ZoteroJS request', () => { }); it('should post updated library settings', () => { - fetchMock.mock( (url, opts) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/settings')); - assert.strictEqual(opts.method, 'POST'); - assert.equal(opts.body, JSON.stringify(newSettings)); - return true; - }, { - status: 204, - headers: { - 'Last-Modified-Version': 3483 - } + fetchMock.mock((url, opts) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/settings')); + assert.strictEqual(opts.method, 'POST'); + assert.equal(opts.body, JSON.stringify(newSettings)); + return true; + }, { + status: 204, + headers: { + 'Last-Modified-Version': 3483 + } }); - const newSettings = { tagColors: { value: [ { - "name": "test-tag", - "color": "#ffcc00" - }]}}; + const newSettings = { + tagColors: { + value: [{ + "name": "test-tag", + "color": "#ffcc00" + }] + } + }; return request({ method: 'post', @@ -1310,22 +1317,24 @@ describe('ZoteroJS request', () => { }); it('should put individual updated keys into library settings', () => { - fetchMock.mock( (url, opts) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/settings/tagColors')); - assert.strictEqual(opts.method, 'PUT'); - assert.equal(opts.body, JSON.stringify(newSettings)); - return true; - }, { - status: 204, - headers: { - 'Last-Modified-Version': 3483 - } + fetchMock.mock((url, opts) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/settings/tagColors')); + assert.strictEqual(opts.method, 'PUT'); + assert.equal(opts.body, JSON.stringify(newSettings)); + return true; + }, { + status: 204, + headers: { + 'Last-Modified-Version': 3483 + } }); - const newSettings = { value: [ { - "name": "test-tag", - "color": "#ffcc00" - }]}; + const newSettings = { + value: [{ + "name": "test-tag", + "color": "#ffcc00" + }] + }; return request({ method: 'put', @@ -1342,15 +1351,15 @@ describe('ZoteroJS request', () => { }); it('should delete individual keys from library settings', () => { - fetchMock.mock( (url, opts) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/settings/tagColors')); - assert.strictEqual(opts.method, 'DELETE'); - return true; - }, { - status: 204, - headers: { - 'Last-Modified-Version': 1234 - } + fetchMock.mock((url, opts) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/settings/tagColors')); + assert.strictEqual(opts.method, 'DELETE'); + return true; + }, { + status: 204, + headers: { + 'Last-Modified-Version': 1234 + } }); return request({ @@ -1391,11 +1400,11 @@ describe('ZoteroJS request', () => { describe('Failing write & delete requests', () => { it('should throw ErrorResponse for error post responses', () => { - fetchMock.mock( (url, opts) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/ABCD1111')); - assert.strictEqual(opts.method, 'PUT'); - return true; - }, { + fetchMock.mock((url, opts) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/ABCD1111')); + assert.strictEqual(opts.method, 'PUT'); + return true; + }, { status: 400, body: 'Uploaded data must be a JSON array' }); @@ -1417,11 +1426,11 @@ describe('ZoteroJS request', () => { }); it('should throw ErrorResponse for error put responses', () => { - fetchMock.mock( (url, opts) => { - assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/ABCD1111')); - assert.strictEqual(opts.method, 'PUT'); - return true; - }, { + fetchMock.mock((url, opts) => { + assert.isOk(url.startsWith('https://api.zotero.org/users/475425/items/ABCD1111')); + assert.strictEqual(opts.method, 'PUT'); + return true; + }, { status: 412, body: 'Item has been modified since specified version (expected 42, found 41)' }); @@ -1471,20 +1480,53 @@ describe('ZoteroJS request', () => { fileName: FILE_NAME, contentType: 'application/x-www-form-urlencoded', }; - it('should perform 3-step file upload procedure ', () => { - let counter = 0; + + let filePatchFullRequest = { + method: 'post', + resource: { + library: 'u475425', + items: 'ABCD1111', + file: null + }, + format: null, + file: NEW_FILE, + ifMatch: FILE_MD5, // old file's md5 + body: undefined, + fileName: FILE_NAME, + contentType: 'application/x-www-form-urlencoded', + }; + + let filePatchPartialRequest = { + method: 'patch', + resource: { + library: 'u475425', + items: 'ABCD1111', + file: null + }, + format: null, + oldFile: FILE, + file: NEW_FILE, + body: undefined, + fileName: FILE_NAME, + contentType: 'application/x-www-form-urlencoded', + algorithm: 'xdelta', + zoteroApiKey: API_KEY + }; + it('should upload a new file', () => { + let counter = 0; fetchMock.mock('https://api.zotero.org/users/475425/items/ABCD1111/file', (url, options) => { var config = options.body.split('&').reduce( - (acc, val) => { + (acc, val) => { acc[val.split('=')[0]] = val.split('=')[1]; return acc }, {} ); - switch(counter++) { + switch (counter++) { case 0: + // first request: upload authorization assert.propertyVal(config, 'md5', FILE_MD5); assert.propertyVal(config, 'filename', FILE_NAME); - assert.propertyVal(config, 'filesize', FILE_SIZE.toString()); + assert.propertyVal(config, 'filesize', FILE.byteLength.toString()); assert.property(config, 'mtime'); return { 'url': 'https://storage.zotero.org', @@ -1494,6 +1536,7 @@ describe('ZoteroJS request', () => { 'uploadKey': 'some key', }; case 1: + // final request: register upload assert.propertyVal(config, 'upload', 'some key'); return { status: 204, @@ -1502,9 +1545,10 @@ describe('ZoteroJS request', () => { } }; default: - throw(new Error(`This is ${counter + 1} request to ${url}. Only expected 2 requests.`)); + throw (new Error(`This is ${counter + 1} request to ${url}. Only expected 2 requests.`)); } }); + // second request: upload file to storage fetchMock.once('https://storage.zotero.org', (url, options) => { assert.strictEqual(counter, 1); assert.strictEqual(options.body.byteLength, 33); @@ -1519,6 +1563,115 @@ describe('ZoteroJS request', () => { assert.isNotOk(response.getData().exists); }); }); + + it('should update a file, using partial upload', () => { + let counter = 0; + fetchMock.mock('begin:https://api.zotero.org/users/475425/items/ABCD1111/file', (url, options) => { + const config = (counter === 0 || counter == 2) && options.body.split('&').reduce( + (acc, val) => { + acc[val.split('=')[0]] = val.split('=')[1]; + return acc + }, {} + ); + const parsedUrl = new URL(url); + assert.strictEqual(options.headers['Zotero-API-Key'], API_KEY); + switch (counter++) { + case 0: + // first request: upload authorization + assert.strictEqual(options.method, 'POST'); + assert.strictEqual(options.headers['If-Match'], FILE_MD5); + assert.strictEqual(options.headers['Content-Type'], 'application/x-www-form-urlencoded'); + assert.propertyVal(config, 'md5', NEW_FILE_MD5); + assert.propertyVal(config, 'filename', FILE_NAME); + assert.propertyVal(config, 'filesize', NEW_FILE.byteLength.toString()); + assert.property(config, 'mtime'); + return { + headers: { + 'Last-Modified-Version': 42 + }, + body: { + 'url': 'https://storage.zotero.org', + 'contentType': 'text/plain', + 'prefix': 'some prefix', + 'suffix': 'some suffix', + 'uploadKey': 'some key', + } + }; + case 1: + // second (last) request: upload file patch + assert.strictEqual(options.method, 'PATCH'); + assert.strictEqual(parsedUrl.searchParams.get('algorithm'), 'xdelta'); + assert.strictEqual(parsedUrl.searchParams.get('upload'), 'some key'); + assert.strictEqual(options.headers['If-Match'], FILE_MD5); + assert.strictEqual(options.body.byteLength, 28); // xdelta patch size + return { + status: 204 + }; + default: + throw (new Error(`Counted ${counter} requests to ${url}. Only expected 2 requests.`)); + } + }); + return request(filePatchPartialRequest).then(response => { + assert.instanceOf(response, FileUploadResponse); + assert.strictEqual(response.getResponseType(), 'FileUploadResponse'); + assert.strictEqual(response.getVersion(), 42); + assert.isNotOk(response.getData().exists); + }); + }); + + it('should update a file, using full upload', () => { + let counter = 0; + fetchMock.mock('https://api.zotero.org/users/475425/items/ABCD1111/file', (url, options) => { + var config = options.body.split('&').reduce( + (acc, val) => { + acc[val.split('=')[0]] = val.split('=')[1]; + return acc + }, {} + ); + switch (counter++) { + case 0: + // first request: upload authorization + assert.strictEqual(options.method, 'POST'); + assert.strictEqual(options.headers['If-Match'], FILE_MD5); + assert.strictEqual(options.headers['Content-Type'], 'application/x-www-form-urlencoded'); + assert.propertyVal(config, 'md5', NEW_FILE_MD5); + assert.propertyVal(config, 'filename', FILE_NAME); + assert.propertyVal(config, 'filesize', NEW_FILE.byteLength.toString()); + assert.property(config, 'mtime'); + return { + 'url': 'https://storage.zotero.org', + 'contentType': 'text/plain', + 'prefix': 'some prefix', + 'suffix': 'some suffix', + 'uploadKey': 'some key', + }; + case 1: + assert.propertyVal(config, 'upload', 'some key'); + return { + status: 204, + headers: { + 'Last-Modified-Version': 42 + } + }; + default: + throw (new Error(`Counted ${counter} requests to ${url}. Only expected 2 requests.`)); + } + }); + fetchMock.once('https://storage.zotero.org', (url, options) => { + assert.strictEqual(counter, 1); + assert.strictEqual(options.body.byteLength, NEW_FILE.byteLength + 'some prefix'.length + 'some suffix'.length); + return { + status: 201 + }; + }); + return request({ ...filePatchFullRequest }).then(response => { + assert.instanceOf(response, FileUploadResponse); + assert.strictEqual(response.getResponseType(), 'FileUploadResponse'); + assert.strictEqual(response.getVersion(), 42); + assert.isNotOk(response.getData().exists); + }); + }); + it('should handle { exists: 1 } response in stage 1', () => { fetchMock.once('https://api.zotero.org/users/475425/items/ABCD1111/file', { headers: { @@ -1533,7 +1686,7 @@ describe('ZoteroJS request', () => { assert.isOk(response.getData().exists); }); }); - it('should detect invalid config', () => { + it('should detect invalid config: body and file', () => { return request({ ...fileUploadRequest, body: 'should not be here' @@ -1541,9 +1694,10 @@ describe('ZoteroJS request', () => { throw new Error('fail'); }).catch(error => { assert.instanceOf(error, Error); - assert.include(error.toString(), 'Cannot use both'); + assert.match(error.toString(), /Cannot use both "file" and "body" in a single request./); }) }); + it('should handle error reponse in stage 1', () => { fetchMock.mock('https://api.zotero.org/users/475425/items/ABCD1111/file', { status: 409, @@ -1639,7 +1793,7 @@ describe('ZoteroJS request', () => { assert.instanceOf(response, FileUploadResponse); assert.strictEqual(response.getVersion(), 42); assert.isOk(response.getData().exists); - }); + }); }); it('should handle error if attempting to register file that does not exist', () => { @@ -1683,8 +1837,8 @@ describe('ZoteroJS request', () => { assert.strictEqual( Array.from( (new Uint8ClampedArray(response.getData()))) - .map(b => String.fromCharCode(b)) - .join(''), + .map(b => String.fromCharCode(b)) + .join(''), 'lorem ipsum' ); });