Add support for partial upload

package.json

@@ -42,7 +42,8 @@
     "@babel/runtime": "^7.22.10",
     "@babel/runtime-corejs3": "^7.22.10",
     "cross-fetch": "^4.0.0",
-    "spark-md5": "^3.0.2"
+    "spark-md5": "^3.0.2",
+    "xdelta3-wasm": "^1.0.0"
   },
   "devDependencies": {
     "@babel/core": "^7.22.10",

src/api.js

@@ -362,32 +362,44 @@ const api = function() {
 
 	/**
 	 * Configure api to upload or download an attachment file
-	 * Can be only used in conjuction with items() and post()/get()
+	 * Can be only used in conjuction with items() and post()/get()/patch()
+	 * Method patch() can only be used to upload a file patch, in this case algorithm must be provided
 	 * Use items() to select attachment item for which file is uploaded/downloaded
 	 * Will populate format on download as well as Content-Type, If-None-Match headers
 	 * in case of an upload
-	 * @param  {String} fileName  - name of the file, should match values in attachment
-	 *                              item entry
-	 * @param  {ArrayBuffer} file - file to be uploaded
-	 * @param  {Number} mtime     - file's mtime, if not provided current time is used
-	 * @param  {Number} md5sum    - existing file md5sum, if matches will override existing file. Leave empty to perform new upload.
+	 * @param  {String} fileName      - name of the file, should match values in attachment item entry
+	 * @param  {ArrayBuffer} file     - file to be uploaded
+	 * @param  {Number} md5OrNewFile  - existing file or existing file's md5sum. Former will calculate
+	 * 									diff and trigger partial update, latter will replace existing
+	 * 									file with a full upload if md5sum matches previous file.
+	 * 									Leave empty to perform new upload.
 	 * @return {Object} Partially configured api functions
 	 * @chainable
 	 */
-	const attachment = function(fileName, file, mtime = null, md5sum = null) {
+	const attachment = function (fileName, file, oldFileOrMd5 = null) {
 		let resource = {
 			...this.resource,
 			file: null
 		};
+
+		let bindParams = {};
+
+		if(typeof(oldFileOrMd5) === 'string' && oldFileOrMd5.length === 32) {
+			bindParams.ifMatch = oldFileOrMd5;
+		} else if(oldFileOrMd5) {
+			bindParams.oldFile = oldFileOrMd5;
+		} else {
+			bindParams.ifNoneMatch = '*';
+		}
+
 		if(fileName && file) {
 			return ef.bind(this)({
 				format: null,
-				[md5sum ? 'ifMatch' : 'ifNoneMatch']: md5sum || '*',
 				contentType: 'application/x-www-form-urlencoded',
 				fileName,
-				file,
 				resource,
-				mtime
+				file,
+				...bindParams
 			})	
 		} else {
 			return ef.bind(this)({ format: null, resource });

src/request.js

@@ -1,10 +1,15 @@
 import SparkMD5 from 'spark-md5';
 
+
+//todo move to worker
+import { init, xd3_encode_memory, xd3_smatch_cfg } from "xdelta3-wasm";
+
+
 import { ApiResponse, DeleteResponse, ErrorResponse, FileDownloadResponse, FileUploadResponse,
 	FileUrlResponse, MultiReadResponse, MultiWriteResponse, PretendResponse, RawApiResponse,
 	SchemaResponse, SingleReadResponse, SingleWriteResponse, } from './response.js';
 
-const headerNames = {
+const acceptedHeaderNames = {
 	authorization: 'Authorization',
 	contentType: 'Content-Type',
 	ifMatch: 'If-Match',
@@ -43,6 +48,11 @@ const queryParamNames = [
 	'tag',
 ];
 
+const filePatchQueryParamNames = [
+	'algorithm',
+	'upload',
+];
+
 const fetchParamNames = [
 	'body',
 	'cache',
@@ -118,9 +128,9 @@ const makeUrlPath = resource => {
 	return path.join('/');
 };
 
-const makeUrlQuery = options => {
+const makeUrlQuery = (options, paramNames) => {
 	let params = [];
-	for(let name of queryParamNames) {
+	for(let name of paramNames) {
 		if(options[name]) {
 			if(queryParamsWithArraySupport.includes(name) && Array.isArray(options[name])) {
 				params.push(...options[name].map(k => `${name}=${encodeURIComponent(k)}`));
@@ -132,6 +142,16 @@ const makeUrlQuery = options => {
 	return params.length ? '?' + params.join('&') : '';
 };
 
+const makeHeaders = (options, headerNames) => {
+	let headers = {};
+	for (let header of Object.keys(headerNames)) {
+		if (header in options) {
+			headers[headerNames[header]] = options[header];
+		}
+	}
+	return headers;
+}
+
 const hasDefinedKey = (object, key) => {
 	return key in object && object[key] !== null && typeof(object[key]) !== 'undefined';
 }
@@ -152,6 +172,8 @@ const sleep = seconds => {
 	});
 };
 
+
+
 /**
  * Executes request and returns a response. Not meant to be called directly, instead use {@link
    module:zotero-api-client~api}.
@@ -224,20 +246,38 @@ const request = async config => {
 	}
 
 	const options = {...defaults, ...config};
-	
-	if (['POST', 'PUT', 'PATCH'].includes(options.method.toUpperCase()) && !('contentType' in config)) {
-		options.contentType = 'application/json';
+
+	if (hasDefinedKey(options, 'body') && (hasDefinedKey(options, 'file') || hasDefinedKey(options, 'oldFile'))) {
+		throw new Error('Cannot use both "file" and "body" in a single request.');
 	}
 
-	const headers = {};
+	if (hasDefinedKey(options, 'oldFile')) {
+		// for partial file upload uses patch(), however file authorisation still uses POST
+		options.method = 'POST';
+		const fileView = new Uint8Array(options.file);
+		const oldFileView = new Uint8Array(options.oldFile);
+
+		// clamp max size to 1K - 10M
+		const maxSize = Math.max(Math.min(fileView.byteLength, oldFileView.byteLength, 10 * 1024 * 1024), 1024);
 
-	for(let header of Object.keys(headerNames)) {
-		if(header in options) {
-			headers[headerNames[header]] = options[header];
+		//todo move to worker
+		await init();
+		const { str, output } = xd3_encode_memory(fileView, oldFileView, maxSize, xd3_smatch_cfg.DEFAULT);
+		if(str !== 'SUCCESS') {
+			throw new Error(`Failed to calculate diff: ${str}`);
 		}
+		options.ifMatch = SparkMD5.ArrayBuffer.hash(options.oldFile);
+		options.filePatch = output;
+		options.algorithm = 'xdelta';
+	}
+
+	if (['POST', 'PUT', 'PATCH'].includes(options.method.toUpperCase()) && !('contentType' in config)) {
+		options.contentType = 'application/json';
 	}
+
+	const headers = makeHeaders(options, acceptedHeaderNames);
 	const path = makeUrlPath(options.resource);
-	const query = makeUrlQuery(options);
+	const query = makeUrlQuery(options, queryParamNames);
 	const url = `https://${options.apiAuthorityPart}/${path}${query}`;
 	const fetchConfig = {};
 
@@ -249,17 +289,12 @@ const request = async config => {
 		}
 	}
 
-	// build pre-upload (authorisation) request body based on the file provided
-	if(hasDefinedKey(options, 'body') && hasDefinedKey(options, 'file')) {
-		throw new Error('Cannot use both "file" and "body" in a single request.');
-	}
-
 	// process the request for file upload authorisation request
-	if(hasDefinedKey(options, 'file') && hasDefinedKey(options, 'fileName')) {
+	if ((hasDefinedKey(options, 'filePatch') || hasDefinedKey(options, 'file')) && hasDefinedKey(options, 'fileName')) {
 		let fileName = options.fileName;
 		let md5sum = SparkMD5.ArrayBuffer.hash(options.file);
+		let mtime = Date.now();
 		let filesize = options.file.byteLength;
-		let mtime = options.mtime || Date.now();
 		fetchConfig['body'] = `md5=${md5sum}&filename=${fileName}&filesize=${filesize}&mtime=${mtime}`;
 	}
 
@@ -296,7 +331,7 @@ const request = async config => {
 		}
 	}
 
-	if((hasDefinedKey(options, 'file') && hasDefinedKey(options, 'fileName')) || options.uploadRegisterOnly === true) {
+	if (((hasDefinedKey(options, 'file') || hasDefinedKey(options, 'filePatch')) && hasDefinedKey(options, 'fileName')) || options.uploadRegisterOnly === true) {
 		if(rawResponse.ok) {
 			let authData = await rawResponse.json();
 			if('exists' in authData && authData.exists) {
@@ -309,30 +344,47 @@ const request = async config => {
 						rawResponse, options
 					);
 				}
-				let prefix = new Uint8ClampedArray(authData.prefix.split('').map(e => e.charCodeAt(0)));
-				let suffix = new Uint8ClampedArray(authData.suffix.split('').map(e => e.charCodeAt(0)));
-				let body = new Uint8ClampedArray(prefix.byteLength + options.file.byteLength + suffix.byteLength);
-				body.set(prefix, 0);
-				body.set(new Uint8ClampedArray(options.file), prefix.byteLength);
-				body.set(suffix, prefix.byteLength + options.file.byteLength);
-
-				// follow-up request
-				let uploadResponse = await fetch(authData.url, {
-					headers: {
-						[headerNames['contentType']]: authData.contentType,
-					},
-					method: 'post',
-					body: body.buffer
-				});
-
-				if(uploadResponse.status === 201) {
-					let registerResponse = await fetch(url, {
+				let uploadResponse, isUploadSuccessful, registerResponse;
+				if(hasDefinedKey(options, 'filePatch')) {
+					const uploadQuery = makeUrlQuery({ ...options, upload: authData.uploadKey }, filePatchQueryParamNames);
+					const uploadUrl = `https://${options.apiAuthorityPart}/${path}${uploadQuery}`;
+
+					// upload file patch request
+					uploadResponse = await fetch(uploadUrl, {
+						...fetchConfig,
+						method: 'PATCH',
+						body: options.filePatch,
+					});
+					isUploadSuccessful = uploadResponse.status === 204;
+				} else {
+					let prefix = new Uint8ClampedArray(authData.prefix.split('').map(e => e.charCodeAt(0)));
+					let suffix = new Uint8ClampedArray(authData.suffix.split('').map(e => e.charCodeAt(0)));
+					let body = new Uint8ClampedArray(prefix.byteLength + options.file.byteLength + suffix.byteLength);
+					body.set(prefix, 0);
+					body.set(new Uint8ClampedArray(options.file), prefix.byteLength);
+					body.set(suffix, prefix.byteLength + options.file.byteLength);
+
+					// full file upload request
+					uploadResponse = await fetch(authData.url, {
+						headers: {
+							[acceptedHeaderNames['contentType']]: authData.contentType,
+						},
+						method: 'POST',
+						body: body.buffer
+					});
+					isUploadSuccessful = uploadResponse.status === 201;
+
+					// register file request
+					registerResponse = await fetch(url, {
 						...fetchConfig,
 						body: `upload=${authData.uploadKey}`
 					});
-					if(!registerResponse.ok) {
+					if (!registerResponse.ok) {
 						return await throwErrorResponse(registerResponse, options, 'Upload stage 3: ');
 					}
+				}
+
+				if (isUploadSuccessful) {
 					response = new FileUploadResponse({}, options, rawResponse, uploadResponse, registerResponse);
 				} else {
 					return await throwErrorResponse(uploadResponse, options, 'Upload stage 2: ');

src/response.js

@@ -1,5 +1,5 @@
 const parseIntHeaders = (headers, headerName) => {
-	const value = headers && headers.get(headerName);
+	const value = (headers && headers.get(headerName)) || null;
 	return value === null ? null : parseInt(value, 10);
 }
 
@@ -371,8 +371,13 @@ class FileUploadResponse extends ApiResponse {
 	* @see {@link module:zotero-api-client~ApiResponse#getVersion}
 	*/
 	getVersion() {
-		return this.registerResponse ?
-			parseIntHeaders(this.registerResponse?.headers, 'Last-Modified-Version') :
+		// full upload will have latest version in the final register response,
+		// partial upload could have latest version in the upload response
+		// (because that goes through API), however currently that's not the case.
+		// If file existed before, and currently for partial uploads, latest version
+		// will be in obtained from the initial response
+		return parseIntHeaders(this.registerResponse?.headers, 'Last-Modified-Version') ??
+			parseIntHeaders(this.uploadResponse?.headers, 'Last-Modified-Version') ??
 			parseIntHeaders(this.response?.headers, 'Last-Modified-Version');
 	}
 }

test/api.spec.js

@@ -11,10 +11,10 @@ const SEARCH_KEY = 'SEARCH_KEY';
 const SETTING_KEY = 'SETTING_KEY';
 const URL_ENCODED_TAGS = 'URL_ENCODED_TAGS';
 const FILE = Uint8ClampedArray.from('lorem ipsum'.split('').map(e => e.charCodeAt(0))).buffer;
+const NEW_FILE = Uint8ClampedArray.from('lorem dolot ipsum'.split('').map(e => e.charCodeAt(0))).buffer;
 const FILE_NAME = 'test.txt';
 const MD5 = '9edb2ca32f7b57662acbc112a80cc59d';
 
-
 describe('Zotero Api Client', () => {
 	var lrc;
 	// mock api so it never calls request(), instead
@@ -449,7 +449,7 @@ describe('Zotero Api Client', () => {
 	});
 
 	describe('Construct attachment requests', () => {
-		it('handles api.library.items(I).attachment(Fname, F).post()', () => {
+		it('handles api.library.items(I).attachment(Fname, F).post() to upload new file', () => {
 			api(KEY).library(LIBRARY_KEY).items(ITEM_KEY).attachment(FILE_NAME, FILE).post();
 			assert.equal(lrc.method, 'post');
 			assert.equal(lrc.resource.library, LIBRARY_KEY);
@@ -463,20 +463,33 @@ describe('Zotero Api Client', () => {
 			assert.isUndefined(lrc.body);
 		});
 
-		it('handles api.library.items(I).attachment(Fname, F, Fmtime, F2md5sum).post()', () => {
-			api(KEY).library(LIBRARY_KEY).items(ITEM_KEY).attachment(FILE_NAME, FILE, 22, MD5).post();
+		it('handles api.library.items(I).attachment(Fname, F, OLD_MD5).post() to update file by performing full upload', () => {
+			api(KEY).library(LIBRARY_KEY).items(ITEM_KEY).attachment(FILE_NAME, FILE, MD5).post();
 			assert.equal(lrc.method, 'post');
 			assert.equal(lrc.resource.library, LIBRARY_KEY);
 			assert.equal(lrc.resource.items, ITEM_KEY);
 			assert.isNull(lrc.resource.file);
 			assert.equal(lrc.file.byteLength, FILE.byteLength);
 			assert.equal(lrc.fileName, FILE_NAME);
-			assert.equal(lrc.mtime, 22);
 			assert.equal(lrc.ifMatch, MD5);
 			assert.equal(lrc.contentType, 'application/x-www-form-urlencoded');
 			assert.isNull(lrc.format);
 			assert.isUndefined(lrc.body);
 		});
+
+		it('handles api.library.items(I).attachment(Fname, F, OLD_FILE).patch() to update file by performing partial upload', () => {
+			api(KEY).library(LIBRARY_KEY).items(ITEM_KEY).attachment(FILE_NAME, NEW_FILE, FILE).patch();
+			assert.equal(lrc.method, 'patch');
+			assert.equal(lrc.resource.library, LIBRARY_KEY);
+			assert.equal(lrc.resource.items, ITEM_KEY);
+			assert.isNull(lrc.resource.file);
+			assert.equal(lrc.fileName, FILE_NAME);
+			assert.equal(lrc.oldFile.byteLength, FILE.byteLength);
+			assert.equal(lrc.file.byteLength, NEW_FILE.byteLength);
+			assert.equal(lrc.contentType, 'application/x-www-form-urlencoded');
+			assert.isNull(lrc.format);
+			assert.isUndefined(lrc.body);
+		});
 
 		it('handles api.library.items(I).attachment().get()', () => {
 			api(KEY).library(LIBRARY_KEY).items(ITEM_KEY).attachment().get();