diff --git a/package.json b/package.json index 4c6763b9ef2e..1e22ba871421 100644 --- a/package.json +++ b/package.json @@ -98,6 +98,7 @@ "classnames": "^2.2.5", "dateformat": "^1.0.12", "deep-equal": "^1.0.1", + "fetch-mock": "^5.12.1", "fuzzy": "^0.1.1", "history": "^2.1.2", "immutability-helper": "^2.0.0", diff --git a/src/backends/backend.js b/src/backends/backend.js index 1f0276288bf8..9ff9e50b8914 100644 --- a/src/backends/backend.js +++ b/src/backends/backend.js @@ -1,6 +1,7 @@ import { attempt, isError } from 'lodash'; import TestRepoBackend from "./test-repo/implementation"; import GitHubBackend from "./github/implementation"; +import GitLabBackend from "./gitlab/implementation"; import NetlifyAuthBackend from "./netlify-auth/implementation"; import { resolveFormat } from "../formats/formats"; import { selectListMethod, selectEntrySlug, selectEntryPath, selectAllowNewEntries, selectFolderEntryExtension } from "../reducers/collections"; @@ -292,6 +293,8 @@ export function resolveBackend(config) { return new Backend(new TestRepoBackend(config), authStore); case "github": return new Backend(new GitHubBackend(config), authStore); + case "gitlab": + return new Backend(new GitLabBackend(config), authStore); case "netlify-auth": return new Backend(new NetlifyAuthBackend(config), authStore); default: diff --git a/src/backends/github/API.js b/src/backends/github/API.js index a4157bb80a4d..9d248cddf934 100644 --- a/src/backends/github/API.js +++ b/src/backends/github/API.js @@ -1,9 +1,10 @@ import LocalForage from "localforage"; import { Base64 } from "js-base64"; -import _ from "lodash"; +import { entries, isString, partition, pick, uniq, zipObject } from "lodash"; +import { Map } from 'immutable'; import { filterPromises, resolvePromiseProperties } from "../../lib/promiseHelper"; import AssetProxy from "../../valueObjects/AssetProxy"; -import { SIMPLE, EDITORIAL_WORKFLOW, status } from "../../constants/publishModes"; +import { EDITORIAL_WORKFLOW, status } from "../../constants/publishModes"; import { APIError, EditorialWorkflowError } from "../../valueObjects/errors"; export default class API { @@ -20,70 +21,47 @@ export default class API { } isCollaborator(user) { - return this.request('/user/repos').then((repos) => { - let contributor = false - for (const repo of repos) { - if (repo.full_name === this.repo && repo.permissions.push) contributor = true; - } - return contributor; - }).catch((error) => { - console.error("Problem with response of /user/repos from GitHub"); - throw error; - }) + return this.request('/user/repos').then(repos => + Object.keys(repos).some( + key => (repos[key].full_name === this.repo) && repos[key].permissions.push + ) + ); } requestHeaders(headers = {}) { - const baseHeader = { - "Content-Type": "application/json", + return { ...headers, + ...(this.token ? { Authorization: `token ${ this.token }` } : {}), + "Content-Type": "application/json", }; - - if (this.token) { - baseHeader.Authorization = `token ${ this.token }`; - return baseHeader; - } - - return baseHeader; - } - - parseJsonResponse(response) { - return response.json().then((json) => { - if (!response.ok) { - return Promise.reject(json); - } - - return json; - }); } urlFor(path, options) { - const cacheBuster = new Date().getTime(); - const params = [`ts=${cacheBuster}`]; - if (options.params) { - for (const key in options.params) { - params.push(`${ key }=${ encodeURIComponent(options.params[key]) }`); - } - } - if (params.length) { - path += `?${ params.join("&") }`; - } - return this.api_root + path; + const cacheBuster = `ts=${ new Date().getTime() }`; + const encodedParams = options.params + ? Object.entries(options.params).map( + ([key, val]) => `${ key }=${ encodeURIComponent(val) }`) + : []; + return `${ this.api_root }${ path }?${ [cacheBuster, ...encodedParams].join("&") }`; } request(path, options = {}) { const headers = this.requestHeaders(options.headers || {}); const url = this.urlFor(path, options); - let responseStatus; - return fetch(url, { ...options, headers }).then((response) => { - responseStatus = response.status; + return fetch(url, { ...options, headers }) + .then((response) => { const contentType = response.headers.get("Content-Type"); if (contentType && contentType.match(/json/)) { - return this.parseJsonResponse(response); + return Promise.all([response, response.json()]); } - return response.text(); + return Promise.all([response, response.text()]); }) - .catch((error) => { - throw new APIError(error.message, responseStatus, 'GitHub'); + .catch(err => [err, null]) + .then(([response, value]) => (response.ok ? value : Promise.reject([value, response]))) + .catch(([errorValue, response]) => { + const errorMessageProp = (errorValue && errorValue.message) ? errorValue.message : null; + const message = errorMessageProp || (isString(errorValue) ? errorValue : ""); + throw new APIError(message, response && response.status, 'GitHub', { response, errorValue }); }); } @@ -95,7 +73,10 @@ export default class API { .catch((error) => { // Meta ref doesn't exist const readme = { - raw: "# Netlify CMS\n\nThis tree is used by the Netlify CMS to store metadata information for specific files and branches.", + raw: `\ +# Netlify CMS + +This tree is used by the Netlify CMS to store metadata information for specific files and branches.`, }; return this.uploadBlob(readme) @@ -112,16 +93,14 @@ export default class API { storeMetadata(key, data) { return this.checkMetadataRef() .then((branchData) => { - const fileTree = { - [`${ key }.json`]: { - path: `${ key }.json`, - raw: JSON.stringify(data), - file: true, - }, + const fileName = `${ key }.json`; + const file = { + path: fileName, + raw: JSON.stringify(data), + file: true, }; - - return this.uploadBlob(fileTree[`${ key }.json`]) - .then(item => this.updateTree(branchData.sha, "/", fileTree)) + return this.uploadBlob(file) + .then(uploadedFile => this.updateTree(branchData.sha, "/", { [fileName]: uploadedFile })) .then(changeTree => this.commit(`Updating “${ key }” metadata`, changeTree)) .then(response => this.patchRef("meta", "_netlify_cms", response.sha)) .then(() => { @@ -170,9 +149,9 @@ export default class API { return this.request(`${ this.repoURL }/contents/${ path }`, { params: { ref: this.branch }, }) - .then(files => { + .then((files) => { if (!Array.isArray(files)) { - throw new Error(`Cannot list files, path ${path} is not a directory but a ${files.type}`); + throw new Error(`Cannot list files, path ${ path } is not a directory but a ${ files.type }`); } return files; }) @@ -196,7 +175,7 @@ export default class API { isUnpublishedEntryModification(path, branch) { return this.readFile(path, null, branch) - .then(data => true) + .then(() => true) .catch((err) => { if (err.message && err.message === "Not Found") { return false; @@ -229,50 +208,26 @@ export default class API { } composeFileTree(files) { - let filename; - let part; - let parts; - let subtree; - const fileTree = {}; - - files.forEach((file) => { - if (file.uploaded) { return; } - parts = file.path.split("/").filter(part => part); - filename = parts.pop(); - subtree = fileTree; - while (part = parts.shift()) { - subtree[part] = subtree[part] || {}; - subtree = subtree[part]; - } - subtree[filename] = file; - file.file = true; - }); - - return fileTree; + return files + .map(file => ({ ...file, file: true })) + .reduce((tree, file) => tree.setIn(file.path.split("/"), file), Map()) + .toJS(); } persistFiles(entry, mediaFiles, options) { - const uploadPromises = []; - const files = mediaFiles.concat(entry); - - files.forEach((file) => { - if (file.uploaded) { return; } - uploadPromises.push(this.uploadBlob(file)); - }); - - const fileTree = this.composeFileTree(files); + const newFiles = [...mediaFiles, entry].filter(file => !file.uploaded); + const uploadsPromise = Promise.all(newFiles.map(file => this.uploadBlob(file))); + const fileTreePromise = uploadsPromise.then(files => this.composeFileTree(files)); + + if (options.mode === EDITORIAL_WORKFLOW) { + const mediaFilesList = mediaFiles.map(file => ({ path: file.path, sha: file.sha })); + return fileTreePromise + .then(fileTree => this.editorialWorkflowGit(fileTree, entry, mediaFilesList, options)); + } - return Promise.all(uploadPromises).then(() => { - if (!options.mode || (options.mode && options.mode === SIMPLE)) { - return this.getBranch() - .then(branchData => this.updateTree(branchData.commit.sha, "/", fileTree)) - .then(changeTree => this.commit(options.commitMessage, changeTree)) - .then(response => this.patchBranch(this.branch, response.sha)); - } else if (options.mode && options.mode === EDITORIAL_WORKFLOW) { - const mediaFilesList = mediaFiles.map(file => ({ path: file.path, sha: file.sha })); - return this.editorialWorkflowGit(fileTree, entry, mediaFilesList, options); - } - }); + return Promise.all([fileTreePromise, this.getBranch()]) + .then(([fileTree, branchData]) => this.updateTree(branchData.commit.sha, "/", fileTree)) + .then(changeTree => this.commit(options.commitMessage, changeTree)); } deleteFile(path, message, options={}) { @@ -282,11 +237,7 @@ export default class API { return this.request(fileURL) .then(({ sha }) => this.request(fileURL, { method: "DELETE", - params: { - sha, - message, - branch, - }, + params: { sha, message, branch }, })); } @@ -294,82 +245,62 @@ export default class API { const contentKey = entry.slug; const branchName = `cms/${ contentKey }`; const unpublished = options.unpublished || false; - if (!unpublished) { - // Open new editorial review workflow for this entry - Create new metadata and commit to new branch` - const contentKey = entry.slug; - const branchName = `cms/${ contentKey }`; - return this.getBranch() - .then(branchData => this.updateTree(branchData.commit.sha, "/", fileTree)) - .then(changeTree => this.commit(options.commitMessage, changeTree)) - .then(commitResponse => this.createBranch(branchName, commitResponse.sha)) - .then(branchResponse => this.createPR(options.commitMessage, branchName)) - .then(prResponse => this.user().then(user => user.name ? user.name : user.login) - .then(username => this.storeMetadata(contentKey, { - type: "PR", - pr: { - number: prResponse.number, - head: prResponse.head && prResponse.head.sha, - }, - user: username, - status: status.first(), - branch: branchName, - collection: options.collectionName, - title: options.parsedData && options.parsedData.title, - description: options.parsedData && options.parsedData.description, - objects: { - entry: { - path: entry.path, - sha: entry.sha, - }, - files: filesList, - }, - timeStamp: new Date().toISOString(), - } - ))); - } else { - // Entry is already on editorial review workflow - just update metadata and commit to existing branch - return this.getBranch(branchName) + const commitPromise = this.getBranch() .then(branchData => this.updateTree(branchData.commit.sha, "/", fileTree)) - .then(changeTree => this.commit(options.commitMessage, changeTree)) - .then((response) => { - const contentKey = entry.slug; - const branchName = `cms/${ contentKey }`; - return this.user().then(user => user.name ? user.name : user.login) - .then(username => this.retrieveMetadata(contentKey)) - .then((metadata) => { - let files = metadata.objects && metadata.objects.files || []; - files = files.concat(filesList); - const updatedPR = metadata.pr; - updatedPR.head = response.sha; - return { - ...metadata, - pr: updatedPR, - title: options.parsedData && options.parsedData.title, - description: options.parsedData && options.parsedData.description, - objects: { - entry: { - path: entry.path, - sha: entry.sha, - }, - files: _.uniq(files), - }, - timeStamp: new Date().toISOString(), - }; - }) - .then(updatedMetadata => this.storeMetadata(contentKey, updatedMetadata)) - .then(this.patchBranch(branchName, response.sha)); - }); - } + .then(changeTree => this.commit(options.commitMessage, changeTree)); + + const usernamePromise = this.user().then(user => (user.name ? user.name : user.login)); + + const initialMetadata = { + title: options.parsedData && options.parsedData.title, + description: options.parsedData && options.parsedData.description, + timeStamp: new Date().toISOString(), + }; + + return (unpublished + + // Entry is already on editorial review workflow - just update + // metadata and commit to existing branch + ? this.retrieveMetadata(contentKey).then(existingMetadata => resolvePromiseProperties({ + ...existingMetadata, + ...initialMetadata, + pr: commitPromise.then(commit => ({ ...existingMetadata.pr, head: commit.sha })), + objects: { + entry: pick(entry, ['path', 'sha']), + files: uniq( + ((existingMetadata.objects && existingMetadata.objects.files) || []).concat(filesList) + ), + }, + })) + .then(newMetadata => Promise.all([newMetadata, commitPromise])) + .then(([newMetadata, commit]) => + this.patchBranch(branchName, commit.sha).then(() => newMetadata) + ) + + // Open new editorial review workflow for this entry - Create new + // metadata and commit to new branch + : resolvePromiseProperties({ + ...initialMetadata, + type: "PR", + pr: commitPromise + .then(commit => this.createBranch(branchName, commit.sha)) + .then(() => this.createPR(options.commitMessage, branchName)) + .then(pr => ({ number: pr.number, head: pr.head && pr.head.sha })), + user: usernamePromise, + status: status.first(), + branch: branchName, + collection: options.collectionName, + objects: { entry: pick(entry, ['path', 'sha']), files: filesList }, + }) + + ).then(metadata => this.storeMetadata(contentKey, metadata)); } updateUnpublishedEntryStatus(collection, slug, status) { const contentKey = slug; return this.retrieveMetadata(contentKey) - .then(metadata => ({ - ...metadata, - status, - })) + .then(metadata => ({ ...metadata, status })) .then(updatedMetadata => this.storeMetadata(contentKey, updatedMetadata)); } @@ -391,7 +322,6 @@ export default class API { publishUnpublishedEntry(collection, slug) { const contentKey = slug; - let prNumber; return this.retrieveMetadata(contentKey) .then(metadata => this.mergePR(metadata.pr, metadata.objects)) .then(() => this.deleteBranch(`cms/${ contentKey }`)); @@ -443,7 +373,6 @@ export default class API { } closePR(pullrequest, objects) { - const headSha = pullrequest.head; const prNumber = pullrequest.number; console.log("%c Deleting PR", "line-height: 30px;text-align: center;font-weight: bold"); // eslint-disable-line return this.request(`${ this.repoURL }/pulls/${ prNumber }`, { @@ -477,10 +406,12 @@ export default class API { forceMergePR(pullrequest, objects) { const files = objects.files.concat(objects.entry); const fileTree = this.composeFileTree(files); - let commitMessage = "Automatically generated. Merged on Netlify CMS\n\nForce merge of:"; - files.forEach((file) => { - commitMessage += `\n* "${ file.path }"`; - }); + const commitMessage = `\ +Automatically generated. Merged on Netlify CMS + +Force merge of: +${ files.map(file => `* "${ file.path }"`).join("\n") }\ +`; console.log("%c Automatic merge not possible - Forcing merge.", "line-height: 30px;text-align: center;font-weight: bold"); // eslint-disable-line return this.getBranch() .then(branchData => this.updateTree(branchData.commit.sha, "/", fileTree)) @@ -493,9 +424,7 @@ export default class API { } toBase64(str) { - return Promise.resolve( - Base64.encode(str) - ); + return Promise.resolve(Base64.encode(str)); } uploadBlob(item) { @@ -507,44 +436,37 @@ export default class API { content: contentBase64, encoding: "base64", }), - }).then((response) => { - item.sha = response.sha; - item.uploaded = true; - return item; + })).then(response => Object.assign({}, item, { + sha: response.sha, + uploaded: true, })); } + isFile(obj) { + return obj.file; + } + updateTree(sha, path, fileTree) { return this.getTree(sha) - .then((tree) => { - let obj; - let filename; - let fileOrDir; - const updates = []; - const added = {}; - - for (let i = 0, len = tree.tree.length; i < len; i++) { - obj = tree.tree[i]; - if (fileOrDir = fileTree[obj.path]) { - added[obj.path] = true; - if (fileOrDir.file) { - updates.push({ path: obj.path, mode: obj.mode, type: obj.type, sha: fileOrDir.sha }); - } else { - updates.push(this.updateTree(obj.sha, obj.path, fileOrDir)); - } - } - } - for (filename in fileTree) { - fileOrDir = fileTree[filename]; - if (added[filename]) { continue; } - updates.push( - fileOrDir.file ? - { path: filename, mode: "100644", type: "blob", sha: fileOrDir.sha } : - this.updateTree(null, filename, fileOrDir) - ); - } + .then(({ tree: dirContents }) => { + const updatedItems = dirContents.filter(item => fileTree[item.path]); + const added = zipObject(updatedItems.map(item => item.path), Array(updatedItems.length).fill(true)); + const [updatedFiles, updatedDirs] = partition(updatedItems, this.isFile); + const updatePromises = [ + ...updatedDirs.map(dir => this.updateTree(dir.sha, dir.path, fileTree[dir.path])), + ...updatedFiles.map(file => ({ ...pick(file, ['path', 'mode', 'type']), sha: fileTree[file.path].sha })), + ]; + + const newItems = entries(fileTree).filter(([filename]) => !added[filename]); + const [newFiles, newDirs] = partition(newItems, ([, file]) => this.isFile(file)); + const newPromises = [ + ...newDirs.map(([dirName, dir]) => this.updateTree(null, dirName, dir)), + ...newFiles.map(([fileName, file]) => ({ path: fileName, mode: "100644", type: "blob", sha: file.sha })), + ]; + + const updates = [...updatePromises, ...newPromises]; return Promise.all(updates) - .then(updates => this.request(`${ this.repoURL }/git/trees`, { + .then(resolvedUpdates => this.request(`${ this.repoURL }/git/trees`, { method: "POST", body: JSON.stringify({ base_tree: sha, tree: updates }), })).then(response => ({ path, mode: "040000", type: "tree", sha: response.sha, parentSha: sha })); diff --git a/src/backends/github/implementation.js b/src/backends/github/implementation.js index 116f678971b4..f929c4b4a98b 100644 --- a/src/backends/github/implementation.js +++ b/src/backends/github/implementation.js @@ -1,7 +1,7 @@ import semaphore from "semaphore"; import AuthenticationPage from "./AuthenticationPage"; import API from "./API"; -import { fileExtension } from '../../lib/pathHelper' +import { fileExtension } from '../../lib/pathHelper'; const MAX_CONCURRENT_DOWNLOADS = 10; @@ -36,8 +36,7 @@ export default class GitHub { // Unauthorized user if (!isCollab) throw new Error("Your GitHub user account does not have access to this repo."); // Authorized user - user.token = state.token; - return user; + return Object.assign({}, user, { token: state.token }); }) ); } @@ -96,31 +95,28 @@ export default class GitHub { unpublishedEntries() { return this.api.listUnpublishedBranches().then((branches) => { const sem = semaphore(MAX_CONCURRENT_DOWNLOADS); - const promises = []; - branches.map((branch) => { - promises.push(new Promise((resolve, reject) => { - const slug = branch.ref.split("refs/heads/cms/").pop(); - return sem.take(() => this.api.readUnpublishedBranchFile(slug).then((data) => { - if (data === null || data === undefined) { - resolve(null); - sem.leave(); - } else { - const path = data.metaData.objects.entry.path; - resolve({ - slug, - file: { path }, - data: data.fileData, - metaData: data.metaData, - isModification: data.isModification, - }); - sem.leave(); - } - }).catch((err) => { - sem.leave(); + const promises = branches.map(branch => new Promise((resolve, reject) => { + const slug = branch.ref.split("refs/heads/cms/").pop(); + return sem.take(() => this.api.readUnpublishedBranchFile(slug).then((data) => { + if (data === null || data === undefined) { resolve(null); - })); + sem.leave(); + } else { + const path = data.metaData.objects.entry.path; + resolve({ + slug, + file: { path }, + data: data.fileData, + metaData: data.metaData, + isModification: data.isModification, + }); + sem.leave(); + } + }).catch((err) => { + sem.leave(); + resolve(null); })); - }); + })); return Promise.all(promises); }) .catch((error) => { diff --git a/src/backends/gitlab/API.js b/src/backends/gitlab/API.js new file mode 100644 index 000000000000..32f580edc0c3 --- /dev/null +++ b/src/backends/gitlab/API.js @@ -0,0 +1,185 @@ +import LocalForage from "localforage"; +import { Base64 } from "js-base64"; +import { isString } from "lodash"; +import AssetProxy from "../../valueObjects/AssetProxy"; +import { APIError } from "../../valueObjects/errors"; + +export default class API { + constructor(config) { + this.api_root = config.api_root || "https://gitlab.com/api/v4"; + this.token = config.token || false; + this.branch = config.branch || "master"; + this.repo = config.repo || ""; + this.repoURL = `/projects/${ encodeURIComponent(this.repo) }`; + } + + user() { + return this.request("/user"); + } + + isCollaborator(user) { + const WRITE_ACCESS = 30; + return this.request(`${ this.repoURL }/members/${ user.id }`) + .then(member => (member.access_level >= WRITE_ACCESS)) + .catch((err) => { + // Member does not have any access. We cannot just check for 404, + // because a 404 is also returned if we have the wrong URI, + // just with an "error" key instead of a "message" key. + if (err.status === 404 && err.meta.errorValue["message"] === "404 Not found") { + return false; + } else { + // Otherwise, it is actually an API error. + throw err; + } + }); + } + + requestHeaders(headers = {}) { + return { + ...headers, + ...(this.token ? { Authorization: `Bearer ${ this.token }` } : {}), + }; + } + + urlFor(path, options) { + const cacheBuster = `ts=${ new Date().getTime() }`; + const encodedParams = options.params + ? Object.entries(options.params).map( + ([key, val]) => `${ key }=${ encodeURIComponent(val) }`) + : []; + return `${ this.api_root }${ path }?${ [cacheBuster, ...encodedParams].join("&") }`; + } + + request(path, options = {}) { + const headers = this.requestHeaders(options.headers || {}); + const url = this.urlFor(path, options); + return fetch(url, { ...options, headers }) + .then((response) => { + const contentType = response.headers.get("Content-Type"); + if (options.method === "HEAD") { + return Promise.all([response]); + } + if (contentType && contentType.match(/json/)) { + return Promise.all([response, response.json()]); + } + return Promise.all([response, response.text()]); + }) + .catch(err => [err, null]) + .then(([response, value]) => { + if (!response.ok) return Promise.reject([value, response]); + /* TODO: remove magic. */ + if (value === undefined) return response; + /* OK */ + return value; + }) + .catch(([errorValue, response]) => { + const errorMessageProp = (errorValue && errorValue.message) ? errorValue.message : null; + const message = errorMessageProp || (isString(errorValue) ? errorValue : ""); + throw new APIError(message, response && response.status, 'GitHub', { response, errorValue }); + }); + } + + readFile(path, sha, branch = this.branch) { + const cache = sha ? LocalForage.getItem(`gh.${ sha }`) : Promise.resolve(null); + return cache.then((cached) => { + if (cached) { return cached; } + + // Files must be downloaded from GitLab as base64 due to this bug: + // https://gitlab.com/gitlab-org/gitlab-ce/issues/31470 + return this.request(`${ this.repoURL }/repository/files/${ encodeURIComponent(path) }`, { + params: { ref: branch }, + cache: "no-store", + }).then(response => this.fromBase64(response.content)) + .then((result) => { + if (sha) { + LocalForage.setItem(`gh.${ sha }`, result); + } + return result; + }); + }); + } + + fileExists(path, branch = this.branch) { + return this.request(`${ this.repoURL }/repository/files/${ encodeURIComponent(path) }`, { + method: "HEAD", + params: { ref: branch }, + cache: "no-store", + }).then(() => true).catch((err) => { + // TODO: 404 can mean either the file does not exist, or if an API + // endpoint doesn't exist. Is there a better way to check for this? + if (err.status === 404) {return false;} else {throw err;} + }); + } + + listFiles(path) { + return this.request(`${ this.repoURL }/repository/tree`, { + params: { path, ref: this.branch }, + }) + .then((files) => { + if (!Array.isArray(files)) { + throw new Error(`Cannot list files, path ${path} is not a directory but a ${files.type}`); + } + return files; + }) + .then(files => files.filter(file => file.type === "blob")); + } + + persistFiles(entry, mediaFiles, options) { + const newMedia = mediaFiles.filter(file => !file.uploaded); + const mediaUploads = newMedia.map(file => this.fileExists(file.path).then(exists => { + return this.uploadAndCommit(file, { + commitMessage: `${ options.commitMessage }: create ${ file.value }.`, + newFile: !exists + }); + })); + + // Wait until media files are uploaded before we commit the main entry. + // This should help avoid inconsistent repository/website state. + return Promise.all(mediaUploads) + .then(() => this.uploadAndCommit(entry, { + commitMessage: options.commitMessage, + newFile: options.newEntry + })); + } + + deleteFile(path, commit_message, options={}) { + const branch = options.branch || this.branch; + return this.request(`${ this.repoURL }/repository/files/${ encodeURIComponent(path) }`, { + method: "DELETE", + params: { commit_message, branch }, + }); + } + + toBase64(str) { + return Promise.resolve(Base64.encode(str)); + } + + fromBase64(str) { + return Base64.decode(str); + } + + uploadAndCommit(item, {commitMessage, newFile = true, branch = this.branch}) { + const content = item instanceof AssetProxy ? item.toBase64() : this.toBase64(item.raw); + // Remove leading slash from path if exists. + const file_path = item.path.replace(/^\//, ''); + + // We cannot use the `/repository/files/:file_path` format here because the file content has to go + // in the URI as a parameter. This overloads the OPTIONS pre-request (at least in Chrome 61 beta). + return content.then(contentBase64 => this.request(`${ this.repoURL }/repository/commits`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + branch, + commit_message: commitMessage, + actions: [{ + action: (newFile ? "create" : "update"), + file_path, + content: contentBase64, + encoding: "base64", + }] + }), + })).then(response => Object.assign({}, item, { uploaded: true })); + } +} \ No newline at end of file diff --git a/src/backends/gitlab/AuthenticationPage.css b/src/backends/gitlab/AuthenticationPage.css new file mode 100644 index 000000000000..949d9b7dfc53 --- /dev/null +++ b/src/backends/gitlab/AuthenticationPage.css @@ -0,0 +1,12 @@ +.root { + display: flex; + flex-flow: column nowrap; + align-items: center; + justify-content: center; + height: 100vh; +} + +.button { + padding: .25em 1em; + height: auto; +} diff --git a/src/backends/gitlab/AuthenticationPage.js b/src/backends/gitlab/AuthenticationPage.js new file mode 100644 index 000000000000..036cf420fb88 --- /dev/null +++ b/src/backends/gitlab/AuthenticationPage.js @@ -0,0 +1,50 @@ +import React from 'react'; +import Button from 'react-toolbox/lib/button'; +import Authenticator from '../../lib/netlify-auth'; +import { Icon } from '../../components/UI'; +import { Notifs } from 'redux-notifications'; +import { Toast } from '../../components/UI/index'; +import styles from './AuthenticationPage.css'; + +export default class AuthenticationPage extends React.Component { + static propTypes = { + onLogin: React.PropTypes.func.isRequired, + }; + + state = {}; + + handleLogin = (e) => { + e.preventDefault(); + const cfg = { + base_url: this.props.base_url, + site_id: (document.location.host.split(':')[0] === 'localhost') ? 'cms.netlify.com' : this.props.siteId + }; + const auth = new Authenticator(cfg); + + auth.authenticate({ provider: 'gitlab', scope: 'repo' }, (err, data) => { + if (err) { + this.setState({ loginError: err.toString() }); + return; + } + this.props.onLogin(data); + }); + }; + + render() { + const { loginError } = this.state; + + return ( +
+ + {loginError &&

{loginError}

} + +
+ ); + } +} diff --git a/src/backends/gitlab/__tests__/API.spec.js b/src/backends/gitlab/__tests__/API.spec.js new file mode 100644 index 000000000000..acebe2d387c6 --- /dev/null +++ b/src/backends/gitlab/__tests__/API.spec.js @@ -0,0 +1,100 @@ +import fetchMock from 'fetch-mock'; +import { curry, escapeRegExp, isMatch, merge } from 'lodash'; +import { Map } from 'immutable'; + +import API from '../API'; + +const compose = (...fns) => val => fns.reduceRight((newVal, fn) => fn(newVal), val); +const pipe = (...fns) => compose(...fns.reverse()); + +const regExpOrString = rOrS => (rOrS instanceof RegExp ? rOrS.toString() : escapeRegExp(rOrS)); + +const mockForAllParams = url => `${url}(\\?.*)?`; +const prependRoot = urlRoot => url => `${urlRoot}${url}`; +const matchWholeURL = str => `^${str}$`; +const strToRegex = str => new RegExp(str); +const matchURL = curry((urlRoot, forAllParams, url) => pipe( + regExpOrString, + ...(forAllParams ? [mockForAllParams] : []), + pipe(regExpOrString, prependRoot)(urlRoot), + matchWholeURL, + strToRegex, +)(url)); + +// `mock` gives us a few advantages over using the standard +// `fetchMock.mock`: +// - Routes can have a root specified that is prepended to the path +// - By default, routes swallow URL parameters (the GitHub API code +// uses a `ts` parameter on _every_ request) +const mockRequest = curry((urlRoot, url, response, options={}) => { + const mergedOptions = merge({}, { + forAllParams: true, + fetchMockOptions: {}, + }, options); + return fetchMock.mock( + matchURL(urlRoot, mergedOptions.forAllParams, url), + response, + options.fetchMockOptions, + ); +}); + +const defaultResponseHeaders = { "Content-Type": "application/json" }; + +afterEach(() => fetchMock.restore()); + +describe('gitlab API', () => { + it('should correctly detect a contributor', () => { + const api = new API({ branch: 'test-branch', repo: 'test-user/test-repo' }); + const user = { id: 1 }; + mockRequest(api.api_root)(`${ api.repoURL }/members/${ user.id }`, { + headers: defaultResponseHeaders, + body: { + id: 1, + access_level: 30, + }, + }); + return expect(api.isCollaborator(user)).resolves.toBe(true); + }); + + it('should correctly detect a non-contributor', () => { + const api = new API({ branch: 'test-branch', repo: 'test-user/test-repo' }); + const user = { id: 1 }; + mockRequest(api.api_root)(`${ api.repoURL }/members/${ user.id }`, { + headers: defaultResponseHeaders, + body: { + id: 1, + access_level: 10, + }, + }); + return expect(api.isCollaborator(user)).resolves.toBe(false); + }); + + it('should list the files in a directory', () => { + const api = new API({ branch: 'test-branch', repo: 'test-user/test-repo' }); + mockRequest(api.api_root)(`${ api.repoURL }/repository/tree`, { + headers: defaultResponseHeaders, + body: [ + { + id: "fff6fe3a23bf1c8ea0692b4a883af99bee26fd3b", + name: "octokit.rb", + type: "blob", + path: "test-directory/octokit.rb", + }, + { + id: "a84d88e7554fc1fa21bcbc4efae3c782a70d2b9d", + name: "octokit", + type: "tree", + path: "test-directory/octokit", + }, + ], + }); + return expect(api.listFiles('test-directory')).resolves.toMatchObject([ + { + id: "fff6fe3a23bf1c8ea0692b4a883af99bee26fd3b", + name: "octokit.rb", + type: "blob", + path: "test-directory/octokit.rb", + }, + ]); + }); +}); diff --git a/src/backends/gitlab/implementation.js b/src/backends/gitlab/implementation.js new file mode 100644 index 000000000000..0dce5eabe3ba --- /dev/null +++ b/src/backends/gitlab/implementation.js @@ -0,0 +1,99 @@ +import semaphore from "semaphore"; +import AuthenticationPage from "./AuthenticationPage"; +import API from "./API"; +import { fileExtension } from '../../lib/pathHelper'; +import { EDITORIAL_WORKFLOW } from "../../constants/publishModes"; + +const MAX_CONCURRENT_DOWNLOADS = 10; + +export default class GitLab { + constructor(config, proxied = false) { + this.config = config; + + if (config.getIn(["publish_mode"]) === EDITORIAL_WORKFLOW) { + throw new Error("The GitLab backend does not support the Editorial Workflow.") + } + + if (!proxied && config.getIn(["backend", "repo"]) == null) { + throw new Error("The GitLab backend needs a \"repo\" in the backend configuration."); + } + + this.repo = config.getIn(["backend", "repo"], ""); + this.branch = config.getIn(["backend", "branch"], "master"); + this.api_root = config.getIn(["backend", "api_root"], "https://gitlab.com/api/v4"); + this.token = ''; + } + + authComponent() { + return AuthenticationPage; + } + + setUser(user) { + this.token = user.token; + this.api = new API({ token: this.token, branch: this.branch, repo: this.repo }); + } + + authenticate(state) { + this.token = state.token; + this.api = new API({ token: this.token, branch: this.branch, repo: this.repo, api_root: this.api_root }); + return this.api.user().then(user => + this.api.isCollaborator(user).then((isCollab) => { + // Unauthorized user + if (!isCollab) throw new Error("Your GitLab user account does not have access to this repo."); + // Authorized user + return Object.assign({}, user, { token: state.token }); + }) + ); + } + + getToken() { + return Promise.resolve(this.token); + } + + entriesByFolder(collection, extension) { + return this.api.listFiles(collection.get("folder")) + .then(files => files.filter(file => fileExtension(file.name) === extension)) + .then(this.fetchFiles); + } + + entriesByFiles(collection) { + const files = collection.get("files").map(collectionFile => ({ + path: collectionFile.get("file"), + label: collectionFile.get("label"), + })); + return this.fetchFiles(files); + } + + fetchFiles = (files) => { + const sem = semaphore(MAX_CONCURRENT_DOWNLOADS); + const promises = []; + files.forEach((file) => { + promises.push(new Promise((resolve, reject) => ( + sem.take(() => this.api.readFile(file.path, file.sha).then((data) => { + resolve({ file, data }); + sem.leave(); + }).catch((err) => { + sem.leave(); + reject(err); + })) + ))); + }); + return Promise.all(promises); + }; + + // Fetches a single entry. + getEntry(collection, slug, path) { + return this.api.readFile(path).then(data => ({ + file: { path }, + data, + })); + } + + persistEntry(entry, mediaFiles = [], options = {}) { + return this.api.persistFiles(entry, mediaFiles, options); + } + + deleteFile(path, commitMessage, options) { + return this.api.deleteFile(path, commitMessage, options); + } +} diff --git a/src/lib/promiseHelper.js b/src/lib/promiseHelper.js index 0d16bd5ec8d4..ee353d36a8fd 100644 --- a/src/lib/promiseHelper.js +++ b/src/lib/promiseHelper.js @@ -7,7 +7,7 @@ export const filterPromises = (arr, filter) => export const resolvePromiseProperties = (obj) => { // Get the keys which represent promises const promiseKeys = Object.keys(obj).filter( - key => typeof obj[key].then === "function"); + key => obj[key] && typeof obj[key].then === "function"); const promises = promiseKeys.map(key => obj[key]); diff --git a/src/valueObjects/errors/APIError.js b/src/valueObjects/errors/APIError.js index 05db8b5cc4c0..fc45bea2d108 100644 --- a/src/valueObjects/errors/APIError.js +++ b/src/valueObjects/errors/APIError.js @@ -1,11 +1,12 @@ export const API_ERROR = 'API_ERROR'; export default class APIError extends Error { - constructor(message, status, api) { + constructor(message, status, api, meta={}) { super(message); this.message = message; this.status = status; this.api = api; this.name = API_ERROR; + this.meta = meta; } } diff --git a/yarn.lock b/yarn.lock index e9e831e8213f..65ccde77ea67 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2940,6 +2940,14 @@ fbjs@^0.8.4, fbjs@^0.8.9: setimmediate "^1.0.5" ua-parser-js "^0.7.9" +fetch-mock@^5.12.1: + version "5.12.1" + resolved "https://registry.yarnpkg.com/fetch-mock/-/fetch-mock-5.12.1.tgz#b1808310f51ab286d1b487a8cb39dfbecb0d097a" + dependencies: + glob-to-regexp "^0.3.0" + node-fetch "^1.3.3" + path-to-regexp "^1.7.0" + figures@^1.3.5, figures@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e" @@ -3217,6 +3225,10 @@ glob-parent@^2.0.0: dependencies: is-glob "^2.0.0" +glob-to-regexp@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" + glob@^6.0.1: version "6.0.4" resolved "https://registry.yarnpkg.com/glob/-/glob-6.0.4.tgz#0f08860f6a155127b2fadd4f9ce24b1aab6e4d22" @@ -5085,7 +5097,7 @@ node-emoji@^1.0.3: dependencies: string.prototype.codepointat "^0.2.0" -node-fetch@^1.0.1: +node-fetch@^1.0.1, node-fetch@^1.3.3: version "1.6.3" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.6.3.tgz#dc234edd6489982d58e8f0db4f695029abcd8c04" dependencies: @@ -5667,6 +5679,12 @@ path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" +path-to-regexp@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.7.0.tgz#59fde0f435badacba103a84e9d3bc64e96b9937d" + dependencies: + isarray "0.0.1" + path-type@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" @@ -5747,10 +5765,6 @@ pluralize@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-1.2.1.tgz#d1a21483fd22bb41e58a12fa3421823140897c45" -pluralize@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-3.1.0.tgz#84213d0a12356069daa84060c559242633161368" - portfinder@^1.0.9: version "1.0.13" resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.13.tgz#bb32ecd87c27104ae6ee44b5a3ccbf0ebb1aede9"