From 79d7c4ca8ad0035921db1187fad5c273c86e5704 Mon Sep 17 00:00:00 2001 From: Caleb Chase Date: Fri, 29 Oct 2021 11:43:06 -0500 Subject: [PATCH 01/19] WebDAV sign in: toggle form visibility As opposed to always showing once clicked. Preparatory work for #733 --- src/components/SyncServiceSignIn/index.js | 108 ++++++++++------------ 1 file changed, 51 insertions(+), 57 deletions(-) diff --git a/src/components/SyncServiceSignIn/index.js b/src/components/SyncServiceSignIn/index.js index 27ab4aab8..3186223db 100644 --- a/src/components/SyncServiceSignIn/index.js +++ b/src/components/SyncServiceSignIn/index.js @@ -15,68 +15,62 @@ import _ from 'lodash'; function WebDAVForm() { const [isVisible, setIsVisible] = useState(false); + const toggleVisible = () => setIsVisible(!isVisible); const [url, setUrl] = useState(''); const [username, setUsername] = useState(''); const [password, setPassword] = useState(''); - return !isVisible ? ( -
{ - setIsVisible(true); - }} - > -

WebDAV

-
- ) : ( + return (
-

WebDAV

-
{ - event.preventDefault(); - persistField('authenticatedSyncService', 'WebDAV'); - persistField('webdavEndpoint', url); - persistField('webdavUsername', username); - persistField('webdavPassword', password); - window.location = window.location.origin + '/'; - }} - > -

- - { - setUrl(e.target.value); - }} - /> -

-

- - { - setUsername(e.target.value); - }} - /> -

-

- - { - setPassword(e.target.value); - }} - /> -

- -
+

WebDAV

+ {isVisible && ( +
{ + event.preventDefault(); + persistField('authenticatedSyncService', 'WebDAV'); + persistField('webdavEndpoint', url); + persistField('webdavUsername', username); + persistField('webdavPassword', password); + window.location = window.location.origin + '/'; + }} + > +

+ + { + setUrl(e.target.value); + }} + /> +

+

+ + { + setUsername(e.target.value); + }} + /> +

+

+ + { + setPassword(e.target.value); + }} + /> +

+ +
+ )}
); } From a137320bc1ea2c30ec05b3a32089c0ebaecb1ecd Mon Sep 17 00:00:00 2001 From: Caleb Chase Date: Fri, 29 Oct 2021 12:06:25 -0500 Subject: [PATCH 02/19] CSS: refactor sign in form styles Make them usable for any sign in form (such as the WIP GitLab form), as opposed to specific to WebDAV. Preparatory work for #733 --- src/components/SyncServiceSignIn/stylesheet.css | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/components/SyncServiceSignIn/stylesheet.css b/src/components/SyncServiceSignIn/stylesheet.css index ed781de52..29f650808 100644 --- a/src/components/SyncServiceSignIn/stylesheet.css +++ b/src/components/SyncServiceSignIn/stylesheet.css @@ -5,16 +5,16 @@ margin: 0; } -#webdavLogin form { +.sync-service-container form { display: table; } -#webdavLogin p { +.sync-service-container form p { display: table-row; } -#webdavLogin label { +.sync-service-container form label { display: table-cell; } -#webdavLogin input { +.sync-service-container form input { display: table-cell; } From 1e468da85f5065e8e2f16767c6fe83d46a7e3322 Mon Sep 17 00:00:00 2001 From: Caleb Chase Date: Fri, 29 Oct 2021 12:34:06 -0500 Subject: [PATCH 03/19] CSS: make sign-in form inputs full width Preparatory work for #733 --- src/components/SyncServiceSignIn/stylesheet.css | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/components/SyncServiceSignIn/stylesheet.css b/src/components/SyncServiceSignIn/stylesheet.css index 29f650808..5f81e7c70 100644 --- a/src/components/SyncServiceSignIn/stylesheet.css +++ b/src/components/SyncServiceSignIn/stylesheet.css @@ -7,6 +7,7 @@ .sync-service-container form { display: table; + width: 100%; } .sync-service-container form p { display: table-row; @@ -14,8 +15,9 @@ .sync-service-container form label { display: table-cell; } -.sync-service-container form input { +.sync-service-container form input:not([type='submit']) { display: table-cell; + width: calc(100% - 10px); } .sync-service-sign-in-container { From 3da3747a120ee65989e71d5999b126f33fdbd1df Mon Sep 17 00:00:00 2001 From: Caleb Chase Date: Thu, 28 Oct 2021 15:17:36 -0500 Subject: [PATCH 04/19] GitLab: implement OAuth sign in For #733 --- .env.sample | 4 +- .gitignore | 1 + package.json | 1 + src/App.js | 39 ++++ src/actions/base.js | 1 + src/actions/sync_backend.js | 6 + src/components/FileBrowser/index.js | 1 + src/components/SyncServiceSignIn/gitlab.svg | 78 ++++++++ src/components/SyncServiceSignIn/index.js | 49 ++++- .../gitlab_sync_backend_client.js | 169 ++++++++++++++++++ .../gitlab_sync_backend_client.test.js | 13 ++ src/util/settings_persister.js | 2 + yarn.lock | 5 + 13 files changed, 367 insertions(+), 2 deletions(-) create mode 100644 src/components/SyncServiceSignIn/gitlab.svg create mode 100644 src/sync_backend_clients/gitlab_sync_backend_client.js create mode 100644 src/sync_backend_clients/gitlab_sync_backend_client.test.js diff --git a/.env.sample b/.env.sample index 0f6f90c14..28ce57726 100644 --- a/.env.sample +++ b/.env.sample @@ -1,3 +1,5 @@ REACT_APP_DROPBOX_CLIENT_ID=your_dropbox_client_id REACT_APP_GOOGLE_DRIVE_API_KEY=your_google_drive_api_key -REACT_APP_GOOGLE_DRIVE_CLIENT_ID=your_google_drive_oauth_client_id \ No newline at end of file +REACT_APP_GOOGLE_DRIVE_CLIENT_ID=your_google_drive_oauth_client_id +REACT_APP_GITLAB_CLIENT_ID=your_gitlab_client_id +REACT_APP_GITLAB_SECRET=your_gitlab_secret diff --git a/.gitignore b/.gitignore index 9c9a468b8..a8fa9f9d2 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ tags npm-debug.log* yarn-debug.log* yarn-error.log* +/.log/ *~ diff --git a/package.json b/package.json index 28e0b62a6..196279d58 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "node": "^12.13.1" }, "dependencies": { + "@bity/oauth2-auth-code-pkce": "^2.13.0", "bowser": "^2.11.0", "classnames": "^2.2.6", "date-fns": "^2.16.1", diff --git a/src/App.js b/src/App.js index 8fb9a2fba..fe18aee11 100644 --- a/src/App.js +++ b/src/App.js @@ -23,6 +23,9 @@ import { setDisappearingLoadingMessage, restoreStaticFile } from './actions/base import createDropboxSyncBackendClient from './sync_backend_clients/dropbox_sync_backend_client'; import createGoogleDriveSyncBackendClient from './sync_backend_clients/google_drive_sync_backend_client'; import createWebDAVSyncBackendClient from './sync_backend_clients/webdav_sync_backend_client'; +import createGitLabSyncBackendClient, { + createGitlabOAuth, +} from './sync_backend_clients/gitlab_sync_backend_client'; import './base.css'; @@ -37,6 +40,30 @@ import { import _ from 'lodash'; import { Map } from 'immutable'; +const handleGitLabAuthResponse = async (oauthClient) => { + let success = false; + try { + success = await oauthClient.isReturningFromAuthServer(); + await oauthClient.getAccessToken(); + } catch { + success = false; + } + if (!success) { + // Edge case: somehow OAuth success redirect occurred but there isn't a code in + // the current location's search params. This /shouldn't/ happen in practice. + alert('Unexpected sign in error, please try again'); + return; + } + + const syncClient = createGitLabSyncBackendClient(oauthClient); + const isAccessible = await syncClient.isProjectAccessible(); + if (!isAccessible) { + alert('Failed to access GitLab project - is the URL is correct?'); + } else { + window.location.search = ''; + } +}; + export default class App extends PureComponent { constructor(props) { super(props); @@ -80,6 +107,18 @@ export default class App extends PureComponent { client, }); break; + case 'GitLab': + const gitlabOAuth = createGitlabOAuth(); + if (gitlabOAuth.isAuthorized()) { + client = createGitLabSyncBackendClient(gitlabOAuth); + initialState.syncBackend = Map({ + isAuthenticated: true, + client, + }); + } else { + handleGitLabAuthResponse(gitlabOAuth); + } + break; case 'WebDAV': client = createWebDAVSyncBackendClient( getPersistedField('webdavEndpoint'), diff --git a/src/actions/base.js b/src/actions/base.js index 741720f04..a425b7416 100644 --- a/src/actions/base.js +++ b/src/actions/base.js @@ -128,6 +128,7 @@ export const setShouldStoreSettingsInSyncBackend = (newShouldStoreSettingsInSync const client = getState().syncBackend.get('client'); switch (client.type) { case 'Dropbox': + case 'GitLab': case 'WebDAV': client .deleteFile('/.organice-config.json') diff --git a/src/actions/sync_backend.js b/src/actions/sync_backend.js index fe6951c13..ea5964c04 100644 --- a/src/actions/sync_backend.js +++ b/src/actions/sync_backend.js @@ -4,6 +4,7 @@ import { ActionCreators } from 'redux-undo'; import { setLoadingMessage, hideLoadingMessage, clearModalStack, setIsLoading } from './base'; import { parseFile, setDirty, setLastSyncAt, setOrgFileErrorMessage } from './org'; import { localStorageAvailable, persistField } from '../util/settings_persister'; +import { createGitlabOAuth } from '../sync_backend_clients/gitlab_sync_backend_client'; import { addSeconds } from 'date-fns'; @@ -20,6 +21,10 @@ export const signOut = () => (dispatch, getState) => { case 'Google Drive': gapi.auth2.getAuthInstance().signOut(); break; + case 'GitLab': + persistField('gitLabProject', null); + createGitlabOAuth().reset(); + break; default: } @@ -90,6 +95,7 @@ export const pushBackup = (pathOrFileId, contents) => { const client = getState().syncBackend.get('client'); switch (client.type) { case 'Dropbox': + case 'GitLab': case 'WebDAV': client.createFile(`${pathOrFileId}.organice-bak`, contents); break; diff --git a/src/components/FileBrowser/index.js b/src/components/FileBrowser/index.js index 88c994ead..b0682f17a 100644 --- a/src/components/FileBrowser/index.js +++ b/src/components/FileBrowser/index.js @@ -26,6 +26,7 @@ const FileBrowser = ({ const getParentDirectoryPath = () => { switch (syncBackendType) { case 'Dropbox': + case 'GitLab': case 'WebDAV': const pathParts = path.split('/'); return pathParts.slice(0, pathParts.length - 1).join('/'); diff --git a/src/components/SyncServiceSignIn/gitlab.svg b/src/components/SyncServiceSignIn/gitlab.svg new file mode 100644 index 000000000..77f5437d3 --- /dev/null +++ b/src/components/SyncServiceSignIn/gitlab.svg @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + H: 2.5 x + 1/2 x + 1x + 1x + + 1x + + 1x + + \ No newline at end of file diff --git a/src/components/SyncServiceSignIn/index.js b/src/components/SyncServiceSignIn/index.js index 3186223db..16c07aea6 100644 --- a/src/components/SyncServiceSignIn/index.js +++ b/src/components/SyncServiceSignIn/index.js @@ -6,11 +6,15 @@ import './stylesheet.css'; import DropboxLogo from './dropbox.svg'; import GoogleDriveLogo from './google_drive.png'; +import GitLabLogo from './gitlab.svg'; import { persistField } from '../../util/settings_persister'; +import { + createGitlabOAuth, + gitLabProjectIdFromURL, +} from '../../sync_backend_clients/gitlab_sync_backend_client'; import { Dropbox } from 'dropbox'; - import _ from 'lodash'; function WebDAVForm() { @@ -108,6 +112,45 @@ function GoogleDriveNote() { ); } +function GitLab() { + const [isVisible, setIsVisible] = useState(false); + const toggleVisible = () => setIsVisible(!isVisible); + + const [project, setProject] = useState(''); + const handleSubmit = (evt) => { + const projectId = gitLabProjectIdFromURL(project); + if (projectId) { + persistField('authenticatedSyncService', 'GitLab'); + persistField('gitLabProject', projectId); + createGitlabOAuth().fetchAuthorizationCode(); + } else { + evt.preventDefault(); + alert('Project does not appear to be a valid gitlab.com URL'); + } + }; + + return ( + <> + GitLab logo + {isVisible && ( +
+

+ + setProject(e.target.value)} + /> +

+ +
+ )} + + ); +} + export default class SyncServiceSignIn extends PureComponent { constructor(props) { super(props); @@ -176,6 +219,10 @@ export default class SyncServiceSignIn extends PureComponent { +
+ +
+
diff --git a/src/sync_backend_clients/gitlab_sync_backend_client.js b/src/sync_backend_clients/gitlab_sync_backend_client.js new file mode 100644 index 000000000..061f06bbd --- /dev/null +++ b/src/sync_backend_clients/gitlab_sync_backend_client.js @@ -0,0 +1,169 @@ +/* global process */ +import { OAuth2AuthCodePKCE } from '@bity/oauth2-auth-code-pkce'; +import { isEmpty } from 'lodash'; +import { getPersistedField } from '../util/settings_persister'; + +export const createGitlabOAuth = () => + new OAuth2AuthCodePKCE({ + authorizationUrl: 'https://gitlab.com/oauth/authorize', + tokenUrl: 'https://gitlab.com/oauth/token', + clientId: process.env.REACT_APP_GITLAB_CLIENT_ID, + redirectUrl: window.location.origin, + scopes: ['api'], + extraAuthorizationParams: { + clientSecret: process.env.REACT_APP_GITLAB_SECRET, + }, + onAccessTokenExpiry: (refreshToken) => refreshToken(), + onInvalidGrant: (refreshAuthCodeOrToken) => refreshAuthCodeOrToken(), + }); + +/** + * Convert project URL to identifier for use with API, since explaining how to find the ID of a + * project is unnecessarily confusing for users. + * + * @see https://docs.gitlab.com/ee/api/projects.html#get-single-project + * @param {string} projectURL Full URL for project, such as gitlab.com/foo/bar/baz. Leading https:// is + * optional. + * @returns {string|undefined} Project ID if `projectURL` appears to be a gitlab.com project and + * parsing succeeded, otherwise undefined. + */ +export const gitLabProjectIdFromURL = (projectURL) => { + if (!projectURL.includes('://')) { + // URL() class requires protocol. + projectURL = `https://${projectURL}`; + } + try { + const url = new URL(projectURL); + const path = url.pathname.replace(/(^\/)|(\/$)/g, ''); + // Rough heuristic to check that url at least *potentially* refers to a project. + // Reminder: a project path is not necessarily /user/project because it may be under one or more + // groups such as /user/group/subgroup/project. + if (url.hostname === 'gitlab.com' && path.split('/').length > 1) { + return encodeURIComponent(path); + } else { + return undefined; + } + } catch { + return undefined; + } +}; + +// TODO implement parsing +// TODO apparently want to use immutable's fromJS +// const treeToDirectoryListing = (tree) + +const API_URL = 'https://gitlab.com/api/v4'; + +/** + * @param {OAuth2AuthCodePKCE} oauthClient + */ +export default (oauthClient) => { + const decoratedFetch = oauthClient.decorateFetchHTTPClient(fetch); + + const getProjectId = () => getPersistedField('gitLabProject'); + + const isSignedIn = async () => { + if (!oauthClient.isAuthorized()) { + return false; + } + // Verify that we have an OAuth token (and refresh if needed). Don't care about return value, + // because the library handles persisting for us. + try { + await oauthClient.getAccessToken(); + return true; + } catch { + return false; + } + }; + + /** + * Check that project exists and user is *probably* able to commit to it. This doesn't take branch + * protection into consideration, so it's not perfect... but who uses protected branches for the + * org files? + * + * This is separate from `isSignedIn` to avoid the overhead of multiple API calls every time the + * page is loaded. + */ + const isProjectAccessible = async () => { + const projectId = getProjectId(); + // Check project exists and user is a member who can *probably* commit. + const [userResponse, membersResponse] = await Promise.all([ + decoratedFetch(`${API_URL}/user`), + decoratedFetch(`${API_URL}/projects/${projectId}/members`), + ]); + if (!userResponse.ok || !membersResponse.ok) { + return false; + } + const [user, members] = await Promise.all([userResponse.json(), membersResponse.json()]); + const matched = members.find((m) => m.id === user.id); + // Access levels: https://docs.gitlab.com/ee/api/members.html#valid-access-levels + // Permissions: https://docs.gitlab.com/ee/user/permissions.html#project-members-permissions + // 30 is developer, which is the minimum to commit to a non-protected branch. If branch is + // protected then they still might be be able to commit, but this is good enough. + return matched && matched.access_level >= 30; + }; + + const getDirectoryListing = async (path) => { + // FIXME use path param/finish implementing this + const project = getProjectId(); + const response = await decoratedFetch( + `${API_URL}/projects/${project}/repository/tree?pagination=keyset` + ); + const data = await response.json(); + console.log(data); + return { + listing: [], + hasMore: false, + }; + }; + + // FIXME stub + const getMoreDirectoryListing = (additionalSyncBackendState) => + new Promise((resolve) => + resolve({ + listing: [], + hasMore: false, + }) + ); + + // FIXME stub + const updateFile = (path, contents) => new Promise((resolve) => resolve()); + + // FIXME stub + const createFile = (path, contents) => new Promise((resolve) => resolve()); + + // FIXME stub + const getFileContentsAndMetadata = (path) => + new Promise((resolve) => + resolve({ + contents: '* FIXME', + lastModifiedAt: new Date(), + }) + ); + + // FIXME stub + const getFileContents = (path) => { + if (isEmpty(path)) return Promise.reject('No path given'); + return new Promise((resolve, reject) => + getFileContentsAndMetadata(path) + .then(({ contents }) => resolve(contents)) + .catch(reject) + ); + }; + + // FIXME stub + const deleteFile = (path) => new Promise((resolve) => resolve()); + + return { + type: 'GitLab', + isSignedIn, + isProjectAccessible, + getDirectoryListing, + getMoreDirectoryListing, + updateFile, + createFile, + getFileContentsAndMetadata, + getFileContents, + deleteFile, + }; +}; diff --git a/src/sync_backend_clients/gitlab_sync_backend_client.test.js b/src/sync_backend_clients/gitlab_sync_backend_client.test.js new file mode 100644 index 000000000..9cbe598af --- /dev/null +++ b/src/sync_backend_clients/gitlab_sync_backend_client.test.js @@ -0,0 +1,13 @@ +import { gitLabProjectIdFromURL } from './gitlab_sync_backend_client'; + +test('Parses GitLab project from URL', () => { + [ + ['https://gitlab.com/user/foo', 'user%2Ffoo'], + ['https://gitlab.com/group/subgroup/project', 'group%2Fsubgroup%2Fproject'], + ['gitlab.com/foo/bar', 'foo%2Fbar'], + ['gitlab.com/user-but-no-project', undefined], + ['', undefined], + ].forEach(([input, expected]) => { + expect(gitLabProjectIdFromURL(input)).toEqual(expected); + }); +}); diff --git a/src/util/settings_persister.js b/src/util/settings_persister.js index d0d29e0f2..335d80359 100644 --- a/src/util/settings_persister.js +++ b/src/util/settings_persister.js @@ -25,6 +25,7 @@ const debouncedPushConfigToSyncBackend = _.debounce( (syncBackendClient, contents) => { switch (syncBackendClient.type) { case 'Dropbox': + case 'GitLab': case 'WebDAV': syncBackendClient .createFile('/.organice-config.json', contents) @@ -333,6 +334,7 @@ export const loadSettingsFromConfigFile = (dispatch, getState) => { let fileContentsPromise = null; switch (syncBackendClient.type) { case 'Dropbox': + case 'GitLab': case 'WebDAV': fileContentsPromise = syncBackendClient.getFileContents('/.organice-config.json'); break; diff --git a/yarn.lock b/yarn.lock index 665d090ec..303df9db2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1752,6 +1752,11 @@ lodash "^4.17.13" to-fast-properties "^2.0.0" +"@bity/oauth2-auth-code-pkce@^2.13.0": + version "2.13.0" + resolved "https://registry.yarnpkg.com/@bity/oauth2-auth-code-pkce/-/oauth2-auth-code-pkce-2.13.0.tgz#cee077db75182b01e9b576a52ee408480d39bdc5" + integrity sha512-jAVsbLIVHvo0Lm6sj2t3ltiLbWKvTlkVflHCiV9DRk1w9FBEOjpJD7OJXkeWBY0Vgr+XiGIm80yJa+sa8XpnWQ== + "@cnakazawa/watch@^1.0.3": version "1.0.3" resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.3.tgz#099139eaec7ebf07a27c1786a3ff64f39464d2ef" From 201d28fb0ae211fc6b72d36fc53d3c1ea32622c0 Mon Sep 17 00:00:00 2001 From: Caleb Chase Date: Sat, 30 Oct 2021 15:25:14 -0500 Subject: [PATCH 05/19] GitLab: implement directory listing methods For #733 --- .../gitlab_sync_backend_client.js | 119 ++++++++++++++---- .../gitlab_sync_backend_client.test.js | 112 ++++++++++++++++- 2 files changed, 207 insertions(+), 24 deletions(-) diff --git a/src/sync_backend_clients/gitlab_sync_backend_client.js b/src/sync_backend_clients/gitlab_sync_backend_client.js index 061f06bbd..083abf777 100644 --- a/src/sync_backend_clients/gitlab_sync_backend_client.js +++ b/src/sync_backend_clients/gitlab_sync_backend_client.js @@ -1,8 +1,11 @@ /* global process */ import { OAuth2AuthCodePKCE } from '@bity/oauth2-auth-code-pkce'; import { isEmpty } from 'lodash'; +import { orgFileExtensions } from '../lib/org_utils'; import { getPersistedField } from '../util/settings_persister'; +import { fromJS, Map } from 'immutable'; + export const createGitlabOAuth = () => new OAuth2AuthCodePKCE({ authorizationUrl: 'https://gitlab.com/oauth/authorize', @@ -48,9 +51,57 @@ export const gitLabProjectIdFromURL = (projectURL) => { } }; -// TODO implement parsing -// TODO apparently want to use immutable's fromJS -// const treeToDirectoryListing = (tree) +/** + * Parse 'link' pagination response header. + * + * @see https://docs.gitlab.com/ee/api/index.html#keyset-based-pagination + * @param {string|null} links raw header value + * @return {Object.} Key-value mapping of link name to url. Empty object if none. + */ +export const parseLinkHeader = (links) => { + if (!links) { + return {}; + } + // Based on https://stackoverflow.com/a/48109741 + return links.split(',').reduce((acc, link) => { + const match = link.match(/<(.*)>; rel="(\w*)"/); + const url = match[1]; + const rel = match[2]; + acc[rel] = url; + return acc; + }, {}); +}; + +/** + * Converts response from GitLab's list repo tree API into organice format. + * + * @see https://docs.gitlab.com/ee/api/repositories.html#list-repository-tree + */ +export const treeToDirectoryListing = (tree) => { + const isDirectory = (it) => it.type === 'tree'; + return fromJS( + tree + .filter((it) => isDirectory(it) || it.name.match(orgFileExtensions)) + .map((it) => ({ + id: it.id, + name: it.name, + // Organice requires a leading "/", whereas GitLab API doesn't use one. + path: `/${it.path}`, + isDirectory: isDirectory(it), + })) + .sort((a, b) => { + // Folders first. + if (a.isDirectory && !b.isDirectory) { + return -1; + } else if (!a.isDirectory && b.isDirectory) { + return 1; + } else { + // Can't have same name, so don't need to check if equal/return 0. + return a.name > b.name ? 1 : -1; + } + }) + ); +}; const API_URL = 'https://gitlab.com/api/v4'; @@ -60,7 +111,7 @@ const API_URL = 'https://gitlab.com/api/v4'; export default (oauthClient) => { const decoratedFetch = oauthClient.decorateFetchHTTPClient(fetch); - const getProjectId = () => getPersistedField('gitLabProject'); + const getProjectApi = () => `${API_URL}/projects/${getPersistedField('gitLabProject')}`; const isSignedIn = async () => { if (!oauthClient.isAuthorized()) { @@ -85,11 +136,12 @@ export default (oauthClient) => { * page is loaded. */ const isProjectAccessible = async () => { - const projectId = getProjectId(); // Check project exists and user is a member who can *probably* commit. const [userResponse, membersResponse] = await Promise.all([ + // https://docs.gitlab.com/ee/api/users.html#list-current-user-for-normal-users decoratedFetch(`${API_URL}/user`), - decoratedFetch(`${API_URL}/projects/${projectId}/members`), + // https://docs.gitlab.com/ee/api/members.html#list-all-members-of-a-group-or-project + decoratedFetch(`${getProjectApi()}/members`), ]); if (!userResponse.ok || !membersResponse.ok) { return false; @@ -103,28 +155,49 @@ export default (oauthClient) => { return matched && matched.access_level >= 30; }; - const getDirectoryListing = async (path) => { - // FIXME use path param/finish implementing this - const project = getProjectId(); - const response = await decoratedFetch( - `${API_URL}/projects/${project}/repository/tree?pagination=keyset` - ); + let cachedDefaultBranch; + const getDefaultBranch = async () => { + if (!cachedDefaultBranch) { + // https://docs.gitlab.com/ee/api/projects.html#get-single-project + const response = await decoratedFetch(getProjectApi()); + if (!response.ok) { + throw new Error(`Unexpected response from project API. Status code: ${response.status}`); + } + const body = await response.json(); + cachedDefaultBranch = body.default_branch; + } + return cachedDefaultBranch; + }; + + const fetchDirectory = async (url) => { + const response = await decoratedFetch(url); + if (!response.ok) { + throw new Error(`Unexpected response from directory API. Status code: ${response.status}`); + } + const pages = parseLinkHeader(response.headers.get('link')); const data = await response.json(); - console.log(data); return { - listing: [], - hasMore: false, + listing: treeToDirectoryListing(data), + hasMore: !!pages.next, + additionalSyncBackendState: Map({ + cursor: pages.next, + }), }; }; - // FIXME stub - const getMoreDirectoryListing = (additionalSyncBackendState) => - new Promise((resolve) => - resolve({ - listing: [], - hasMore: false, - }) - ); + const getDirectoryListing = async (path) => { + const params = new URLSearchParams({ + pagination: 'keyset', + ref: await getDefaultBranch(), + // Organice requires a leading "/", whereas GitLab API requires there *not* be one. + path: path.replace(/^\//, ''), + }); + // https://docs.gitlab.com/ee/api/repositories.html#list-repository-tree + return await fetchDirectory(`${getProjectApi()}/repository/tree?${params}`); + }; + + const getMoreDirectoryListing = async (additionalSyncBackendState) => + await fetchDirectory(additionalSyncBackendState.get('cursor')); // FIXME stub const updateFile = (path, contents) => new Promise((resolve) => resolve()); diff --git a/src/sync_backend_clients/gitlab_sync_backend_client.test.js b/src/sync_backend_clients/gitlab_sync_backend_client.test.js index 9cbe598af..24d285916 100644 --- a/src/sync_backend_clients/gitlab_sync_backend_client.test.js +++ b/src/sync_backend_clients/gitlab_sync_backend_client.test.js @@ -1,4 +1,9 @@ -import { gitLabProjectIdFromURL } from './gitlab_sync_backend_client'; +import { fromJS } from 'immutable'; +import { + gitLabProjectIdFromURL, + parseLinkHeader, + treeToDirectoryListing, +} from './gitlab_sync_backend_client'; test('Parses GitLab project from URL', () => { [ @@ -11,3 +16,108 @@ test('Parses GitLab project from URL', () => { expect(gitLabProjectIdFromURL(input)).toEqual(expected); }); }); + +test('Parses Link pagination header', () => { + [ + [null, {}], + [ + `; rel="first", ; rel="last"`, + { first: 'https://foo.local', last: 'https://bar.local' }, + ], + ].forEach(([input, expected]) => { + expect(parseLinkHeader(input)).toEqual(expected); + }); +}); + +describe('Converts file tree to directory listing', () => { + test('Handles a file', () => { + expect( + treeToDirectoryListing([ + { + id: '1eabc98234098234', + name: 'foo.org', + type: 'blob', + // Notice that API response has no leading "/" in path, whereas result does. + path: 'somewhere/foo.org', + mode: '100644', + }, + ]) + ).toEqual( + fromJS([ + { + id: '1eabc98234098234', + name: 'foo.org', + path: '/somewhere/foo.org', + isDirectory: false, + }, + ]) + ); + }); + + test('Handles a directory', () => { + expect( + treeToDirectoryListing([ + { + id: '1e8eb8723398', + name: 'somedir', + type: 'tree', + path: 'somedir', + mode: '040000', + }, + ]) + ).toEqual( + fromJS([ + { + id: '1e8eb8723398', + name: 'somedir', + path: '/somedir', + isDirectory: true, + }, + ]) + ); + }); + + test('Filters a non-org file', () => { + expect( + treeToDirectoryListing([ + { + id: '1eabc98234098234', + name: 'foo.txt', + type: 'blob', + path: 'foo.txt', + mode: '100644', + }, + ]) + ).toEqual(fromJS([])); + }); + + test('Sorts correctly', () => { + const data = [ + { + id: '93842093', + name: 'mno', + type: 'tree', + path: 'mno', + mode: '040000', + }, + { + id: '0921384', + name: 'xyz.org', + type: 'blob', + path: 'xyz.org', + mode: '100644', + }, + { + id: '123abc', + name: 'abc.org', + type: 'blob', + path: 'abc.org', + mode: '100644', + }, + ]; + const names = treeToDirectoryListing(data) + .toJS() + .map((it) => it.name); + expect(names).toEqual(['mno', 'abc.org', 'xyz.org']); + }); +}); From aa4d898f74e917b54cb39b4561c851fb0869f5bb Mon Sep 17 00:00:00 2001 From: Caleb Chase Date: Sat, 30 Oct 2021 18:55:47 -0500 Subject: [PATCH 06/19] GitLab: implement file read methods For #733 --- .../gitlab_sync_backend_client.js | 61 ++++++++++++++----- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/src/sync_backend_clients/gitlab_sync_backend_client.js b/src/sync_backend_clients/gitlab_sync_backend_client.js index 083abf777..1a592719c 100644 --- a/src/sync_backend_clients/gitlab_sync_backend_client.js +++ b/src/sync_backend_clients/gitlab_sync_backend_client.js @@ -1,6 +1,5 @@ /* global process */ import { OAuth2AuthCodePKCE } from '@bity/oauth2-auth-code-pkce'; -import { isEmpty } from 'lodash'; import { orgFileExtensions } from '../lib/org_utils'; import { getPersistedField } from '../util/settings_persister'; @@ -106,6 +105,9 @@ export const treeToDirectoryListing = (tree) => { const API_URL = 'https://gitlab.com/api/v4'; /** + * GitLab sync backend, implemented using their REST API. + * + * @see https://docs.gitlab.com/ee/api/api_resources.html * @param {OAuth2AuthCodePKCE} oauthClient */ export default (oauthClient) => { @@ -205,23 +207,52 @@ export default (oauthClient) => { // FIXME stub const createFile = (path, contents) => new Promise((resolve) => resolve()); - // FIXME stub - const getFileContentsAndMetadata = (path) => - new Promise((resolve) => - resolve({ - contents: '* FIXME', - lastModifiedAt: new Date(), - }) + const getRawFile = async (path) => { + const params = new URLSearchParams({ + ref: await getDefaultBranch(), + }); + const encodedPath = encodeURIComponent(path.replace(/^\//, '')); + // https://docs.gitlab.com/ee/api/repository_files.html#get-raw-file-from-repository + const response = await decoratedFetch( + `${getProjectApi()}/repository/files/${encodedPath}/raw?${params}` ); + if (!response.ok) { + throw new Error(`Unexpected response from file API. Status code: ${response.status}`); + } + return { + contents: await response.text(), + commit: response.headers.get('x-gitlab-last-commit-id'), + }; + }; - // FIXME stub - const getFileContents = (path) => { - if (isEmpty(path)) return Promise.reject('No path given'); - return new Promise((resolve, reject) => - getFileContentsAndMetadata(path) - .then(({ contents }) => resolve(contents)) - .catch(reject) + const getCommitDate = async (sha) => { + // https://docs.gitlab.com/ee/api/commits.html#get-a-single-commit + const response = await decoratedFetch( + `${getProjectApi()}/repository/commits/${sha}?stats=false` ); + if (!response.ok) { + throw new Error(`Unexpected response from commit API. Status code: ${response.status}`); + } + const body = await response.json(); + // Dates are ISO-8601. + // Note: while commit date *should* generally be the same as or later than the author date, + // that isn't guaranteed since history can be rewritten at will. So we pick the newer of the + // two. + const committed = new Date(body.committed_date); + const authored = new Date(body.authored_date); + return committed > authored ? committed : authored; + }; + + const getFileContentsAndMetadata = async (path) => { + const file = await getRawFile(path); + return { + contents: file.contents, + lastModifiedAt: await getCommitDate(file.commit), + }; + }; + + // Parentheses are necessarily to await the actual promise. Yay, foot-guns. + const getFileContents = async (path) => (await getRawFile(path)).contents; }; // FIXME stub From 317a090801df15685aae2c78e6652c9cf01bb9e6 Mon Sep 17 00:00:00 2001 From: Caleb Chase Date: Sat, 30 Oct 2021 19:12:34 -0500 Subject: [PATCH 07/19] GitLab: don't backup files Version control makes this redundant. For #733 --- src/actions/sync_backend.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/actions/sync_backend.js b/src/actions/sync_backend.js index ea5964c04..4c3be68df 100644 --- a/src/actions/sync_backend.js +++ b/src/actions/sync_backend.js @@ -95,7 +95,6 @@ export const pushBackup = (pathOrFileId, contents) => { const client = getState().syncBackend.get('client'); switch (client.type) { case 'Dropbox': - case 'GitLab': case 'WebDAV': client.createFile(`${pathOrFileId}.organice-bak`, contents); break; @@ -103,6 +102,9 @@ export const pushBackup = (pathOrFileId, contents) => { pathOrFileId = pathOrFileId.startsWith('/') ? pathOrFileId.substr(1) : pathOrFileId; client.duplicateFile(pathOrFileId, (fileName) => `${fileName}.organice-bak`); break; + case 'GitLab': + // No-op for GitLab, because the beauty of version control makes backup files redundant. + break; default: } }; From c06da04d3f5791b7fc86bff4d67095e5a343e295 Mon Sep 17 00:00:00 2001 From: Caleb Chase Date: Sat, 30 Oct 2021 22:51:26 -0500 Subject: [PATCH 08/19] GitLab: implement create/update/delete methods For #733 --- .../gitlab_sync_backend_client.js | 57 ++++++++++++++++--- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/src/sync_backend_clients/gitlab_sync_backend_client.js b/src/sync_backend_clients/gitlab_sync_backend_client.js index 1a592719c..556d21cf6 100644 --- a/src/sync_backend_clients/gitlab_sync_backend_client.js +++ b/src/sync_backend_clients/gitlab_sync_backend_client.js @@ -201,12 +201,6 @@ export default (oauthClient) => { const getMoreDirectoryListing = async (additionalSyncBackendState) => await fetchDirectory(additionalSyncBackendState.get('cursor')); - // FIXME stub - const updateFile = (path, contents) => new Promise((resolve) => resolve()); - - // FIXME stub - const createFile = (path, contents) => new Promise((resolve) => resolve()); - const getRawFile = async (path) => { const params = new URLSearchParams({ ref: await getDefaultBranch(), @@ -253,10 +247,57 @@ export default (oauthClient) => { // Parentheses are necessarily to await the actual promise. Yay, foot-guns. const getFileContents = async (path) => (await getRawFile(path)).contents; + + const doCommit = async (action) => { + const capitalizedAction = action.action.charAt(0).toUpperCase() + action.action.slice(1); + const message = + `[organice] ${capitalizedAction} ${action.file_path}\n` + + 'Automatic commit from organice app.'; + // It's also possible to modify files using the files API instead of commits API. For this use + // case they're about equal, but I picked commits because it doesn't require non-standard + // encoding for file paths, whereas the files API requires URI encoding plus converting period + // characters to %2E. Also, the commits API is more flexible in case of future changes, since it + // allows modifying multiple files at a time. + // + // https://docs.gitlab.com/ee/api/commits.html#create-a-commit-with-multiple-files-and-actions + // https://docs.gitlab.com/ee/api/repository_files.html + await decoratedFetch(`${getProjectApi()}/repository/commits`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + branch: await getDefaultBranch(), + commit_message: message, + // Organice only modifies a single file at a time, so only one action. + actions: [action], + stats: false, + }), + }); }; - // FIXME stub - const deleteFile = (path) => new Promise((resolve) => resolve()); + const createFile = async (path, content) => { + await doCommit({ + action: 'create', + file_path: path.replace(/^\//, ''), + content, + }); + }; + + const updateFile = async (path, content) => { + await doCommit({ + action: 'update', + file_path: path.replace(/^\//, ''), + content, + }); + }; + + const deleteFile = async (path) => { + await doCommit({ + action: 'delete', + file_path: path.replace(/^\//, ''), + }); + }; return { type: 'GitLab', From 1445b3c17d0ea82b4934de547799642d36e6cfa8 Mon Sep 17 00:00:00 2001 From: Caleb Chase Date: Sat, 30 Oct 2021 22:51:43 -0500 Subject: [PATCH 09/19] GitLab: persist settings file properly For #733 --- src/util/settings_persister.js | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/util/settings_persister.js b/src/util/settings_persister.js index 335d80359..06a21552f 100644 --- a/src/util/settings_persister.js +++ b/src/util/settings_persister.js @@ -21,11 +21,33 @@ export const localStorageAvailable = (() => { } })(); +const updateConfigForGitLab = async (client, contents) => { + const filename = '/.organice-config.json'; + // GitLab doesn't allow updating a file that doesn't exist or creating one that already exists, so + // need to figure out whether or not it exists. + let exists = false; + try { + const existingContents = await client.getFileContents(filename); + exists = true; + if (existingContents === contents) { + // Also avoid a pointless empty commit if file didn't actually change. + return; + } + } catch { + // Error is (presumably) because file doesn't exist. Unfortunately this isn't perfect because it + // could have failed due to e.g. a network issue. + } + if (exists) { + await client.updateFile(filename, contents); + } else { + await client.createFile(filename, contents); + } +}; + const debouncedPushConfigToSyncBackend = _.debounce( (syncBackendClient, contents) => { switch (syncBackendClient.type) { case 'Dropbox': - case 'GitLab': case 'WebDAV': syncBackendClient .createFile('/.organice-config.json', contents) @@ -33,6 +55,11 @@ const debouncedPushConfigToSyncBackend = _.debounce( alert(`There was an error trying to push settings to your sync backend: ${error}`) ); break; + case 'GitLab': + updateConfigForGitLab(syncBackendClient, contents).catch((error) => + alert(`There was an error trying to push settings to your sync backend: ${error}`) + ); + break; case 'Google Drive': syncBackendClient .createFile('.organice-config.json', 'root', contents) From 4f0d14d108ec1c8a9aaf964868ee7f8417d15dd9 Mon Sep 17 00:00:00 2001 From: Caleb Chase Date: Sun, 31 Oct 2021 11:00:50 -0500 Subject: [PATCH 10/19] GitLab: add setup info to README.org For #733 --- README.org | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/README.org b/README.org index 2418ca1c7..45227686a 100644 --- a/README.org +++ b/README.org @@ -1,3 +1,4 @@ +# -*- org-adapt-indentation: nil; fill-column: 70; -*- #+title: organice documentation #+html:

organice - /'ɔ:gənaɪz/

@@ -385,8 +386,8 @@ yarn install --production=false *** Setup any of the synchronization back-ends -organice can sync your Org files using Dropbox, Google Drive and -WebDAV as back-ends. +organice can sync your Org files using Dropbox, Google Drive, GitLab, +and WebDAV as back-ends. If you want to develop a feature that needs synchronization, then you will have to set up any of those options. If you want to work on a @@ -406,7 +407,7 @@ your files with either of them anyway and use WebDAV all the way. In any case, [[#faq_webdav][here's how to get running locally with a WebDAV setup]]. -**** Dropbox or Google Drive +**** Dropbox, Google Drive, or GitLab To test against your own Dropbox or Google Drive application, you'll need to create a ~.env~ file by copying [[file:.env.sample][.env.sample]] to just ~.env~. @@ -415,8 +416,8 @@ need to create a ~.env~ file by copying [[file:.env.sample][.env.sample]] to jus cp .env.sample .env #+END_SRC -Then, fill in the blanks in ~.env~ with your Dropbox or Google Drive -credentials. More information about that is in the section +Then, fill in the blanks in ~.env~ with your Dropbox, Google Drive, or +GitLab credentials. More information about that is in the section [[#synchronization_back_ends][Synchronization back-ends]]. **** Running the application @@ -706,6 +707,33 @@ Client ID. Then, you will create a new ~.env~ file (analogous to =REACT_APP_GOOGLE_DRIVE_API_KEY= and =REACT_APP_GOOGLE_DRIVE_CLIENT_ID=. +*** GitLab + :PROPERTIES: + :CUSTOM_ID: gitlab + :END: + +To configure your own instance of organice for GitLab, please create +an OAuth application by going to [[https://gitlab.com/-/profile/applications][GitLab's application settings for +your profile]] and filling out the form with the following details: + +- Name: "organice test" (or whatever you prefer) +- Redirect URI: ~http://localhost:3000/~ for local development, or + whatever domain you are hosting it with. +- Confidential: /uncheck/ this +- Expire access tokens: leave checked +- Scopes: =api= only + +Once filled out, click "save application" and keep this page open. +Then, create a new ~.env~ file (analogous to ~.env.sample~) and set +the following variables: + +- =REACT_APP_GITLAB_CLIENT_ID=: The value that GitLab provides for + =Application ID= +- =REACT_APP_GITLAB_SECRET=: The value that GitLab provides for =Secret=. + +You may also refer to [[https://docs.gitlab.com/ee/integration/oauth_provider.html#user-owned-applications][GitLab's documentation]] for more information +regarding OAuth applications, if interested. + *** Encryption :PROPERTIES: :CUSTOM_ID: encryption From b84332233a6fc2f4e5033d4f9ee789dc5eec5273 Mon Sep 17 00:00:00 2001 From: Caleb Chase Date: Sun, 31 Oct 2021 14:08:32 -0500 Subject: [PATCH 11/19] GitLab: fix last modified date type Should be a string, not Date object. --- src/sync_backend_clients/gitlab_sync_backend_client.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sync_backend_clients/gitlab_sync_backend_client.js b/src/sync_backend_clients/gitlab_sync_backend_client.js index 556d21cf6..4aa88c7ce 100644 --- a/src/sync_backend_clients/gitlab_sync_backend_client.js +++ b/src/sync_backend_clients/gitlab_sync_backend_client.js @@ -234,7 +234,8 @@ export default (oauthClient) => { // two. const committed = new Date(body.committed_date); const authored = new Date(body.authored_date); - return committed > authored ? committed : authored; + // Use Date objects for comparison, but need to return as strings. + return committed > authored ? committed.toISOString() : authored.toISOString(); }; const getFileContentsAndMetadata = async (path) => { From 3493edfe9984abc9a5536a9522fd39fd47fdfe2c Mon Sep 17 00:00:00 2001 From: "Alain M. Lafon" Date: Mon, 1 Nov 2021 16:17:44 +0100 Subject: [PATCH 12/19] chore: Style, prioritize back-ends and document sign_in page --- src/components/SyncServiceSignIn/gitlab.svg | 226 ++++++++++++++---- src/components/SyncServiceSignIn/index.js | 28 ++- .../SyncServiceSignIn/stylesheet.css | 7 + 3 files changed, 213 insertions(+), 48 deletions(-) diff --git a/src/components/SyncServiceSignIn/gitlab.svg b/src/components/SyncServiceSignIn/gitlab.svg index 77f5437d3..c29c5b86b 100644 --- a/src/components/SyncServiceSignIn/gitlab.svg +++ b/src/components/SyncServiceSignIn/gitlab.svg @@ -1,7 +1,53 @@ - - - - diff --git a/src/components/SyncServiceSignIn/index.js b/src/components/SyncServiceSignIn/index.js index 16c07aea6..95f0f4470 100644 --- a/src/components/SyncServiceSignIn/index.js +++ b/src/components/SyncServiceSignIn/index.js @@ -201,7 +201,7 @@ export default class SyncServiceSignIn extends PureComponent { return (

- organice syncs your files with Dropbox, Google Drive and WebDAV. + organice syncs your files with Dropbox, GitLab, WebDAV and Google Drive.

Click to sign in with:

@@ -209,6 +209,14 @@ export default class SyncServiceSignIn extends PureComponent { Dropbox logo
+
+ +
+ +
+ +
+
-
- -
- -
- -
+
+ For questions regarding synchronization back-ends, please consult the{' '} + + documentation + + . +
); } diff --git a/src/components/SyncServiceSignIn/stylesheet.css b/src/components/SyncServiceSignIn/stylesheet.css index 5f81e7c70..03c719b06 100644 --- a/src/components/SyncServiceSignIn/stylesheet.css +++ b/src/components/SyncServiceSignIn/stylesheet.css @@ -42,6 +42,13 @@ cursor: pointer; padding: 20px 10px; + text-align: center; +} + +/* Sync back-end logos or titles are the same height. */ +.sync-service-container img, +.sync-service-container h2 { + max-height: 87px; } .sync-service-container:last-of-type { From 34e38a12757291511d3a3a60534b54ac9bbeb981 Mon Sep 17 00:00:00 2001 From: "Alain M. Lafon" Date: Mon, 1 Nov 2021 16:34:50 +0100 Subject: [PATCH 13/19] doc: Rationale for custom function vs. existing strategy pattern --- src/util/settings_persister.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/util/settings_persister.js b/src/util/settings_persister.js index 06a21552f..35d298b98 100644 --- a/src/util/settings_persister.js +++ b/src/util/settings_persister.js @@ -56,6 +56,9 @@ const debouncedPushConfigToSyncBackend = _.debounce( ); break; case 'GitLab': + // INFO: Not calling syncBackendClient.createFile is a + // workaround for + // https://github.com/200ok-ch/organice/issues/736 updateConfigForGitLab(syncBackendClient, contents).catch((error) => alert(`There was an error trying to push settings to your sync backend: ${error}`) ); From 016d07132658ce8472889f4f3b646b41bc7ab1c8 Mon Sep 17 00:00:00 2001 From: "Alain M. Lafon" Date: Mon, 1 Nov 2021 16:36:38 +0100 Subject: [PATCH 14/19] chore: Remove linter warning --- src/components/SyncServiceSignIn/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/SyncServiceSignIn/index.js b/src/components/SyncServiceSignIn/index.js index 95f0f4470..63036cb51 100644 --- a/src/components/SyncServiceSignIn/index.js +++ b/src/components/SyncServiceSignIn/index.js @@ -232,7 +232,7 @@ export default class SyncServiceSignIn extends PureComponent { documentation From 9f051bb68445f634358c6ac0065a3c8cbac6c308 Mon Sep 17 00:00:00 2001 From: "Alain M. Lafon" Date: Mon, 1 Nov 2021 16:41:05 +0100 Subject: [PATCH 15/19] chore: Increase `per_page` from the default of 20 to 100 Otherwise it's likely that the user has to click "Load more..." everytime she goes to the file browser. At least I would have to(; --- src/sync_backend_clients/gitlab_sync_backend_client.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sync_backend_clients/gitlab_sync_backend_client.js b/src/sync_backend_clients/gitlab_sync_backend_client.js index 4aa88c7ce..665cbfb3d 100644 --- a/src/sync_backend_clients/gitlab_sync_backend_client.js +++ b/src/sync_backend_clients/gitlab_sync_backend_client.js @@ -193,6 +193,7 @@ export default (oauthClient) => { ref: await getDefaultBranch(), // Organice requires a leading "/", whereas GitLab API requires there *not* be one. path: path.replace(/^\//, ''), + per_page: 100, }); // https://docs.gitlab.com/ee/api/repositories.html#list-repository-tree return await fetchDirectory(`${getProjectApi()}/repository/tree?${params}`); From 2d881f9df5f4eaff1c6c08339c0acc39867ec5c3 Mon Sep 17 00:00:00 2001 From: "Alain M. Lafon" Date: Mon, 1 Nov 2021 16:52:30 +0100 Subject: [PATCH 16/19] chore: Find all places where a list of sync back-ends are mentioned Add GitLab to all of them and show it as the second option after Dropbox. --- README.org | 36 +++++++++++++++++++----------------- sample.org | 6 +++--- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/README.org b/README.org index 45227686a..d923fb21f 100644 --- a/README.org +++ b/README.org @@ -27,13 +27,13 @@ Community chat: #organice on IRC [[https://libera.chat/][Libera.Chat]], or [[htt organice is an implementation of [[http://orgmode.org/][Org mode]] without the dependency of [[https://www.gnu.org/software/emacs/][Emacs]]. It is built for mobile and desktop browsers and syncs with -Dropbox, Google Drive and WebDAV. +[[https://www.dropbox.com/][Dropbox]], [[https://gitlab.com/][GitLab]], [[https://en.wikipedia.org/wiki/WebDAV][WebDAV]] and [[https://drive.google.com][Google Drive]]. At [[https://200ok.ch/][200ok]], we run an instance of organice at https://organice.200ok.ch, which is open for anyone to use! organice does not have a back-end -(it's just a front-end application, which uses either Dropbox, Google -Drive or WebDAV as back-end storage). We don't store any kind of data -on our servers - we also don't use analytics on organice.200ok.ch. +(it's just a front-end application, which uses different back-end +storage providers). We don't store any kind of data on our servers - +we also don't use analytics on organice.200ok.ch. [[https://raw.githubusercontent.com/200ok-ch/organice/master/images/screenshot-overview.png]] @@ -386,8 +386,8 @@ yarn install --production=false *** Setup any of the synchronization back-ends -organice can sync your Org files using Dropbox, Google Drive, GitLab, -and WebDAV as back-ends. +organice can sync your Org files using Dropbox, GitLab, WebDAV or +Google Drive as back-ends. If you want to develop a feature that needs synchronization, then you will have to set up any of those options. If you want to work on a @@ -730,7 +730,7 @@ the following variables: - =REACT_APP_GITLAB_CLIENT_ID=: The value that GitLab provides for =Application ID= - =REACT_APP_GITLAB_SECRET=: The value that GitLab provides for =Secret=. - + You may also refer to [[https://docs.gitlab.com/ee/integration/oauth_provider.html#user-owned-applications][GitLab's documentation]] for more information regarding OAuth applications, if interested. @@ -740,16 +740,18 @@ regarding OAuth applications, if interested. :END: If you do not trust your data with Dropbox or Google, you are free to -host your own [[#webdav][WebDAV]] server and take any number of precautionary -measures. For example, you can encrypt your data on disk. organice -itself is just a front-end application, requires no server and has no -tracking system. Therefore, the data within any organice instance -(self hosted or not) is already only accessible to you, your browser -and the network between your browser and your chosen back-end. -Therefore, if have a strong SSL certificate configured on your WebDAV -server and organice instance, then organice will communicate securely -via HTTPS to your server where your data is as secure as you make it. -Then, your data will be encrypted and inaccessible to any third party. +use Gitlab ([[https://about.gitlab.com/solutions/open-source/][which is open-source]]) or host your own [[#webdav][WebDAV]] server and +take any number of precautionary measures. + +For example, you can encrypt your data on disk. organice itself is +just a front-end application, requires no server and has no tracking +system. Therefore, the data within any organice instance (self hosted +or not) is already only accessible to you, your browser and the +network between your browser and your chosen back-end. Therefore, if +have a strong SSL certificate configured on your WebDAV server and +organice instance, then organice will communicate securely via HTTPS +to your server where your data is as secure as you make it. Then, your +data will be encrypted and inaccessible to any third party. Of course, security is hard. So the above statement is not a guarantee, but a guideline. You're responsible to ensure that the diff --git a/sample.org b/sample.org index fb8cca00e..00d8b7554 100644 --- a/sample.org +++ b/sample.org @@ -100,7 +100,7 @@ Give them a try on these nested headers to get a feel for how they operate: ***** Clooney ** Syncing The "cloud" button in the lower left hand corner syncs changes to your -chosen sync service (Dropbox, Google Drive or WebDAV). +chosen sync service (Dropbox, GitLab, WebDAV or Google Drive). If there's a newer version on the server and no local changes, it'll pull. @@ -412,7 +412,7 @@ SCHEDULED: <2018-09-17 Mon> ** This entry also only shows on exactly one day <2020-02-17 Mon> * Syncing -organice pulls down your org files from Dropbox, Google Drive or WebDAV. Click the "Sign in" button in the upper right hand corner to sign in with either of them and authenticate organice. +organice pulls down your org files from Dropbox, GitLab, WebDAV or Google Drive. Click the "Sign in" button in the upper right hand corner to sign in with either of them and authenticate organice. ** Backups The first time you push changes from organice back up to your chosen sync service, organice will make a backup of the original file first. It'll be named {your-file-name}.organice-bak. Dropbox and Google Drive also both keep a full version history of your files for you, but this is an additional precaution in case something goes wrong pushing the file back up. @@ -430,7 +430,7 @@ Default behaviour: You can adjust these defaults on a file per file basis by creating file settings in the [[/settings][settings menu]]. * organice operates completely client side -You don't log in to organice directly because organice doesn't have a back end - it operates completely client side using Dropbox, Google Drive or WebDAV as back-ends. +You don't log in to organice directly because organice doesn't have a back end - it operates completely client side using Dropbox, GitLab, WebDAV or Google Drive as back-ends for storage. * Capture URL params and Siri support organice supports a flexible mechanism for capturing using URL parameters. This mechanism integrates very nicely with the new [[https://support.apple.com/guide/shortcuts/welcome/ios][Siri Shortcuts]] feature in iOS 12, allowing you to use Siri to execute capture templates. From 007d5683cb378d1fdf9e58987bfd212b14133e29 Mon Sep 17 00:00:00 2001 From: "Alain M. Lafon" Date: Mon, 1 Nov 2021 16:54:47 +0100 Subject: [PATCH 17/19] doc: Update changelog --- changelog.org | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog.org b/changelog.org index 76b383c99..32b76cca0 100644 --- a/changelog.org +++ b/changelog.org @@ -7,6 +7,11 @@ All user visible changes to organice will be documented in this file. When there are updates to the changelog, you will be notified and see a 'gift' icon appear on the top right corner. +* [2021-11-01 Mon] +** Added + - *EPIC* Add GitLab as a sync back-end + - PR: https://github.com/200ok-ch/organice/pull/734 + - Thank you [[https://github.com/chasecaleb][chasecaleb]] for the PR🙏 * [2021-10-29 Fri] ** Fixed - Loading settings when using WebDAV as synchronization back-end From 8cbeecb2866e2db5e6b966f02430dbec208fefed Mon Sep 17 00:00:00 2001 From: "Alain M. Lafon" Date: Mon, 1 Nov 2021 17:12:49 +0100 Subject: [PATCH 18/19] chore: Add GitLab application credentials to deployment automation --- bin/compile_and_upload.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/compile_and_upload.sh b/bin/compile_and_upload.sh index 2b1cf55f3..45ff3f4a1 100755 --- a/bin/compile_and_upload.sh +++ b/bin/compile_and_upload.sh @@ -10,9 +10,13 @@ set -e # Configure available back-end API keys cp .env.sample .env [ -z ${REACT_APP_DROPBOX_CLIENT_ID+x} ] || sed -i "s/your_dropbox_client_id/${REACT_APP_DROPBOX_CLIENT_ID}/" .env + [ -z ${REACT_APP_GOOGLE_DRIVE_API_KEY+x} ] || sed -i "s/your_google_drive_api_key/${REACT_APP_GOOGLE_DRIVE_API_KEY}/" .env [ -z ${REACT_APP_GOOGLE_DRIVE_CLIENT_ID+x} ] || sed -i "s/your_google_drive_oauth_client_id/${REACT_APP_GOOGLE_DRIVE_CLIENT_ID}/" .env +[ -z ${REACT_APP_GITLAB_CLIENT_ID+x} ] || sed -i "s/your_gitlab_client_id/${REACT_APP_GITLAB_CLIENT_ID}/" .env +[ -z ${REACT_APP_GITLAB_SECRET+x} ] || sed -i "s/your_gitlab_secret/${REACT_APP_GITLAB_SECRET}/" .env + yarn install yarn run build cd build From 53455e3d1a747e4f463a3366eaf6c23b98fbc308 Mon Sep 17 00:00:00 2001 From: Caleb Chase Date: Mon, 1 Nov 2021 11:37:22 -0500 Subject: [PATCH 19/19] Reorganize comments for clarity --- src/util/settings_persister.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/util/settings_persister.js b/src/util/settings_persister.js index 35d298b98..3eed8e7cd 100644 --- a/src/util/settings_persister.js +++ b/src/util/settings_persister.js @@ -21,16 +21,20 @@ export const localStorageAvailable = (() => { } })(); +/** + * GitLab doesn't allow updating a file that doesn't exist or creating one that already exists, so + * need to figure out which to do. + */ const updateConfigForGitLab = async (client, contents) => { const filename = '/.organice-config.json'; - // GitLab doesn't allow updating a file that doesn't exist or creating one that already exists, so - // need to figure out whether or not it exists. let exists = false; try { const existingContents = await client.getFileContents(filename); exists = true; + // INFO: Not calling syncBackendClient.createFile is a + // workaround for + // https://github.com/200ok-ch/organice/issues/736 if (existingContents === contents) { - // Also avoid a pointless empty commit if file didn't actually change. return; } } catch { @@ -56,9 +60,6 @@ const debouncedPushConfigToSyncBackend = _.debounce( ); break; case 'GitLab': - // INFO: Not calling syncBackendClient.createFile is a - // workaround for - // https://github.com/200ok-ch/organice/issues/736 updateConfigForGitLab(syncBackendClient, contents).catch((error) => alert(`There was an error trying to push settings to your sync backend: ${error}`) );