Skip to content

Commit

Permalink
feat: Version control mvp (#6271)
Browse files Browse the repository at this point in the history
* implement basic git service

* cleanup connected prop

* add skeleton of git functions

* initial import/export setup

* split out export service

* refactor and improve export

* begin import

* more commands and basic import

* clean up imports with transactions

* work folder import functions

* reintroduce versionid

* add missing import to pull workfolder

* add get-status endpoint

* add cleanup to disconnect

* add initRepo options

* add more checks and cleanup

* minor cleanup

* refactor prefs

* fix server.ts

* fix sending deleted files

* rename files to ee

* add variable override and fix critical cred import bug

* fix mkdir race condition

* make initRepo default to true

* fix front back integration

* improve connect flow

* add comment to generated ssh key

* fix(editor): use useToast composable

* fix buttons size

* commenting out repo init for now

* fix(editor): update UI logic

* fix(editor): remove console.log

* fix(editor): remove unused ref

* adjust endpoints for improved UI

* fix(editor): add push and pull buttons

* keep or not ssh key

* switching file name to id

* fix(editor): add success messages, fix save button

* fixed faulty diff preventing pull

* fix build

* fix(editor): adding loader to VC components

* removing duplicate exports

* improve conflict finding on push pull

* manage pull conflict

* alternate push pull

* fix pull confirmation

* fix rm and credential export/import

* switch to alternative pull implementation

* fix initial commit

* fix(editor): subscribing to VC store action to refresh lists

* fix(editor): wrap VC store actions with try

* feat: add fine-grained file selection for push action

* fix: close modal after successful push

* fix(editor): VC preferences validation

* fix confirm

* fix: update endpoint to /get-status

* feat: update pull modal override changes message

* fix missing wf error

* undo

* removing connect endpoint

* fix(editor): add button titles

* fix(editor): cleaning up store action

* add version-control/set-read-only protection

* fix(editor): adding set branch readonly

* fix(editor): remove Push button if branch set to readonly

* fix(editor): fix some styles

* fix(editor): remove duplicate and delete actions in WF list when branch is readonly

* fix: load status before opening selective push modal

* fix(editor): extend readonly logic

* add cleanup after failed initRepo

* fix deleted files crashing get-status

* fix n8n-checkbox in staging dialog

* fix(editor): fix loading

* fix(editor): resize buttons

* fix(editor): fix translation

* fix(editor): fix copy text size

* fix(editor): fix copy text size

* fix(editor): add disconnection confirmation

* fix(editor): add disconnection confirmation

* fix(editor): set large buttons

* add public api Pull endpoint

* feat: add refresh ssh key

* return prefs when new keys are generated

* fix(editor): adding readOnly mode to main header

* fix(editor): adding readOnly mode to workflow settings

* improve credential owner import

* add middleware to endpoints

* improve public api error/doc

* do not create branch if one already exists

* update wordings for connect toasts

* fix(editor): updating and separating readonly modes

* fix(editor): fix readonly mode in WF list

* fix(editor): disable elements dragging on canvas in readonly mode (WIP: not working when NodeView page is loaded first)

* fix(editor): fix canvas draggables in readonly env

* fix(editor): remove unused variables

* fix(editor): hide actions in node connections when readonly

* fix(editor): hide actions in node connections when readonly

* fix(editor): disable Save button when readonly

* fix(editor): disable Save settings if no branch is selected

* fix(editor): lint fix

* fix(editor): update snapshots

* fix(editor): replace Loading... text

* fix(editor): reset Loading... text

* reset branchname on disconnect

* fix(editor): adding some translations

* fix(editor): fix unit test

* fix(editor): fix loading

* fix(editor): set settings saved message

* fix(editor): update connection flag

* fix branchName not returning after connect

* temporary (but still breaking) fix for postgres

* fix(editor): adding tooltip to Push/Pull buttons when they're collapsed

* fix(editor): enabled activator in readonly mode

* fix test

* fix(editor): disabling new item addition for workflows in readonly mode

* fix(editor): modify Pull/Push button tooltips

* do not commit empty variables file

---------

Co-authored-by: Michael Auerswald <michael.auerswald@gmail.com>
Co-authored-by: Romain Minaud <romain.minaud@gmail.com>
Co-authored-by: Alex Grozav <alex@grozav.com>
  • Loading branch information
4 people authored May 31, 2023
1 parent 04cfa54 commit 1b32141
Show file tree
Hide file tree
Showing 75 changed files with 3,710 additions and 450 deletions.
5 changes: 5 additions & 0 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
"@types/jsonwebtoken": "^9.0.1",
"@types/localtunnel": "^1.9.0",
"@types/lodash.debounce": "^4.0.7",
"@types/lodash.difference": "^4",
"@types/lodash.get": "^4.4.6",
"@types/lodash.intersection": "^4.4.7",
"@types/lodash.iteratee": "^4.7.7",
Expand All @@ -91,6 +92,7 @@
"@types/lodash.uniq": "^4.5.7",
"@types/lodash.uniqby": "^4.7.7",
"@types/lodash.unset": "^4.5.7",
"@types/lodash.without": "^4.4.7",
"@types/parseurl": "^1.3.1",
"@types/passport-jwt": "^3.0.6",
"@types/psl": "^1.1.0",
Expand Down Expand Up @@ -159,6 +161,7 @@
"jwks-rsa": "^3.0.1",
"ldapts": "^4.2.6",
"localtunnel": "^2.0.0",
"lodash.difference": "^4",
"lodash.get": "^4.4.2",
"lodash.intersection": "^4.4.0",
"lodash.iteratee": "^4.7.0",
Expand All @@ -172,6 +175,7 @@
"lodash.uniq": "^4.5.0",
"lodash.uniqby": "^4.7.0",
"lodash.unset": "^4.5.2",
"lodash.without": "^4.4.0",
"luxon": "^3.3.0",
"mysql2": "~2.3.3",
"n8n-core": "workspace:*",
Expand All @@ -197,6 +201,7 @@
"samlify": "^2.8.9",
"semver": "^7.3.8",
"shelljs": "^0.8.5",
"simple-git": "^3.17.0",
"source-map-support": "^0.5.21",
"sqlite3": "^5.1.6",
"sse-channel": "^4.0.0",
Expand Down
10 changes: 10 additions & 0 deletions packages/cli/src/PublicApi/types.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,16 @@ export interface IJsonSchema {
required: string[];
}

export class VersionControlPull {
force?: boolean;

variables?: { [key: string]: string };
}

export declare namespace PublicVersionControlRequest {
type Pull = AuthenticatedRequest<{}, {}, VersionControlPull, {}>;
}

// ----------------------------------
// /audit
// ----------------------------------
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
post:
x-eov-operation-id: pull
x-eov-operation-handler: v1/handlers/versionControl/versionControl.handler
tags:
- VersionControl
summary: Pull changes from the remote repository
description: Requires the Version Control feature to be licensed and connected to a repository.
requestBody:
description: Pull options
required: true
content:
application/json:
schema:
$ref: "../schemas/pull.yml"
responses:
"200":
description: Import result
content:
application/json:
schema:
$ref: "../schemas/importResult.yml"
"400":
$ref: "../../../../shared/spec/responses/badRequest.yml"
"409":
$ref: "../../../../shared/spec/responses/conflict.yml"
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
type: object
additionalProperties: true
properties:
variables:
type: object
properties:
added:
type: array
items:
type: string
changed:
type: array
items:
type: string
credentials:
type: array
items:
type: object
properties:
id:
type: string
name:
type: string
type:
type: string
workflows:
type: array
items:
type: object
properties:
id:
type: string
name:
type: string
tags:
type: object
properties:
tags:
type: array
items:
type: object
properties:
id:
type: string
name:
type: string
mappings:
type: array
items:
type: object
properties:
workflowId:
type: string
tagId:
type: string
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
type: object
properties:
force:
type: boolean
example: true
variables:
type: object
example: { "foo": "bar" }
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import type express from 'express';
import type { StatusResult } from 'simple-git';
import type { PublicVersionControlRequest } from '../../../types';
import { authorize } from '../../shared/middlewares/global.middleware';
import type { ImportResult } from '@/environments/versionControl/types/importResult';
import Container from 'typedi';
import { VersionControlService } from '@/environments/versionControl/versionControl.service.ee';
import { VersionControlPreferencesService } from '@/environments/versionControl/versionControlPreferences.service.ee';
import { isVersionControlLicensed } from '@/environments/versionControl/versionControlHelper.ee';

export = {
pull: [
authorize(['owner', 'member']),
async (
req: PublicVersionControlRequest.Pull,
res: express.Response,
): Promise<ImportResult | StatusResult | Promise<express.Response>> => {
const versionControlPreferencesService = Container.get(VersionControlPreferencesService);
if (!isVersionControlLicensed()) {
return res
.status(401)
.json({ status: 'Error', message: 'Version Control feature is not licensed' });
}
if (!versionControlPreferencesService.isVersionControlConnected()) {
return res
.status(400)
.json({ status: 'Error', message: 'Version Control is not connected to a repository' });
}
try {
const versionControlService = Container.get(VersionControlService);
const result = await versionControlService.pullWorkfolder({
force: req.body.force,
variables: req.body.variables,
userId: req.user.id,
importAfterPull: true,
});
if ((result as ImportResult)?.workflows) {
return res.status(200).send(result as ImportResult);
} else {
return res.status(409).send(result);
}
} catch (error) {
return res.status(400).send((error as { message: string }).message);
}
},
],
};
32 changes: 18 additions & 14 deletions packages/cli/src/PublicApi/v1/openapi.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
---
openapi: 3.0.0
info:
title: n8n Public API
title: n8n Public API11
description: n8n Public API
termsOfService: https://n8n.io/legal/terms
contact:
Expand All @@ -24,35 +24,39 @@ tags:
description: Operations about workflows
- name: Credential
description: Operations about credentials
- name: VersionControl
description: Operations about version control

paths:
/audit:
$ref: './handlers/audit/spec/paths/audit.yml'
$ref: "./handlers/audit/spec/paths/audit.yml"
/credentials:
$ref: './handlers/credentials/spec/paths/credentials.yml'
$ref: "./handlers/credentials/spec/paths/credentials.yml"
/credentials/{id}:
$ref: './handlers/credentials/spec/paths/credentials.id.yml'
$ref: "./handlers/credentials/spec/paths/credentials.id.yml"
/credentials/schema/{credentialTypeName}:
$ref: './handlers/credentials/spec/paths/credentials.schema.id.yml'
$ref: "./handlers/credentials/spec/paths/credentials.schema.id.yml"
/executions:
$ref: './handlers/executions/spec/paths/executions.yml'
$ref: "./handlers/executions/spec/paths/executions.yml"
/executions/{id}:
$ref: './handlers/executions/spec/paths/executions.id.yml'
$ref: "./handlers/executions/spec/paths/executions.id.yml"
/workflows:
$ref: './handlers/workflows/spec/paths/workflows.yml'
$ref: "./handlers/workflows/spec/paths/workflows.yml"
/workflows/{id}:
$ref: './handlers/workflows/spec/paths/workflows.id.yml'
$ref: "./handlers/workflows/spec/paths/workflows.id.yml"
/workflows/{id}/activate:
$ref: './handlers/workflows/spec/paths/workflows.id.activate.yml'
$ref: "./handlers/workflows/spec/paths/workflows.id.activate.yml"
/workflows/{id}/deactivate:
$ref: './handlers/workflows/spec/paths/workflows.id.deactivate.yml'
$ref: "./handlers/workflows/spec/paths/workflows.id.deactivate.yml"
/version-control/pull:
$ref: "./handlers/versionControl/spec/paths/versionControl.yml"
components:
schemas:
$ref: './shared/spec/schemas/_index.yml'
$ref: "./shared/spec/schemas/_index.yml"
responses:
$ref: './shared/spec/responses/_index.yml'
$ref: "./shared/spec/responses/_index.yml"
parameters:
$ref: './shared/spec/parameters/_index.yml'
$ref: "./shared/spec/parameters/_index.yml"
securitySchemes:
ApiKeyAuth:
type: apiKey
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
NotFound:
$ref: './notFound.yml'
$ref: "./notFound.yml"
Unauthorized:
$ref: './unauthorized.yml'
$ref: "./unauthorized.yml"
BadRequest:
$ref: './badRequest.yml'
$ref: "./badRequest.yml"
Conflict:
$ref: "./conflict.yml"
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
description: Conflict
26 changes: 15 additions & 11 deletions packages/cli/src/PublicApi/v1/shared/spec/schemas/_index.yml
Original file line number Diff line number Diff line change
@@ -1,22 +1,26 @@
Error:
$ref: './error.yml'
$ref: "./error.yml"
Execution:
$ref: './../../../handlers/executions/spec/schemas/execution.yml'
$ref: "./../../../handlers/executions/spec/schemas/execution.yml"
Node:
$ref: './../../../handlers/workflows/spec/schemas/node.yml'
$ref: "./../../../handlers/workflows/spec/schemas/node.yml"
Tag:
$ref: './../../../handlers/workflows/spec/schemas/tag.yml'
$ref: "./../../../handlers/workflows/spec/schemas/tag.yml"
Workflow:
$ref: './../../../handlers/workflows/spec/schemas/workflow.yml'
$ref: "./../../../handlers/workflows/spec/schemas/workflow.yml"
WorkflowSettings:
$ref: './../../../handlers/workflows/spec/schemas/workflowSettings.yml'
$ref: "./../../../handlers/workflows/spec/schemas/workflowSettings.yml"
ExecutionList:
$ref: './../../../handlers/executions/spec/schemas/executionList.yml'
$ref: "./../../../handlers/executions/spec/schemas/executionList.yml"
WorkflowList:
$ref: './../../../handlers/workflows/spec/schemas/workflowList.yml'
$ref: "./../../../handlers/workflows/spec/schemas/workflowList.yml"
Credential:
$ref: './../../../handlers/credentials/spec/schemas/credential.yml'
$ref: "./../../../handlers/credentials/spec/schemas/credential.yml"
CredentialType:
$ref: './../../../handlers/credentials/spec/schemas/credentialType.yml'
$ref: "./../../../handlers/credentials/spec/schemas/credentialType.yml"
Audit:
$ref: './../../../handlers/audit/spec/schemas/audit.yml'
$ref: "./../../../handlers/audit/spec/schemas/audit.yml"
Pull:
$ref: "./../../../handlers/versionControl/spec/schemas/pull.yml"
ImportResult:
$ref: "./../../../handlers/versionControl/spec/schemas/importResult.yml"
6 changes: 4 additions & 2 deletions packages/cli/src/Server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -167,9 +167,10 @@ import {
isLdapCurrentAuthenticationMethod,
isSamlCurrentAuthenticationMethod,
} from './sso/ssoHelpers';
import { isVersionControlLicensed } from '@/environments/versionControl/versionControlHelper';
import { isVersionControlLicensed } from '@/environments/versionControl/versionControlHelper.ee';
import { VersionControlService } from '@/environments/versionControl/versionControl.service.ee';
import { VersionControlController } from '@/environments/versionControl/versionControl.controller.ee';
import { VersionControlPreferencesService } from './environments/versionControl/versionControlPreferences.service.ee';

const exec = promisify(callbackExec);

Expand Down Expand Up @@ -468,6 +469,7 @@ export class Server extends AbstractServer {
const postHog = this.postHog;
const samlService = Container.get(SamlService);
const versionControlService = Container.get(VersionControlService);
const versionControlPreferencesService = Container.get(VersionControlPreferencesService);

const controllers: object[] = [
new EventBusController(),
Expand Down Expand Up @@ -496,7 +498,7 @@ export class Server extends AbstractServer {
postHog,
}),
new SamlController(samlService),
new VersionControlController(versionControlService),
new VersionControlController(versionControlService, versionControlPreferencesService),
];

if (isLdapEnabled()) {
Expand Down
14 changes: 14 additions & 0 deletions packages/cli/src/environments/versionControl/constants.ts
Original file line number Diff line number Diff line change
@@ -1 +1,15 @@
export const VERSION_CONTROL_PREFERENCES_DB_KEY = 'features.versionControl';
export const VERSION_CONTROL_GIT_FOLDER = 'git';
export const VERSION_CONTROL_GIT_KEY_COMMENT = 'n8n deploy key';
export const VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER = 'workflows';
export const VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER = 'credentials';
export const VERSION_CONTROL_VARIABLES_EXPORT_FILE = 'variables.json';
export const VERSION_CONTROL_TAGS_EXPORT_FILE = 'tags.json';
export const VERSION_CONTROL_SSH_FOLDER = 'ssh';
export const VERSION_CONTROL_SSH_KEY_NAME = 'key';
export const VERSION_CONTROL_DEFAULT_BRANCH = 'main';
export const VERSION_CONTROL_ORIGIN = 'origin';
export const VERSION_CONTROL_API_ROOT = 'version-control';
export const VERSION_CONTROL_README = `
# n8n Version Control
`;
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import type { RequestHandler } from 'express';
import {
isVersionControlLicensed,
isVersionControlLicensedAndEnabled,
} from '../versionControlHelper';
import { isVersionControlLicensed } from '../versionControlHelper.ee';
import Container from 'typedi';
import { VersionControlPreferencesService } from '../versionControlPreferences.service.ee';

export const versionControlLicensedAndEnabledMiddleware: RequestHandler = (req, res, next) => {
if (isVersionControlLicensedAndEnabled()) {
const versionControlPreferencesService = Container.get(VersionControlPreferencesService);
if (versionControlPreferencesService.isVersionControlLicensedAndEnabled()) {
next();
} else {
res.status(401).json({ status: 'error', message: 'Unauthorized' });
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
export interface ExportResult {
count: number;
folder: string;
files: Array<{
id: string;
name: string;
}>;
removedFiles?: string[];
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import type { ICredentialDataDecryptedObject } from 'n8n-workflow';

export interface ExportableCredential {
id: string;
name: string;
type: string;
data: ICredentialDataDecryptedObject;
}
Loading

0 comments on commit 1b32141

Please sign in to comment.