diff --git a/.eslintrc.yml b/.eslintrc.yml index 0e3f34210..04317f68d 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -60,7 +60,7 @@ rules: '@typescript-eslint/naming-convention': ['error', { 'selector': 'function', 'format': ['camelCase'], 'leadingUnderscore': 'allow' }] comma-dangle: [error, only-multiline] - curly: [error, multi-or-nest] +# curly: [error, multi-or-nest] eol-last: ['error', 'always'] func-names: off import/extensions: off diff --git a/docs/gitops.md b/docs/gitops.md index 64f2e8610..54d45788a 100644 --- a/docs/gitops.md +++ b/docs/gitops.md @@ -18,7 +18,7 @@ sequenceDiagram participant SC as Session Controller participant MC as Master Session Controller participant ACA as APL Core API - participant RR as Remote Repo + participant RR as Remote Git participant IMDB as In-Memory DB activate MC activate IMDB @@ -80,7 +80,7 @@ sequenceDiagram participant API as Express API participant SC as Session Controller participant MC as Master Session Controller - participant RR as Remote Repo + participant RR as Remote Git participant ACA as APL Core API participant IMDB as In-Memory DB activate MC @@ -131,7 +131,7 @@ sequenceDiagram participant Client as Client participant API as Express API participant SC as Git Handler - participant RR as Remote Repo + participant RR as Remote Git participant ACA as APL Core API participant IMDB as In-Memory DB activate IMDB @@ -179,7 +179,7 @@ sequenceDiagram participant Client as Client participant API as Express API participant SC as Git Handler - participant RR as Remote Repo + participant RR as Remote Git participant ACA as APL Core API Client->>API: HTTP POST/PUT/PATCH/DELETE activate API diff --git a/package-lock.json b/package-lock.json index 95f9ab86d..e8873d579 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,7 @@ "glob": "^11.0.1", "http-signature": "^1.3.6", "json-stable-stringify": "^1.0.1", + "jsonpath": "^1.1.1", "jsonwebtoken": "^9.0.0", "jwt-decode": "^2.2.0", "lightship": "^6.7.2", @@ -59,6 +60,7 @@ "@types/express": "4.17.13", "@types/fs-extra": "^9.0.13", "@types/jest": "^29.5.14", + "@types/jsonpath": "^0.2.4", "@types/lodash": "4.14.182", "@types/lowdb": "1.0.11", "@types/node": "^16.18.125", @@ -105,8 +107,7 @@ "watch": "0.19.2" }, "engines": { - "node": ">=20 <21", - "npm": "^10" + "node": ">=20 <21" } }, "node_modules/@ampproject/remapping": { @@ -4635,6 +4636,13 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "node_modules/@types/jsonpath": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@types/jsonpath/-/jsonpath-0.2.4.tgz", + "integrity": "sha512-K3hxB8Blw0qgW6ExKgMbXQv2UPZBoE2GqLpVY+yr7nMD2Pq86lsuIzyAaiQ7eMqFL5B6di6pxSkogLJEyEHoGA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/jsonwebtoken": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.1.tgz", @@ -7946,8 +7954,7 @@ "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==" }, "node_modules/deepmerge": { "version": "4.3.1", @@ -8769,6 +8776,78 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", + "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", + "license": "BSD-2-Clause", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^4.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=4.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", + "license": "MIT", + "dependencies": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/escodegen/node_modules/optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "license": "MIT", + "dependencies": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/escodegen/node_modules/prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/escodegen/node_modules/type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", + "license": "MIT", + "dependencies": { + "prelude-ls": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/eslint": { "version": "8.26.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.26.0.tgz", @@ -9340,7 +9419,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", - "dev": true, "engines": { "node": ">=4.0" } @@ -9349,7 +9427,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -9914,8 +9991,7 @@ "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", - "dev": true + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" }, "node_modules/fast-printf": { "version": "1.6.9", @@ -10900,9 +10976,9 @@ } }, "node_modules/glob/node_modules/jackspeak": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.0.3.tgz", - "integrity": "sha512-oSwM7q8PTHQWuZAlp995iPpPJ4Vkl7qT0ZRD+9duL9j2oBy6KcTfyxc8mEuHJYC+z/kbps80aJLkaNzTOrf/kw==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.0.2.tgz", + "integrity": "sha512-bZsjR/iRjl1Nk1UkjGpAzLNfQtzuijhn2g+pbZb98HQ1Gk8vM9hfbxeMBP+M2/UUdwj0RqGG3mlvk2MsAqwvEw==", "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/cliui": "^8.0.2" @@ -13524,6 +13600,17 @@ "node >= 0.2.0" ] }, + "node_modules/jsonpath": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/jsonpath/-/jsonpath-1.1.1.tgz", + "integrity": "sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==", + "license": "MIT", + "dependencies": { + "esprima": "1.2.2", + "static-eval": "2.0.2", + "underscore": "1.12.1" + } + }, "node_modules/jsonpath-plus": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-10.1.0.tgz", @@ -13541,6 +13628,18 @@ "node": ">=18.0.0" } }, + "node_modules/jsonpath/node_modules/esprima": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.2.2.tgz", + "integrity": "sha512-+JpPZam9w5DuJ3Q67SqsMGtiHKENSMRVoxvArfJZK01/BfLEObtZ6orJa/MtoGNR/rfMgp5837T41PAmTwAv/A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/JSONStream": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", @@ -21936,7 +22035,7 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, + "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -22227,6 +22326,15 @@ "node": ">=10" } }, + "node_modules/static-eval": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/static-eval/-/static-eval-2.0.2.tgz", + "integrity": "sha512-N/D219Hcr2bPjLxPiV+TQE++Tsmrady7TqAJugLy7Xk1EumfDWS/f5dtBbkRCGE7wKKXuYockQoj8Rm2/pVKyg==", + "license": "MIT", + "dependencies": { + "escodegen": "^1.8.1" + } + }, "node_modules/static-extend": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", @@ -23834,6 +23942,12 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, + "node_modules/underscore": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz", + "integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==", + "license": "MIT" + }, "node_modules/undici": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/undici/-/undici-5.1.1.tgz", @@ -24359,7 +24473,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", - "dev": true, "engines": { "node": ">=0.10.0" } diff --git a/package.json b/package.json index d20c27533..0ffffbec6 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "glob": "^11.0.1", "http-signature": "^1.3.6", "json-stable-stringify": "^1.0.1", + "jsonpath": "^1.1.1", "jsonwebtoken": "^9.0.0", "jwt-decode": "^2.2.0", "lightship": "^6.7.2", @@ -59,6 +60,7 @@ "@types/express": "4.17.13", "@types/fs-extra": "^9.0.13", "@types/jest": "^29.5.14", + "@types/jsonpath": "^0.2.4", "@types/lodash": "4.14.182", "@types/lowdb": "1.0.11", "@types/node": "^16.18.125", @@ -105,8 +107,7 @@ "watch": "0.19.2" }, "engines": { - "node": ">=20 <21", - "npm": "^10" + "node": ">=20 <21" }, "engineStrict": true, "homepage": "https://github.com/redkubes/otomi-api#readme", diff --git a/src/api.authz.test.ts b/src/api.authz.test.ts index 327c16cf3..cfcdfbaff 100644 --- a/src/api.authz.test.ts +++ b/src/api.authz.test.ts @@ -8,9 +8,9 @@ import OtomiStack from 'src/otomi-stack' import request, { SuperAgentTest } from 'supertest' import { HttpError } from './error' import { getSessionStack } from './middleware' -import { App, Coderepo, SealedSecret } from './otomi-models' -import { Repo } from './repo' +import { App, CodeRepo, SealedSecret } from './otomi-models' import * as getValuesSchemaModule from './utils' +import { Git } from './git' const platformAdminToken = getToken(['platform-admin']) const teamAdminToken = getToken(['team-admin', 'team-team1']) @@ -39,15 +39,19 @@ describe('API authz tests', () => { beforeAll(async () => { const _otomiStack = await getSessionStack() - _otomiStack.repo = mockDeep() + _otomiStack.git = mockDeep() _otomiStack.doDeployment = jest.fn().mockImplementation(() => Promise.resolve()) - await _otomiStack.createTeam({ name: 'team1' }) + _otomiStack.transformApps = jest.fn().mockReturnValue([]) + await _otomiStack.initRepo() otomiStack = _otomiStack as jest.Mocked - otomiStack.createTeam = jest.fn().mockResolvedValue(undefined) - otomiStack.saveTeams = jest.fn().mockResolvedValue(undefined) + otomiStack.saveTeam = jest.fn().mockResolvedValue(undefined) otomiStack.doDeployment = jest.fn().mockImplementation(() => Promise.resolve()) - await otomiStack.init() + otomiStack.doRepoDeployment = jest.fn().mockImplementation(() => Promise.resolve()) + otomiStack.doTeamDeployment = jest.fn().mockImplementation(() => Promise.resolve()) + await otomiStack.loadValues() + await otomiStack.createTeam({ name: 'team1' }) + await otomiStack.createTeam({ name: 'team2' }) app = await initApp(otomiStack) agent = request.agent(app) agent.set('Accept', 'application/json') @@ -482,10 +486,6 @@ describe('API authz tests', () => { .expect(403) }) - test('team member cannot get all secrets', async () => { - await agent.get('/v1/secrets').set('Authorization', `Bearer ${teamMemberToken}`).expect(403) - }) - test('team member cannot get all sealedsecrets', async () => { await agent.get('/v1/sealedsecrets').set('Authorization', `Bearer ${teamMemberToken}`).expect(403) }) @@ -640,14 +640,14 @@ describe('API authz tests', () => { describe('Code repository endpoints tests', () => { const data = { - label: 'demo', + name: 'demo', gitService: 'github' as 'gitea' | 'github' | 'gitlab', repositoryUrl: 'https://github.com/buildpacks/samples', private: true, secret: 'demo', } - test('team member can create its own coderepo', async () => { - jest.spyOn(otomiStack, 'createCoderepo').mockResolvedValue({} as Coderepo) + test('team member can create its own codeRepo', async () => { + jest.spyOn(otomiStack, 'createCodeRepo').mockResolvedValue({} as CodeRepo) await agent .post(`/v1/teams/${teamId}/coderepos`) .send(data) @@ -655,16 +655,16 @@ describe('API authz tests', () => { .expect(200) }) - test('team member can read its own coderepo', async () => { - jest.spyOn(otomiStack, 'getCoderepo').mockResolvedValue({} as never) + test('team member can read its own codeRepo', async () => { + jest.spyOn(otomiStack, 'getCodeRepo').mockResolvedValue({} as never) await agent .get(`/v1/teams/${teamId}/coderepos/my-uuid`) .set('Authorization', `Bearer ${teamMemberToken}`) .expect(200) }) - test('team member can update its own coderepo', async () => { - jest.spyOn(otomiStack, 'editCoderepo').mockResolvedValue({} as Coderepo) + test('team member can update its own codeRepo', async () => { + jest.spyOn(otomiStack, 'editCodeRepo').mockResolvedValue({} as CodeRepo) await agent .put(`/v1/teams/${teamId}/coderepos/my-uuid`) @@ -673,8 +673,8 @@ describe('API authz tests', () => { .expect(200) }) - test('team member can delete its own coderepo', async () => { - jest.spyOn(otomiStack, 'deleteCoderepo').mockResolvedValue() + test('team member can delete its own codeRepo', async () => { + jest.spyOn(otomiStack, 'deleteCodeRepo').mockResolvedValue() await agent .delete(`/v1/teams/${teamId}/coderepos/my-uuid`) @@ -684,7 +684,7 @@ describe('API authz tests', () => { .expect('Content-Type', /json/) }) - test('team member cannot create others coderepo', async () => { + test('team member cannot create others codeRepo', async () => { await agent .post(`/v1/teams/${otherTeamId}/coderepos`) .send(data) @@ -692,14 +692,14 @@ describe('API authz tests', () => { .expect(403) }) - test('team member cannot read others coderepo', async () => { + test('team member cannot read others codeRepo', async () => { await agent .get(`/v1/teams/${otherTeamId}/coderepos/my-uuid`) .set('Authorization', `Bearer ${teamMemberToken}`) .expect(403) }) - test('team member cannot update others coderepo', async () => { + test('team member cannot update others codeRepo', async () => { await agent .put(`/v1/teams/${otherTeamId}/coderepos/my-uuid`) .send(data) @@ -707,7 +707,7 @@ describe('API authz tests', () => { .expect(403) }) - test('team member cannot delete others coderepo', async () => { + test('team member cannot delete others codeRepo', async () => { await agent .delete(`/v1/teams/${otherTeamId}/coderepos/my-uuid`) .set('Content-Type', 'application/json') diff --git a/src/api/apps/{teamId}/{appId}.ts b/src/api/apps/{teamId}/{appId}.ts index c4d70736b..a72377c25 100644 --- a/src/api/apps/{teamId}/{appId}.ts +++ b/src/api/apps/{teamId}/{appId}.ts @@ -4,7 +4,7 @@ import { App, OpenApiRequestExt } from 'src/otomi-models' export default function (): OperationHandlerArray { const get: Operation = [ ({ otomi, params: { teamId, appId } }: OpenApiRequestExt, res): void => { - res.json(otomi.getApp(teamId, appId)) + res.json(otomi.getTeamApp(teamId, appId)) }, ] const put: Operation = [ diff --git a/src/api/coderepos.ts b/src/api/coderepos.ts index b316fedd4..2f74422dc 100644 --- a/src/api/coderepos.ts +++ b/src/api/coderepos.ts @@ -2,13 +2,13 @@ import Debug from 'debug' import { Operation, OperationHandlerArray } from 'express-openapi' import { OpenApiRequestExt } from 'src/otomi-models' -const debug = Debug('otomi:api:coderepos') +const debug = Debug('otomi:api:codeRepos') export default function (): OperationHandlerArray { const get: Operation = [ ({ otomi }: OpenApiRequestExt, res): void => { - debug('getAllCoderepos') - const v = otomi.getAllCoderepos() + debug('getAllCodeRepos') + const v = otomi.getAllCodeRepos() res.json(v) }, ] diff --git a/src/api/secrets.ts b/src/api/secrets.ts deleted file mode 100644 index b972c21cb..000000000 --- a/src/api/secrets.ts +++ /dev/null @@ -1,19 +0,0 @@ -import Debug from 'debug' -import { Operation, OperationHandlerArray } from 'express-openapi' -import { OpenApiRequestExt } from 'src/otomi-models' - -const debug = Debug('otomi:api:secrets') - -export default function (): OperationHandlerArray { - const get: Operation = [ - ({ otomi }: OpenApiRequestExt, res): void => { - debug('getAllSecrets') - const v = otomi.getAllSecrets() - res.json(v) - }, - ] - const api = { - get, - } - return api -} diff --git a/src/api/teams/{teamId}/backups/{backupId}.ts b/src/api/teams/{teamId}/backups/{backupId}.ts deleted file mode 100644 index 584f65ca2..000000000 --- a/src/api/teams/{teamId}/backups/{backupId}.ts +++ /dev/null @@ -1,38 +0,0 @@ -import Debug from 'debug' -import { Operation, OperationHandlerArray } from 'express-openapi' -import { Backup, OpenApiRequestExt } from 'src/otomi-models' - -const debug = Debug('otomi:api:teams:backups') - -export default function (): OperationHandlerArray { - const del: Operation = [ - async ({ otomi, params: { backupId } }: OpenApiRequestExt, res): Promise => { - debug(`deleteBackup(${backupId})`) - await otomi.deleteBackup(decodeURIComponent(backupId)) - res.json({}) - }, - ] - const get: Operation = [ - ({ otomi, params: { backupId } }: OpenApiRequestExt, res): void => { - debug(`getBackup(${backupId})`) - const data = otomi.getBackup(decodeURIComponent(backupId)) - res.json(data) - }, - ] - const put: Operation = [ - async ({ otomi, params: { teamId, backupId }, body }: OpenApiRequestExt, res): Promise => { - debug(`editBackup(${backupId})`) - const data = await otomi.editBackup(decodeURIComponent(backupId), { - ...body, - teamId: decodeURIComponent(teamId), - } as Backup) - res.json(data) - }, - ] - const api = { - delete: del, - get, - put, - } - return api -} diff --git a/src/api/teams/{teamId}/backups/{backupName}.ts b/src/api/teams/{teamId}/backups/{backupName}.ts new file mode 100644 index 000000000..235685233 --- /dev/null +++ b/src/api/teams/{teamId}/backups/{backupName}.ts @@ -0,0 +1,37 @@ +import Debug from 'debug' +import { Operation, OperationHandlerArray } from 'express-openapi' +import { Backup, OpenApiRequestExt } from 'src/otomi-models' + +const debug = Debug('otomi:api:teams:backups') + +export default function (): OperationHandlerArray { + const del: Operation = [ + async ({ otomi, params: { teamId, backupName } }: OpenApiRequestExt, res): Promise => { + debug(`deleteBackup(${backupName})`) + await otomi.deleteBackup(decodeURIComponent(teamId), decodeURIComponent(backupName)) + res.json({}) + }, + ] + const get: Operation = [ + ({ otomi, params: { teamId, backupName } }: OpenApiRequestExt, res): void => { + debug(`getBackup(${backupName})`) + const data = otomi.getBackup(decodeURIComponent(teamId), decodeURIComponent(backupName)) + res.json(data) + }, + ] + const put: Operation = [ + async ({ otomi, params: { teamId, backupName }, body }: OpenApiRequestExt, res): Promise => { + debug(`editBackup(${backupName})`) + const data = await otomi.editBackup(decodeURIComponent(teamId), decodeURIComponent(backupName), { + ...body, + } as Backup) + res.json(data) + }, + ] + const api = { + delete: del, + get, + put, + } + return api +} diff --git a/src/api/teams/{teamId}/builds/{buildId}.ts b/src/api/teams/{teamId}/builds/{buildId}.ts deleted file mode 100644 index 82429f1ed..000000000 --- a/src/api/teams/{teamId}/builds/{buildId}.ts +++ /dev/null @@ -1,38 +0,0 @@ -import Debug from 'debug' -import { Operation, OperationHandlerArray } from 'express-openapi' -import { Build, OpenApiRequestExt } from 'src/otomi-models' - -const debug = Debug('otomi:api:teams:builds') - -export default function (): OperationHandlerArray { - const del: Operation = [ - async ({ otomi, params: { buildId } }: OpenApiRequestExt, res): Promise => { - debug(`deleteBuild(${buildId})`) - await otomi.deleteBuild(decodeURIComponent(buildId)) - res.json({}) - }, - ] - const get: Operation = [ - ({ otomi, params: { buildId } }: OpenApiRequestExt, res): void => { - debug(`getBuild(${buildId})`) - const data = otomi.getBuild(decodeURIComponent(buildId)) - res.json(data) - }, - ] - const put: Operation = [ - ({ otomi, params: { teamId, buildId }, body }: OpenApiRequestExt, res): void => { - debug(`editBuild(${buildId})`) - const data = otomi.editBuild(decodeURIComponent(buildId), { - ...body, - teamId: decodeURIComponent(teamId), - } as Build) - res.json(data) - }, - ] - const api = { - delete: del, - get, - put, - } - return api -} diff --git a/src/api/teams/{teamId}/builds/{buildName}.ts b/src/api/teams/{teamId}/builds/{buildName}.ts new file mode 100644 index 000000000..3eca03781 --- /dev/null +++ b/src/api/teams/{teamId}/builds/{buildName}.ts @@ -0,0 +1,37 @@ +import Debug from 'debug' +import { Operation, OperationHandlerArray } from 'express-openapi' +import { Build, OpenApiRequestExt } from 'src/otomi-models' + +const debug = Debug('otomi:api:teams:builds') + +export default function (): OperationHandlerArray { + const del: Operation = [ + async ({ otomi, params: { teamId, buildName } }: OpenApiRequestExt, res): Promise => { + debug(`deleteBuild(${buildName})`) + await otomi.deleteBuild(decodeURIComponent(teamId), decodeURIComponent(buildName)) + res.json({}) + }, + ] + const get: Operation = [ + ({ otomi, params: { teamId, buildName } }: OpenApiRequestExt, res): void => { + debug(`getBuild(${buildName})`) + const data = otomi.getBuild(decodeURIComponent(teamId), decodeURIComponent(buildName)) + res.json(data) + }, + ] + const put: Operation = [ + ({ otomi, params: { teamId, buildName }, body }: OpenApiRequestExt, res): void => { + debug(`editBuild(${buildName})`) + const data = otomi.editBuild(decodeURIComponent(teamId), decodeURIComponent(buildName), { + ...body, + } as Build) + res.json(data) + }, + ] + const api = { + delete: del, + get, + put, + } + return api +} diff --git a/src/api/teams/{teamId}/coderepos.ts b/src/api/teams/{teamId}/coderepos.ts index e60f3a0ed..d54f52d66 100644 --- a/src/api/teams/{teamId}/coderepos.ts +++ b/src/api/teams/{teamId}/coderepos.ts @@ -1,21 +1,21 @@ import Debug from 'debug' import { Operation, OperationHandlerArray } from 'express-openapi' -import { Coderepo, OpenApiRequestExt } from 'src/otomi-models' +import { CodeRepo, OpenApiRequestExt } from 'src/otomi-models' -const debug = Debug('otomi:api:teams:coderepos') +const debug = Debug('otomi:api:teams:codeRepos') export default function (): OperationHandlerArray { const get: Operation = [ ({ otomi, params: { teamId } }: OpenApiRequestExt, res): void => { - debug(`getTeamCoderepos(${teamId})`) - const v = otomi.getTeamCoderepos(teamId) + debug(`getTeamCodeRepos(${teamId})`) + const v = otomi.getTeamCodeRepos(teamId) res.json(v) }, ] const post: Operation = [ async ({ otomi, params: { teamId }, body }: OpenApiRequestExt, res): Promise => { - debug(`createCoderepos(${teamId}, ...)`) - const v = await otomi.createCoderepo(teamId, body as Coderepo) + debug(`createCodeRepos(${teamId}, ...)`) + const v = await otomi.createCodeRepo(teamId, body as CodeRepo) res.json(v) }, ] diff --git a/src/api/teams/{teamId}/coderepos/{codeRepositoryName}.ts b/src/api/teams/{teamId}/coderepos/{codeRepositoryName}.ts new file mode 100644 index 000000000..6299272f5 --- /dev/null +++ b/src/api/teams/{teamId}/coderepos/{codeRepositoryName}.ts @@ -0,0 +1,37 @@ +import Debug from 'debug' +import { Operation, OperationHandlerArray } from 'express-openapi' +import { CodeRepo, OpenApiRequestExt } from 'src/otomi-models' + +const debug = Debug('otomi:api:teams:codeRepos') + +export default function (): OperationHandlerArray { + const del: Operation = [ + async ({ otomi, params: { teamId, codeRepositoryName } }: OpenApiRequestExt, res): Promise => { + debug(`deleteCodeRepo(${codeRepositoryName})`) + await otomi.deleteCodeRepo(decodeURIComponent(teamId), decodeURIComponent(codeRepositoryName)) + res.json({}) + }, + ] + const get: Operation = [ + ({ otomi, params: { teamId, codeRepositoryName } }: OpenApiRequestExt, res): void => { + debug(`getCodeRepo(${codeRepositoryName})`) + const data = otomi.getCodeRepo(decodeURIComponent(teamId), decodeURIComponent(codeRepositoryName)) + res.json(data) + }, + ] + const put: Operation = [ + async ({ otomi, params: { teamId, codeRepositoryName }, body }: OpenApiRequestExt, res): Promise => { + debug(`editCodeRepo(${codeRepositoryName})`) + const data = await otomi.editCodeRepo(decodeURIComponent(teamId), decodeURIComponent(codeRepositoryName), { + ...body, + } as CodeRepo) + res.json(data) + }, + ] + const api = { + delete: del, + get, + put, + } + return api +} diff --git a/src/api/teams/{teamId}/coderepos/{coderepoId}.ts b/src/api/teams/{teamId}/coderepos/{coderepoId}.ts deleted file mode 100644 index 78590afb9..000000000 --- a/src/api/teams/{teamId}/coderepos/{coderepoId}.ts +++ /dev/null @@ -1,38 +0,0 @@ -import Debug from 'debug' -import { Operation, OperationHandlerArray } from 'express-openapi' -import { Coderepo, OpenApiRequestExt } from 'src/otomi-models' - -const debug = Debug('otomi:api:teams:coderepos') - -export default function (): OperationHandlerArray { - const del: Operation = [ - async ({ otomi, params: { coderepoId } }: OpenApiRequestExt, res): Promise => { - debug(`deleteCoderepo(${coderepoId})`) - await otomi.deleteCoderepo(decodeURIComponent(coderepoId)) - res.json({}) - }, - ] - const get: Operation = [ - ({ otomi, params: { coderepoId } }: OpenApiRequestExt, res): void => { - debug(`getCoderepo(${coderepoId})`) - const data = otomi.getCoderepo(decodeURIComponent(coderepoId)) - res.json(data) - }, - ] - const put: Operation = [ - async ({ otomi, params: { teamId, coderepoId }, body }: OpenApiRequestExt, res): Promise => { - debug(`editCoderepo(${coderepoId})`) - const data = await otomi.editCoderepo(decodeURIComponent(coderepoId), { - ...body, - teamId: decodeURIComponent(teamId), - } as Coderepo) - res.json(data) - }, - ] - const api = { - delete: del, - get, - put, - } - return api -} diff --git a/src/api/teams/{teamId}/netpols/{netpolId}.ts b/src/api/teams/{teamId}/netpols/{netpolId}.ts deleted file mode 100644 index 163e5c7ee..000000000 --- a/src/api/teams/{teamId}/netpols/{netpolId}.ts +++ /dev/null @@ -1,38 +0,0 @@ -import Debug from 'debug' -import { Operation, OperationHandlerArray } from 'express-openapi' -import { Netpol, OpenApiRequestExt } from 'src/otomi-models' - -const debug = Debug('otomi:api:teams:netpols') - -export default function (): OperationHandlerArray { - const del: Operation = [ - async ({ otomi, params: { netpolId } }: OpenApiRequestExt, res): Promise => { - debug(`deleteNetpol(${netpolId})`) - await otomi.deleteNetpol(decodeURIComponent(netpolId)) - res.json({}) - }, - ] - const get: Operation = [ - ({ otomi, params: { netpolId } }: OpenApiRequestExt, res): void => { - debug(`getNetpol(${netpolId})`) - const data = otomi.getNetpol(decodeURIComponent(netpolId)) - res.json(data) - }, - ] - const put: Operation = [ - async ({ otomi, params: { teamId, netpolId }, body }: OpenApiRequestExt, res): Promise => { - debug(`editNetpol(${netpolId})`) - const data = await otomi.editNetpol(decodeURIComponent(netpolId), { - ...body, - teamId: decodeURIComponent(teamId), - } as Netpol) - res.json(data) - }, - ] - const api = { - delete: del, - get, - put, - } - return api -} diff --git a/src/api/teams/{teamId}/netpols/{netpolName}.ts b/src/api/teams/{teamId}/netpols/{netpolName}.ts new file mode 100644 index 000000000..fc79833c8 --- /dev/null +++ b/src/api/teams/{teamId}/netpols/{netpolName}.ts @@ -0,0 +1,37 @@ +import Debug from 'debug' +import { Operation, OperationHandlerArray } from 'express-openapi' +import { Netpol, OpenApiRequestExt } from 'src/otomi-models' + +const debug = Debug('otomi:api:teams:netpols') + +export default function (): OperationHandlerArray { + const del: Operation = [ + async ({ otomi, params: { teamId, netpolName } }: OpenApiRequestExt, res): Promise => { + debug(`deleteNetpol(${netpolName})`) + await otomi.deleteNetpol(decodeURIComponent(teamId), decodeURIComponent(netpolName)) + res.json({}) + }, + ] + const get: Operation = [ + ({ otomi, params: { teamId, netpolName } }: OpenApiRequestExt, res): void => { + debug(`getNetpol(${netpolName})`) + const data = otomi.getNetpol(decodeURIComponent(teamId), decodeURIComponent(netpolName)) + res.json(data) + }, + ] + const put: Operation = [ + async ({ otomi, params: { teamId, netpolName }, body }: OpenApiRequestExt, res): Promise => { + debug(`editNetpol(${netpolName})`) + const data = await otomi.editNetpol(decodeURIComponent(teamId), decodeURIComponent(netpolName), { + ...body, + } as Netpol) + res.json(data) + }, + ] + const api = { + delete: del, + get, + put, + } + return api +} diff --git a/src/api/teams/{teamId}/projects/{projectId}.ts b/src/api/teams/{teamId}/projects/{projectId}.ts deleted file mode 100644 index f8560176b..000000000 --- a/src/api/teams/{teamId}/projects/{projectId}.ts +++ /dev/null @@ -1,38 +0,0 @@ -import Debug from 'debug' -import { Operation, OperationHandlerArray } from 'express-openapi' -import { OpenApiRequestExt, Project } from 'src/otomi-models' - -const debug = Debug('otomi:api:teams:projects') - -export default function (): OperationHandlerArray { - const del: Operation = [ - async ({ otomi, params: { projectId } }: OpenApiRequestExt, res): Promise => { - debug(`deleteProject(${projectId})`) - await otomi.deleteProject(decodeURIComponent(projectId)) - res.json({}) - }, - ] - const get: Operation = [ - ({ otomi, params: { projectId } }: OpenApiRequestExt, res): void => { - debug(`getProject(${projectId})`) - const data = otomi.getProject(decodeURIComponent(projectId)) - res.json(data) - }, - ] - const put: Operation = [ - async ({ otomi, params: { teamId, projectId }, body }: OpenApiRequestExt, res): Promise => { - debug(`editProject(${projectId})`) - const data = await otomi.editProject(decodeURIComponent(projectId), { - ...body, - teamId: decodeURIComponent(teamId), - } as Project) - res.json(data) - }, - ] - const api = { - delete: del, - get, - put, - } - return api -} diff --git a/src/api/teams/{teamId}/projects/{projectName}.ts b/src/api/teams/{teamId}/projects/{projectName}.ts new file mode 100644 index 000000000..e63509c57 --- /dev/null +++ b/src/api/teams/{teamId}/projects/{projectName}.ts @@ -0,0 +1,38 @@ +import Debug from 'debug' +import { Operation, OperationHandlerArray } from 'express-openapi' +import { OpenApiRequestExt, Project } from 'src/otomi-models' + +const debug = Debug('otomi:api:teams:projects') + +export default function (): OperationHandlerArray { + const del: Operation = [ + async ({ otomi, params: { teamId, projectName } }: OpenApiRequestExt, res): Promise => { + debug(`deleteProject(${projectName})`) + await otomi.deleteProject(decodeURIComponent(teamId), decodeURIComponent(projectName)) + res.json({}) + }, + ] + const get: Operation = [ + ({ otomi, params: { teamId, projectName } }: OpenApiRequestExt, res): void => { + debug(`getProject(${projectName})`) + const data = otomi.getProject(decodeURIComponent(teamId), decodeURIComponent(projectName)) + res.json(data) + }, + ] + const put: Operation = [ + async ({ otomi, params: { teamId, projectName }, body }: OpenApiRequestExt, res): Promise => { + debug(`editProject(${projectName})`) + const data = await otomi.editProject(decodeURIComponent(teamId), decodeURIComponent(projectName), { + ...body, + teamId: decodeURIComponent(teamId), + } as Project) + res.json(data) + }, + ] + const api = { + delete: del, + get, + put, + } + return api +} diff --git a/src/api/teams/{teamId}/sealedsecrets/{secretId}.ts b/src/api/teams/{teamId}/sealedsecrets/{sealedSecretName}.ts similarity index 52% rename from src/api/teams/{teamId}/sealedsecrets/{secretId}.ts rename to src/api/teams/{teamId}/sealedsecrets/{sealedSecretName}.ts index 370ef7fe7..bbbd2bd6e 100644 --- a/src/api/teams/{teamId}/sealedsecrets/{secretId}.ts +++ b/src/api/teams/{teamId}/sealedsecrets/{sealedSecretName}.ts @@ -6,23 +6,23 @@ const debug = Debug('otomi:api:teams:sealedsecrets') export default function (): OperationHandlerArray { const del: Operation = [ - async ({ otomi, params: { secretId } }: OpenApiRequestExt, res): Promise => { - debug(`deleteSealedSecret(${secretId})`) - await otomi.deleteSealedSecret(decodeURIComponent(secretId)) + async ({ otomi, params: { teamId, sealedSecretName } }: OpenApiRequestExt, res): Promise => { + debug(`deleteSealedSecret(${sealedSecretName})`) + await otomi.deleteSealedSecret(decodeURIComponent(teamId), decodeURIComponent(sealedSecretName)) res.json({}) }, ] const get: Operation = [ - async ({ otomi, params: { secretId } }: OpenApiRequestExt, res): Promise => { - debug(`getSealedSecret(${secretId})`) - const data = await otomi.getSealedSecret(decodeURIComponent(secretId)) + async ({ otomi, params: { teamId, sealedSecretName } }: OpenApiRequestExt, res): Promise => { + debug(`getSealedSecret(${sealedSecretName})`) + const data = await otomi.getSealedSecret(decodeURIComponent(teamId), decodeURIComponent(sealedSecretName)) res.json(data) }, ] const put: Operation = [ - async ({ otomi, params: { teamId, secretId }, body }: OpenApiRequestExt, res): Promise => { - debug(`editSealedSecret(${secretId})`) - const data = await otomi.editSealedSecret(decodeURIComponent(secretId), { + async ({ otomi, params: { teamId, sealedSecretName }, body }: OpenApiRequestExt, res): Promise => { + debug(`editSealedSecret(${sealedSecretName})`) + const data = await otomi.editSealedSecret(decodeURIComponent(sealedSecretName), { ...body, teamId: decodeURIComponent(teamId), } as SealedSecret) diff --git a/src/api/teams/{teamId}/secrets.ts b/src/api/teams/{teamId}/secrets.ts deleted file mode 100644 index eeb6265a4..000000000 --- a/src/api/teams/{teamId}/secrets.ts +++ /dev/null @@ -1,27 +0,0 @@ -import Debug from 'debug' -import { Operation, OperationHandlerArray } from 'express-openapi' -import { OpenApiRequestExt, Secret } from 'src/otomi-models' - -const debug = Debug('otomi:api:teams:secrets') - -export default function (): OperationHandlerArray { - const get: Operation = [ - ({ otomi, params: { teamId } }: OpenApiRequestExt, res): void => { - debug(`getSecrets(${teamId})`) - const v = otomi.getSecrets(teamId) - res.json(v) - }, - ] - const post: Operation = [ - ({ otomi, params: { teamId }, body }: OpenApiRequestExt, res): void => { - debug(`createSecret(${teamId}, ...)`) - const v = otomi.createSecret(teamId, body as Secret) - res.json(v) - }, - ] - const api = { - get, - post, - } - return api -} diff --git a/src/api/teams/{teamId}/secrets/{secretId}.ts b/src/api/teams/{teamId}/secrets/{secretId}.ts deleted file mode 100644 index dfe8dffed..000000000 --- a/src/api/teams/{teamId}/secrets/{secretId}.ts +++ /dev/null @@ -1,38 +0,0 @@ -import Debug from 'debug' -import { Operation, OperationHandlerArray } from 'express-openapi' -import { OpenApiRequestExt, Secret } from 'src/otomi-models' - -const debug = Debug('otomi:api:teams:secrets') - -export default function (): OperationHandlerArray { - const del: Operation = [ - ({ otomi, params: { secretId } }: OpenApiRequestExt, res): void => { - debug(`deleteSecret(${secretId})`) - otomi.deleteSecret(decodeURIComponent(secretId)) - res.json({}) - }, - ] - const get: Operation = [ - ({ otomi, params: { secretId } }: OpenApiRequestExt, res): void => { - debug(`getSecret(${secretId})`) - const data = otomi.getSecret(decodeURIComponent(secretId)) - res.json(data) - }, - ] - const put: Operation = [ - ({ otomi, params: { teamId, secretId }, body }: OpenApiRequestExt, res): void => { - debug(`editSecret(${secretId})`) - const data = otomi.editSecret(decodeURIComponent(secretId), { - ...body, - teamId: decodeURIComponent(teamId), - } as Secret) - res.json(data) - }, - ] - const api = { - delete: del, - get, - put, - } - return api -} diff --git a/src/api/teams/{teamId}/services/{serviceId}.ts b/src/api/teams/{teamId}/services/{serviceId}.ts deleted file mode 100644 index 5a7c5f1cc..000000000 --- a/src/api/teams/{teamId}/services/{serviceId}.ts +++ /dev/null @@ -1,38 +0,0 @@ -import Debug from 'debug' -import { Operation, OperationHandlerArray } from 'express-openapi' -import { OpenApiRequestExt, Service } from 'src/otomi-models' - -const debug = Debug('otomi:api:teams:services') - -export default function (): OperationHandlerArray { - const del: Operation = [ - async ({ otomi, params: { serviceId } }: OpenApiRequestExt, res): Promise => { - debug(`deleteService(${serviceId})`) - await otomi.deleteService(decodeURIComponent(serviceId)) - res.json({}) - }, - ] - const get: Operation = [ - ({ otomi, params: { serviceId } }: OpenApiRequestExt, res): void => { - debug(`getService(${serviceId})`) - const data = otomi.getService(decodeURIComponent(serviceId)) - res.json(data) - }, - ] - const put: Operation = [ - async ({ otomi, params: { teamId, serviceId }, body }: OpenApiRequestExt, res): Promise => { - debug(`editService(${serviceId})`) - const data = await otomi.editService(decodeURIComponent(serviceId), { - ...body, - teamId: decodeURIComponent(teamId), - } as Service) - res.json(data) - }, - ] - const api = { - delete: del, - get, - put, - } - return api -} diff --git a/src/api/teams/{teamId}/services/{serviceName}.ts b/src/api/teams/{teamId}/services/{serviceName}.ts new file mode 100644 index 000000000..86b61ffd3 --- /dev/null +++ b/src/api/teams/{teamId}/services/{serviceName}.ts @@ -0,0 +1,37 @@ +import Debug from 'debug' +import { Operation, OperationHandlerArray } from 'express-openapi' +import { OpenApiRequestExt, Service } from 'src/otomi-models' + +const debug = Debug('otomi:api:teams:services') + +export default function (): OperationHandlerArray { + const del: Operation = [ + async ({ otomi, params: { teamId, serviceName } }: OpenApiRequestExt, res): Promise => { + debug(`deleteService(${serviceName})`) + await otomi.deleteService(decodeURIComponent(teamId), decodeURIComponent(serviceName)) + res.json({}) + }, + ] + const get: Operation = [ + ({ otomi, params: { teamId, serviceName } }: OpenApiRequestExt, res): void => { + debug(`getService(${serviceName})`) + const data = otomi.getService(decodeURIComponent(teamId), decodeURIComponent(serviceName)) + res.json(data) + }, + ] + const put: Operation = [ + async ({ otomi, params: { teamId, serviceName }, body }: OpenApiRequestExt, res): Promise => { + debug(`editService(${serviceName})`) + const data = await otomi.editService(decodeURIComponent(teamId), decodeURIComponent(serviceName), { + ...body, + } as Service) + res.json(data) + }, + ] + const api = { + delete: del, + get, + put, + } + return api +} diff --git a/src/api/teams/{teamId}/workloads/{workloadId}.ts b/src/api/teams/{teamId}/workloads/{workloadId}.ts deleted file mode 100644 index 64e469a42..000000000 --- a/src/api/teams/{teamId}/workloads/{workloadId}.ts +++ /dev/null @@ -1,38 +0,0 @@ -import Debug from 'debug' -import { Operation, OperationHandlerArray } from 'express-openapi' -import { OpenApiRequestExt, Workload } from 'src/otomi-models' - -const debug = Debug('otomi:api:teams:workloads') - -export default function (): OperationHandlerArray { - const del: Operation = [ - async ({ otomi, params: { workloadId } }: OpenApiRequestExt, res): Promise => { - debug(`deleteWorkload(${workloadId})`) - await otomi.deleteWorkload(decodeURIComponent(workloadId)) - res.json({}) - }, - ] - const get: Operation = [ - ({ otomi, params: { workloadId } }: OpenApiRequestExt, res): void => { - debug(`getWorkload(${workloadId})`) - const data = otomi.getWorkload(decodeURIComponent(workloadId)) - res.json(data) - }, - ] - const put: Operation = [ - async ({ otomi, params: { teamId, workloadId }, body }: OpenApiRequestExt, res): Promise => { - debug(`editWorkload(${workloadId})`) - const data = await otomi.editWorkload(decodeURIComponent(workloadId), { - ...body, - teamId: decodeURIComponent(teamId), - } as Workload) - res.json(data) - }, - ] - const api = { - delete: del, - get, - put, - } - return api -} diff --git a/src/api/teams/{teamId}/workloads/{workloadName}.ts b/src/api/teams/{teamId}/workloads/{workloadName}.ts new file mode 100644 index 000000000..f7e8c9b08 --- /dev/null +++ b/src/api/teams/{teamId}/workloads/{workloadName}.ts @@ -0,0 +1,37 @@ +import Debug from 'debug' +import { Operation, OperationHandlerArray } from 'express-openapi' +import { OpenApiRequestExt, Workload } from 'src/otomi-models' + +const debug = Debug('otomi:api:teams:workloads') + +export default function (): OperationHandlerArray { + const del: Operation = [ + async ({ otomi, params: { teamId, workloadName } }: OpenApiRequestExt, res): Promise => { + debug(`deleteWorkload(${workloadName})`) + await otomi.deleteWorkload(decodeURIComponent(teamId), decodeURIComponent(workloadName)) + res.json({}) + }, + ] + const get: Operation = [ + ({ otomi, params: { teamId, workloadName } }: OpenApiRequestExt, res): void => { + debug(`getWorkload(${workloadName})`) + const data = otomi.getWorkload(decodeURIComponent(teamId), decodeURIComponent(workloadName)) + res.json(data) + }, + ] + const put: Operation = [ + async ({ otomi, params: { teamId, workloadName }, body }: OpenApiRequestExt, res): Promise => { + debug(`editWorkload(${workloadName})`) + const data = await otomi.editWorkload(decodeURIComponent(teamId), decodeURIComponent(workloadName), { + ...body, + } as Workload) + res.json(data) + }, + ] + const api = { + delete: del, + get, + put, + } + return api +} diff --git a/src/api/teams/{teamId}/workloads/{workloadId}/values.ts b/src/api/teams/{teamId}/workloads/{workloadName}/values.ts similarity index 55% rename from src/api/teams/{teamId}/workloads/{workloadId}/values.ts rename to src/api/teams/{teamId}/workloads/{workloadName}/values.ts index ba2b2e47f..32541e97b 100644 --- a/src/api/teams/{teamId}/workloads/{workloadId}/values.ts +++ b/src/api/teams/{teamId}/workloads/{workloadName}/values.ts @@ -6,19 +6,18 @@ const debug = Debug('otomi:api:teams:workloadValues') export default function (): OperationHandlerArray { const get: Operation = [ - ({ otomi, params: { workloadId } }: OpenApiRequestExt, res): void => { - debug(`getWorkloadValues(${workloadId})`) - const data = otomi.getWorkloadValues(decodeURIComponent(workloadId)) + ({ otomi, params: { teamId, workloadName } }: OpenApiRequestExt, res): void => { + debug(`getWorkloadValues(${workloadName})`) + const data = otomi.getWorkloadValues(decodeURIComponent(teamId), decodeURIComponent(workloadName)) res.json(data) }, ] const put: Operation = [ - async ({ otomi, params: { teamId, workloadId }, body }: OpenApiRequestExt, res): Promise => { - debug(`putWorkloadValues(${workloadId})`) - const data = await otomi.editWorkloadValues(decodeURIComponent(workloadId), { + async ({ otomi, params: { teamId, workloadName }, body }: OpenApiRequestExt, res): Promise => { + debug(`putWorkloadValues(${workloadName})`) + const data = await otomi.editWorkloadValues(decodeURIComponent(teamId), decodeURIComponent(workloadName), { ...body, - teamId: decodeURIComponent(teamId), } as WorkloadValues) res.json(data) }, diff --git a/src/app.ts b/src/app.ts index 79c374028..704903db1 100644 --- a/src/app.ts +++ b/src/app.ts @@ -13,8 +13,8 @@ import logger from 'morgan' import path from 'path' import { default as Authz } from 'src/authz' import { - DbMessage, authzMiddleware, + DbMessage, errorMiddleware, getIo, getSessionStack, @@ -22,19 +22,18 @@ import { sessionMiddleware, } from 'src/middleware' import { setMockIdx } from 'src/mocks' -import { Build, OpenAPIDoc, OpenApiRequestExt, Schema, SealedSecret, Service, Workload } from 'src/otomi-models' +import { OpenAPIDoc, OpenApiRequestExt, Schema } from 'src/otomi-models' import { default as OtomiStack } from 'src/otomi-stack' import { extract, getPaths, getValuesSchema } from 'src/utils' import { CHECK_LATEST_COMMIT_INTERVAL, + cleanEnv, DRONE_WEBHOOK_SECRET, EXPRESS_PAYLOAD_LIMIT, GIT_PASSWORD, GIT_USER, - cleanEnv, } from 'src/validators' import swaggerUi from 'swagger-ui-express' -import Db from './db' import giteaCheckLatest from './gitea/connect' import { getBuildStatus, getSealedSecretStatus, getServiceStatus, getWorkloadStatus } from './k8s_operations' @@ -63,13 +62,12 @@ const checkAgainstGitea = async () => { const latestOtomiVersion = await giteaCheckLatest(encodedToken, clusterInfo) // check the local version against the latest online version // if the latest online is newer it will be pulled locally - if (latestOtomiVersion && latestOtomiVersion.data[0].sha !== otomiStack.repo.commitSha) { + if (latestOtomiVersion && latestOtomiVersion.data[0].sha !== otomiStack.git.commitSha) { debug('Local values differentiate from Git repository, retrieving latest values') - await otomiStack.repo.pull() + await otomiStack.git.pull() // inflate new db - otomiStack.db = new Db() await otomiStack.loadValues() - const sha = await otomiStack.repo.getCommitSha() + const sha = await otomiStack.git.getCommitSha() const msg: DbMessage = { state: 'clean', editor: 'system', sha, reason: 'conflict' } getIo().emit('db', msg) } @@ -77,13 +75,17 @@ const checkAgainstGitea = async () => { const resourceStatus = async (errorSet) => { const otomiStack = await getSessionStack() + if (!otomiStack.isLoaded) { + debug('Values are not loaded yet') + return + } const { cluster } = otomiStack.getSettings(['cluster']) const domainSuffix = cluster?.domainSuffix const resources = { - workloads: otomiStack.db.getCollection('workloads') as Array, - builds: otomiStack.db.getCollection('builds') as Array, - services: otomiStack.db.getCollection('services') as Array, - sealedSecrets: otomiStack.db.getCollection('sealedsecrets') as Array, + workloads: otomiStack.repoService.getAllWorkloads(), + builds: otomiStack.repoService.getAllBuilds(), + services: otomiStack.repoService.getAllServices(), + sealedSecrets: otomiStack.repoService.getAllSealedSecrets(), } const statusFunctions = { workloads: getWorkloadStatus, @@ -97,7 +99,7 @@ const resourceStatus = async (errorSet) => { const promises = resources[resourceType].map(async (resource) => { try { const res = await statusFunctions[resourceType](resource, domainSuffix) - return { [resource.id]: res } + return { [resource.name]: res } } catch (error) { const errorMessage = `${resourceType}-${resource.name}-${error.message}` if (!errorSet.has(errorMessage)) { @@ -176,7 +178,7 @@ export async function initApp(inOtomiStack?: OtomiStack | undefined) { if (status === 'success') { const stack = await getSessionStack() debug('Drone deployed, root pull') - await stack.repo.pull() + await stack.git.pull() } }) let server: Server | undefined @@ -188,7 +190,7 @@ export async function initApp(inOtomiStack?: OtomiStack | undefined) { debug(`Listening on :::${PORT}`) lightship.signalReady() // Clone repo after the application is ready to avoid Pod NotReady phenomenon, and thus infinite Pod crash loopback - ;(await getSessionStack()).initRepo() + ;(await getSessionStack()).initGit() }) .on('error', (e) => { console.error(e) @@ -203,7 +205,11 @@ export async function initApp(inOtomiStack?: OtomiStack | undefined) { const emitResourceStatusInterval = 10 * 1000 const errorSet = new Set() setInterval(async function () { - await resourceStatus(errorSet) + try { + await resourceStatus(errorSet) + } catch (e) { + debug(e) + } }, emitResourceStatusInterval) // and register session middleware diff --git a/src/db.test.ts b/src/db.test.ts deleted file mode 100644 index 95d7efeee..000000000 --- a/src/db.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -import Db from 'src/db' -jest.mock('src/otomi-stack') - -describe('Db', () => { - let testDb: Db - - beforeEach(() => { - testDb = new Db() - }) - - test('can store with id', () => { - const v = testDb.createItem('teams', { name: 'n1', k: '1' }, undefined, '1') - expect(v).toEqual({ name: 'n1', k: '1', id: '1' }) - }) - - test('can store without id', () => { - const v = testDb.createItem('teams', { name: 'n1', k: '1' }) - expect(v).toMatchObject({ name: 'n1', k: '1' }) - // @ts-ignore - expect(v.id).toBeDefined() - }) - - test('cannot store resource with existing selector', () => { - testDb.createItem('teams', { name: 'n1' }) - expect(() => testDb.createItem('teams', { name: 'n1' }, { name: 'n1' })).toThrow() - }) - - test('can remove item', () => { - testDb.createItem('teams', { name: 'name1', k: 'a' }, undefined, '1') - testDb.createItem('teams', { name: 'name2', k: 'b' }, undefined, '2') - - testDb.deleteItem('teams', { id: '1' }) - expect(() => testDb.getItem('teams', { id: '1' })).toThrow() - - const v = testDb.getItem('teams', { id: '2' }) - expect(v).not.toBeUndefined() - }) - - test('can update item', () => { - testDb.createItem('teams', { name: 'n1', k: '1' }, undefined, '1') - testDb.updateItem('teams', { name: 'n1', k: '2' }, { id: '1' }) - const v = testDb.getItem('teams', { id: '1' }) - expect(v).toEqual({ name: 'n1', k: '2', id: '1' }) - }) - - test('can obtain item', () => { - testDb.createItem('teams', { name: 'n1', k: '1' }, undefined, '1') - const v = testDb.getItem('teams', { name: 'n1' }) - expect(v).toEqual({ name: 'n1', k: '1', id: '1' }) - }) - - test('can obtain collection', () => { - testDb.createItem('teams', { name: 'n1', k: '1' }) - testDb.createItem('teams', { name: 'n2', k: '1' }) - const v = testDb.getCollection('teams') - expect(v.length).toBe(2) - }) - - test('can obtain collection by selector', () => { - testDb.createItem('teams', { name: 'n1', k: '1' }) - testDb.createItem('teams', { name: 'n2', k: '1' }) - const v = testDb.getCollection('teams', { k: '1' }) - expect(v.length).toBe(2) - const v2 = testDb.getCollection('teams', { name: 'n1' }) - expect(v2.length).toBe(1) - }) - - test('throws error if item does not exist', () => { - expect(() => testDb.getItem('teams', { teamId: 'n1' })).toThrow() - }) -}) diff --git a/src/db.ts b/src/db.ts deleted file mode 100644 index d9a77418f..000000000 --- a/src/db.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { debug } from 'console' -import cloneDeep from 'lodash/cloneDeep' -import findIndex from 'lodash/findIndex' -import low from 'lowdb' -import FileSync from 'lowdb/adapters/FileSync' -import Memory from 'lowdb/adapters/Memory' -import { AlreadyExists, NotExistError } from 'src/error' -import { - App, - Backup, - Build, - Cloudtty, - Cluster, - Coderepo, - Netpol, - Policies, - Project, - SealedSecret, - Secret, - Service, - Settings, - Team, - User, - Workload, - WorkloadValues, -} from 'src/otomi-models' -import { mergeData } from 'src/utils' -import { v4 as uuidv4 } from 'uuid' - -export type DbType = - | Backup - | Build - | Cloudtty - | Cluster - | Netpol - | SealedSecret - | Secret - | Service - | Team - | Settings - | App - | Workload - | WorkloadValues - | User - | Project - | Policies - | Coderepo -export type Schema = { - apps: App[] - sealedsecrets: SealedSecret[] - secrets: Secret[] - services: Service[] - netpols: Netpol[] - settings: Settings - teams: Team[] - workloads: Workload[] - workloadValues: WorkloadValues[] - builds: Build[] - policies: Record - cloudttys: Cloudtty[] - users: User[] - projects: Project[] - coderepos: Coderepo[] -} - -export default class Db { - db: low.LowdbSync - - constructor(path?: string) { - // @ts-ignore - this.db = low(path === undefined ? new Memory('') : new FileSync(path)) - this.db._.mixin({ - replaceRecord(arr: Record[], currentObject: Record, newObject: Record) { - return arr.splice(findIndex(arr, currentObject), 1, newObject) - }, - }) - // Set some defaults (required if your JSON file is empty) - this.db - .defaults({ - apps: [], - backups: [], - builds: [], - cloudttys: [], - netpols: [], - users: [], - projects: [], - coderepos: [], - policies: {}, - sealedsecrets: [], - secrets: [], - services: [], - settings: {}, - teams: [], - workloads: [], - workloadValues: [], - }) - .write() - } - - getItem(name: string, selector: any): DbType { - // By default data is returned by reference, this means that modifications to returned objects may change the database. - // To avoid such behavior, we use .cloneDeep(). - const data = this.getItemReference(name, selector) - return cloneDeep(data) - } - - getItemReference(type: string, selector: any, mustThrow = true): DbType | undefined { - const coll = this.db.get(type) - // @ts-ignore - const data = coll.find(selector).value() - if (data === undefined) { - debug(`Selector props do not exist in '${type}': ${JSON.stringify(selector)}`) - if (mustThrow) throw new NotExistError() - else return - } - if (data?.length) { - debug(`More than one item found for '${type}' with selector: ${JSON.stringify(selector)}`) - if (mustThrow) throw new NotExistError() - else return - } - return data - } - - getCollection(type: string, selector?: any): Array { - // @ts-ignore - return this.db.get(type).filter(selector).value() - } - - populateItem(type: string, data: DbType, selector?: any, id?: string): DbType | undefined { - // @ts-ignore - if (selector && this.db.get(type).find(selector).value()) return undefined - return ( - this.db - .get(type) - // @ts-ignore - .push(data) - .last() - .assign({ id: id || uuidv4() }) - .write() - ) - } - - createItem(type: string, data: Record, selector?: Record, id?: string): DbType { - // @ts-ignore - if (selector && this.db.get(type).find(selector).value()) - throw new AlreadyExists(`Item already exists in '${type}' collection: ${JSON.stringify(selector)}`) - const cleanData = { ...data, ...selector } - const ret = this.populateItem(type, cleanData, selector, id) - return ret - } - - deleteItem(type: string, selector: any): void { - this.getItemReference(type, selector) - // @ts-ignore - this.db.get(type).remove(selector).write() - } - - updateItem(type: string, data: Record, selector: Record, merge = false): DbType { - const prev = this.getItemReference(type, selector) - const col = this.db.get(type) - // @ts-ignore - const idx = col.findIndex(selector).value() - const merged = (merge && prev ? mergeData(prev, data) : data) as Record - const newData = { ...merged, ...selector } - col.value().splice(idx, 1, newData) - return newData - } -} diff --git a/src/git.ts b/src/git.ts new file mode 100644 index 000000000..0f35314c7 --- /dev/null +++ b/src/git.ts @@ -0,0 +1,487 @@ +import axios, { AxiosResponse } from 'axios' +import Debug from 'debug' +import diff from 'deep-diff' +import { rmSync } from 'fs' +import { copy, ensureDir, pathExists, readFile, writeFile } from 'fs-extra' +import { unlink } from 'fs/promises' +import { glob } from 'glob' +import stringifyJson from 'json-stable-stringify' +import jsonpath from 'jsonpath' +import { cloneDeep, get, isEmpty, merge, set, unset } from 'lodash' +import { basename, dirname, join } from 'path' +import simpleGit, { CheckRepoActions, CleanOptions, CommitResult, ResetMode, SimpleGit } from 'simple-git' +import { + cleanEnv, + GIT_BRANCH, + GIT_LOCAL_PATH, + GIT_PASSWORD, + GIT_PUSH_RETRIES, + GIT_REPO_URL, + GIT_USER, + TOOLS_HOST, +} from 'src/validators' +import { parse as parseYaml, stringify as stringifyYaml } from 'yaml' +import { BASEURL } from './constants' +import { GitPullError, HttpError, ValidationError } from './error' +import { DbMessage, getIo } from './middleware' +import { Core } from './otomi-models' +import { FileMap, getFilePath, renderManifest, renderManifestForSecrets } from './repo' +import { removeBlankAttributes } from './utils' + +const debug = Debug('otomi:repo') + +const env = cleanEnv({ + GIT_BRANCH, + GIT_LOCAL_PATH, + GIT_PASSWORD, + GIT_REPO_URL, + GIT_USER, + GIT_PUSH_RETRIES, + TOOLS_HOST, +}) + +const baseUrl = BASEURL +const prepareUrl = `${baseUrl}/prepare` +const initUrl = `${baseUrl}/init` +const valuesUrl = `${baseUrl}/otomi/values` + +const getProtocol = (url): string => (url && url.includes('://') ? url.split('://')[0] : 'https') + +const getUrl = (url): string => (!url || url.includes('://') ? url : `${getProtocol(url)}://${url}`) + +function getUrlAuth(url, user, password): string | undefined { + if (!url) return + const protocol = getProtocol(url) + const [_, bareUrl] = url.split('://') + const encodedUser = encodeURIComponent(user as string) + const encodedPassword = encodeURIComponent(password as string) + return protocol === 'file' ? `${protocol}://${bareUrl}` : `${protocol}://${encodedUser}:${encodedPassword}@${bareUrl}` +} + +const secretFileRegex = new RegExp(`^(.*/)?secrets.*.yaml(.dec)?$`) +export class Git { + branch: string + commitSha: string + corrupt = false + email: string + git: SimpleGit + password: string + path: string + remote: string + remoteBranch: string + secretFilePostfix = '' + url: string | undefined + urlAuth: string | undefined + user: string + + constructor( + path: string, + url: string | undefined, + user: string, + email: string, + urlAuth: string | undefined, + branch: string | undefined, + ) { + this.branch = branch || 'main' + this.email = email + this.path = path + this.remote = 'origin' + this.remoteBranch = join(this.remote, this.branch) + this.urlAuth = urlAuth + this.user = user + this.url = url + this.git = simpleGit(this.path) + } + + getProtocol() { + return getProtocol(this.url) + } + + async requestInitValues(): Promise { + debug(`Tools: requesting "init" on values repo path ${this.path}`) + const res = await axios.get(initUrl, { params: { envDir: this.path } }) + return res + } + + async requestPrepareValues(): Promise { + debug(`Tools: requesting "prepare" on values repo path ${this.path}`) + const res = await axios.get(prepareUrl, { params: { envDir: this.path } }) + return res + } + + async requestValues(params): Promise { + debug(`Tools: requesting "otomi/values" ${this.path}`) + const res = await axios.get(valuesUrl, { params: { envDir: this.path, ...params } }) + return res + } + async addConfig(): Promise { + debug(`Adding git config`) + await this.git.addConfig('user.name', this.user) + await this.git.addConfig('user.email', this.email) + if (this.isRootClone()) { + if (this.getProtocol() === 'file') { + // tell the the git repo there to accept updates even when it is checked out + const _git = simpleGit(this.url!.replace('file://', '')) + await _git.addConfig('receive.denyCurrentBranch', 'updateInstead') + } + // same for the root repo, which needs to accept pushes from children + await this.git.addConfig('receive.denyCurrentBranch', 'updateInstead') + } + } + + async init(bare = true): Promise { + await this.git.init(bare !== undefined ? bare : this.isRootClone()) + await this.git.addRemote(this.remote, this.url!) + } + + async initSops(): Promise { + if (this.secretFilePostfix === '.dec') return + this.secretFilePostfix = (await pathExists(join(this.path, '.sops.yaml'))) ? '.dec' : '' + } + + getSafePath(file: string): string { + if (this.secretFilePostfix === '') return file + // otherwise we might have to give *.dec variant for secrets + if (file.match(secretFileRegex) && !file.endsWith(this.secretFilePostfix)) return `${file}${this.secretFilePostfix}` + return file + } + + async removeFile(file: string): Promise { + const absolutePath = join(this.path, file) + const exists = await this.fileExists(file) + if (exists) { + debug(`Removing file: ${absolutePath}`) + // Remove empty secret file due to https://github.com/mozilla/sops/issues/926 issue + await unlink(absolutePath) + } + if (file.match(secretFileRegex)) { + // also remove the encrypted file as they are operated on in pairs + const encFile = `${file}${this.secretFilePostfix}` + if (await this.fileExists(encFile)) { + const absolutePathEnc = join(this.path, encFile) + debug(`Removing enc file: ${absolutePathEnc}`) + await unlink(absolutePathEnc) + } + } + } + + async removeDir(dir: string): Promise { + const absolutePath = join(this.path, dir) + const exists = await this.fileExists(dir) + if (exists) { + debug(`Removing directory: ${absolutePath}`) + rmSync(absolutePath, { recursive: true, force: true }) + } + } + + async diffFile(file: string, data: Record): Promise { + const repoFile: string = this.getSafePath(file) + const oldData = await this.readFile(repoFile) + const deepDiff = diff(data, oldData) + debug(`Diff for ${file}: `, deepDiff) + return deepDiff + } + + async writeFile(file: string, data: Record, unsetBlankAttributes = true): Promise { + let cleanedData = data + if (unsetBlankAttributes) cleanedData = removeBlankAttributes(data, { emptyArrays: true }) + if (isEmpty(cleanedData) && file.match(secretFileRegex)) { + // remove empty secrets file which sops can't handle + return this.removeFile(file) + } + // we also bail when no changes found + const hasDiff = await this.diffFile(file, data) + if (!hasDiff) return + // ok, write new content + const absolutePath = join(this.path, file) + debug(`Writing to file: ${absolutePath}`) + const sortedData = JSON.parse(stringifyJson(data) as string) + const content = isEmpty(sortedData) ? '' : stringifyYaml(sortedData, undefined, 4) + const dir = dirname(absolutePath) + await ensureDir(dir) + await writeFile(absolutePath, content, 'utf8') + } + + async fileExists(relativePath: string): Promise { + const absolutePath = join(this.path, relativePath) + return await pathExists(absolutePath) + } + + async readDir(relativePath: string): Promise { + const absolutePath = join(this.path, relativePath) + const files = await glob([`${absolutePath}/**/*.yaml`]) + const filenames = files.map((file) => basename(file)) + return filenames + } + + async readFile(file: string, checkSuffix = false): Promise> { + if (!(await this.fileExists(file))) return {} + const safeFile = checkSuffix ? this.getSafePath(file) : file + const absolutePath = join(this.path, safeFile) + debug(`Reading from file: ${absolutePath}`) + const doc = parseYaml(await readFile(absolutePath, 'utf8')) || {} + return doc + } + + async loadConfig(file: string, secretFile: string): Promise { + const data = await this.readFile(file) + const secretData = await this.readFile(secretFile, true) + return merge(data, secretData) as Core + } + + async saveConfig( + config: Record, + fileMap: FileMap, + unsetBlankAttributes?: boolean, + ): Promise> { + const jsonPathsValuesPublic = jsonpath.nodes(config, fileMap.jsonPathExpression) + await Promise.all( + jsonPathsValuesPublic.map(async (node) => { + const nodePath = node.path + const nodeValue = node.value + try { + const filePath = getFilePath(fileMap, nodePath, nodeValue, '') + const manifest = renderManifest(fileMap, nodePath, nodeValue) + await this.writeFile(filePath, manifest, unsetBlankAttributes) + } catch (e) { + console.log(nodePath) + console.log(fileMap) + throw e + } + }), + ) + } + + async saveConfigWithSecrets( + config: Record, + secretJsonPaths: string[], + fileMap: FileMap, + ): Promise> { + const secretData = {} + const plainData = cloneDeep(config) + secretJsonPaths.forEach((objectPath) => { + const val = get(config, objectPath) + if (val) { + set(secretData, objectPath, val) + unset(plainData, objectPath) + } + }) + + await this.saveConfig(plainData, fileMap) + await this.saveSecretConfig(secretData, fileMap) + } + + async saveSecretConfig(secretConfig: Record, fileMap: FileMap, unsetBlankAttributes?: boolean) { + const jsonPathsValuesSecrets = jsonpath.nodes(secretConfig, fileMap.jsonPathExpression) + await Promise.all( + jsonPathsValuesSecrets.map(async (node) => { + const nodePath = node.path + const nodeValue = node.value + try { + const filePath = getFilePath(fileMap, nodePath, nodeValue, 'secrets.') + const manifest = renderManifestForSecrets(fileMap, nodeValue) + await this.writeFile(filePath, manifest, unsetBlankAttributes) + } catch (e) { + console.log(nodePath) + console.log(fileMap) + throw e + } + }), + ) + } + + async deleteConfig(config: Record, fileMap: FileMap, fileNamePrefix = '') { + const jsonPathsValuesSecrets = jsonpath.nodes(config, fileMap.jsonPathExpression) + await Promise.all( + jsonPathsValuesSecrets.map(async (node) => { + const nodePath = node.path + const nodeValue = node.value + try { + const filePath = getFilePath(fileMap, nodePath, nodeValue, fileNamePrefix) + await this.removeFile(filePath) + } catch (e) { + console.log(nodePath) + console.log(fileMap) + throw e + } + }), + ) + } + + isRootClone(): boolean { + return this.path === env.GIT_LOCAL_PATH + } + + hasRemote(): boolean { + return !!env.GIT_REPO_URL + } + + async initFromTestFolder(): Promise { + // we inflate GIT_LOCAL_PATH from the ./test folder + debug(`DEV mode: using local folder values`) + await copy(join(process.cwd(), 'test'), env.GIT_LOCAL_PATH, { + recursive: true, + overwrite: false, + }) + await this.init(false) + await this.git.checkoutLocalBranch(this.branch) + await this.git.add('.') + await this.addConfig() + await this.git.commit('initial commit', undefined, this.getOptions()) + } + + async clone(): Promise { + debug(`Checking if local git repository exists at: ${this.path}`) + const isRepo = await this.git.checkIsRepo(CheckRepoActions.IS_REPO_ROOT) + // remote root url + this.url = getUrl(`${env.GIT_REPO_URL}`) + if (!isRepo) { + debug(`Initializing repo...`) + if (!this.hasRemote() && this.isRootClone()) return await this.initFromTestFolder() + else if (!this.isRootClone()) { + // child clone, point to remote root + this.urlAuth = getUrlAuth(this.url, env.GIT_USER, env.GIT_PASSWORD) + } + debug(`Cloning from '${this.url}' to '${this.path}'`) + await this.git.clone(this.urlAuth!, this.path) + await this.addConfig() + await this.git.checkout(this.branch) + } else if (this.url) { + debug('Repo already exists. Checking out correct branch.') + // Git fetch ensures that local git repository is synced with remote repository + await this.git.fetch({}) + await this.git.checkout(this.branch) + } + this.commitSha = await this.getCommitSha() + } + + getOptions() { + const options = {} + if (env.isDev) options['--no-verify'] = null // only for dev do we have git hooks blocking direct commit + return options + } + + async commit(editor: string): Promise { + await this.git.add('./*') + const summary = await this.git.commit(`otomi-api commit by ${editor}`, undefined, this.getOptions()) + debug(`Commit summary: ${JSON.stringify(summary)}`) + return summary + } + + async pull(skipRequest = false, skipMsg = false): Promise { + // test root can't pull as it has no remote + if (!this.url) return + debug('Pulling') + try { + const summary = await this.git.pull(this.remote, this.branch, { '--rebase': 'true', '--depth': '5' }) + const summJson = JSON.stringify(summary) + debug(`Pull summary: ${summJson}`) + this.commitSha = await this.getCommitSha() + if (!skipRequest) await this.requestInitValues() + await this.initSops() + } catch (e) { + debug('Could not pull from remote. Upstream commits? Marked db as corrupt.', e) + this.corrupt = true + if (!skipMsg) { + const msg: DbMessage = { editor: 'system', state: 'corrupt', reason: 'conflict' } + getIo().emit('db', msg) + } + try { + // Remove local changes so that no conflict can happen + debug('Removing local changes.') + await this.git.reset(ResetMode.HARD) + debug(`Go to ${this.branch} branch`) + await this.git.checkout(this.branch) + debug('Removing local changes.') + await this.git.reset(ResetMode.HARD) + debug('Cleaning local values and directories.') + await this.git.clean(CleanOptions.FORCE, ['-d']) + debug('Get the latest branch from:', this.remote) + await this.git.fetch(this.remote, this.branch) + debug('Reconciling divergent branches.') + await this.git.merge([`${this.remote}/${this.branch}`, '--strategy-option=theirs']) + debug('Trying to remove upstream commits: ', this.remote) + await this.git.push([this.remote, this.branch, '--force']) + } catch (error) { + debug('Failed to remove upstream commits: ', error) + throw new GitPullError('Failed to remove upstream commits!') + } + debug('Removed upstream commits!') + const cleanMsg: DbMessage = { editor: 'system', state: 'clean', reason: 'restored' } + getIo().emit('db', cleanMsg) + this.corrupt = false + } + } + + async push(): Promise { + if (!this.url && this.isRootClone()) return + debug('Pushing') + const summary = await this.git.push(this.remote, this.branch) + debug('Pushed. Summary: ', summary) + return + } + + async getCommitSha(): Promise { + return this.git.revparse('HEAD') + } + + async save(editor: string, encryptSecrets = true): Promise { + // prepare values first + try { + if (encryptSecrets) { + await this.requestPrepareValues() + } else { + debug(`Data does not need to be encrypted`) + } + } catch (e) { + debug(`ERROR: ${JSON.stringify(e)}`) + if (e.response) { + const { status } = e.response as AxiosResponse + if (status === 422) throw new ValidationError() + throw HttpError.fromCode(status) + } + throw new HttpError(500, `${e}`) + } + // all good? commit + await this.commit(editor) + try { + // we are in a unique developer branch, so we can pull, push, and merge + // with the remote root, which might have been modified by another developer + // since this is a child branch, we don't need to re-init + // retry up to 10 times to pull and push if there are conflicts + const retries = env.GIT_PUSH_RETRIES + for (let attempt = 1; attempt <= retries; attempt++) { + await this.git.pull(this.remote, this.branch, { '--rebase': 'true', '--depth': '5' }) + try { + await this.push() + break + } catch (error) { + if (attempt === retries) throw error + debug(`Attempt ${attempt} failed. Retrying...`) + await new Promise((resolve) => setTimeout(resolve, 50)) + } + } + } catch (e) { + debug(`${e.message.trim()} for command ${JSON.stringify(e.task?.commands)}`) + debug(`Merge error: ${JSON.stringify(e)}`) + throw new GitPullError() + } + } +} + +export default async function getRepo( + path: string, + url: string, + user: string, + email: string, + password: string, + branch: string, + method: 'clone' | 'init' = 'clone', +): Promise { + await ensureDir(path, { mode: 0o744 }) + const urlNormalized = getUrl(url) + const urlAuth = getUrlAuth(urlNormalized, user, password) + const repo = new Git(path, urlNormalized, user, email, urlAuth, branch) + await repo[method]() + return repo +} diff --git a/src/k8s_operations.ts b/src/k8s_operations.ts index b8ae5aa39..630904f0a 100644 --- a/src/k8s_operations.ts +++ b/src/k8s_operations.ts @@ -391,7 +391,9 @@ export async function getSecretValues(name: string, namespace: string): Promise< } return decodedData } catch (error) { - if (process.env.NODE_ENV !== 'development') debug(`Failed to get secret values for ${name} in ${namespace}.`) + if (process.env.NODE_ENV !== 'development') { + debug(`Failed to get secret values for ${name} in ${namespace}.`) + } } } @@ -417,9 +419,9 @@ export async function getSealedSecretSyncedStatus(name: string, namespace: strin } return 'NotFound' } catch (error) { - if (process.env.NODE_ENV !== 'development') + if (process.env.NODE_ENV !== 'development') { debug(`Failed to get SealedSecret synced status for ${name} in ${namespace}.`) - + } return 'NotFound' } } diff --git a/src/middleware/authz.ts b/src/middleware/authz.ts index d1ca3b930..14da84060 100644 --- a/src/middleware/authz.ts +++ b/src/middleware/authz.ts @@ -2,11 +2,13 @@ import { RequestHandler } from 'express' import get from 'lodash/get' import Authz, { getTeamSelfServiceAuthz } from 'src/authz' -import Db from 'src/db' import { OpenApiRequestExt, PermissionSchema, TeamSelfService } from 'src/otomi-models' import OtomiStack from 'src/otomi-stack' import { cleanEnv } from 'src/validators' import { getSessionStack } from './session' +import { RepoService } from '../services/RepoService' +import { debug } from 'console' +import { find } from 'lodash' const HttpMethodMapping: Record = { DELETE: 'delete', @@ -38,7 +40,7 @@ function renameKeys(obj: Record) { // } // } -export function authorize(req: OpenApiRequestExt, res, next, authz: Authz, db: Db): RequestHandler { +export function authorize(req: OpenApiRequestExt, res, next, authz: Authz, repoService: RepoService): RequestHandler { const { params: { teamId }, body, @@ -72,17 +74,31 @@ export function authorize(req: OpenApiRequestExt, res, next, authz: Authz, db: D } } - const schemaToDbMap: Record = { - Secret: 'secrets', + const schemaToRepoMap: Record = { Service: 'services', - Team: 'teams', + Team: 'teamConfig', + App: 'apps', + Build: 'builds', + Workload: 'workloads', + Settings: 'otomi', + Project: 'projects', + Netpol: 'netpols', Policy: 'policies', + SealedSecret: 'sealedSecrets', } + const teamSpecificCollections = [ + 'builds', + 'services', + 'workloads', + 'netpols', + 'projects', + 'policies', + 'sealedSecrets', + ] // <-- These are fetched per team const selector = renameKeys(req.params) - - if (['create', 'update'].includes(action)) { - const collection = schemaToDbMap[schemaName] + const collectionId = schemaToRepoMap[schemaName] + if (collectionId && ['create', 'update'].includes(action)) { let dataOrig = get( req, `apiDoc.components.schemas.TeamSelfService.properties.${schemaName.toLowerCase()}.x-allow-values`, @@ -90,12 +106,19 @@ export function authorize(req: OpenApiRequestExt, res, next, authz: Authz, db: D ) if (action === 'update') { - if (collection === 'policies') { - const policies = db.db.get(['policies']).value() - const id = req.params.policyId - dataOrig = policies[teamId][id] - } else dataOrig = db.getItemReference(collection, selector, false) as Record + try { + let collection + if (teamSpecificCollections.includes(collectionId)) { + collection = repoService.getTeamConfigService(teamId).getCollection(collectionId) + } else { + collection = repoService.getCollection(collectionId) + } + dataOrig = find(collection, selector) || {} + } catch (error) { + debug('Error in authzMiddleware', error) + } } + const violatedAttributes = authz.validateWithAbac(action, schemaName, teamId, req.body, dataOrig) if (violatedAttributes.length > 0) { return res.status(403).send({ @@ -125,6 +148,6 @@ export function authzMiddleware(authz: Authz): RequestHandler { req.apiDoc.components.schemas.TeamSelfService as TeamSelfService as PermissionSchema, otomi, ) - return authorize(req, res, next, authz, otomi.db) + return authorize(req, res, next, authz, otomi.repoService) } } diff --git a/src/middleware/error.ts b/src/middleware/error.ts index e41618758..cedaa3a93 100644 --- a/src/middleware/error.ts +++ b/src/middleware/error.ts @@ -28,6 +28,6 @@ export function errorMiddleware(e, req: OpenApiRequest, res: Response, next): vo msg = `Required property missing! '${requiredProperties}'` } const { otomi } = req as any - if (otomi?.sessionId) cleanSession(otomi.sessionId as string) + if (otomi?.sessionId && otomi?.sessionId !== 'main') cleanSession(otomi.sessionId as string) res.status(code).json({ error: msg }) } diff --git a/src/middleware/jwt.test.ts b/src/middleware/jwt.test.ts index 27de72ba1..13ee7a6dd 100644 --- a/src/middleware/jwt.test.ts +++ b/src/middleware/jwt.test.ts @@ -3,6 +3,8 @@ import OtomiStack from 'src/otomi-stack' import { getUser } from './jwt' import * as getValuesSchemaModule from '../utils' import { loadSpec } from '../app' +import { mockDeep } from 'jest-mock-extended' +import { Git } from '../git' const email = 'test@user.net' const platformAdminGroups = ['platform-admin', 'all-teams-admin'] @@ -31,7 +33,11 @@ describe('JWT claims mapping', () => { beforeEach(async () => { otomiStack = new OtomiStack() + otomiStack.git = mockDeep() + jest.spyOn(otomiStack, 'transformApps').mockReturnValue([]) + await otomiStack.init() + await otomiStack.loadValues() }) test('A user in either platform-admin or all-teams-admin group should get platformAdmin role and have isPlatformAdmin', () => { diff --git a/src/middleware/jwt.ts b/src/middleware/jwt.ts index 75de37531..ac1a6117f 100644 --- a/src/middleware/jwt.ts +++ b/src/middleware/jwt.ts @@ -37,8 +37,12 @@ export function getUser(user: JWT, otomi: OtomiStack): SessionUser { const teamId = group.substring(5) if (group.substring(0, 5) === 'team-' && !sessionUser.teams.includes(teamId)) { // we might be assigned team-* without that team yet existing in the values, so ignore those - const existing = otomi.db.getItemReference('teams', { id: teamId }, false) - if (existing) sessionUser.teams.push(teamId) + if (otomi.isLoaded) { + const existing = otomi.repoService.getTeamConfig(teamId) + if (existing) { + sessionUser.teams.push(teamId) + } + } } }) return sessionUser diff --git a/src/middleware/session.ts b/src/middleware/session.ts index 6b921df13..bcf4776c5 100644 --- a/src/middleware/session.ts +++ b/src/middleware/session.ts @@ -10,7 +10,7 @@ import { Server } from 'socket.io' import { ApiNotReadyError } from 'src/error' import { OpenApiRequestExt } from 'src/otomi-models' import { default as OtomiStack, rootPath } from 'src/otomi-stack' -import { EDITOR_INACTIVITY_TIMEOUT, cleanEnv } from 'src/validators' +import { cleanEnv, EDITOR_INACTIVITY_TIMEOUT } from 'src/validators' import { v4 as uuidv4 } from 'uuid' const debug = Debug('otomi:session') @@ -41,10 +41,10 @@ export const setSessionStack = async (editor: string, sessionId: string): Promis if (env.isTest) return readOnlyStack if (!sessions[sessionId]) { debug(`Creating session ${sessionId} for user ${editor}`) - sessions[sessionId] = new OtomiStack(editor, sessionId, readOnlyStack.db) - // init repo without inflating db from files as its slow and we just need a copy of the db - await sessions[sessionId].initRepo(true) - sessions[sessionId].db = cloneDeep(readOnlyStack.db) + sessions[sessionId] = new OtomiStack(editor, sessionId) + // init repo without inflating values from files as its faster to copy the values + await sessions[sessionId].initGit(false) + sessions[sessionId].repoService = cloneDeep(readOnlyStack.repoService) } else sessions[sessionId].sessionId = sessionId return sessions[sessionId] } diff --git a/src/openapi/api.yaml b/src/openapi/api.yaml index 0398179ff..f305dbda1 100644 --- a/src/openapi/api.yaml +++ b/src/openapi/api.yaml @@ -76,23 +76,6 @@ paths: schema: $ref: '#/components/schemas/OtomiCliValues' - /secrets: - get: - operationId: getAllSecrets - description: Get all secrets - x-aclSchema: Secret - responses: - '200': - description: Successfully obtained all secrets - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/Secret' - '400': - <<: *BadRequest - /services: get: operationId: getAllServices @@ -244,7 +227,7 @@ paths: '400': <<: *BadRequest - '/teams/{teamId}/services/{serviceId}': + '/teams/{teamId}/services/{serviceName}': parameters: - $ref: '#/components/parameters/teamParams' - $ref: '#/components/parameters/serviceParams' @@ -389,10 +372,10 @@ paths: schema: $ref: '#/components/schemas/SealedSecret' - '/teams/{teamId}/sealedsecrets/{secretId}': + '/teams/{teamId}/sealedsecrets/{sealedSecretName}': parameters: - $ref: '#/components/parameters/teamParams' - - $ref: '#/components/parameters/secretParams' + - $ref: '#/components/parameters/sealedSecretParams' get: operationId: getSealedSecret description: Get a sealed secret from a given team @@ -432,98 +415,6 @@ paths: <<: *DefaultGetResponses '200': description: Successfully deleted a team sealed secret - '/teams/{teamId}/secrets': - parameters: - - $ref: '#/components/parameters/teamParams' - get: - operationId: getSecrets - description: Get secrets from a given team - x-aclSchema: Secret - responses: - '200': - description: Successfully obtained secrets - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/Secret' - '400': - description: Bad Request - content: - application/json: - schema: - $ref: '#/components/schemas/OpenApiValidationError' - post: - operationId: createSecret - description: Create a team secret - x-aclSchema: Secret - parameters: - - name: teamId - in: path - description: ID of team - required: true - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/Secret' - description: Service object - required: true - responses: - <<: *DefaultPostResponses - '200': - description: Successfully stored secret configuration - content: - application/json: - schema: - $ref: '#/components/schemas/Secret' - - '/teams/{teamId}/secrets/{secretId}': - parameters: - - $ref: '#/components/parameters/teamParams' - - $ref: '#/components/parameters/secretParams' - get: - operationId: getSecret - description: Get a secret from a given team - x-aclSchema: Secret - responses: - <<: *DefaultGetResponses - '200': - description: Successfully obtained secret configuration - content: - application/json: - schema: - $ref: '#/components/schemas/Secret' - put: - operationId: editSecret - description: Edit a secret from a given team - x-aclSchema: Secret - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/Secret' - description: Secret object that contains updated values - required: true - responses: - <<: *DefaultGetResponses - '200': - description: Successfully edited a team secret - content: - application/json: - schema: - $ref: '#/components/schemas/Secret' - delete: - operationId: deleteSecret - description: Delete a secret from a given team - x-aclSchema: Secret - responses: - <<: *DefaultGetResponses - '200': - description: Successfully deleted a team secret '/netpols': get: @@ -576,7 +467,7 @@ paths: schema: $ref: '#/components/schemas/Netpol' - '/teams/{teamId}/netpols/{netpolId}': + '/teams/{teamId}/netpols/{netpolName}': parameters: - $ref: '#/components/parameters/teamParams' - $ref: '#/components/parameters/netpolParams' @@ -671,7 +562,7 @@ paths: schema: $ref: '#/components/schemas/Backup' - '/teams/{teamId}/backups/{backupId}': + '/teams/{teamId}/backups/{backupName}': parameters: - $ref: '#/components/parameters/teamParams' - $ref: '#/components/parameters/backupParams' @@ -780,7 +671,7 @@ paths: schema: $ref: '#/components/schemas/Build' - '/teams/{teamId}/builds/{buildId}': + '/teams/{teamId}/builds/{buildName}': parameters: - $ref: '#/components/parameters/teamParams' - $ref: '#/components/parameters/buildParams' @@ -1101,7 +992,7 @@ paths: schema: $ref: '#/components/schemas/Project' - '/teams/{teamId}/projects/{projectId}': + '/teams/{teamId}/projects/{projectName}': parameters: - $ref: '#/components/parameters/teamParams' - $ref: '#/components/parameters/projectParams' @@ -1147,9 +1038,9 @@ paths: /coderepos: get: - operationId: getAllCoderepos + operationId: getAllCodeRepos description: Get all code repositories - x-aclSchema: Coderepo + x-aclSchema: CodeRepo responses: '200': description: Successfully obtained all code repositories @@ -1158,7 +1049,7 @@ paths: schema: type: array items: - $ref: '#/components/schemas/Coderepo' + $ref: '#/components/schemas/CodeRepo' '400': <<: *BadRequest @@ -1166,9 +1057,9 @@ paths: parameters: - $ref: '#/components/parameters/teamParams' get: - operationId: getTeamCoderepos + operationId: getTeamCodeRepos description: Get code repos from a given team - x-aclSchema: Coderepo + x-aclSchema: CodeRepo responses: '200': description: Successfully obtained code repositories @@ -1177,7 +1068,7 @@ paths: schema: type: array items: - $ref: '#/components/schemas/Coderepo' + $ref: '#/components/schemas/CodeRepo' '400': description: Bad Request content: @@ -1185,9 +1076,9 @@ paths: schema: $ref: '#/components/schemas/OpenApiValidationError' post: - operationId: createCoderepo + operationId: createCodeRepo description: Create a team code repo - x-aclSchema: Coderepo + x-aclSchema: CodeRepo parameters: - name: teamId in: path @@ -1199,8 +1090,8 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Coderepo' - description: Coderepo object + $ref: '#/components/schemas/CodeRepo' + description: CodeRepo object required: true responses: <<: *DefaultPostResponses @@ -1209,16 +1100,16 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Coderepo' + $ref: '#/components/schemas/CodeRepo' - '/teams/{teamId}/coderepos/{coderepoId}': + '/teams/{teamId}/coderepos/{codeRepositoryName}': parameters: - $ref: '#/components/parameters/teamParams' - - $ref: '#/components/parameters/coderepoParams' + - $ref: '#/components/parameters/codeRepoParams' get: - operationId: getCoderepo + operationId: getCodeRepo description: Get a code repo from a given team - x-aclSchema: Coderepo + x-aclSchema: CodeRepo responses: <<: *DefaultGetResponses '200': @@ -1226,17 +1117,17 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Coderepo' + $ref: '#/components/schemas/CodeRepo' put: - operationId: editCoderepo + operationId: editCodeRepo description: Edit a code repo from a given team - x-aclSchema: Coderepo + x-aclSchema: CodeRepo requestBody: content: application/json: schema: - $ref: '#/components/schemas/Coderepo' - description: Coderepo object that contains updated values + $ref: '#/components/schemas/CodeRepo' + description: CodeRepo object that contains updated values required: true responses: <<: *DefaultGetResponses @@ -1245,11 +1136,11 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/Coderepo' + $ref: '#/components/schemas/CodeRepo' delete: - operationId: deleteCoderepo + operationId: deleteCodeRepo description: Delete a code repo from a given team - x-aclSchema: Coderepo + x-aclSchema: CodeRepo responses: <<: *DefaultGetResponses '200': @@ -1366,7 +1257,7 @@ paths: schema: $ref: '#/components/schemas/Workload' - '/teams/{teamId}/workloads/{workloadId}': + '/teams/{teamId}/workloads/{workloadName}': parameters: - $ref: '#/components/parameters/teamParams' - $ref: '#/components/parameters/workloadParams' @@ -1410,7 +1301,7 @@ paths: schema: $ref: '#/components/schemas/Workload' - '/teams/{teamId}/workloads/{workloadId}/values': + '/teams/{teamId}/workloads/{workloadName}/values': parameters: - $ref: '#/components/parameters/teamParams' - $ref: '#/components/parameters/workloadParams' @@ -1724,16 +1615,16 @@ servers: components: parameters: backupParams: - name: backupId + name: backupName in: path - description: ID of the backup + description: Name of the backup required: true schema: type: string buildParams: - name: buildId + name: buildName in: path - description: ID of the build + description: Name of the build required: true schema: type: string @@ -1752,16 +1643,16 @@ components: schema: type: string projectParams: - name: projectId + name: projectName in: path - description: ID of the project + description: Name of the project required: true schema: type: string - coderepoParams: - name: coderepoId + codeRepoParams: + name: codeRepositoryName in: path - description: ID of the code repo + description: Name of the code repository required: true schema: type: string @@ -1773,23 +1664,23 @@ components: schema: type: string netpolParams: - name: netpolId + name: netpolName in: path - description: ID of the network policy + description: Name of the network policy required: true schema: type: string serviceParams: - name: serviceId + name: serviceName in: path - description: ID of the service + description: Name of the service required: true schema: type: string - secretParams: - name: secretId + sealedSecretParams: + name: sealedSecretName in: path - description: ID of the secret + description: Name of the sealed secret required: true schema: type: string @@ -1801,9 +1692,9 @@ components: schema: type: string workloadParams: - name: workloadId + name: workloadName in: path - description: ID of the workload + description: Name of the workload required: true schema: type: string @@ -1836,8 +1727,8 @@ components: $ref: cloudtty.yaml#/Cloudtty Cluster: $ref: cluster.yaml#/Cluster - Coderepo: - $ref: coderepo.yaml#/Coderepo + CodeRepo: + $ref: codeRepo.yaml#/CodeRepo Ingress: $ref: service.yaml#/Ingress IngressCluster: @@ -1870,14 +1761,6 @@ components: $ref: sealedsecretskeys.yaml#/SealedSecretsKeys K8sSecret: $ref: k8sSecret.yaml#/K8sSecret - Secret: - $ref: secret.yaml#/Secret - SecretDockerRegistry: - $ref: secret.yaml#/SecretDockerRegistry - SecretGeneric: - $ref: secret.yaml#/SecretGeneric - SecretTLS: - $ref: secret.yaml#/SecretTLS Service: $ref: service.yaml#/Service Session: diff --git a/src/openapi/coderepo.yaml b/src/openapi/codeRepo.yaml similarity index 95% rename from src/openapi/coderepo.yaml rename to src/openapi/codeRepo.yaml index 45d6d2412..449ecb9d5 100644 --- a/src/openapi/coderepo.yaml +++ b/src/openapi/codeRepo.yaml @@ -1,4 +1,4 @@ -Coderepo: +CodeRepo: x-acl: platformAdmin: - create-any @@ -20,7 +20,7 @@ Coderepo: type: string teamId: $ref: definitions.yaml#/idName - label: + name: $ref: 'definitions.yaml#/idName' gitService: type: string @@ -37,7 +37,7 @@ Coderepo: secret: type: string required: - - label + - name - gitService - repositoryUrl type: object diff --git a/src/openapi/secret.yaml b/src/openapi/secret.yaml deleted file mode 100644 index 086fda894..000000000 --- a/src/openapi/secret.yaml +++ /dev/null @@ -1,110 +0,0 @@ -Secret: - x-acl: - platformAdmin: - - create-any - - read-any - - update-any - - delete-any - teamAdmin: - - create - - read - - update - - delete - teamMember: - - create - - read - - update - - delete - # additionalProperties: false - properties: - id: - # readOnly: true - type: string - name: - type: string - # A lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com') - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - description: A secret name - namespace: - $ref: definitions.yaml#/idName - description: A Kubernetes namespace. - x-acl: - platformAdmin: - - create-any - - read-any - - update-any - - delete-any - teamAdmin: [] - teamMember: [] - secret: - title: Type - description: Type of secret used in Otomi. - oneOf: - - $ref: '#/SecretGeneric' - - $ref: '#/SecretDockerRegistry' - - $ref: '#/SecretTLS' - type: object - required: - - name - - secret - type: object - x-externalDocsPath: docs/for-devs/console/secrets - -SecretGeneric: - title: Generic - description: The secret's entries listed here must exist in the corresponding vault secret. - properties: - type: - type: string - enum: - - generic - default: generic - entries: - type: array - items: - description: A property in a vault secret. - # a valid k8s secret key must consist of alphanumeric characters, '-', '_' or '.' (e.g. 'key.name', or 'KEY_NAME', or 'key-name' - pattern: ^[-._a-zA-Z0-9]+$ - type: string - uniqueItems: true - minItems: 1 - type: object - required: - - type - - entries - -SecretDockerRegistry: - title: Docker registry - description: A secret in vault with one property ".dockerconfigjson" that is an encoded dockerconfigjson blob. - properties: - type: - type: string - enum: - - docker-registry - default: docker-registry - type: object - required: - - type - -SecretTLS: - title: TLS - description: The TLS fields must exist in the corresponding vault secret. - properties: - type: - type: string - enum: - - tls - default: tls - crt: - type: string - default: tls.crt - key: - type: string - default: tls.key - ca: - type: string - default: ca.crt - type: object - required: - - type - - crt diff --git a/src/openapi/settings.yaml b/src/openapi/settings.yaml index 2efbdf738..5f60aa681 100644 --- a/src/openapi/settings.yaml +++ b/src/openapi/settings.yaml @@ -326,6 +326,25 @@ Settings: nodeSelector: $ref: definitions.yaml#/labels description: 'One or more label/value pairs of one or more nodes. This will enforce scheduling of all platform services on these nodes.' + #TODO move this one out of otomi and in to versions + version: + title: Version + default: latest + description: | + Set the version to a valid release found in the linode/apl-core Github repository. + type: string + required: + - version + x-externalDocsPath: docs/for-ops/console/settings/platform + x-acl: + platformAdmin: [read-any, update-any] + teamAdmin: [read] + teamMember: [read] + versions: + title: Platform + type: object + additionalProperties: false + properties: version: title: Version default: latest diff --git a/src/openapi/settingsinfo.yaml b/src/openapi/settingsinfo.yaml index 7261b2d9f..876a2aadc 100644 --- a/src/openapi/settingsinfo.yaml +++ b/src/openapi/settingsinfo.yaml @@ -33,6 +33,10 @@ SettingsInfo: hasExternalIDP: type: boolean default: false + smtp: + properties: + smarthost: + type: string ingressClassNames: description: Ingress class names that are used by the cluster. items: diff --git a/src/otomi-models.ts b/src/otomi-models.ts index 407edfe20..ea1c0b91d 100644 --- a/src/otomi-models.ts +++ b/src/otomi-models.ts @@ -2,13 +2,13 @@ import { Request } from 'express' import { JSONSchema4 } from 'json-schema' import { components, external, operations, paths } from 'src/generated-schema' import OtomiStack from 'src/otomi-stack' + export type App = components['schemas']['App'] export type AppList = components['schemas']['AppList'] export type Backup = components['schemas']['Backup'] export type Kubecfg = components['schemas']['Kubecfg'] export type K8sService = components['schemas']['K8sService'] export type Netpol = components['schemas']['Netpol'] -export type Secret = components['schemas']['Secret'] & { teamId?: string } export type SealedSecret = components['schemas']['SealedSecret'] & { teamId?: string } export type SealedSecretsKeys = components['schemas']['SealedSecretsKeys'] export type K8sSecret = components['schemas']['K8sSecret'] @@ -27,7 +27,7 @@ export type Workload = components['schemas']['Workload'] export type WorkloadValues = components['schemas']['WorkloadValues'] export type User = components['schemas']['User'] export type Project = components['schemas']['Project'] -export type Coderepo = components['schemas']['Coderepo'] +export type CodeRepo = components['schemas']['CodeRepo'] export type Build = components['schemas']['Build'] export type Policy = components['schemas']['Policy'] export type Policies = components['schemas']['Policies'] @@ -37,9 +37,12 @@ export type TeamAuthz = components['schemas']['TeamAuthz'] export type Alerts = Settings['alerts'] export type Cluster = Settings['cluster'] export type Dns = Settings['dns'] +export type Ingress = Settings['ingress'] +export type Smtp = Settings['smtp'] export type Kms = Settings['kms'] export type Oidc = Settings['oidc'] export type Otomi = Settings['otomi'] +export type Versions = Settings['versions'] export interface OpenApiRequest extends Request { operationDoc: { @@ -145,3 +148,36 @@ export interface Core { teamConfig: Record version: number } + +export interface Repo { + apps: App[] + alerts: Alerts + cluster: Cluster + databases: Record + dns: Dns + ingress: Ingress + kms: Kms + obj: Record + oidc: Oidc + otomi: Otomi + platformBackups: Record + smtp: Smtp + users: User[] + versions: Versions + teamConfig: Record +} + +export interface TeamConfig { + apps: App[] + backups: Backup[] + builds: Build[] + codeRepos: CodeRepo[] + netpols: Netpol[] + policies: Policies + projects: Project[] + sealedsecrets: SealedSecret[] + services: Service[] + settings: Team + workloads: Workload[] + workloadValues: WorkloadValues[] +} diff --git a/src/otomi-stack.test.ts b/src/otomi-stack.test.ts index 75e567aab..76ca61a85 100644 --- a/src/otomi-stack.test.ts +++ b/src/otomi-stack.test.ts @@ -1,10 +1,12 @@ /* eslint-disable @typescript-eslint/no-empty-function */ -import { mockDeep } from 'jest-mock-extended' -import { App, Coderepo, User, Workload } from 'src/otomi-models' +import { App, CodeRepo, Policies, Team, TeamConfig, User } from 'src/otomi-models' import OtomiStack from 'src/otomi-stack' -import { Repo } from 'src/repo' -import { loadSpec } from './app' +import { mockDeep } from 'jest-mock-extended' import { PublicUrlExists } from './error' +import { loadSpec } from './app' +import { Git } from './git' +import { RepoService } from './services/RepoService' +import { TeamConfigService } from './services/TeamConfigService' jest.mock('src/utils', () => { const originalModule = jest.requireActual('src/utils') @@ -17,6 +19,16 @@ jest.mock('src/utils', () => { } }) +jest.mock('src/utils/userUtils', () => { + const originalModule = jest.requireActual('src/utils/userUtils') + + return { + __esModule: true, + ...originalModule, + getKeycloakUsers: jest.fn().mockResolvedValue([]), + } +}) + beforeAll(async () => { jest.spyOn(console, 'log').mockImplementation(() => {}) jest.spyOn(console, 'debug').mockImplementation(() => {}) @@ -28,61 +40,99 @@ beforeAll(async () => { describe('Data validation', () => { let otomiStack: OtomiStack + const teamId = 'aa' + let mockRepoService: jest.Mocked + let mockTeamConfigService: jest.Mocked beforeEach(async () => { otomiStack = new OtomiStack() await otomiStack.init() - otomiStack.repo = mockDeep() + otomiStack.git = mockDeep() + mockRepoService = mockDeep() + otomiStack.repoService = mockRepoService + + // Mock TeamConfigService + mockTeamConfigService = mockDeep() + + // Mocking getServices() to return a list of services + mockTeamConfigService.getServices.mockReturnValue([ + { + name: 'svc', + ingress: { domain: 'a.com', subdomain: 'b' }, + }, + { name: 'svc', ingress: { domain: 'a.com', subdomain: 'b', paths: ['/test/'] } }, + ]) + + // Ensure getTeamConfigService() returns our mocked TeamConfigService + mockRepoService.getTeamConfigService.mockReturnValue(mockTeamConfigService) jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doRepoDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() }) - test('should throw exception on duplicated domain', async () => { + test('should throw exception on duplicated domain', () => { const svc = { name: 'svc', ingress: { domain: 'a.com', subdomain: 'b' } } - const svc1 = { ...svc } - await otomiStack.createService('aa', svc) - expect(() => otomiStack.checkPublicUrlInUse(svc1)).toThrow(new PublicUrlExists()) + expect(() => otomiStack.checkPublicUrlInUse(teamId, svc)).toThrow(new PublicUrlExists()) }) - test('should throw exception on duplicated url with path', async () => { + test('should throw exception on duplicated url with path', () => { const svc = { name: 'svc', ingress: { domain: 'a.com', subdomain: 'b', paths: ['/test/'] } } - await otomiStack.createService('aa', svc) - expect(() => otomiStack.checkPublicUrlInUse(svc)).toThrow(new PublicUrlExists()) + expect(() => otomiStack.checkPublicUrlInUse(teamId, svc)).toThrow(new PublicUrlExists()) }) - test('should not throw exception on unique url', async () => { - const svc1 = { name: 'svc', ingress: { domain: 'a.com', subdomain: 'b', paths: ['/test/'] } } - await otomiStack.createService('aa', svc1) - const svc2 = { name: 'svc', ingress: { domain: 'a.com', subdomain: 'b' } } + test('should not throw exception on unique url', () => { const svc3 = { name: 'svc', ingress: { domain: 'a.com', subdomain: 'b', paths: ['/bla'] } } - expect(() => otomiStack.checkPublicUrlInUse(svc2)).not.toThrow() - expect(() => otomiStack.checkPublicUrlInUse(svc3)).not.toThrow() + expect(() => otomiStack.checkPublicUrlInUse(teamId, svc3)).not.toThrow() }) - test('should not throw exception when of type cluster', async () => { + test('should not throw exception when of type cluster', () => { const svc = { name: 'svc', ingress: { type: 'cluster' } } - // @ts-ignore - await otomiStack.createService('aa', svc) - expect(() => otomiStack.checkPublicUrlInUse(svc)).not.toThrow() + expect(() => otomiStack.checkPublicUrlInUse(teamId, svc)).not.toThrow() }) - test('should not throw exception when editing', async () => { + test('should not throw exception when editing', () => { const svc = { id: 'x1', name: 'svc', ingress: { domain: 'a.com', subdomain: 'b', paths: ['/test/'] } } - await otomiStack.createService('aa', svc) const svc1 = { id: 'x1', name: 'svc', ingress: { domain: 'a.com', subdomain: 'c' } } - expect(() => otomiStack.checkPublicUrlInUse(svc1)).not.toThrow() + expect(() => otomiStack.checkPublicUrlInUse(teamId, svc1)).not.toThrow() }) test('should create a password when password is not specified', async () => { - const createItemSpy = jest.spyOn(otomiStack.db, 'createItem') - await otomiStack.createTeam({ name: 'test' }) + const createItemSpy = jest.spyOn(otomiStack.repoService, 'createTeamConfig').mockReturnValue({ + builds: [], + codeRepos: [], + workloads: [], + services: [], + sealedsecrets: [], + backups: [], + projects: [], + netpols: [], + settings: {} as Team, + apps: [], + policies: {} as Policies, + workloadValues: [], + } as TeamConfig) + await otomiStack.createTeam({ name: 'test' }, false) expect(createItemSpy.mock.calls[0][1].password).not.toEqual('') createItemSpy.mockRestore() }) test('should not create a password when password is specified', async () => { - const createItemSpy = jest.spyOn(otomiStack.db, 'createItem') + const createItemSpy = jest.spyOn(otomiStack.repoService, 'createTeamConfig').mockReturnValue({ + builds: [], + codeRepos: [], + workloads: [], + services: [], + sealedsecrets: [], + backups: [], + projects: [], + netpols: [], + settings: {} as Team, + apps: [], + policies: {} as Policies, + workloadValues: [], + } as TeamConfig) const myPassword = 'someAwesomePassword' - await otomiStack.createTeam({ name: 'test', password: myPassword }) + await otomiStack.createTeam({ name: 'test', password: myPassword }, false) expect(createItemSpy.mock.calls[0][1].password).toEqual(myPassword) createItemSpy.mockRestore() }) @@ -92,10 +142,13 @@ describe('Work with values', () => { let otomiStack: OtomiStack beforeEach(async () => { otomiStack = new OtomiStack() + jest.spyOn(otomiStack, 'transformApps').mockReturnValue([]) + await otomiStack.init() - otomiStack.repo = new Repo('./test', undefined, 'someuser', 'some@ema.il', undefined, undefined) + otomiStack.git = new Git('./test', undefined, 'someuser', 'some@ema.il', undefined, undefined) jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() - jest.spyOn(otomiStack, 'loadApp').mockResolvedValue() + jest.spyOn(otomiStack, 'doRepoDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() }) test('can load from configuration to database and back', async () => { @@ -108,35 +161,10 @@ describe('Workload values', () => { beforeEach(async () => { otomiStack = new OtomiStack() await otomiStack.init() - otomiStack.repo = new Repo('./test', undefined, 'someuser', 'some@ema.il', undefined, undefined) + otomiStack.git = new Git('./test', undefined, 'someuser', 'some@ema.il', undefined, undefined) jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() - }) - - test('can load workload values (empty dict)', async () => { - const w: Workload = { id: '1', teamId: '2', name: 'name', url: 'https://test.local' } - - otomiStack.repo.fileExists = jest.fn().mockReturnValue(true) - otomiStack.repo.readFile = jest.fn().mockReturnValue({}) - const res = await otomiStack.loadWorkloadValues(w) - expect(res).toEqual({ id: '1', teamId: '2', name: 'name', values: {} }) - }) - - test('can load workload values (dict)', async () => { - const w: Workload = { id: '1', teamId: '2', name: 'name', url: 'https://test.local' } - - otomiStack.repo.fileExists = jest.fn().mockReturnValue(true) - otomiStack.repo.readFile = jest.fn().mockReturnValue({ values: 'test: 1' }) - const res = await otomiStack.loadWorkloadValues(w) - expect(res).toEqual({ id: '1', teamId: '2', name: 'name', values: { test: 1 } }) - }) - - test('can load workload values (empty string)', async () => { - const w: Workload = { id: '1', teamId: '2', name: 'name', url: 'https://test.local' } - - otomiStack.repo.fileExists = jest.fn().mockReturnValue(true) - otomiStack.repo.readFile = jest.fn().mockReturnValue({ values: '' }) - const res = await otomiStack.loadWorkloadValues(w) - expect(res).toEqual({ id: '1', teamId: '2', name: 'name', values: {} }) + jest.spyOn(otomiStack, 'doRepoDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() }) test('returns filtered apps if App array is submitted isPreinstalled flag is true', () => { @@ -181,21 +209,20 @@ describe('Users tests', () => { beforeEach(async () => { otomiStack = new OtomiStack() await otomiStack.init() - otomiStack.repo = mockDeep() - otomiStack.loadApps = jest.fn().mockResolvedValue([]) - const dbMock = mockDeep() - dbMock.getItem.mockImplementation((collection, query) => { - return [defaultPlatformAdmin, anyPlatformAdmin].find((user) => user.id === query.id) - }) - dbMock.deleteItem.mockReturnValue() - - otomiStack.db = dbMock + otomiStack.git = mockDeep() jest.spyOn(otomiStack, 'getSettings').mockReturnValue({ cluster: { domainSuffix }, }) - jest.spyOn(otomiStack, 'saveUsers').mockResolvedValue() + jest.spyOn(otomiStack, 'saveUser').mockResolvedValue() jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doRepoDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'transformApps').mockReturnValue([]) + jest.spyOn(otomiStack, 'getApp').mockReturnValue({ id: 'keycloak' }) + await otomiStack.initRepo() + await otomiStack.createUser(defaultPlatformAdmin) + await otomiStack.createUser(anyPlatformAdmin) }) afterEach(() => { @@ -277,13 +304,31 @@ describe('Users tests', () => { describe('Code repositories tests', () => { let otomiStack: OtomiStack + let teamConfigService: TeamConfigService beforeEach(async () => { otomiStack = new OtomiStack() + jest.spyOn(otomiStack, 'transformApps').mockReturnValue([]) await otomiStack.init() - const dbMock = mockDeep() - dbMock.deleteItem.mockReturnValue() - otomiStack.db = dbMock + await otomiStack.initRepo() + otomiStack.git = mockDeep() + + try { + otomiStack.repoService.createTeamConfig('demo', { name: 'demo' }) + } catch { + // ignore + } + teamConfigService = otomiStack.repoService.getTeamConfigService('demo') + const codeRepo = { + id: '1', + teamId: 'demo', + name: 'code-1', + gitService: 'gitea', + repositoryUrl: 'https://gitea.test.com', + } as CodeRepo + + jest.spyOn(teamConfigService, 'getCodeRepo').mockReturnValue(codeRepo) + jest.spyOn(otomiStack.git, 'deleteConfig').mockResolvedValue() }) afterEach(() => { @@ -291,397 +336,347 @@ describe('Code repositories tests', () => { }) test('should create an internal code repository', async () => { - const createItemSpy = jest.spyOn(otomiStack.db, 'createItem').mockReturnValue({ + const createItemSpy = jest.spyOn(teamConfigService, 'createCodeRepo').mockReturnValue({ teamId: 'demo', - label: 'code-1', + name: 'code-1', gitService: 'gitea', repositoryUrl: 'https://gitea.test.com', - } as Coderepo) + } as CodeRepo) - const saveTeamCodereposSpy = jest.spyOn(otomiStack, 'saveTeamCoderepos').mockResolvedValue() - const doDeploymentSpy = jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() + const saveTeamCodeReposSpy = jest.spyOn(otomiStack, 'saveTeamCodeRepo').mockResolvedValue() + const doDeploymentSpy = jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() - const coderepo = await otomiStack.createCoderepo('demo', { - label: 'code-1', + const codeRepo = await otomiStack.createCodeRepo('demo', { + name: 'code-1', gitService: 'gitea', repositoryUrl: 'https://gitea.test.com', }) - expect(coderepo).toEqual({ + expect(codeRepo).toEqual({ teamId: 'demo', - label: 'code-1', + name: 'code-1', gitService: 'gitea', repositoryUrl: 'https://gitea.test.com', }) - expect(createItemSpy).toHaveBeenCalledWith( - 'coderepos', - { - teamId: 'demo', - label: 'code-1', - gitService: 'gitea', - repositoryUrl: 'https://gitea.test.com', - }, - { teamId: 'demo', label: 'code-1' }, - ) - expect(saveTeamCodereposSpy).toHaveBeenCalledWith('demo') - expect(doDeploymentSpy).toHaveBeenCalledWith(['coderepos']) + expect(createItemSpy).toHaveBeenCalledWith({ + teamId: 'demo', + name: 'code-1', + gitService: 'gitea', + repositoryUrl: 'https://gitea.test.com', + }) + expect(saveTeamCodeReposSpy).toHaveBeenCalledWith('demo', codeRepo) + expect(doDeploymentSpy).toHaveBeenCalledWith('demo', expect.any(Function), false) createItemSpy.mockRestore() - saveTeamCodereposSpy.mockRestore() + saveTeamCodeReposSpy.mockRestore() doDeploymentSpy.mockRestore() }) test('should get an existing internal code repository', () => { - const coderepo = { + const codeRepo = { id: '1', teamId: 'demo', - label: 'code-1', + name: 'code-1', gitService: 'gitea', repositoryUrl: 'https://gitea.test.com', - } as Coderepo + } as CodeRepo - jest.spyOn(otomiStack.db, 'getItem').mockReturnValue(coderepo) + jest.spyOn(teamConfigService, 'getCodeRepo').mockReturnValue(codeRepo) - const result = otomiStack.getCoderepo('1') - expect(result).toEqual(coderepo) + const result = otomiStack.getCodeRepo('demo', '1') + expect(result).toEqual(codeRepo) }) test('should edit an existing internal code repository', async () => { - const updateItemSpy = jest.spyOn(otomiStack.db, 'updateItem').mockReturnValue({ + const updateItemSpy = jest.spyOn(teamConfigService, 'updateCodeRepo').mockReturnValue({ id: '1', teamId: 'demo', - label: 'code-1-updated', + name: 'code-1-updated', gitService: 'gitea', repositoryUrl: 'https://gitea.test.com', - } as Coderepo) + } as CodeRepo) - const saveTeamCodereposSpy = jest.spyOn(otomiStack, 'saveTeamCoderepos').mockResolvedValue() - const doDeploymentSpy = jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() + const saveTeamCodeReposSpy = jest.spyOn(otomiStack, 'saveTeamCodeRepo').mockResolvedValue() + const doDeploymentSpy = jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() - const coderepo = await otomiStack.editCoderepo('1', { + const codeRepo = await otomiStack.editCodeRepo('demo', '1', { teamId: 'demo', - label: 'code-1-updated', + name: 'code-1-updated', gitService: 'gitea', repositoryUrl: 'https://gitea.test.com', }) - expect(coderepo).toEqual({ + expect(codeRepo).toEqual({ id: '1', teamId: 'demo', - label: 'code-1-updated', + name: 'code-1-updated', gitService: 'gitea', repositoryUrl: 'https://gitea.test.com', }) - expect(updateItemSpy).toHaveBeenCalledWith( - 'coderepos', - { - teamId: 'demo', - label: 'code-1-updated', - gitService: 'gitea', - repositoryUrl: 'https://gitea.test.com', - }, - { id: '1' }, - ) - expect(saveTeamCodereposSpy).toHaveBeenCalledWith('demo') - expect(doDeploymentSpy).toHaveBeenCalledWith(['coderepos']) + expect(updateItemSpy).toHaveBeenCalledWith('1', { + teamId: 'demo', + name: 'code-1-updated', + gitService: 'gitea', + repositoryUrl: 'https://gitea.test.com', + }) + expect(saveTeamCodeReposSpy).toHaveBeenCalledWith('demo', codeRepo) + expect(doDeploymentSpy).toHaveBeenCalledWith('demo', expect.any(Function), false) updateItemSpy.mockRestore() - saveTeamCodereposSpy.mockRestore() + saveTeamCodeReposSpy.mockRestore() doDeploymentSpy.mockRestore() }) test('should delete an existing internal code repository', async () => { - const coderepo = { + const codeRepo = { id: '1', teamId: 'demo', - label: 'code-1', + name: 'code-1', gitService: 'gitea', repositoryUrl: 'https://gitea.test.com', - } as Coderepo + } as CodeRepo - jest.spyOn(otomiStack, 'getCoderepo').mockReturnValue(coderepo) - const deleteItemSpy = jest.spyOn(otomiStack.db, 'deleteItem').mockResolvedValue({} as never) - const saveTeamCodereposSpy = jest.spyOn(otomiStack, 'saveTeamCoderepos').mockResolvedValue() - const doDeploymentSpy = jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'getCodeRepo').mockReturnValue(codeRepo) + const deleteItemSpy = jest.spyOn(teamConfigService, 'deleteCodeRepo').mockResolvedValue({} as never) + const doDeploymentSpy = jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() - await otomiStack.deleteCoderepo('1') + await otomiStack.deleteCodeRepo('demo', '1') - expect(deleteItemSpy).toHaveBeenCalledWith('coderepos', { id: '1' }) - expect(saveTeamCodereposSpy).toHaveBeenCalledWith('demo') - expect(doDeploymentSpy).toHaveBeenCalledWith(['coderepos']) + expect(deleteItemSpy).toHaveBeenCalledWith('1') + expect(doDeploymentSpy).toHaveBeenCalledWith('demo', expect.any(Function), false) deleteItemSpy.mockRestore() - saveTeamCodereposSpy.mockRestore() doDeploymentSpy.mockRestore() }) test('should create an external public code repository', async () => { - const createItemSpy = jest.spyOn(otomiStack.db, 'createItem').mockReturnValue({ + const createItemSpy = jest.spyOn(teamConfigService, 'createCodeRepo').mockReturnValue({ teamId: 'demo', - label: 'code-1', + name: 'code-1', gitService: 'github', repositoryUrl: 'https://github.test.com', - } as Coderepo) + } as CodeRepo) - const saveTeamCodereposSpy = jest.spyOn(otomiStack, 'saveTeamCoderepos').mockResolvedValue() - const doDeploymentSpy = jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() + const saveTeamCodeReposSpy = jest.spyOn(otomiStack, 'saveTeamCodeRepo').mockResolvedValue() + const doDeploymentSpy = jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() - const coderepo = await otomiStack.createCoderepo('demo', { - label: 'code-1', + const codeRepo = await otomiStack.createCodeRepo('demo', { + name: 'code-1', gitService: 'github', repositoryUrl: 'https://github.test.com', }) - expect(coderepo).toEqual({ + expect(codeRepo).toEqual({ teamId: 'demo', - label: 'code-1', + name: 'code-1', gitService: 'github', repositoryUrl: 'https://github.test.com', }) - expect(createItemSpy).toHaveBeenCalledWith( - 'coderepos', - { - teamId: 'demo', - label: 'code-1', - gitService: 'github', - repositoryUrl: 'https://github.test.com', - }, - { teamId: 'demo', label: 'code-1' }, - ) - expect(saveTeamCodereposSpy).toHaveBeenCalledWith('demo') - expect(doDeploymentSpy).toHaveBeenCalledWith(['coderepos']) + expect(createItemSpy).toHaveBeenCalledWith({ + teamId: 'demo', + name: 'code-1', + gitService: 'github', + repositoryUrl: 'https://github.test.com', + }) + expect(saveTeamCodeReposSpy).toHaveBeenCalledWith('demo', codeRepo) + expect(doDeploymentSpy).toHaveBeenCalledWith('demo', expect.any(Function), false) createItemSpy.mockRestore() - saveTeamCodereposSpy.mockRestore() + saveTeamCodeReposSpy.mockRestore() doDeploymentSpy.mockRestore() }) test('should get an existing external public code repository', () => { - const coderepo = { + const codeRepo = { id: '1', teamId: 'demo', - label: 'code-1', + name: 'code-1', gitService: 'github', repositoryUrl: 'https://github.test.com', - } as Coderepo + } as CodeRepo - jest.spyOn(otomiStack.db, 'getItem').mockReturnValue(coderepo) + jest.spyOn(teamConfigService, 'getCodeRepo').mockReturnValue(codeRepo) - const result = otomiStack.getCoderepo('1') - expect(result).toEqual(coderepo) + const result = otomiStack.getCodeRepo('demo', '1') + expect(result).toEqual(codeRepo) }) test('should edit an existing external public code repository', async () => { - const updateItemSpy = jest.spyOn(otomiStack.db, 'updateItem').mockReturnValue({ + const updateItemSpy = jest.spyOn(teamConfigService, 'updateCodeRepo').mockReturnValue({ id: '1', teamId: 'demo', - label: 'code-1-updated', + name: 'code-1-updated', gitService: 'github', repositoryUrl: 'https://github.test.com', - } as Coderepo) + } as CodeRepo) - const saveTeamCodereposSpy = jest.spyOn(otomiStack, 'saveTeamCoderepos').mockResolvedValue() - const doDeploymentSpy = jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() + const saveTeamCodeReposSpy = jest.spyOn(otomiStack, 'saveTeamCodeRepo').mockResolvedValue() + const doDeploymentSpy = jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() - const coderepo = await otomiStack.editCoderepo('1', { + const codeRepo = await otomiStack.editCodeRepo('demo', '1', { teamId: 'demo', - label: 'code-1-updated', + name: 'code-1-updated', gitService: 'github', repositoryUrl: 'https://github.test.com', }) - expect(coderepo).toEqual({ + expect(codeRepo).toEqual({ id: '1', teamId: 'demo', - label: 'code-1-updated', + name: 'code-1-updated', gitService: 'github', repositoryUrl: 'https://github.test.com', }) - expect(updateItemSpy).toHaveBeenCalledWith( - 'coderepos', - { - teamId: 'demo', - label: 'code-1-updated', - gitService: 'github', - repositoryUrl: 'https://github.test.com', - }, - { id: '1' }, - ) - expect(saveTeamCodereposSpy).toHaveBeenCalledWith('demo') - expect(doDeploymentSpy).toHaveBeenCalledWith(['coderepos']) + expect(updateItemSpy).toHaveBeenCalledWith('1', { + teamId: 'demo', + name: 'code-1-updated', + gitService: 'github', + repositoryUrl: 'https://github.test.com', + }) + expect(saveTeamCodeReposSpy).toHaveBeenCalledWith('demo', codeRepo) + expect(doDeploymentSpy).toHaveBeenCalledWith('demo', expect.any(Function), false) updateItemSpy.mockRestore() - saveTeamCodereposSpy.mockRestore() + saveTeamCodeReposSpy.mockRestore() doDeploymentSpy.mockRestore() }) test('should delete an existing external public code repository', async () => { - const coderepo = { + const codeRepo = { id: '1', teamId: 'demo', - label: 'code-1', + name: 'code-1', gitService: 'github', repositoryUrl: 'https://github.test.com', - } as Coderepo + } as CodeRepo - jest.spyOn(otomiStack, 'getCoderepo').mockReturnValue(coderepo) - const deleteItemSpy = jest.spyOn(otomiStack.db, 'deleteItem').mockResolvedValue({} as never) - const saveTeamCodereposSpy = jest.spyOn(otomiStack, 'saveTeamCoderepos').mockResolvedValue() - const doDeploymentSpy = jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'getCodeRepo').mockReturnValue(codeRepo) + const deleteItemSpy = jest.spyOn(teamConfigService, 'deleteCodeRepo').mockResolvedValue({} as never) + const doDeploymentSpy = jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() - await otomiStack.deleteCoderepo('1') + await otomiStack.deleteCodeRepo('demo', '1') - expect(deleteItemSpy).toHaveBeenCalledWith('coderepos', { id: '1' }) - expect(saveTeamCodereposSpy).toHaveBeenCalledWith('demo') - expect(doDeploymentSpy).toHaveBeenCalledWith(['coderepos']) + expect(deleteItemSpy).toHaveBeenCalledWith('1') + expect(doDeploymentSpy).toHaveBeenCalledWith('demo', expect.any(Function), false) deleteItemSpy.mockRestore() - saveTeamCodereposSpy.mockRestore() doDeploymentSpy.mockRestore() }) test('should create an external private code repository', async () => { - const createItemSpy = jest.spyOn(otomiStack.db, 'createItem').mockReturnValue({ + const createItemSpy = jest.spyOn(teamConfigService, 'createCodeRepo').mockReturnValue({ teamId: 'demo', - label: 'code-1', + name: 'code-1', gitService: 'github', repositoryUrl: 'https://github.test.com', private: true, secret: 'test', - } as Coderepo) + } as CodeRepo) - const saveTeamCodereposSpy = jest.spyOn(otomiStack, 'saveTeamCoderepos').mockResolvedValue() - const doDeploymentSpy = jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() + const saveTeamCodeReposSpy = jest.spyOn(otomiStack, 'saveTeamCodeRepo').mockResolvedValue() + const doDeploymentSpy = jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() - const coderepo = await otomiStack.createCoderepo('demo', { - label: 'code-1', + const codeRepo = await otomiStack.createCodeRepo('demo', { + name: 'code-1', gitService: 'github', repositoryUrl: 'https://github.test.com', private: true, secret: 'test', }) - expect(coderepo).toEqual({ + expect(codeRepo).toEqual({ teamId: 'demo', - label: 'code-1', + name: 'code-1', gitService: 'github', repositoryUrl: 'https://github.test.com', private: true, secret: 'test', }) - expect(createItemSpy).toHaveBeenCalledWith( - 'coderepos', - { - teamId: 'demo', - label: 'code-1', - gitService: 'github', - repositoryUrl: 'https://github.test.com', - private: true, - secret: 'test', - }, - { teamId: 'demo', label: 'code-1' }, - ) - expect(saveTeamCodereposSpy).toHaveBeenCalledWith('demo') - expect(doDeploymentSpy).toHaveBeenCalledWith(['coderepos']) - - createItemSpy.mockRestore() - saveTeamCodereposSpy.mockRestore() - doDeploymentSpy.mockRestore() - }) - - test('should get an existing external private code repository', () => { - const coderepo = { - id: '1', + expect(createItemSpy).toHaveBeenCalledWith({ teamId: 'demo', - label: 'code-1', + name: 'code-1', gitService: 'github', repositoryUrl: 'https://github.test.com', private: true, secret: 'test', - } as Coderepo - - jest.spyOn(otomiStack.db, 'getItem').mockReturnValue(coderepo) + }) + expect(saveTeamCodeReposSpy).toHaveBeenCalledWith('demo', codeRepo) + expect(doDeploymentSpy).toHaveBeenCalledWith('demo', expect.any(Function), false) - const result = otomiStack.getCoderepo('1') - expect(result).toEqual(coderepo) + createItemSpy.mockRestore() + saveTeamCodeReposSpy.mockRestore() + doDeploymentSpy.mockRestore() }) test('should edit an existing external private code repository', async () => { - const updateItemSpy = jest.spyOn(otomiStack.db, 'updateItem').mockReturnValue({ + const updateItemSpy = jest.spyOn(teamConfigService, 'updateCodeRepo').mockReturnValue({ id: '1', teamId: 'demo', - label: 'code-1-updated', + name: 'code-1-updated', gitService: 'github', repositoryUrl: 'https://github.test.com', private: true, secret: 'test', - } as Coderepo) + } as CodeRepo) - const saveTeamCodereposSpy = jest.spyOn(otomiStack, 'saveTeamCoderepos').mockResolvedValue() - const doDeploymentSpy = jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() + const saveTeamCodeReposSpy = jest.spyOn(otomiStack, 'saveTeamCodeRepo').mockResolvedValue() + const doDeploymentSpy = jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() - const coderepo = await otomiStack.editCoderepo('1', { + const codeRepo = await otomiStack.editCodeRepo('demo', '1', { teamId: 'demo', - label: 'code-1-updated', + name: 'code-1-updated', gitService: 'github', repositoryUrl: 'https://github.test.com', private: true, secret: 'test', }) - expect(coderepo).toEqual({ + expect(codeRepo).toEqual({ id: '1', teamId: 'demo', - label: 'code-1-updated', + name: 'code-1-updated', gitService: 'github', repositoryUrl: 'https://github.test.com', private: true, secret: 'test', }) - expect(updateItemSpy).toHaveBeenCalledWith( - 'coderepos', - { - teamId: 'demo', - label: 'code-1-updated', - gitService: 'github', - repositoryUrl: 'https://github.test.com', - private: true, - secret: 'test', - }, - { id: '1' }, - ) - expect(saveTeamCodereposSpy).toHaveBeenCalledWith('demo') - expect(doDeploymentSpy).toHaveBeenCalledWith(['coderepos']) + expect(updateItemSpy).toHaveBeenCalledWith('1', { + teamId: 'demo', + name: 'code-1-updated', + gitService: 'github', + repositoryUrl: 'https://github.test.com', + private: true, + secret: 'test', + }) + expect(saveTeamCodeReposSpy).toHaveBeenCalledWith('demo', codeRepo) + expect(doDeploymentSpy).toHaveBeenCalledWith('demo', expect.any(Function), false) updateItemSpy.mockRestore() - saveTeamCodereposSpy.mockRestore() + saveTeamCodeReposSpy.mockRestore() doDeploymentSpy.mockRestore() }) test('should delete an existing external private code repository', async () => { - const coderepo = { + const codeRepo = { id: '1', teamId: 'demo', - label: 'code-1', + name: 'code-1', gitService: 'github', repositoryUrl: 'https://github.test.com', private: true, secret: 'test', - } as Coderepo + } as CodeRepo - jest.spyOn(otomiStack, 'getCoderepo').mockReturnValue(coderepo) - const deleteItemSpy = jest.spyOn(otomiStack.db, 'deleteItem').mockResolvedValue({} as never) - const saveTeamCodereposSpy = jest.spyOn(otomiStack, 'saveTeamCoderepos').mockResolvedValue() - const doDeploymentSpy = jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'getCodeRepo').mockReturnValue(codeRepo) + const deleteItemSpy = jest.spyOn(teamConfigService, 'deleteCodeRepo').mockResolvedValue({} as never) + const doDeploymentSpy = jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() - await otomiStack.deleteCoderepo('1') + await otomiStack.deleteCodeRepo('demo', '1') - expect(deleteItemSpy).toHaveBeenCalledWith('coderepos', { id: '1' }) - expect(saveTeamCodereposSpy).toHaveBeenCalledWith('demo') - expect(doDeploymentSpy).toHaveBeenCalledWith(['coderepos']) + expect(deleteItemSpy).toHaveBeenCalledWith('1') + expect(doDeploymentSpy).toHaveBeenCalledWith('demo', expect.any(Function), false) deleteItemSpy.mockRestore() - saveTeamCodereposSpy.mockRestore() doDeploymentSpy.mockRestore() }) }) diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index dcac044ab..5c0d74ac1 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -7,17 +7,17 @@ import { getRegions, ObjectStorageKeyRegions } from '@linode/api-v4' import { emptyDir, existsSync, pathExists, rmSync, unlink } from 'fs-extra' import { readdir, readFile, writeFile } from 'fs/promises' import { generate as generatePassword } from 'generate-password' -import { cloneDeep, filter, get, isArray, isEmpty, map, omit, pick, set, unset } from 'lodash' +import { cloneDeep, filter, isArray, isEmpty, map, mapValues, omit, pick, unset } from 'lodash' import { getAppList, getAppSchema, getSpec } from 'src/app' -import Db from 'src/db' import { AlreadyExists, HttpError, OtomiError, PublicUrlExists, ValidationError } from 'src/error' +import getRepo, { Git } from 'src/git' import { cleanAllSessions, cleanSession, DbMessage, getIo, getSessionStack } from 'src/middleware' import { App, Backup, Build, Cloudtty, - Coderepo, + CodeRepo, Core, K8sService, Netpol, @@ -25,8 +25,8 @@ import { Policies, Policy, Project, + Repo, SealedSecret, - Secret, Service, Session, SessionUser, @@ -39,8 +39,7 @@ import { Workload, WorkloadValues, } from 'src/otomi-models' -import getRepo, { Repo } from 'src/repo' -import { arrayToObject, getServiceUrl, getValuesSchema, objectToArray, removeBlankAttributes } from 'src/utils' +import { arrayToObject, getServiceUrl, getValuesSchema, removeBlankAttributes } from 'src/utils' import { cleanEnv, CUSTOM_ROOT_CA, @@ -72,13 +71,16 @@ import { k8sdelete, watchPodUntilRunning, } from './k8s_operations' +import { getFileMaps, loadValues } from './repo' +import { RepoService } from './services/RepoService' +import { TeamConfigService } from './services/TeamConfigService' import { validateBackupFields } from './utils/backupUtils' import { getGiteaRepoUrls, normalizeRepoUrl, testPrivateRepoConnect, testPublicRepoConnect, -} from './utils/coderepoUtils' +} from './utils/codeRepoUtils' import { getPolicies } from './utils/policiesUtils' import { EncryptedDataRecord, encryptSecretItem, sealedSecretManifest } from './utils/sealedSecretUtils' import { getKeycloakUsers, isValidUsername } from './utils/userUtils' @@ -90,8 +92,6 @@ interface ExcludedApp extends App { } const debug = Debug('otomi:otomi-stack') -const secretTransferProps = ['type', 'ca', 'crt', 'key', 'entries', 'dockerconfig'] - const env = cleanEnv({ CUSTOM_ROOT_CA, DEFAULT_PLATFORM_ADMIN_EMAIL, @@ -109,100 +109,24 @@ const env = cleanEnv({ OBJ_STORAGE_APPS, }) -export function getTeamBackupsFilePath(teamId: string): string { - return `env/teams/backups.${teamId}.yaml` -} -export function getTeamBackupsJsonPath(teamId: string): string { - return `teamConfig.${teamId}.backups` -} - -export function getTeamNetpolsFilePath(teamId: string): string { - return `env/teams/netpols.${teamId}.yaml` -} -export function getTeamNetpolsJsonPath(teamId: string): string { - return `teamConfig.${teamId}.netpols` -} - -export function getTeamSealedSecretsValuesRootPath(teamId: string): string { - return `env/teams/${teamId}/sealedsecrets` -} -export function getTeamSealedSecretsValuesFilePath(teamId: string, sealedSecretsName: string): string { +export const rootPath = '/tmp/otomi/values' +//TODO Move this to the repo.ts +const clusterSettingsFilePath = 'env/settings/cluster.yaml' +function getTeamSealedSecretsValuesFilePath(teamId: string, sealedSecretsName: string): string { return `env/teams/${teamId}/sealedsecrets/${sealedSecretsName}` } -export function getTeamWorkloadsFilePath(teamId: string): string { - return `env/teams/workloads.${teamId}.yaml` -} -export function getTeamWorkloadValuesFilePath(teamId: string, workloadName): string { - return `env/teams/workloads/${teamId}/${workloadName}.yaml` -} - -export function getTeamProjectsFilePath(teamId: string): string { - return `env/teams/projects.${teamId}.yaml` -} - -export function getTeamCodereposFilePath(teamId: string): string { - return `env/teams/coderepos.${teamId}.yaml` -} - -export function getTeamBuildsFilePath(teamId: string): string { - return `env/teams/builds.${teamId}.yaml` -} - -export function getTeamPoliciesFilePath(teamId: string): string { - return `env/teams/policies.${teamId}.yaml` -} - -export function getTeamWorkloadsJsonPath(teamId: string): string { - return `teamConfig.${teamId}.workloads` -} - -export function getTeamProjectsJsonPath(teamId: string): string { - return `teamConfig.${teamId}.projects` -} - -export function getTeamCodereposJsonPath(teamId: string): string { - return `teamConfig.${teamId}.coderepos` -} - -export function getTeamBuildsJsonPath(teamId: string): string { - return `teamConfig.${teamId}.builds` -} - -export function getTeamPoliciesJsonPath(teamId: string): string { - return `teamConfig.${teamId}.policies` -} - -export function getTeamSealedSecretsJsonPath(teamId: string): string { - return `teamConfig.${teamId}.sealedsecrets` -} - -export function getTeamSecretsJsonPath(teamId: string): string { - return `teamConfig.${teamId}.secrets` -} - -export function getTeamServicesFilePath(teamId: string): string { - return `env/teams/services.${teamId}.yaml` -} - -export function getTeamServicesJsonPath(teamId: string): string { - return `teamConfig.${teamId}.services` -} - -export const rootPath = '/tmp/otomi/values' export default class OtomiStack { private coreValues: Core - - db: Db editor?: string sessionId?: string isLoaded = false - repo: Repo + git: Git + repoService: RepoService - constructor(editor?: string, sessionId?: string, inDb?: Db) { + constructor(editor?: string, sessionId?: string) { this.editor = editor this.sessionId = sessionId ?? 'main' - this.db = inDb ?? new Db() } getAppList() { @@ -215,12 +139,11 @@ export default class OtomiStack { } async getValues(query): Promise> { - return (await this.repo.requestValues(query)).data + return (await this.git.requestValues(query)).data } getRepoPath() { if (env.isTest || this.sessionId === undefined) return env.GIT_LOCAL_PATH - const folder = `${rootPath}/${this.sessionId}` - return folder + return `${rootPath}/${this.sessionId}` } async init(): Promise { @@ -235,7 +158,98 @@ export default class OtomiStack { } } - async initRepo(skipDbInflation = false): Promise { + transformServices(servicesArray: any[] = [], teamId: string): any[] { + return servicesArray.map((service) => { + const { cluster, dns } = this.getSettings(['cluster', 'dns']) + const url = getServiceUrl({ domain: service.domain, name: service.name, teamId, cluster, dns }) + + const headers = isArray(service.headers) ? undefined : service.headers + + const inService = omit(service, [ + 'certArn', + 'certName', + 'domain', + 'forwardPath', + 'hasCert', + 'paths', + 'type', + 'ownHost', + 'tlsPass', + 'ingressClassName', + 'headers', + 'useCname', + 'cname', + ]) + const svc = { + ...inService, + name: service.name, + teamId, + ingress: + service.type === 'cluster' + ? { type: 'cluster' } + : { + certArn: service.certArn || undefined, + certName: service.certName || undefined, + domain: url.domain, + headers, + forwardPath: 'forwardPath' in service, + hasCert: 'hasCert' in service, + paths: service.paths || [], + subdomain: url.subdomain, + tlsPass: 'tlsPass' in service, + type: service.type, + useDefaultHost: !service.domain && service.ownHost, + ingressClassName: service.ingressClassName || undefined, + useCname: service.useCname, + cname: service.cname, + }, + } + return removeBlankAttributes(svc) + }) + } + + transformApps(appsObj: Record): App[] { + if (!appsObj || typeof appsObj !== 'object') return [] + + return Object.entries(appsObj).map(([appId, appData]) => { + // Retrieve schema to check if the `enabled` flag should be considered + const appSchema = getAppSchema(appId) + const isEnabled = appSchema?.properties?.enabled ? !!appData.enabled : undefined + + return { + id: appId, + enabled: isEnabled, + values: omit(appData, ['enabled']), + rawValues: {}, + } + }) + } + + async initRepo(repoService?: RepoService): Promise { + if (repoService) { + this.repoService = repoService + return + } else { + // We need to map the app values, so it adheres the App interface + const rawRepo = await loadValues(this.getRepoPath()) + + rawRepo.apps = this.transformApps(rawRepo.apps) + rawRepo.teamConfig = mapValues(rawRepo.teamConfig, (teamConfig) => ({ + ...teamConfig, + apps: this.transformApps(teamConfig.apps), + })) + + const repo = rawRepo as Repo + this.repoService = new RepoService(repo) + //TODO fix this transforming of the services + this.repoService.getRepo().teamConfig = mapValues(repo.teamConfig, (teamConfig, teamName) => ({ + ...teamConfig, + services: this.transformServices(teamConfig.services, teamName), + })) + } + } + + async initGit(inflateValues = true): Promise { await this.init() // every editor gets their own folder to detect conflicts upon deploy const path = this.getRepoPath() @@ -243,9 +257,10 @@ export default class OtomiStack { const url = env.GIT_REPO_URL for (;;) { try { - this.repo = await getRepo(path, url, env.GIT_USER, env.GIT_EMAIL, env.GIT_PASSWORD, branch) - await this.repo.pull() - if (await this.repo.fileExists('env/cluster.yaml')) break + this.git = await getRepo(path, url, env.GIT_USER, env.GIT_EMAIL, env.GIT_PASSWORD, branch) + await this.git.pull() + //TODO fetch this url from the repo + if (await this.git.fileExists(clusterSettingsFilePath)) break debug(`Values are not present at ${url}:${branch}`) } catch (e) { // Remove password from error message @@ -257,8 +272,10 @@ export default class OtomiStack { debug(`Trying again in ${timeoutMs} ms`) await new Promise((resolve) => setTimeout(resolve, timeoutMs)) } - // branches get a copy of the "main" branch db, so we don't need to inflate - if (!skipDbInflation) await this.loadValues() + + if (inflateValues) { + await this.loadValues() + } debug(`Values are loaded for ${this.editor} in ${this.sessionId}`) } @@ -281,24 +298,14 @@ export default class OtomiStack { } getSettingsInfo(): SettingsInfo { - const settings = this.db.db.get(['settings']).value() as Settings - const { cluster, dns, obj, otomi, ingress, smtp } = pick(settings, [ - 'cluster', - 'dns', - 'obj', - 'otomi', - 'ingress', - 'smtp', - ]) as Settings - const settingsInfo = { - cluster: pick(cluster, ['name', 'domainSuffix', 'provider']), - dns: pick(dns, ['zones']), - obj: pick(obj, ['provider']), - otomi: pick(otomi, ['hasExternalDNS', 'hasExternalIDP', 'isPreInstalled']), - smtp: pick(smtp, ['smarthost']), - ingressClassNames: map(ingress?.classes, 'className') ?? [], + return { + cluster: pick(this.repoService.getCluster(), ['name', 'domainSuffix', 'provider']), + dns: pick(this.repoService.getDns(), ['zones']), + obj: pick(this.repoService.getObj(), ['provider']), + otomi: pick(this.repoService.getOtomi(), ['hasExternalDNS', 'hasExternalIDP', 'isPreInstalled']), + smtp: pick(this.repoService.getSmtp(), ['smarthost']), + ingressClassNames: map(this.repoService.getIngress()?.classes, 'className') ?? [], } as SettingsInfo - return settingsInfo } async createObjWizard(data: ObjWizard): Promise { @@ -378,7 +385,7 @@ export default class OtomiStack { } getSettings(keys?: string[]): Settings { - const settings = this.db.db.get(['settings']).value() + const settings = this.repoService.getSettings() if (!keys) return settings return pick(settings, keys) as Settings } @@ -386,10 +393,10 @@ export default class OtomiStack { async loadIngressApps(id: string): Promise { try { debug(`Loading ingress apps for ${id}`) - const content = await this.repo.loadConfig('env/apps/ingress-nginx.yaml', 'env/apps/secrets.ingress-nginx.yaml') + const content = await this.git.loadConfig('env/apps/ingress-nginx.yaml', 'env/apps/secrets.ingress-nginx.yaml') const values = content?.apps?.['ingress-nginx'] ?? {} const teamId = 'admin' - this.db.createItem('apps', { enabled: true, values, rawValues: {}, teamId }, { teamId, id }, id) + this.repoService.getTeamConfigService(teamId).createApp({ enabled: true, values, rawValues: {}, id }) debug(`Ingress app loaded for ${id}`) } catch (error) { debug(`Failed to load ingress apps for ${id}:`) @@ -401,9 +408,9 @@ export default class OtomiStack { debug(`Removing ingress apps for ${id}`) const path = `env/apps/${id}.yaml` const secretsPath = `env/apps/secrets.${id}.yaml` - this.db.deleteItem('apps', { teamId: 'admin', id }) - await this.repo.removeFile(path) - await this.repo.removeFile(secretsPath) + this.repoService.deleteApp(id) + await this.git.removeFile(path) + await this.git.removeFile(secretsPath) debug(`Ingress app removed for ${id}`) } catch (error) { debug(`Failed to remove ingress app for ${id}:`) @@ -433,7 +440,7 @@ export default class OtomiStack { } async editSettings(data: Settings, settingId: string): Promise { - const settings = this.db.db.get('settings').value() as Settings + const settings = this.repoService.getSettings() await this.editIngressApps(settings, data, settingId) const updatedSettingsData: any = { ...data } // Preserve the otomi.adminPassword when editing otomi settings @@ -444,18 +451,12 @@ export default class OtomiStack { } } settings[settingId] = removeBlankAttributes(updatedSettingsData[settingId] as Record) - this.db.db.set('settings', settings).write() - const secretPaths = this.getSecretPaths() - await this.saveSettings(secretPaths) - await this.doDeployment(['settings']) - return settings - } - - // Check if the collection name already exists in any collection - isCollectionNameTaken(collectionName: string, teamId: string, name: string): boolean { - return this.db.getCollection(collectionName).some((e: any) => { - return e.teamId === teamId && e.name === name + this.repoService.updateSettings(settings) + await this.saveSettings() + await this.doRepoDeployment((repoService) => { + repoService.updateSettings(settings) }) + return settings } filterExcludedApp(apps: App | App[]) { @@ -475,31 +476,40 @@ export default class OtomiStack { return apps } - getApp(teamId: string, id: string): App | ExcludedApp { - // @ts-ignore - const app = this.db.getItem('apps', { teamId, id }) as App + getTeamApp(teamId: string, id: string): App | ExcludedApp { + const app = this.getApp(id) this.filterExcludedApp(app) if (teamId === 'admin') return app - const adminApp = this.db.getItem('apps', { teamId: 'admin', id: app.id }) as App + const adminApp = this.repoService.getTeamConfigService(teamId).getApp(id) return { ...cloneDeep(app), enabled: adminApp.enabled } } + getApp(name: string): App { + return this.repoService.getApp(name) + } + getApps(teamId: string, picks?: string[]): Array { - const apps = this.db.getCollection('apps', { teamId }) as Array + const appList = this.getAppList() + const apps = this.repoService.getApps().filter((app) => appList.includes(app.id)) const providerSpecificApps = this.filterExcludedApp(apps) as App[] if (teamId === 'admin') return providerSpecificApps - let teamApps = providerSpecificApps.map((app: App) => { - const adminApp = this.db.getItem('apps', { teamId: 'admin', id: app.id }) as App - return { ...cloneDeep(app), enabled: adminApp.enabled } - }) + // If not team admin load available teamApps + const core = this.getCore() + let teamApps = providerSpecificApps + .map((app: App) => { + const isShared = !!core.adminApps.find((a) => a.name === app.id)?.isShared + const inTeamApps = !!core.teamApps.find((a) => a.name === app.id) + if (isShared || inTeamApps) return app + }) + .filter((app): app is App => app !== undefined) // Ensures no `undefined` elements if (!picks) return teamApps if (picks.includes('enabled')) { - const adminApps = this.db.getCollection('apps', { teamId: 'admin' }) as Array + const adminApps = this.repoService.getApps() teamApps = adminApps.map((adminApp) => { const teamApp = teamApps.find((app) => app.id === adminApp.id) @@ -510,17 +520,16 @@ export default class OtomiStack { return teamApps.map((app) => pick(app, picks)) as Array } - async editApp(teamId, id, data: App): Promise { - // @ts-ignore - let app: App = this.db.getItem('apps', { teamId, id }) + async editApp(teamId: string, id: string, data: App): Promise { + let app: App = this.repoService.getApp(id) // Shallow merge, so only first level attributes can be replaced (values, rawValues, etc.) app = { ...app, ...data } - const updatedApp = this.db.updateItem('apps', app as Record, { teamId, id }) as App - const secretPaths = this.getSecretPaths() - // also save admin apps - await this.saveAdminApps(secretPaths) - await this.doDeployment(['apps']) - return updatedApp + app = this.repoService.updateApp(id, app) + await this.saveAdminApp(app) + await this.doRepoDeployment((repoService) => { + repoService.updateApp(id, app) + }) + return this.repoService.getApp(id) } canToggleApp(id: string): boolean { @@ -529,58 +538,28 @@ export default class OtomiStack { } async toggleApps(teamId: string, ids: string[], enabled: boolean): Promise { - ids.map((id) => { - // we might be given a dep that is only relevant to core, or - // which is essential, so skip it - const orig = this.db.getItemReference('apps', { teamId, id }, false) as App - if (orig && this.canToggleApp(id)) this.db.updateItem('apps', { enabled }, { teamId, id }, true) - }) - const secretPaths = this.getSecretPaths() - // also save admin apps - await this.saveAdminApps(secretPaths) - await this.doDeployment(['apps']) - } - - async loadApp(appInstanceId: string): Promise { - const isIngressApp = appInstanceId.startsWith('ingress-nginx-') - const appId = isIngressApp ? 'ingress-nginx' : appInstanceId - const path = `env/apps/${appInstanceId}.yaml` - const secretsPath = `env/apps/secrets.${appInstanceId}.yaml` - const content = await this.repo.loadConfig(path, secretsPath) - let values = content?.apps?.[appInstanceId] ?? {} - if (appInstanceId === 'ingress-nginx-platform') { - const isIngressNginxPlatformAppExists = await this.repo.fileExists(`env/apps/ingress-nginx-platform.yaml`) - if (!isIngressNginxPlatformAppExists) { - const defaultIngressNginxContent = await this.repo.loadConfig( - `env/apps/ingress-nginx.yaml`, - `env/apps/secrets.ingress-nginx.yaml`, - ) - values = defaultIngressNginxContent?.apps?.['ingress-nginx'] ?? {} - } - } - const rawValues = {} - - let enabled - const app = getAppSchema(appId) - if (app?.properties?.enabled) enabled = !!values.enabled - - // we do not want to send enabled flag to the input forms - delete values.enabled - const teamId = 'admin' - this.db.createItem('apps', { enabled, values, rawValues, teamId }, { teamId, id: appInstanceId }, appInstanceId) - } - - async loadApps(): Promise { - const apps = this.getAppList() await Promise.all( - apps.map(async (appId) => { - await this.loadApp(appId) + ids.map(async (id) => { + const orig = this.repoService.getApp(id) + if (orig && this.canToggleApp(id)) { + const app = this.repoService.updateApp(id, { enabled }) + await this.saveAdminApp(app) + } }), ) + + await this.doRepoDeployment((repoService) => { + ids.map((id) => { + const orig = repoService.getApp(id) + if (orig && this.canToggleApp(id)) { + repoService.updateApp(id, { enabled }) + } + }) + }) } getTeams(): Array { - return this.db.getCollection('teams') as Array + return this.repoService.getAllTeamSettings() } getTeamSelfServiceFlags(id: string): TeamSelfService { @@ -593,11 +572,12 @@ export default class OtomiStack { } getTeam(id: string): Team { - return this.db.getItem('teams', { id }) as Team + const team = this.repoService.getTeamConfigService(id).getSettings() + return { ...team, name: id } } async createTeam(data: Team, deploy = true): Promise { - const id = data.id || data.name + const teamName = data.name if (isEmpty(data.password)) { debug(`creating password for team '${data.name}'`) @@ -612,70 +592,71 @@ export default class OtomiStack { }) } - const team = this.db.createItem('teams', data, { id }, id) as Team + const teamConfig = this.repoService.createTeamConfig(teamName, data) + const team = teamConfig.settings const apps = getAppList() const core = this.getCore() - apps.forEach((appId) => { + const teamApps = apps.flatMap((appId) => { const isShared = !!core.adminApps.find((a) => a.name === appId)?.isShared const inTeamApps = !!core.teamApps.find((a) => a.name === appId) - // Admin apps are loaded by loadApps function - if (id !== 'admin' && (isShared || inTeamApps)) this.db.createItem('apps', {}, { teamId: id, id: appId }, appId) + return teamName !== 'admin' && (isShared || inTeamApps) + ? [this.repoService.getTeamConfigService(teamName).createApp({ id: appId })] + : [] // Empty array removes `undefined` entries }) + const policies = getPolicies() + if (!data.id) { + this.repoService.getTeamConfigService(teamName).updatePolicies(policies) + await this.saveTeamPolicies(teamName) + } if (deploy) { - if (!data.id) { - const policies = getPolicies() - this.db.db.set(`policies[${data.name}]`, policies).write() - await this.saveTeamPolicies(data.name) - } - const secretPaths = this.getSecretPaths() - await this.saveTeams(secretPaths) - await this.doDeployment(['teams', 'policies']) + await this.saveTeam(team) + await this.doRepoDeployment((repoService) => { + repoService.createTeamConfig(teamName, data) + repoService.getTeamConfigService(teamName).setApps(teamApps) + repoService.getTeamConfigService(teamName).updatePolicies(policies) + }) } return team } async editTeam(id: string, data: Team): Promise { - const team = this.db.updateItem('teams', data, { id }) as Team - const secretPaths = this.getSecretPaths() - await this.saveTeams(secretPaths) - await this.doDeployment(['teams']) + const team = this.repoService.getTeamConfigService(id).updateSettings(data) + await this.saveTeam(team) + await this.doTeamDeployment(id, (teamService) => { + teamService.updateSettings(team) + }) return team } async deleteTeam(id: string): Promise { - try { - this.db.deleteItem('services', { id }) - } catch (e) { - // no services found - } - this.db.deleteItem('teams', { id }) - const secretPaths = this.getSecretPaths() - await this.saveTeams(secretPaths) - await this.doDeployment(['teams']) + await this.deleteTeamConfig(id) + await this.doRepoDeployment((repoService) => { + repoService.deleteTeamConfig(id) + }) } getTeamServices(teamId: string): Array { - const ids = { teamId } - return this.db.getCollection('services', ids) as Array + return this.repoService.getTeamConfigService(teamId).getServices() } getTeamBackups(teamId: string): Array { - const ids = { teamId } - return this.db.getCollection('backups', ids) as Array + return this.repoService.getTeamConfigService(teamId).getBackups() } getAllBackups(): Array { - return this.db.getCollection('backups') as Array + return this.repoService.getAllBackups() } async createBackup(teamId: string, data: Backup): Promise { validateBackupFields(data.name, data.ttl) try { - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const backup = this.db.createItem('backups', { ...data, teamId }, { teamId, name: data.name }) as Backup - await this.saveTeamBackups(teamId) - await this.doDeployment(['backups']) + const backup = this.repoService.getTeamConfigService(teamId).createBackup(data) + + await this.saveTeamBackup(teamId, data) + await this.doTeamDeployment(teamId, (teamService) => { + teamService.createBackup(backup) + }) return backup } catch (err) { if (err.code === 409) err.publicMessage = 'Backup name already exists' @@ -683,40 +664,54 @@ export default class OtomiStack { } } - getBackup(id: string): Backup { - return this.db.getItem('backups', { id }) as Backup + getBackup(teamId: string, name: string): Backup { + return this.repoService.getTeamConfigService(teamId).getBackup(name) } - async editBackup(id: string, data: Backup): Promise { + async editBackup(teamId: string, name: string, data: Backup): Promise { validateBackupFields(data.name, data.ttl) - const backup = this.db.updateItem('backups', data, { id }) as Backup - await this.saveTeamBackups(data.teamId!) - await this.doDeployment(['backups']) + const backup = this.repoService.getTeamConfigService(teamId).updateBackup(name, data) + await this.saveTeamBackup(teamId, data) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.updateBackup(name, backup) + }, + false, + ) return backup } - async deleteBackup(id: string): Promise { - const backup = this.getBackup(id) - this.db.deleteItem('backups', { id }) - await this.saveTeamBackups(backup.teamId!) - await this.doDeployment(['backups']) + async deleteBackup(teamId: string, name: string): Promise { + await this.deleteTeamBackup(teamId, name) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.deleteBackup(name) + }, + false, + ) } getTeamNetpols(teamId: string): Array { - const ids = { teamId } - return this.db.getCollection('netpols', ids) as Array + return this.repoService.getTeamConfigService(teamId).getNetpols() } getAllNetpols(): Array { - return this.db.getCollection('netpols') as Array + return this.repoService.getAllNetpols() } async createNetpol(teamId: string, data: Netpol): Promise { try { - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const netpol = this.db.createItem('netpols', { ...data, teamId }, { teamId, name: data.name }) as Netpol - await this.saveTeamNetpols(teamId) - await this.doDeployment(['netpols']) + const netpol = this.repoService.getTeamConfigService(teamId).createNetpol(data) + await this.saveTeamNetpols(teamId, data) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.createNetpol(netpol) + }, + false, + ) return netpol } catch (err) { if (err.code === 409) err.publicMessage = 'Network policy name already exists' @@ -724,27 +719,38 @@ export default class OtomiStack { } } - getNetpol(id: string): Netpol { - return this.db.getItem('netpols', { id }) as Netpol + getNetpol(teamId: string, name: string): Netpol { + return this.repoService.getTeamConfigService(teamId).getNetpol(name) } - async editNetpol(id: string, data: Netpol): Promise { + async editNetpol(teamId: string, name: string, data: Netpol): Promise { // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const netpol = this.db.updateItem('netpols', data, { id }) as Netpol - await this.saveTeamNetpols(netpol.teamId!) - await this.doDeployment(['netpols']) + const netpol = this.repoService.getTeamConfigService(teamId).updateNetpol(name, data) + await this.saveTeamNetpols(teamId, data) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.updateNetpol(name, netpol) + }, + false, + ) return netpol } - async deleteNetpol(id: string): Promise { - const netpol = this.getNetpol(id) - this.db.deleteItem('netpols', { id }) - await this.saveTeamNetpols(netpol.teamId!) - await this.doDeployment(['netpols']) + async deleteNetpol(teamId: string, name: string): Promise { + const netpol = this.repoService.getTeamConfigService(teamId).getNetpol(name) + await this.deleteTeamNetpol(teamId, netpol.name) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.deleteNetpol(name) + }, + false, + ) } getAllUsers(sessionUser: SessionUser): Array { - const users = this.db.getCollection('users') as Array + const users = this.repoService.getUsers() if (sessionUser.isPlatformAdmin) return users else if (sessionUser.isTeamAdmin) { const usersWithBasicInfo = users.map((user) => { @@ -758,8 +764,7 @@ export default class OtomiStack { async createUser(data: User): Promise { const { valid, error } = isValidUsername(data.email.split('@')[0]) if (!valid) { - const err = new HttpError(400, error as string) - throw err + throw new HttpError(400, error as string) } const initialPassword = generatePassword({ length: 16, @@ -770,23 +775,25 @@ export default class OtomiStack { strict: true, }) const user = { ...data, initialPassword } - let existingUsers = this.db.getCollection('users') as any + let existingUsersEmail = this.repoService.getUsersEmail() if (!env.isDev) { const { otomi, cluster } = this.getSettings(['otomi', 'cluster']) - const keycloak = this.getApp('admin', 'keycloak') + const keycloak = this.getApp('keycloak') const keycloakBaseUrl = `https://keycloak.${cluster?.domainSuffix}` const realm = 'otomi' const username = keycloak?.values?.adminUsername as string const password = otomi?.adminPassword as string - existingUsers = await getKeycloakUsers(keycloakBaseUrl, realm, username, password) + existingUsersEmail = await getKeycloakUsers(keycloakBaseUrl, realm, username, password) } try { - if (existingUsers.some((existingUser) => existingUser.email === user.email)) + if (existingUsersEmail.some((existingUser) => existingUser === user.email)) { throw new AlreadyExists('User email already exists') - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const createdUser = this.db.createItem('users', user, { name: user.email }) as User - await this.saveUsers() - await this.doDeployment(['users']) + } + const createdUser = this.repoService.createUser(user) + await this.saveUser(createdUser) + await this.doRepoDeployment((repoService) => { + repoService.createUser(user) + }) return createdUser } catch (err) { if (err.code === 409) err.publicMessage = 'User email already exists' @@ -795,69 +802,84 @@ export default class OtomiStack { } getUser(id: string): User { - return this.db.getItem('users', { id }) as User + return this.repoService.getUser(id) } async editUser(id: string, data: User): Promise { - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const user = this.db.updateItem('users', data, { id }) as User - await this.saveUsers() - await this.doDeployment(['users']) + const user = this.repoService.updateUser(id, data) + await this.saveUser(user) + await this.doRepoDeployment((repoService) => { + repoService.updateUser(id, user) + }) return user } async deleteUser(id: string): Promise { - const user = this.db.getItem('users', { id }) as User + const user = this.repoService.getUser(id) if (user.email === env.DEFAULT_PLATFORM_ADMIN_EMAIL) { const error = new OtomiError('Forbidden') error.code = 403 error.publicMessage = 'Cannot delete the default platform admin user' throw error } - this.db.deleteItem('users', { id }) - await this.saveUsers() - await this.doDeployment(['users']) + await this.deleteUserFile(user) + await this.doRepoDeployment((repoService) => { + repoService.deleteUser(user.email) + }) } async editTeamUsers( data: Pick[], ): Promise> { - data.forEach((user) => { - const existingUser = this.db.getItem('users', { id: user.id }) as User - this.db.updateItem('users', { ...existingUser, teams: user.teams }, { id: user.id }) as User + for (const user of data) { + const existingUser = this.repoService.getUser(user.id!) + const updateUser = this.repoService.updateUser(user.id!, { ...existingUser, teams: user.teams }) + await this.saveUser(updateUser) + } + const users = this.repoService.getUsers() + await this.doRepoDeployment((repoService) => { + for (const user of data) { + const existingUser = repoService.getUser(user.id!) + repoService.updateUser(user.id!, { ...existingUser, teams: user.teams }) + } }) - const users = this.db.getCollection('users') as Array - await this.saveUsers() - await this.doDeployment(['users']) return users } getTeamProjects(teamId: string): Array { - const ids = { teamId } - return this.db.getCollection('projects', ids) as Array + return this.repoService.getTeamConfigService(teamId).getProjects() } getAllProjects(): Array { - return this.db.getCollection('projects') as Array + return this.repoService.getAllProjects() } // Creates a new project and reserves a given name for 'builds', 'workloads' and 'services' resources async createProject(teamId: string, data: Project): Promise { // Check if the project name already exists in any collection - const projectNameTaken = ['builds', 'workloads', 'services'].some((collectionName) => - this.isCollectionNameTaken(collectionName, teamId, data.name), - ) + const projectNameTaken = this.repoService.getTeamConfigService(teamId).doesProjectNameExist(data.name) const projectNameTakenPublicMessage = `In the team '${teamId}' there is already a resource that match the project name '${data.name}'` try { if (projectNameTaken) throw new AlreadyExists(projectNameTakenPublicMessage) - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const project = this.db.createItem('projects', { ...data, teamId }, { teamId, name: data.name }) as Project - await this.saveTeamProjects(teamId) - await this.saveTeamBuilds(teamId) - await this.saveTeamWorkloads(teamId) - await this.saveTeamServices(teamId) - await this.doDeployment(['projects', 'builds', 'workloads', 'workloadValues', 'services']) + const project = this.repoService.getTeamConfigService(teamId).createProject({ ...data, teamId }) + if (data.build) { + await this.createBuild(teamId, data.build) + } + if (data.workload) { + await this.createWorkload(teamId, data.workload) + } + if (data.service) { + await this.createService(teamId, data.service) + } + await this.saveTeamProject(teamId, data) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.createProject(project) + }, + false, + ) return project } catch (err) { if (err.code === 409 && projectNameTaken) err.publicMessage = projectNameTakenPublicMessage @@ -866,129 +888,194 @@ export default class OtomiStack { } } - getProject(id: string): Project { - const p = this.db.getItem('projects', { id }) as Project - let b, w, wv, s + getProject(teamId: string, name: string): Project { + const project = this.repoService.getTeamConfigService(teamId).getProject(name) + let build, workload, workloadValues, services try { - b = this.db.getItem('builds', { id: p.build?.id }) as Build + build = this.repoService.getTeamConfigService(teamId).getBuild(project.build!.name) } catch (err) { - b = {} + build = {} } try { - w = this.db.getItem('workloads', { id: p.workload?.id }) as Workload + workload = this.repoService.getTeamConfigService(teamId).getWorkload(project.workload!.name) } catch (err) { - w = {} + workload = {} } try { - wv = this.db.getItem('workloadValues', { id: p.workloadValues?.id }) as WorkloadValues + workloadValues = this.repoService.getTeamConfigService(teamId).getWorkloadValues(project.workloadValues!.name!) } catch (err) { - wv = {} + workloadValues = {} } try { - s = this.db.getItem('services', { id: p.service?.id }) as Service + services = this.repoService.getTeamConfigService(teamId).getService(project.service!.name) } catch (err) { - s = {} + services = {} + } + return { + teamId, + ...project, + name: project.name, + build, + workload, + workloadValues, + service: services, } - return { ...p, build: b, workload: w, workloadValues: wv, service: s } } - async editProject(id: string, data: Project): Promise { - const { build, workload, workloadValues, service, teamId, name } = data - const { values } = workloadValues as WorkloadValues - + async editProject(teamId: string, name: string, data: Project): Promise { + const { build, workload, workloadValues, service } = data let b, w, wv, s - if (!build?.id && build?.mode) b = this.db.createItem('builds', { ...build, teamId }, { teamId, name }) as Build - else if (build?.id) b = this.db.updateItem('builds', build, { id: build.id }) as Build - if (!workload?.id) w = this.db.createItem('workloads', { ...workload, teamId }, { teamId, name }) as Workload - else w = this.db.updateItem('workloads', workload, { id: workload.id }) as Workload + if (build) { + try { + b = this.repoService.getTeamConfigService(teamId).createBuild({ ...build, teamId }) + } catch (error) { + if (error.code == 409) b = this.repoService.getTeamConfigService(teamId).updateBuild(build.name, build) + } + } - if (!data.workloadValues?.id) - wv = this.db.createItem('workloadValues', { teamId, values }, { teamId, name }, w.id) as WorkloadValues - else wv = this.db.updateItem('workloadValues', { teamId, values }, { id: workloadValues?.id }) as WorkloadValues + if (workload) { + try { + w = this.repoService.getTeamConfigService(teamId).createWorkload(workload) + } catch (error) { + if (error.code === 409) + w = this.repoService.getTeamConfigService(teamId).updateWorkload(workload.name, workload) + } + } - if (!service?.id) s = this.db.createItem('services', { ...service, teamId }, { teamId, name }) as Service - else s = this.db.updateItem('services', service, { id: service.id }) as Service + if (workload && workloadValues) { + try { + wv = this.repoService.getTeamConfigService(teamId).createWorkloadValues({ ...workloadValues, name }) + } catch (error) { + if (error.code === 409) + wv = this.repoService.getTeamConfigService(teamId).updateWorkloadValues(name, { ...workloadValues, name }) + } + } + + if (service) { + try { + s = this.repoService.getTeamConfigService(teamId).createService({ ...service, teamId }) + } catch (error) { + if (error.code === 409) s = this.repoService.getTeamConfigService(teamId).updateService(service.name, service) + } + } const updatedData = { - id, name, teamId, - ...(b && { build: { id: b.id } }), - workload: { id: w.id }, - workloadValues: { id: wv.id }, - service: { id: s.id }, + ...(b && { build: { name: b.name } }), + workload: { name: w.name }, + workloadValues: { name: wv.name }, + service: { name: s.name }, } - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const project = this.db.updateItem('projects', updatedData, { id }) as Project - await this.saveTeamProjects(project.teamId!) - await this.saveTeamBuilds(project.teamId!) - await this.saveTeamWorkloads(project.teamId!) - await this.saveTeamServices(project.teamId!) - await this.doDeployment(['projects', 'builds', 'workloads', 'workloadValues', 'services']) + let project: Project + try { + project = this.repoService.getTeamConfigService(teamId).createProject(updatedData) + } catch (error) { + if (error.code === 409) { + project = this.repoService.getTeamConfigService(teamId).updateProject(name, updatedData) + } else { + throw error + } + } + await this.saveTeamProject(teamId, data) + await this.saveTeamBuild(teamId, b) + await this.saveTeamWorkload(teamId, w) + await this.saveTeamWorkloadValues(teamId, wv) + await this.saveTeamService(teamId, s) + //Leave this for now as we will replace projects + await this.doDeployment(['projects', 'builds', 'workloads', 'workloadValues', 'services'], teamId, false) return project } // Deletes a project and all its related resources - async deleteProject(id: string): Promise { - const p = this.db.getItem('projects', { id }) as Project - if (p.build?.id) this.db.deleteItem('builds', { id: p.build.id }) - if (p.workload?.id) this.db.deleteItem('workloads', { id: p.workload.id }) - if (p.workloadValues?.id) this.db.deleteItem('workloadValues', { id: p.workloadValues.id }) - if (p.service?.id) this.db.deleteItem('services', { id: p.service.id }) - this.db.deleteItem('projects', { id }) - await this.saveTeamProjects(p.teamId!) - await this.saveTeamBuilds(p.teamId!) - await this.saveTeamWorkloads(p.teamId!) - await this.saveTeamServices(p.teamId!) - await this.doDeployment(['projects', 'builds', 'workloads', 'workloadValues', 'services']) + async deleteProject(teamId: string, name: string): Promise { + const p = this.repoService.getTeamConfigService(teamId).getProject(name) + if (p.build?.name) { + await this.deleteTeamBuild(teamId, p.build.name) + } + if (p.workload?.name) { + await this.deleteTeamWorkload(teamId, p.workload.name) + } + if (p.workloadValues?.name) { + await this.deleteTeamWorkloadValues(teamId, p.workloadValues.name) + } + if (p.service?.name) { + await this.deleteTeamService(teamId, p.service.name) + } + await this.deleteTeamProject(teamId, name) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.deleteBuild(name) + teamService.deleteWorkload(name) + teamService.deleteWorkloadValues(name) + teamService.deleteService(name) + teamService.deleteProject(name) + }, + false, + ) } - getTeamCoderepos(teamId: string): Array { - const ids = { teamId } - return this.db.getCollection('coderepos', ids) as Array + getTeamCodeRepos(teamId: string): Array { + return this.repoService.getTeamConfigService(teamId).getCodeRepos() } - getAllCoderepos(): Array { - const allCoderepos = this.db.getCollection('coderepos') as Array - return allCoderepos + getAllCodeRepos(): Array { + const allCodeRepos = this.repoService.getAllCodeRepos() + return allCodeRepos } - async createCoderepo(teamId: string, data: Coderepo): Promise { + async createCodeRepo(teamId: string, data: CodeRepo): Promise { try { const body = { ...data } if (!body.private) unset(body, 'secret') - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const coderepo = this.db.createItem('coderepos', { ...body, teamId }, { teamId, label: body.label }) as Coderepo - await this.saveTeamCoderepos(teamId) - await this.doDeployment(['coderepos']) - return coderepo + const codeRepo = this.repoService.getTeamConfigService(teamId).createCodeRepo({ ...data, teamId }) + await this.saveTeamCodeRepo(teamId, codeRepo) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.createCodeRepo(codeRepo) + }, + false, + ) + return codeRepo } catch (err) { - if (err.code === 409) err.publicMessage = 'Code repe label already exists' + if (err.code === 409) err.publicMessage = 'Code repo label already exists' throw err } } - getCoderepo(id: string): Coderepo { - return this.db.getItem('coderepos', { id }) as Coderepo + getCodeRepo(teamId: string, name: string): CodeRepo { + return this.repoService.getTeamConfigService(teamId).getCodeRepo(name) } - async editCoderepo(id: string, data: Coderepo): Promise { + async editCodeRepo(teamId: string, name: string, data: CodeRepo): Promise { const body = { ...data } if (!body.private) unset(body, 'secret') // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const coderepo = this.db.updateItem('coderepos', body, { id }) as Coderepo - await this.saveTeamCoderepos(coderepo.teamId as string) - await this.doDeployment(['coderepos']) - return coderepo + const codeRepo = this.repoService.getTeamConfigService(teamId).updateCodeRepo(name, body) + await this.saveTeamCodeRepo(teamId, codeRepo) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.updateCodeRepo(name, codeRepo) + }, + false, + ) + return codeRepo } - async deleteCoderepo(id: string): Promise { - const coderepo = this.getCoderepo(id) - this.db.deleteItem('coderepos', { id }) - await this.saveTeamCoderepos(coderepo.teamId as string) - await this.doDeployment(['coderepos']) + async deleteCodeRepo(teamId: string, name: string): Promise { + await this.deleteTeamCodeRepo(teamId, name) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.deleteCodeRepo(name) + }, + false, + ) } async getTestRepoConnect(url: string, teamId: string, secretName: string): Promise { @@ -1022,7 +1109,7 @@ export default class OtomiStack { async getInternalRepoUrls(teamId: string): Promise { if (env.isDev || !teamId || teamId === 'admin') return [] const { cluster, otomi } = this.getSettings(['cluster', 'otomi']) - const gitea = this.getApp('admin', 'gitea') + const gitea = this.getApp('gitea') const username = gitea?.values?.adminUsername as string const password = (gitea?.values?.adminPassword as string) || (otomi?.adminPassword as string) const orgName = `team-${teamId}` @@ -1032,15 +1119,14 @@ export default class OtomiStack { } getDashboard(teamId: string): Array { - const ids = teamId !== 'admin' ? { teamId } : undefined - const projects = this.db.getCollection('projects', ids) as Array - const builds = this.db.getCollection('builds', ids) as Array - const workloads = this.db.getCollection('workloads', ids) as Array - const services = this.db.getCollection('services', ids) as Array - const secrets = this.db.getCollection('sealed-secrets', ids) as Array - const netpols = this.db.getCollection('netpols', ids) as Array - - const inventory = [ + const projects = this.repoService.getTeamConfigService(teamId).getProjects() + const builds = this.repoService.getTeamConfigService(teamId).getBuilds() + const workloads = this.repoService.getTeamConfigService(teamId).getWorkloads() + const services = this.repoService.getTeamConfigService(teamId).getServices() + const secrets = this.repoService.getTeamConfigService(teamId).getSealedSecrets() + const netpols = this.repoService.getTeamConfigService(teamId).getNetpols() + + return [ { name: 'projects', count: projects?.length }, { name: 'builds', count: builds?.length }, { name: 'workloads', count: workloads?.length }, @@ -1048,24 +1134,27 @@ export default class OtomiStack { { name: 'sealed secrets', count: secrets?.length }, { name: 'network policies', count: netpols?.length }, ] - return inventory } getTeamBuilds(teamId: string): Array { - const ids = { teamId } - return this.db.getCollection('builds', ids) as Array + return this.repoService.getTeamConfigService(teamId).getBuilds() } getAllBuilds(): Array { - return this.db.getCollection('builds') as Array + return this.repoService.getAllBuilds() } async createBuild(teamId: string, data: Build): Promise { try { - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const build = this.db.createItem('builds', { ...data, teamId }, { teamId, name: data.name }) as Build - await this.saveTeamBuilds(teamId) - await this.doDeployment(['builds']) + const build = this.repoService.getTeamConfigService(teamId).createBuild({ ...data, teamId }) + await this.saveTeamBuild(teamId, data) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.createBuild(build) + }, + false, + ) return build } catch (err) { if (err.code === 409) err.publicMessage = 'Build name already exists' @@ -1073,44 +1162,52 @@ export default class OtomiStack { } } - getBuild(id: string): Build { - return this.db.getItem('builds', { id }) as Build + getBuild(teamId: string, name: string): Build { + return this.repoService.getTeamConfigService(teamId).getBuild(name) } - async editBuild(id: string, data: Build): Promise { + async editBuild(teamId: string, name: string, data: Build): Promise { // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const build = this.db.updateItem('builds', data, { id }) as Build - await this.saveTeamBuilds(build.teamId!) - await this.doDeployment(['builds']) + const build = this.repoService.getTeamConfigService(teamId).updateBuild(name, data) + await this.saveTeamBuild(teamId, build) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.updateBuild(name, build) + }, + false, + ) return build } - async deleteBuild(id: string): Promise { - const build = this.getBuild(id) - const p = this.db.getCollection('projects') as Array + async deleteBuild(teamId: string, name: string): Promise { + const p = this.repoService.getTeamConfigService(teamId).getProjects() p.forEach((project: Project) => { - if (project?.build?.id === id) { - const updatedData = { ...project, build: null } - this.db.updateItem('projects', updatedData, { id: project.id }) as Project + if (project?.build?.name === name) { + const updatedData = { ...project, build: undefined } + this.repoService.getTeamConfigService(teamId).updateProject(project.name, updatedData) } }) - this.db.deleteItem('builds', { id }) - await this.saveTeamBuilds(build.teamId!) - await this.doDeployment(['builds']) + await this.deleteTeamBuild(teamId, name) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.deleteBuild(name) + }, + false, + ) } getTeamPolicies(teamId: string): Policies { - const policies = this.db.db.get(['policies']).value() - return policies[teamId] + return this.repoService.getTeamConfigService(teamId).getPolicies() } getAllPolicies(): Record { - return this.db.db.get(['policies']).value() + return this.repoService.getAllPolicies() } getPolicy(teamId: string, id: string): Policy { - const policies = this.db.db.get(['policies']).value() - return policies[teamId][id] + return this.repoService.getTeamConfigService(teamId).getPolicy(id) } async editPolicy(teamId: string, policyId: string, data: Policy): Promise { @@ -1118,13 +1215,19 @@ export default class OtomiStack { teamPolicies[policyId] = removeBlankAttributes(data) const policy = this.getPolicy(teamId, policyId) await this.saveTeamPolicies(teamId) - await this.doDeployment(['policies']) + await this.doTeamDeployment( + teamId, + (teamService) => { + const rootStackPolicies = teamService.getPolicies() + rootStackPolicies[policyId] = removeBlankAttributes(data) + }, + false, + ) return policy } async getK8sVersion(): Promise { - const version = (await getKubernetesVersion()) as string - return version + return (await getKubernetesVersion()) as string } async connectCloudtty(data: Cloudtty): Promise { @@ -1208,12 +1311,11 @@ export default class OtomiStack { } getTeamWorkloads(teamId: string): Array { - const ids = { teamId } - return this.db.getCollection('workloads', ids) as Array + return this.repoService.getTeamConfigService(teamId).getWorkloads() } getAllWorkloads(): Array { - return this.db.getCollection('workloads') as Array + return this.repoService.getAllWorkloads() } async getWorkloadCatalog(data: { @@ -1249,7 +1351,7 @@ export default class OtomiStack { const uuid = uuidv4() const localHelmChartsDir = `/tmp/otomi/charts/${uuid}` const helmChartCatalogUrl = env.HELM_CHART_CATALOG - const { user, email } = this.repo + const { user, email } = this.git try { await sparseCloneChart( @@ -1274,15 +1376,20 @@ export default class OtomiStack { async createWorkload(teamId: string, data: Workload): Promise { try { - const workload = this.db.createItem('workloads', { ...data, teamId }, { teamId, name: data.name }) as Workload - this.db.createItem( - 'workloadValues', - { teamId, values: {} }, - { teamId, name: workload.name }, - workload.id, - ) as WorkloadValues - await this.saveTeamWorkloads(teamId) - await this.doDeployment(['workloads', 'workloadValues']) + const workload = this.repoService.getTeamConfigService(teamId).createWorkload({ ...data, teamId }) + const workloadValues = this.repoService + .getTeamConfigService(teamId) + .createWorkloadValues({ teamId, values: {}, id: workload.id, name: workload.name }) + await this.saveTeamWorkload(teamId, data) + await this.saveTeamWorkloadValues(teamId, workloadValues) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.createWorkload(workload) + teamService.createWorkloadValues(workloadValues) + }, + false, + ) return workload } catch (err) { if (err.code === 409) err.publicMessage = 'Workload name already exists' @@ -1290,60 +1397,91 @@ export default class OtomiStack { } } - getWorkload(id: string): Workload { - return this.db.getItem('workloads', { id }) as Workload + getWorkload(teamId: string, name: string): Workload { + return this.repoService.getTeamConfigService(teamId).getWorkload(name) } - async editWorkload(id: string, data: Workload): Promise { - const workload = this.db.updateItem('workloads', data, { id }) as Workload - await this.saveTeamWorkloads(workload.teamId!) - await this.doDeployment(['workloads', 'workloadValues']) + async editWorkload(teamId: string, name: string, data: Workload): Promise { + const workload = this.repoService.getTeamConfigService(teamId).updateWorkload(name, data) + await this.saveTeamWorkload(teamId, workload) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.updateWorkload(name, workload) + }, + false, + ) return workload } - async deleteWorkload(id: string): Promise { - const p = this.db.getCollection('projects') as Array + async deleteWorkload(teamId: string, name: string): Promise { + const p = this.repoService.getTeamConfigService(teamId).getProjects() p.forEach((project: Project) => { - if (project?.workload?.id === id) { - const updatedData = { ...project, workload: null, workloadValues: null } - this.db.updateItem('projects', updatedData, { id: project.id }) as Project + if (project?.workload?.name === name) { + const updatedData = { ...project, workload: undefined, workloadValues: undefined } + this.repoService.getTeamConfigService(teamId).updateProject(project.name, updatedData) } }) - const workloadValues = this.db.getItem('workloadValues', { id }) as WorkloadValues - const path = getTeamWorkloadValuesFilePath(workloadValues.teamId!, workloadValues.name) - await this.repo.removeFile(path) - this.db.deleteItem('workloadValues', { id }) - this.db.deleteItem('workloads', { id }) - await this.saveTeamWorkloads(workloadValues.teamId!) - await this.doDeployment(['workloads', 'workloadValues']) - } - - async editWorkloadValues(id: string, data: WorkloadValues): Promise { - const workloadValues = this.db.updateItem('workloadValues', data, { id }) as WorkloadValues - await this.saveTeamWorkloads(workloadValues.teamId!) - await this.doDeployment(['workloadValues']) + await this.deleteTeamWorkloadValues(teamId, name) + await this.deleteTeamWorkload(teamId, name) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.deleteWorkloadValues(name) + teamService.deleteWorkload(name) + }, + false, + ) + } + + async editWorkloadValues(teamId: string, name: string, data: WorkloadValues): Promise { + let workloadValues + try { + workloadValues = this.repoService.getTeamConfigService(teamId).createWorkloadValues({ ...data, name }) + } catch (error) { + if (error.code === 409) { + debug('Workload values already exists, updating values') + workloadValues = this.repoService.getTeamConfigService(teamId).updateWorkloadValues(name, data) + } + } + await this.saveTeamWorkloadValues(teamId, workloadValues) + await this.doTeamDeployment( + teamId, + (teamService) => { + try { + teamService.createWorkloadValues({ ...data, name }) + } catch (error) { + if (error.code === 409) { + debug('Workload values already exists, updating values') + teamService.updateWorkloadValues(name, data) + } + } + }, + false, + ) return workloadValues } - getWorkloadValues(id: string): WorkloadValues { - return this.db.getItem('workloadValues', { id }) as WorkloadValues + getWorkloadValues(teamId: string, name: string): WorkloadValues { + return this.repoService.getTeamConfigService(teamId).getWorkloadValues(name) } getAllServices(): Array { - return this.db.getCollection('services') as Array + return this.repoService.getAllServices() } async createService(teamId: string, data: Service): Promise { - this.checkPublicUrlInUse(data) + this.checkPublicUrlInUse(teamId, data) try { - const service = this.db.createItem( - 'services', - { ...data, teamId }, - { teamId, name: data.name }, - data?.id, - ) as Service - await this.saveTeamServices(teamId) - await this.doDeployment(['services']) + const service = this.repoService.getTeamConfigService(teamId).createService({ ...data, teamId }) + await this.saveTeamService(teamId, data) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.createService(service) + }, + false, + ) return service } catch (err) { if (err.code === 409) err.publicMessage = 'Service name already exists' @@ -1351,38 +1489,48 @@ export default class OtomiStack { } } - getService(id: string): Service { - return this.db.getItem('services', { id }) as Service + getService(teamId: string, name: string): Service { + return this.repoService.getTeamConfigService(teamId).getService(name) } - async editService(id: string, data: Service): Promise { - const service = this.db.updateItem('services', data, { id }) as Service - await this.saveTeamServices(service.teamId!) - await this.doDeployment(['services']) + async editService(teamId: string, name: string, data: Service): Promise { + const service = this.repoService.getTeamConfigService(teamId).updateService(name, data) + await this.saveTeamService(teamId, service) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.updateService(name, service) + }, + false, + ) return service } - async deleteService(id: string, deleteProjectService = true): Promise { - const service = this.getService(id) + async deleteService(teamId: string, name: string, deleteProjectService = true): Promise { if (deleteProjectService) { - const p = this.db.getCollection('projects') as Array + const p = this.repoService.getTeamConfigService(teamId).getProjects() p.forEach((project: Project) => { - if (project?.service?.id === id) { - const updatedData = { ...project, service: null } - this.db.updateItem('projects', updatedData, { id: project.id }) as Project + if (project?.service?.name === name) { + const updatedData = { ...project, service: undefined } + this.repoService.getTeamConfigService(teamId).updateProject(project.name, updatedData) } }) } - this.db.deleteItem('services', { id }) - await this.saveTeamServices(service.teamId!) - await this.doDeployment(['services']) + await this.deleteTeamService(teamId, name) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.deleteService(name) + }, + false, + ) } - checkPublicUrlInUse(data: any): void { + checkPublicUrlInUse(teamId: string, data: any): void { // skip when editing or when svc is of type "cluster" as it has no url if (data.id || data?.ingress?.type === 'cluster') return const newSvc = data.ingress - const services = this.db.getCollection('services') + const services = this.repoService.getTeamConfigService(teamId).getServices() const servicesFiltered = filter(services, (svc: any) => { if (svc.ingress?.type !== 'cluster') { @@ -1429,20 +1577,88 @@ export default class OtomiStack { } } - async doDeployment(collectionIds?: string[]): Promise { + async doTeamDeployment( + teamId: string, + action: (teamService: TeamConfigService) => void, + encryptSecrets = true, + ): Promise { + const rootStack = await getSessionStack() + + try { + // Commit and push Git changes + await this.git.save(this.editor!, encryptSecrets) + + // Execute the provided action dynamically + action(rootStack.repoService.getTeamConfigService(teamId)) + + debug(`Updated root stack values with ${this.sessionId} changes`) + + // Emit pipeline status + const sha = await rootStack.git.getCommitSha() + this.emitPipelineStatus(sha) + } catch (e) { + const msg: DbMessage = { editor: 'system', state: 'corrupt', reason: 'deploy' } + getIo().emit('db', msg) + throw e + } finally { + // Clean up the session + await cleanSession(this.sessionId!) + } + } + + async doRepoDeployment(action: (repoService: RepoService) => void, encryptSecrets = true): Promise { + const rootStack = await getSessionStack() + + try { + // Commit and push Git changes + await this.git.save(this.editor!, encryptSecrets) + + // Execute the provided action dynamically + action(rootStack.repoService) + + debug(`Updated root stack values with ${this.sessionId} changes`) + + // Emit pipeline status + const sha = await rootStack.git.getCommitSha() + this.emitPipelineStatus(sha) + } catch (e) { + const msg: DbMessage = { editor: 'system', state: 'corrupt', reason: 'deploy' } + getIo().emit('db', msg) + throw e + } finally { + // Clean up the session + await cleanSession(this.sessionId!) + } + } + + //TODO remove this one when we remove projects + async doDeployment(collectionIds?: string[], teamId?: string, encryptSecrets = true): Promise { const rootStack = await getSessionStack() try { - // commit and pull-push remote root - await this.repo.save(this.editor!) - // update db with the new values + // Commit and push Git changes + await this.git.save(this.editor!, encryptSecrets) + if (collectionIds) { collectionIds.forEach((collectionId) => { - const collection = this.db.db.get(collectionId).value() - rootStack.db.db.set(collectionId, collection).write() + if (teamId && collectionId in this.repoService.getRepo().teamConfig[teamId]) { + // If a teamId is provided and collection is inside teamConfig, update the specific team + const collection = this.repoService.getTeamConfigService(teamId).getCollection(collectionId) + rootStack.repoService.getTeamConfigService(teamId).updateCollection(collectionId, collection) + } else { + // Otherwise, update the root repo collection + console.log('collectionId', collectionId) + const collection = this.repoService.getCollection(collectionId) + if (collection) { + rootStack.repoService.updateCollection(collectionId, collection) + } + } }) } + debug(`Updated root stack values with ${this.sessionId} changes`) - const sha = await rootStack.repo.getCommitSha() + + // Emit pipeline status + const sha = await rootStack.git.getCommitSha() this.emitPipelineStatus(sha) } catch (e) { const msg: DbMessage = { editor: 'system', state: 'corrupt', reason: 'deploy' } @@ -1459,7 +1675,7 @@ export default class OtomiStack { await emptyDir(rootPath) // and re-init root const rootStack = await getSessionStack() - await rootStack.initRepo() + await rootStack.initGit() } apiClient?: k8s.CoreV1Api @@ -1566,32 +1782,7 @@ export default class OtomiStack { const secretName = 'harbor-pushsecret' const secretRes = await client.readNamespacedSecret(secretName, namespace) const { body: secret }: { body: k8s.V1Secret } = secretRes - const token = Buffer.from(secret.data!['.dockerconfigjson'], 'base64').toString('ascii') - return token - } - - createSecret(teamId: string, data: Record): Secret { - return this.db.createItem('secrets', { ...data, teamId }, { teamId, name: data.name }) as Secret - } - - editSecret(id: string, data: Secret): Secret { - return this.db.updateItem('secrets', data, { id }) as Secret - } - - deleteSecret(id: string): void { - this.db.deleteItem('secrets', { id }) - } - - getSecret(id: string): Secret { - return this.db.getItem('secrets', { id }) as Secret - } - - getAllSecrets(): Array { - return this.db.getCollection('secrets', {}) as Array - } - - getSecrets(teamId: string): Array { - return this.db.getCollection('secrets', { teamId }) as Array + return Buffer.from(secret.data!['.dockerconfigjson'], 'base64').toString('ascii') } async createSealedSecret(teamId: string, data: SealedSecret): Promise { @@ -1607,15 +1798,19 @@ export default class OtomiStack { const encryptedItem = encryptSecretItem(certificate, data.name, namespace, obj.value, 'namespace-wide') return { [obj.key]: encryptedItem } }) - const encryptedData = Object.assign({}, ...(await Promise.all(encryptedDataPromises))) as EncryptedDataRecord - const sealedSecret = this.db.createItem( - 'sealedsecrets', - { ...data, teamId, encryptedData, namespace }, - { teamId, name: data.name }, - ) as SealedSecret + const encryptedData = Object.assign({}, ...(await Promise.all(encryptedDataPromises))) as EncryptedDataRecord[] + const sealedSecret = this.repoService + .getTeamConfigService(teamId) + .createSealedSecret({ ...data, teamId, encryptedData, namespace }) const sealedSecretChartValues = sealedSecretManifest(data, encryptedData, namespace) - await this.saveTeamSealedSecrets(teamId, sealedSecretChartValues, sealedSecret.id!) - await this.doDeployment(['sealedsecrets']) + await this.saveTeamSealedSecrets(teamId, sealedSecretChartValues, sealedSecret.name) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.createSealedSecret(sealedSecret) + }, + false, + ) return sealedSecret } catch (err) { if (err.code === 409) err.publicMessage = 'SealedSecret name already exists' @@ -1623,7 +1818,7 @@ export default class OtomiStack { } } - async editSealedSecret(id: string, data: SealedSecret): Promise { + async editSealedSecret(name: string, data: SealedSecret): Promise { const namespace = data.namespace ?? `team-${data?.teamId}` const certificate = await getSealedSecretsCertificate() if (!certificate) { @@ -1636,541 +1831,272 @@ export default class OtomiStack { const encryptedItem = encryptSecretItem(certificate, data.name, namespace, obj.value, 'namespace-wide') return { [obj.key]: encryptedItem } }) - const encryptedData = Object.assign({}, ...(await Promise.all(encryptedDataPromises))) as EncryptedDataRecord - const sealedSecret = this.db.updateItem('sealedsecrets', { ...data, encryptedData }, { id }) as SealedSecret + const encryptedData = Object.assign({}, ...(await Promise.all(encryptedDataPromises))) as EncryptedDataRecord[] + const sealedSecret = this.repoService + .getTeamConfigService(data.teamId!) + .updateSealedSecret(name, { ...data, encryptedData }) const sealedSecretChartValues = sealedSecretManifest(data, encryptedData, namespace) - await this.saveTeamSealedSecrets(data.teamId!, sealedSecretChartValues, id) - await this.doDeployment(['sealedsecrets']) + await this.saveTeamSealedSecrets(data.teamId!, sealedSecretChartValues, name) + await this.doTeamDeployment( + data.teamId!, + (teamService) => { + teamService.updateSealedSecret(name, sealedSecret) + }, + false, + ) return sealedSecret } - async deleteSealedSecret(id: string): Promise { - const sealedSecret = await this.getSealedSecret(id) - this.db.deleteItem('sealedsecrets', { id }) - const relativePath = getTeamSealedSecretsValuesFilePath(sealedSecret.teamId!, `${id}.yaml`) - await this.repo.removeFile(relativePath) - await this.doDeployment(['sealedsecrets']) + async deleteSealedSecret(teamId: string, name: string): Promise { + const sealedSecret = await this.getSealedSecret(teamId, name) + this.repoService.getTeamConfigService(teamId).deleteSealedSecret(name) + const relativePath = getTeamSealedSecretsValuesFilePath(sealedSecret.teamId!, `${name}.yaml`) + await this.git.removeFile(relativePath) + await this.doTeamDeployment( + teamId, + (teamService) => { + teamService.deleteSealedSecret(name) + }, + false, + ) } - async getSealedSecret(id: string): Promise { - const sealedSecret = this.db.getItem('sealedsecrets', { id }) as SealedSecret - const namespace = sealedSecret.namespace ?? `team-${sealedSecret.teamId}` + async getSealedSecret(teamId: string, name: string): Promise { + const sealedSecret = this.repoService.getTeamConfigService(teamId).getSealedSecret(name) + const namespace = sealedSecret.namespace ?? `team-${teamId}` const secretValues = (await getSecretValues(sealedSecret.name, namespace)) || {} const isDisabled = isEmpty(secretValues) const encryptedData = Object.entries(sealedSecret.encryptedData).map(([key, value]) => ({ key, value: secretValues?.[key] || value, })) - const res = { ...sealedSecret, encryptedData, isDisabled } as any - return res + return { ...sealedSecret, encryptedData, isDisabled } as any } getAllSealedSecrets(): Array { - return this.db.getCollection('sealedsecrets', {}) as Array + return this.repoService.getAllSealedSecrets() } getSealedSecrets(teamId: string): Array { - return this.db.getCollection('sealedsecrets', { teamId }) as Array + return this.repoService.getTeamConfigService(teamId).getSealedSecrets() } async getSecretsFromK8s(teamId: string): Promise> { if (env.isDev) return [] - const secrets = await getTeamSecretsFromK8s(`team-${teamId}`) - return secrets + return await getTeamSecretsFromK8s(`team-${teamId}`) } async loadValues(): Promise>>>> { debug('Loading values') - await this.repo.initSops() - await this.loadCluster() - await this.loadSettings() - await this.loadUsers() - await this.loadTeams() - await this.loadApps() + await this.git.initSops() + await this.initRepo() this.isLoaded = true } - async loadCluster(): Promise { - const data = await this.repo.loadConfig('env/cluster.yaml', 'env/secrets.cluster.yaml') - // @ts-ignore - this.db.db.get('settings').assign(data).write() - } - - async loadSettings(): Promise { - const data: Record = await this.repo.loadConfig('env/settings.yaml', `env/secrets.settings.yaml`) - data.otomi!.nodeSelector = objectToArray((data.otomi!.nodeSelector ?? {}) as Record) - // @ts-ignore - this.db.db.get('settings').assign(data).write() - } - - async loadUsers(): Promise { - const { secretFilePostfix } = this.repo - const relativePath = `env/secrets.users.yaml` - const secretRelativePath = `${relativePath}${secretFilePostfix}` - if (!(await this.repo.fileExists(relativePath)) || !(await this.repo.fileExists(secretRelativePath))) { - debug(`No users found`) - return + async saveAdminApp(app: App, secretPaths?: string[]): Promise { + const { id, enabled, values, rawValues } = app + const apps = { + [id]: { + ...(values || {}), + }, } - const data = await this.repo.readFile(secretRelativePath) - const inData: Array = get(data, `users`, []) - inData.forEach((inUser) => { - const res: any = this.db.populateItem('users', { ...inUser }, undefined, inUser.id as string) - debug(`Loaded user: email: ${res.name}, id: ${res.id}`) - }) - } - - async loadTeamSealedSecrets(teamId: string): Promise { - const sealedSecretsValuesRootPath = getTeamSealedSecretsValuesRootPath(teamId) - const sealedSecretsFileNames = await this.repo.readDir(sealedSecretsValuesRootPath) - if (sealedSecretsFileNames.length === 0) return - - await Promise.all( - sealedSecretsFileNames.map(async (id: string) => { - const relativePath = getTeamSealedSecretsValuesFilePath(teamId, id) - if (!(await this.repo.fileExists(relativePath))) { - debug(`Team ${teamId} has no sealed secrets yet`) - return - } - const data = await this.repo.readFile(relativePath) - const res: any = this.db.populateItem( - 'sealedsecrets', - { - encryptedData: data.spec.encryptedData, - metadata: data.spec.template.metadata, - type: data.spec.template.type, - name: data.metadata.name, - teamId, - }, - undefined, - id.replace('.yaml', ''), - ) - debug(`Loaded sealed secret: name: ${res.name}, id: ${res.id}, teamId: ${res.teamId}`) - }), - ) - } - - async loadTeamBackups(teamId: string): Promise { - const relativePath = getTeamBackupsFilePath(teamId) - if (!(await this.repo.fileExists(relativePath))) { - debug(`Team ${teamId} has no backups yet`) - return + if (!isEmpty(rawValues)) { + apps[id]._rawValues = rawValues } - const data = await this.repo.readFile(relativePath) - const inData: Array = get(data, getTeamBackupsJsonPath(teamId), []) - inData.forEach((inBackup) => { - const res: any = this.db.populateItem('backups', { ...inBackup, teamId }, undefined, inBackup.id as string) - debug(`Loaded backup: name: ${res.name}, id: ${res.id}, teamId: ${res.teamId}`) - }) - } - async loadTeamNetpols(teamId: string): Promise { - const relativePath = getTeamNetpolsFilePath(teamId) - if (!(await this.repo.fileExists(relativePath))) { - debug(`Team ${teamId} has no network policies yet`) - return + if (this.canToggleApp(id)) { + apps[id].enabled = !!enabled + } else { + delete apps[id].enabled } - const data = await this.repo.readFile(relativePath) - const inData: Array = get(data, getTeamNetpolsJsonPath(teamId), []) - inData.forEach((inNetpol) => { - const res: any = this.db.populateItem('netpols', { ...inNetpol, teamId }, undefined, inNetpol.id as string) - debug(`Loaded network policy: name: ${res.name}, id: ${res.id}, teamId: ${res.teamId}`) - }) - } - async loadTeamProjects(teamId: string): Promise { - const relativePath = getTeamProjectsFilePath(teamId) - if (!(await this.repo.fileExists(relativePath))) { - debug(`Team ${teamId} has no projects yet`) - return - } - const data = await this.repo.readFile(relativePath) - const inData: Array = get(data, getTeamProjectsJsonPath(teamId), []) - inData.forEach((inProject) => { - const res: any = this.db.populateItem('projects', { ...inProject, teamId }, undefined, inProject.id as string) - debug(`Loaded project: name: ${res.name}, id: ${res.id}, teamId: ${res.teamId}`) - }) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplApp')! + await this.git.saveConfigWithSecrets({ apps }, secretPaths ?? this.getSecretPaths(), fileMap) } - async loadTeamCoderepos(teamId: string): Promise { - const relativePath = getTeamCodereposFilePath(teamId) - if (!(await this.repo.fileExists(relativePath))) { - debug(`Team ${teamId} has no coderepos yet`) - return + async saveSettings(secretPaths?: string[]): Promise { + const settings = cloneDeep(this.getSettings()) as Record> + settings.otomi.nodeSelector = arrayToObject(settings.otomi.nodeSelector as []) + const fileMaps = getFileMaps('').filter((fm) => fm.resourceDir === 'settings')! + for (const fileMap of fileMaps) { + await this.git.saveConfigWithSecrets(settings, secretPaths ?? this.getSecretPaths(), fileMap) } - const data = await this.repo.readFile(relativePath) - const inData: Array = get(data, getTeamCodereposJsonPath(teamId), []) - inData.forEach((inCoderepo) => { - const res: any = this.db.populateItem('coderepos', { ...inCoderepo, teamId }, undefined, inCoderepo.id as string) - debug(`Loaded coderepo: name: ${res.name}, id: ${res.id}, teamId: ${res.teamId}`) - }) } - async loadTeamBuilds(teamId: string): Promise { - const relativePath = getTeamBuildsFilePath(teamId) - if (!(await this.repo.fileExists(relativePath))) { - debug(`Team ${teamId} has no builds yet`) - return - } - const data = await this.repo.readFile(relativePath) - const inData: Array = get(data, getTeamBuildsJsonPath(teamId), []) - inData.forEach((inBuild) => { - const res: any = this.db.populateItem('builds', { ...inBuild, teamId }, undefined, inBuild.id as string) - debug(`Loaded build: name: ${res.name}, id: ${res.id}, teamId: ${res.teamId}`) - }) + async saveUser(user: User): Promise { + debug(`Saving user ${user.email}`) + const users: User[] = [] + users.push(user) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplUser')! + await this.git.saveSecretConfig({ users }, fileMap, false) } - async loadTeamPolicies(teamId: string): Promise { - const relativePath = getTeamPoliciesFilePath(teamId) - if (!(await this.repo.fileExists(relativePath))) { - debug(`Team ${teamId} has no policies yet`) - return - } - const data = await this.repo.readFile(relativePath) - const inData: any = get(data, getTeamPoliciesJsonPath(teamId), {}) - this.db.db.set(`policies[${teamId}]`, inData).write() - debug(`Loaded policies of team: ${teamId}`) + async deleteUserFile(user: User): Promise { + debug(`Deleting user ${user.email}`) + this.repoService.deleteUser(user.email) + const users: User[] = [] + users.push(user) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplUser')! + await this.git.deleteConfig({ users }, fileMap, 'secrets.') } - async loadTeamWorkloads(teamId: string): Promise { - const relativePath = getTeamWorkloadsFilePath(teamId) - if (!(await this.repo.fileExists(relativePath))) { - debug(`Team ${teamId} has no workloads yet`) - return - } - const data = await this.repo.readFile(relativePath) - const inData: Array = get(data, getTeamWorkloadsJsonPath(teamId), []) - inData.forEach((inWorkload) => { - const res: any = this.db.populateItem('workloads', { ...inWorkload, teamId }, undefined, inWorkload.id as string) - debug(`Loaded workload: name: ${res.name}, id: ${res.id}, teamId: ${res.teamId}`) - }) - const workloads = this.getTeamWorkloads(teamId) - await Promise.all( - workloads.map((workload) => { - this.loadWorkloadValues(workload) - }), - ) + async saveTeam(team: Team, secretPaths?: string[]): Promise { + debug(`Saving team ${team.name}`) + debug('team', JSON.stringify(team)) + const inTeam = team + //TODO fix this issue where resource quota needs to be saved as an object + inTeam.resourceQuota = arrayToObject((team.resourceQuota as []) ?? []) as any + const repo = this.createTeamConfigInRepo(team.name, 'settings', inTeam) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamSettingSet')! + await this.git.saveConfigWithSecrets(repo, secretPaths ?? this.getSecretPaths(), fileMap) } - async loadWorkloadValues(workload: Workload): Promise { - const relativePath = getTeamWorkloadValuesFilePath(workload.teamId!, workload.name) - let data = { values: {} } as Record - if (!(await this.repo.fileExists(relativePath))) - debug(`The workload values file does not exists at ${relativePath}`) - else data = await this.repo.readFile(relativePath) - - data.id = workload.id! - data.teamId = workload.teamId! - data.name = workload.name! - try { - data.values = parseYaml(data.values as string) || {} - } catch (error) { - debug( - `The values property does not seem to be a YAML formated string at ${relativePath}. Falling back to empty map.`, - ) - data.values = {} - } - - const res = this.db.populateItem('workloadValues', data, undefined, workload.id as string) as WorkloadValues - debug(`Loaded workload values: name: ${res.name} id: ${res.id}, teamId: ${workload.teamId!}`) - return res + async deleteTeamConfig(name: string): Promise { + this.repoService.deleteTeamConfig(name) + const teamDir = `env/teams/${name}` + await this.git.removeDir(teamDir) } - async loadTeams(): Promise { - const mergedData: Core = await this.repo.loadConfig('env/teams.yaml', `env/secrets.teams.yaml`) - const tc = mergedData?.teamConfig || {} - if (!tc.admin) tc.admin = { id: 'admin' } - await Promise.all( - Object.values(tc).map(async (team: Team) => { - await this.loadTeam(team) - this.loadTeamNetpols(team.id!) - this.loadTeamServices(team.id!) - this.loadTeamSealedSecrets(team.id!) - this.loadTeamWorkloads(team.id!) - this.loadTeamBackups(team.id!) - this.loadTeamProjects(team.id!) - this.loadTeamCoderepos(team.id!) - this.loadTeamBuilds(team.id!) - this.loadTeamPolicies(team.id!) - }), - ) + async saveTeamSealedSecrets(teamId: string, data: any, name: string): Promise { + const relativePath = getTeamSealedSecretsValuesFilePath(teamId, `${name}.yaml`) + debug(`Saving sealed secrets of team: ${teamId}`) + await this.git.writeFile(relativePath, data) } - async loadTeamServices(teamId: string): Promise { - const relativePath = getTeamServicesFilePath(teamId) - if (!(await this.repo.fileExists(relativePath))) { - debug(`Team ${teamId} has no services yet`) - return - } - const data = await this.repo.readFile(relativePath) - const services = get(data, getTeamServicesJsonPath(teamId), []) - services.forEach((svc) => { - this.loadService(svc, teamId) - }) - } + async deleteTeamSealedSecrets(teamId: string, id: string): Promise { + const sealedSecret = this.repoService.getTeamConfigService(teamId).getSealedSecret(id) + this.repoService.getTeamConfigService(teamId).deleteSealedSecret(id) - async saveCluster(secretPaths?: string[]): Promise { - await this.repo.saveConfig( - 'env/cluster.yaml', - 'env/secrets.cluster.yaml', - this.getSettings(['cluster']), - secretPaths ?? this.getSecretPaths(), - ) + const repo = this.createTeamConfigInRepo(teamId, 'sealedSecrets', [sealedSecret]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamSecret')! + await this.git.deleteConfig(repo, fileMap) } - async saveAdminApps(secretPaths?: string[]): Promise { - await Promise.all( - this.getApps('admin').map(async (app) => { - const apps = {} - const { id, enabled, values, rawValues } = app - apps[id] = { - ...(values || {}), - } - if (!isEmpty(rawValues)) apps[id]._rawValues = rawValues + async saveTeamBackup(teamId: string, backup: Backup): Promise { + debug(`Saving backup ${backup.name} for team ${teamId}`) + const repo = this.createTeamConfigInRepo(teamId, 'backups', [backup]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamBackup')! + await this.git.saveConfig(repo, fileMap) + } - if (this.canToggleApp(id)) apps[id].enabled = !!enabled - else delete apps[id].enabled + async deleteTeamBackup(teamId: string, name: string): Promise { + const backup = this.repoService.getTeamConfigService(teamId).getBackup(name) + this.repoService.getTeamConfigService(teamId).deleteBackup(name) - await this.repo.saveConfig( - `env/apps/${id}.yaml`, - `env/apps/secrets.${id}.yaml`, - { apps }, - secretPaths ?? this.getSecretPaths(), - ) - }), - ) + const repo = this.createTeamConfigInRepo(teamId, 'backups', [backup]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamBackup')! + await this.git.deleteConfig(repo, fileMap) } - async saveSettings(secretPaths?: string[]): Promise { - const settings = cloneDeep(this.getSettings()) as Record> - settings.otomi.nodeSelector = arrayToObject(settings.otomi.nodeSelector as []) - await this.repo.saveConfig( - 'env/settings.yaml', - `env/secrets.settings.yaml`, - omit(settings, ['cluster']), - secretPaths ?? this.getSecretPaths(), - ) + async saveTeamNetpols(teamId: string, netpol: Netpol): Promise { + debug(`Saving netpols ${netpol.name} for team ${teamId}`) + const repo = this.createTeamConfigInRepo(teamId, 'netpols', [netpol]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamNetworkControl')! + await this.git.saveConfig(repo, fileMap) } - async saveUsers(): Promise { - const users = this.db.getCollection('users') as Array - const relativePath = `env/secrets.users.yaml` - const { secretFilePostfix } = this.repo - let secretRelativePath = `${relativePath}${secretFilePostfix}` - if (secretFilePostfix) { - const secretExists = await this.repo.fileExists(relativePath) - if (!secretExists) secretRelativePath = relativePath - } - const outData: Record = set({}, `users`, users) - debug(`Saving users`) - await this.repo.writeFile(secretRelativePath, outData, false) - if (users.length === 0) { - await this.repo.removeFile(relativePath) - await this.repo.removeFile(secretRelativePath) - } - } + async deleteTeamNetpol(teamId: string, name: string): Promise { + const netpol = this.repoService.getTeamConfigService(teamId).getNetpol(name) + this.repoService.getTeamConfigService(teamId).deleteNetpol(name) - async saveTeams(secretPaths?: string[]): Promise { - const filePath = 'env/teams.yaml' - const secretFilePath = `env/secrets.teams.yaml` - const teamValues = {} - const teams = this.getTeams() - await Promise.all( - teams.map((inTeam) => { - const team: Record = omit(inTeam, 'name') - const teamId = team.id as string - team.resourceQuota = arrayToObject((team.resourceQuota as []) ?? []) - teamValues[teamId] = team - }), - ) - const values = set({}, 'teamConfig', teamValues) - await this.repo.saveConfig(filePath, secretFilePath, values, secretPaths ?? this.getSecretPaths()) + const repo = this.createTeamConfigInRepo(teamId, 'netpols', [netpol]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamNetworkControl')! + await this.git.deleteConfig(repo, fileMap) } - async saveTeamSealedSecrets(teamId: string, data: any, id: string): Promise { - const relativePath = getTeamSealedSecretsValuesFilePath(teamId, `${id}.yaml`) - debug(`Saving sealed secrets of team: ${teamId}`) - await this.repo.writeFile(relativePath, data) + async saveTeamWorkload(teamId: string, workload: Workload): Promise { + debug(`Saving workload ${workload.name} for team ${teamId}`) + const repo = this.createTeamConfigInRepo(teamId, 'workloads', [workload]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamWorkload')! + await this.git.saveConfig(repo, fileMap) } - async saveTeamBackups(teamId: string): Promise { - const backups = this.db.getCollection('backups', { teamId }) as Array - const cleaneBackups: Array> = backups.map((obj) => { - return omit(obj, ['teamId']) - }) - const relativePath = getTeamBackupsFilePath(teamId) - const outData: Record = set({}, getTeamBackupsJsonPath(teamId), cleaneBackups) - debug(`Saving backups of team: ${teamId}`) - await this.repo.writeFile(relativePath, outData) - } + async deleteTeamWorkload(teamId: string, name: string): Promise { + const workload = this.repoService.getTeamConfigService(teamId).getWorkload(name) + this.repoService.getTeamConfigService(teamId).deleteWorkload(name) - async saveTeamNetpols(teamId: string): Promise { - const netpols = this.db.getCollection('netpols', { teamId }) as Array - const cleaneNetpols: Array> = netpols.map((obj) => { - return omit(obj, ['teamId']) - }) - const relativePath = getTeamNetpolsFilePath(teamId) - const outData: Record = set({}, getTeamNetpolsJsonPath(teamId), cleaneNetpols) - debug(`Saving network policies of team: ${teamId}`) - await this.repo.writeFile(relativePath, outData) + const repo = this.createTeamConfigInRepo(teamId, 'workloads', [workload]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamWorkload')! + await this.git.deleteConfig(repo, fileMap) } - async saveTeamWorkloads(teamId: string): Promise { - const workloads = this.db.getCollection('workloads', { teamId }) as Array - const cleaneWorkloads: Array> = workloads.map((obj) => { - return omit(obj, ['teamId']) - }) - const relativePath = getTeamWorkloadsFilePath(teamId) - const outData: Record = set({}, getTeamWorkloadsJsonPath(teamId), cleaneWorkloads) - debug(`Saving workloads of team: ${teamId}`) - await this.repo.writeFile(relativePath, outData) - await Promise.all( - workloads.map((workload) => { - this.saveWorkloadValues(workload) - }), - ) + async saveTeamProject(teamId: string, project: Project): Promise { + debug(`Saving project ${project.name} for team ${teamId}`) + const repo = this.createTeamConfigInRepo(teamId, 'projects', [project]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamProject')! + await this.git.saveConfig(repo, fileMap) } - async saveTeamProjects(teamId: string): Promise { - const projects = this.db.getCollection('projects', { teamId }) as Array - const cleaneProjects: Array> = projects.map((obj) => { - return omit(obj, ['teamId']) - }) - const relativePath = getTeamProjectsFilePath(teamId) - const outData: Record = set({}, getTeamProjectsJsonPath(teamId), cleaneProjects) - debug(`Saving projects of team: ${teamId}`) - await this.repo.writeFile(relativePath, outData) + async deleteTeamProject(teamId: string, name: string): Promise { + const project = this.repoService.getTeamConfigService(teamId).getProject(name) + this.repoService.getTeamConfigService(teamId).deleteProject(name) + + const repo = this.createTeamConfigInRepo(teamId, 'projects', [project]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamProject')! + await this.git.deleteConfig(repo, fileMap) } - async saveTeamCoderepos(teamId: string): Promise { - const coderepos = this.db.getCollection('coderepos', { teamId }) as Array - const cleaneProjects: Array> = coderepos.map((obj) => { - return omit(obj, ['teamId']) - }) - const relativePath = getTeamCodereposFilePath(teamId) - const outData: Record = set({}, getTeamCodereposJsonPath(teamId), cleaneProjects) - debug(`Saving coderepos of team: ${teamId}`) - await this.repo.writeFile(relativePath, outData) + async saveTeamBuild(teamId: string, build: Build): Promise { + debug(`Saving build ${build.name} for team ${teamId}`) + const repo = this.createTeamConfigInRepo(teamId, 'builds', [build]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamBuild')! + await this.git.saveConfig(repo, fileMap) } - async saveTeamBuilds(teamId: string): Promise { - const builds = this.db.getCollection('builds', { teamId }) as Array - const cleaneBuilds: Array> = builds.map((obj) => { - return omit(obj, ['teamId']) - }) - const relativePath = getTeamBuildsFilePath(teamId) - const outData: Record = set({}, getTeamBuildsJsonPath(teamId), cleaneBuilds) - debug(`Saving builds of team: ${teamId}`) - await this.repo.writeFile(relativePath, outData) + async deleteTeamBuild(teamId: string, name: string): Promise { + const build = this.repoService.getTeamConfigService(teamId).getBuild(name) + this.repoService.getTeamConfigService(teamId).deleteBuild(name) + + const repo = this.createTeamConfigInRepo(teamId, 'builds', [build]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamBuild')! + await this.git.deleteConfig(repo, fileMap) } - async saveTeamPolicies(teamId: string): Promise { - const policies = this.getTeamPolicies(teamId) - const relativePath = getTeamPoliciesFilePath(teamId) - const outData: Record = set({}, getTeamPoliciesJsonPath(teamId), policies) - debug(`Saving policies of team: ${teamId}`) - await this.repo.writeFile(relativePath, outData) + async saveTeamCodeRepo(teamId: string, codeRepo: CodeRepo): Promise { + debug(`Saving codeRepo ${codeRepo.name} for team ${teamId}`) + const repo = this.createTeamConfigInRepo(teamId, 'codeRepos', [codeRepo]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamCodeRepo')! + await this.git.saveConfig(repo, fileMap) } - async saveWorkloadValues(workload: Workload): Promise { - debug(`Saving workload values: id: ${workload.id!} teamId: ${workload.teamId!} name: ${workload.name}`) - const data = this.getWorkloadValues(workload.id!) - const outData = omit(data, ['id', 'teamId', 'name']) as Record - outData.values = stringifyYaml(data.values, undefined, 4) - const path = getTeamWorkloadValuesFilePath(workload.teamId!, workload.name) + async deleteTeamCodeRepo(teamId: string, label: string): Promise { + const codeRepo = this.repoService.getTeamConfigService(teamId).getCodeRepo(label) + this.repoService.getTeamConfigService(teamId).deleteCodeRepo(label) - await this.repo.writeFile(path, outData, false) + const repo = this.createTeamConfigInRepo(teamId, 'codeRepos', [codeRepo]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamCodeRepo')! + await this.git.deleteConfig(repo, fileMap) } - async saveTeamServices(teamId: string): Promise { - const services = this.db.getCollection('services', { teamId }) - const data = {} - const values: any[] = [] - services.forEach((service) => { - const value = this.convertDbServiceToValues(service) - values.push(value) - }) + async saveTeamPolicies(teamId: string): Promise { + debug(`Saving team policies ${teamId}`) + const policies = this.getTeamPolicies(teamId) - set(data, getTeamServicesJsonPath(teamId), values) - const filePath = getTeamServicesFilePath(teamId) - await this.repo.writeFile(filePath, data) + const repo = this.createTeamConfigInRepo(teamId, 'policies', policies) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamPolicy')! + await this.git.saveConfig(repo, fileMap) } - async loadTeam(inTeam: Team): Promise { - const team = { ...inTeam, name: inTeam.id } as Record - team.resourceQuota = objectToArray(inTeam.resourceQuota as Record) - const res = await this.createTeam(team as Team, false) - // const res: any = this.db.populateItem('teams', { ...team, name: team.id! }, undefined, team.id as string) - debug(`Loaded team: ${res.id!}`) - } + async saveTeamWorkloadValues(teamId: string, workloadValues: WorkloadValues): Promise { + debug(`Saving workload values teamId: ${teamId} name: ${workloadValues.name}`) + const data = this.getWorkloadValues(teamId, workloadValues.name!) + const updatedWorkloadValues = cloneDeep(data) as Record + updatedWorkloadValues.values = stringifyYaml(data.values, undefined, 4) - loadSecret(inSecret, teamId): void { - const secret: Record = omit(inSecret, ...secretTransferProps) - secret.teamId = teamId - secret.secret = secretTransferProps.reduce((memo: any, prop) => { - if (inSecret[prop] !== undefined) memo[prop] = inSecret[prop] - return memo - }, {}) - const res: any = this.db.populateItem('secrets', secret, { teamId, name: secret.name }, secret.id as string) - debug(`Loaded secret: name: ${res.name}, id: ${res.id}, teamId: ${teamId}`) + const repo = this.createTeamConfigInRepo(teamId, 'workloadValues', [updatedWorkloadValues]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamWorkloadValues')! + await this.git.saveConfig(repo, fileMap) } - convertDbSecretToValues(inSecret: any): any { - const secret: any = omit(inSecret, 'secret') - secretTransferProps.forEach((prop) => { - if (inSecret.secret[prop] !== undefined) secret[prop] = inSecret.secret[prop] - }) - return secret - } - - loadService(svcRaw, teamId): void { - // Create service - const svc = omit( - svcRaw, - 'certArn', - 'certName', - 'domain', - 'forwardPath', - 'hasCert', - 'paths', - 'type', - 'ownHost', - 'tlsPass', - 'ingressClassName', - 'headers', - 'useCname', - 'cname', - ) - svc.teamId = teamId - if (!('name' in svcRaw)) debug('Unknown service structure') - if (svcRaw.type === 'cluster') svc.ingress = { type: 'cluster' } - else { - const { cluster, dns } = this.getSettings(['cluster', 'dns']) - const url = getServiceUrl({ domain: svcRaw.domain, name: svcRaw.name, teamId, cluster, dns }) - // TODO remove the isArray check in 0.5.24 - const headers = isArray(svcRaw.headers) ? undefined : svcRaw.headers - svc.ingress = { - certArn: svcRaw.certArn || undefined, - certName: svcRaw.certName || undefined, - domain: url.domain, - headers, - forwardPath: 'forwardPath' in svcRaw, - hasCert: 'hasCert' in svcRaw, - paths: svcRaw.paths ? svcRaw.paths : [], - subdomain: url.subdomain, - tlsPass: 'tlsPass' in svcRaw, - type: svcRaw.type, - useDefaultHost: !svcRaw.domain && svcRaw.ownHost, - ingressClassName: svcRaw.ingressClassName || undefined, - useCname: svcRaw.useCname, - cname: svcRaw.cname, - } - } + async deleteTeamWorkloadValues(teamId: string, name: string): Promise { + const workloadValues = this.repoService.getTeamConfigService(teamId).getWorkloadValues(name) + this.repoService.getTeamConfigService(teamId).deleteWorkloadValues(name) - const res: any = this.db.populateItem('services', removeBlankAttributes(svc), undefined, svc.id as string) - debug(`Loaded service: name: ${res.name}, id: ${res.id}`) + const repo = this.createTeamConfigInRepo(teamId, 'workloadValues', [workloadValues]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamWorkloadValues')! + await this.git.deleteConfig(repo, fileMap) } - // eslint-disable-next-line class-methods-use-this convertDbServiceToValues(svc: any): any { const svcCloned = omit(svc, ['teamId', 'ingress', 'path']) if (svc.ingress && svc.ingress.type !== 'cluster') { @@ -2192,10 +2118,37 @@ export default class OtomiStack { return svcCloned } + async saveTeamService(teamId: string, service: Service): Promise { + debug(`Saving service: ${service.name} teamId: ${teamId}`) + const newService = this.convertDbServiceToValues(service) + const repo = this.createTeamConfigInRepo(teamId, 'services', [newService]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamService')! + await this.git.saveConfig(repo, fileMap) + } + + async deleteTeamService(teamId: string, name: string): Promise { + const service = this.repoService.getTeamConfigService(teamId).getService(name) + this.repoService.getTeamConfigService(teamId).deleteService(name) + + const repo = this.createTeamConfigInRepo(teamId, 'services', [service]) + const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamService')! + await this.git.deleteConfig(repo, fileMap) + } + + private createTeamConfigInRepo(teamId: string, key: string, value: T): Record { + return { + teamConfig: { + [teamId]: { + [key]: value, + }, + }, + } + } + async getSession(user: k8s.User): Promise { const rootStack = await getSessionStack() const valuesSchema = await getValuesSchema() - const currentSha = rootStack.repo.commitSha + const currentSha = rootStack.git.commitSha const { obj } = this.getSettings(['obj']) const regions = await getRegions() const objStorageRegions = @@ -2206,7 +2159,7 @@ export default class OtomiStack { const data: Session = { ca: env.CUSTOM_ROOT_CA, core: this.getCore() as Record, - corrupt: rootStack.repo.corrupt, + corrupt: rootStack.git.corrupt, editor: this.editor, inactivityTimeout: env.EDITOR_INACTIVITY_TIMEOUT, user: user as SessionUser, diff --git a/src/playground.ts b/src/playground.ts new file mode 100644 index 000000000..7799644c5 --- /dev/null +++ b/src/playground.ts @@ -0,0 +1,26 @@ +#!/usr/bin/env node --nolazy -r ts-node/register -r tsconfig-paths/register + +import { getFileMaps, getFilePath, loadValues } from './repo' +import { Repo } from './otomi-models' + +async function play() { + // Suppose your environment directory is "my-environment" + const envDir = '/private/tmp/otomi-bootstrap-dev' + + const allSpecs = await loadValues(envDir) + + const repo = allSpecs as Repo + + const build = repo.teamConfig['demo'].builds[0] + const jsonPath = ['$', 'teamConfig', 'demo'] + const filePath = getFilePath( + getFileMaps(envDir).find((filemap) => filemap.kind === 'AplTeamBuild')!, + jsonPath, + build, + '', + ) + console.log(allSpecs) + console.log(filePath) +} + +play() diff --git a/src/repo.ts b/src/repo.ts old mode 100644 new mode 100755 index e46ca0b3a..b70996993 --- a/src/repo.ts +++ b/src/repo.ts @@ -1,420 +1,524 @@ -import axios, { AxiosResponse } from 'axios' -import Debug from 'debug' -import diff from 'deep-diff' -import { copy, ensureDir, pathExists, readFile, writeFile } from 'fs-extra' -import { unlink } from 'fs/promises' -import { glob } from 'glob' -import stringifyJson from 'json-stable-stringify' -import { cloneDeep, get, isEmpty, merge, set, unset } from 'lodash' -import { basename, dirname, join } from 'path' -import simpleGit, { CheckRepoActions, CleanOptions, CommitResult, ResetMode, SimpleGit } from 'simple-git' -import { - cleanEnv, - GIT_BRANCH, - GIT_LOCAL_PATH, - GIT_PASSWORD, - GIT_PUSH_RETRIES, - GIT_REPO_URL, - GIT_USER, - TOOLS_HOST, -} from 'src/validators' -import { parse as parseYaml, stringify as stringifyYaml } from 'yaml' -import { BASEURL } from './constants' -import { GitPullError, HttpError, ValidationError } from './error' -import { DbMessage, getIo } from './middleware' -import { Core } from './otomi-models' -import { removeBlankAttributes } from './utils' - -const debug = Debug('otomi:repo') - -const env = cleanEnv({ - GIT_BRANCH, - GIT_LOCAL_PATH, - GIT_PASSWORD, - GIT_REPO_URL, - GIT_USER, - GIT_PUSH_RETRIES, - TOOLS_HOST, -}) - -const baseUrl = BASEURL -const prepareUrl = `${baseUrl}/prepare` -const initUrl = `${baseUrl}/init` -const valuesUrl = `${baseUrl}/otomi/values` - -const getProtocol = (url): string => (url && url.includes('://') ? url.split('://')[0] : 'https') - -const getUrl = (url): string => (!url || url.includes('://') ? url : `${getProtocol(url)}://${url}`) - -function getUrlAuth(url, user, password): string | undefined { - if (!url) return - const protocol = getProtocol(url) - const [_, bareUrl] = url.split('://') - const encodedUser = encodeURIComponent(user as string) - const encodedPassword = encodeURIComponent(password as string) - return protocol === 'file' ? `${protocol}://${bareUrl}` : `${protocol}://${encodedUser}:${encodedPassword}@${bareUrl}` +import { rmSync } from 'fs' +import { rm } from 'fs/promises' +import { globSync } from 'glob' +import jsonpath from 'jsonpath' +import { cloneDeep, get, merge, omit, set } from 'lodash' +import path from 'path' +import { getDirNames, loadYaml } from './utils' + +export async function getTeamNames(envDir: string): Promise> { + const teamsDir = path.join(envDir, 'env', 'teams') + return await getDirNames(teamsDir, { skipHidden: true }) } -const secretFileRegex = new RegExp(`^(.*/)?secrets.*.yaml(.dec)?$`) -export class Repo { - branch: string - commitSha: string - corrupt = false - email: string - git: SimpleGit - password: string - path: string - remote: string - remoteBranch: string - secretFilePostfix = '' - url: string | undefined - urlAuth: string | undefined - user: string - - constructor( - path: string, - url: string | undefined, - user: string, - email: string, - urlAuth: string | undefined, - branch: string | undefined, - ) { - this.branch = branch || 'main' - this.email = email - this.path = path - this.remote = 'origin' - this.remoteBranch = join(this.remote, this.branch) - this.urlAuth = urlAuth - this.user = user - this.url = url - this.git = simpleGit(this.path) - } - - getProtocol() { - return getProtocol(this.url) - } - - async requestInitValues(): Promise { - debug(`Tools: requesting "init" on values repo path ${this.path}`) - const res = await axios.get(initUrl, { params: { envDir: this.path } }) - return res - } - - async requestPrepareValues(): Promise { - debug(`Tools: requesting "prepare" on values repo path ${this.path}`) - const res = await axios.get(prepareUrl, { params: { envDir: this.path } }) - return res - } +export type AplKind = + | 'AplApp' + | 'AplAlertSet' + | 'AplCluster' + | 'AplDatabase' + | 'AplDns' + | 'AplIngress' + | 'AplObjectStorage' + | 'AplKms' + | 'AplIdentityProvider' + | 'AplCapabilitySet' + | 'AplSmtp' + | 'AplBackupCollection' + | 'AplUser' + | 'AplTeamCodeRepo' + | 'AplTeamBuild' + | 'AplTeamPolicy' + | 'AplTeamSettingSet' + | 'AplTeamNetworkControl' + | 'AplTeamProject' + | 'AplTeamBackup' + | 'AplTeamSecret' + | 'AplTeamService' + | 'AplTeamWorkload' + | 'AplTeamWorkloadValues' + | 'AplTeamTool' + | 'AplVersion' + +export interface FileMap { + envDir: string + kind: AplKind + jsonPathExpression: string + pathGlob: string + processAs: 'arrayItem' | 'mapItem' + resourceGroup: 'team' | 'platformSettings' | 'platformApps' | 'platformDatabases' | 'platformBackups' | 'users' + resourceDir: string + loadToSpec: boolean +} - async requestValues(params): Promise { - debug(`Tools: requesting "otomi/values" ${this.path}`) - const res = await axios.get(valuesUrl, { params: { envDir: this.path, ...params } }) - return res - } - async addConfig(): Promise { - debug(`Adding git config`) - await this.git.addConfig('user.name', this.user) - await this.git.addConfig('user.email', this.email) - if (this.isRootClone()) { - if (this.getProtocol() === 'file') { - // tell the the git repo there to accept updates even when it is checked out - const _git = simpleGit(this.url!.replace('file://', '')) - await _git.addConfig('receive.denyCurrentBranch', 'updateInstead') - } - // same for the root repo, which needs to accept pushes from children - await this.git.addConfig('receive.denyCurrentBranch', 'updateInstead') +export function getResourceFileName(fileMap: FileMap, jsonPath: jsonpath.PathComponent[], data: Record) { + let fileName = 'unknown' + if (fileMap.resourceGroup === 'team') { + if (fileMap.processAs === 'arrayItem') { + fileName = data.name || data.id || fileName + } else { + fileName = jsonPath[jsonPath.length - 1].toString() + } + } else { + if (fileMap.processAs === 'arrayItem') { + fileName = data.name || data.id || fileName + } else { + fileName = jsonPath[jsonPath.length - 1].toString() } } + return fileName +} - async init(bare = true): Promise { - await this.git.init(bare !== undefined ? bare : this.isRootClone()) - await this.git.addRemote(this.remote, this.url!) - } - - async initSops(): Promise { - if (this.secretFilePostfix === '.dec') return - this.secretFilePostfix = (await pathExists(join(this.path, '.sops.yaml'))) ? '.dec' : '' - } - - getSafePath(file: string): string { - if (this.secretFilePostfix === '') return file - // otherwise we might have to give *.dec variant for secrets - if (file.match(secretFileRegex) && !file.endsWith(this.secretFilePostfix)) return `${file}${this.secretFilePostfix}` - return file - } +export function getTeamNameFromJsonPath(jsonPath: jsonpath.PathComponent[]): string { + return jsonPath[2].toString() +} - async removeFile(file: string): Promise { - const absolutePath = join(this.path, file) - const exists = await this.fileExists(file) - if (exists) { - debug(`Removing file: ${absolutePath}`) - // Remove empty secret file due to https://github.com/mozilla/sops/issues/926 issue - await unlink(absolutePath) - } - if (file.match(secretFileRegex)) { - // also remove the encrypted file as they are operated on in pairs - const encFile = `${file}${this.secretFilePostfix}` - if (await this.fileExists(encFile)) { - const absolutePathEnc = join(this.path, encFile) - debug(`Removing enc file: ${absolutePathEnc}`) - await unlink(absolutePathEnc) - } - } +export function getResourceName(fileMap: FileMap, jsonPath: jsonpath.PathComponent[], data: Record) { + let resourceName = 'unknown' + if (fileMap.processAs === 'arrayItem') { + resourceName = data.name || data.id || resourceName + return resourceName } - async diffFile(file: string, data: Record): Promise { - const repoFile: string = this.getSafePath(file) - const oldData = await this.readFile(repoFile) - const deepDiff = diff(data, oldData) - debug(`Diff for ${file}: `, deepDiff) - return deepDiff + if (fileMap.resourceGroup === 'team') { + resourceName = getTeamNameFromJsonPath(jsonPath) + return resourceName + } else { + resourceName = jsonPath[jsonPath.length - 1].toString() + return resourceName } +} - async writeFile(file: string, data: Record, unsetBlankAttributes = true): Promise { - let cleanedData = data - if (unsetBlankAttributes) cleanedData = removeBlankAttributes(data, { emptyArrays: true }) - if (isEmpty(cleanedData) && file.match(secretFileRegex)) { - // remove empty secrets file which sops can't handle - return this.removeFile(file) - } - // we also bail when no changes found - const hasDiff = await this.diffFile(file, data) - if (!hasDiff) return - // ok, write new content - const absolutePath = join(this.path, file) - debug(`Writing to file: ${absolutePath}`) - const sortedData = JSON.parse(stringifyJson(data) as string) - const content = isEmpty(sortedData) ? '' : stringifyYaml(sortedData, undefined, 4) - const dir = dirname(absolutePath) - await ensureDir(dir) - await writeFile(absolutePath, content, 'utf8') +export function getFilePath( + fileMap: FileMap, + jsonPath: jsonpath.PathComponent[], + data: Record, + fileNamePrefix: string, +) { + let filePath = '' + const resourceName = getResourceFileName(fileMap, jsonPath, data) + if (fileMap.resourceGroup === 'team') { + const teamName = getTeamNameFromJsonPath(jsonPath) + filePath = `${fileMap.envDir}/env/teams/${teamName}/${fileMap.resourceDir}/${fileNamePrefix}${resourceName}.yaml` + } else { + filePath = `${fileMap.envDir}/env/${fileMap.resourceDir}/${fileNamePrefix}${resourceName}.yaml` } + // normalize paths like /ab/c/./test/yaml + return path.normalize(filePath) +} - async fileExists(relativePath: string): Promise { - const absolutePath = join(this.path, relativePath) - return await pathExists(absolutePath) - } +export function extractTeamDirectory(filePath: string): string { + const match = filePath.match(/\/teams\/([^/]+)/) + if (match === null) throw new Error(`Cannot extract team name from ${filePath} string`) + return match[1] +} - async readDir(relativePath: string): Promise { - const absolutePath = join(this.path, relativePath) - const files = await glob([`${absolutePath}/**/*.yaml`]) - const filenames = files.map((file) => basename(file)) - return filenames - } +export function getJsonPath(fileMap: FileMap, filePath: string): string { + let { jsonPathExpression: jsonPath } = fileMap - async readFile(file: string, checkSuffix = false): Promise> { - if (!(await this.fileExists(file))) return {} - const safeFile = checkSuffix ? this.getSafePath(file) : file - const absolutePath = join(this.path, safeFile) - debug(`Reading from file: ${absolutePath}`) - const doc = parseYaml(await readFile(absolutePath, 'utf8')) || {} - return doc + if (fileMap.resourceGroup === 'team') { + const teamName = extractTeamDirectory(filePath) + jsonPath = jsonPath.replace('teamConfig.*', `teamConfig.${teamName}`) } - async loadConfig(file: string, secretFile: string): Promise { - const data = await this.readFile(file) - const secretData = await this.readFile(secretFile, true) - return merge(data, secretData) as Core + if (jsonPath.includes('.*')) { + const fileName = path.basename(filePath, path.extname(filePath)) + const strippedFileName = fileName.replace(/^secrets\.|\.yaml|\.dec$/g, '') + jsonPath = jsonPath.replace('.*', `.${strippedFileName}`) } + if (jsonPath.includes('[*]')) jsonPath = jsonPath.replace('[*]', '') + jsonPath = jsonPath.replace('$.', '') + return jsonPath +} - async saveConfig( - dataPath: string, - inSecretRelativeFilePath: string, - config: Record, - secretJsonPaths: string[], - ): Promise> { - const secretData = {} - const plainData = cloneDeep(config) - secretJsonPaths.forEach((objectPath) => { - const val = get(config, objectPath) - if (val) { - set(secretData, objectPath, val) - unset(plainData, objectPath) - } - }) +export function getFileMaps(envDir: string): Array { + const maps: Array = [ + { + kind: 'AplApp', + envDir, + jsonPathExpression: '$.apps.*', + pathGlob: `${envDir}/env/apps/*.{yaml,yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'platformApps', + resourceDir: 'apps', + loadToSpec: true, + }, + { + envDir, + kind: 'AplAlertSet', + jsonPathExpression: '$.alerts', + pathGlob: `${envDir}/env/settings/*alerts.{yaml,yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'platformSettings', + resourceDir: 'settings', + loadToSpec: true, + }, + { + kind: 'AplCluster', + envDir, + jsonPathExpression: '$.cluster', + pathGlob: `${envDir}/env/settings/cluster.{yaml,yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'platformSettings', + resourceDir: 'settings', + loadToSpec: true, + }, + { + kind: 'AplDatabase', + envDir, + jsonPathExpression: '$.databases.*', + pathGlob: `${envDir}/env/databases/*.{yaml,yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'platformDatabases', + resourceDir: 'databases', + loadToSpec: true, + }, + { + kind: 'AplDns', + envDir, + jsonPathExpression: '$.dns', + pathGlob: `${envDir}/env/settings/*dns.{yaml,yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'platformSettings', + resourceDir: 'settings', + loadToSpec: true, + }, + { + kind: 'AplIngress', + envDir, + jsonPathExpression: '$.ingress', + pathGlob: `${envDir}/env/settings/ingress.yaml`, + processAs: 'mapItem', + resourceGroup: 'platformSettings', + resourceDir: 'settings', + loadToSpec: true, + }, + { + kind: 'AplKms', + envDir, + jsonPathExpression: '$.kms', + pathGlob: `${envDir}/env/settings/*kms.{yaml,yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'platformSettings', + resourceDir: 'settings', + loadToSpec: true, + }, + { + kind: 'AplObjectStorage', + envDir, + jsonPathExpression: '$.obj', + pathGlob: `${envDir}/env/settings/*obj.{yaml,yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'platformSettings', + resourceDir: 'settings', + loadToSpec: true, + }, + { + kind: 'AplIdentityProvider', + envDir, + jsonPathExpression: '$.oidc', + pathGlob: `${envDir}/env/settings/*oidc.{yaml,yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'platformSettings', + resourceDir: 'settings', + loadToSpec: true, + }, + { + kind: 'AplCapabilitySet', + envDir, + jsonPathExpression: '$.otomi', + pathGlob: `${envDir}/env/settings/*otomi.{yaml,yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'platformSettings', + resourceDir: 'settings', + loadToSpec: true, + }, + { + kind: 'AplBackupCollection', + envDir, + jsonPathExpression: '$.platformBackups', + pathGlob: `${envDir}/env/settings/*platformBackups.{yaml,yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'platformBackups', + resourceDir: 'settings', + loadToSpec: true, + }, + { + kind: 'AplSmtp', + envDir, + jsonPathExpression: '$.smtp', + pathGlob: `${envDir}/env/settings/*smtp.{yaml,yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'platformSettings', + resourceDir: 'settings', + loadToSpec: true, + }, + { + kind: 'AplUser', + envDir, + jsonPathExpression: '$.users[*]', + pathGlob: `${envDir}/env/users/*.{yaml,yaml.dec}`, + processAs: 'arrayItem', + resourceGroup: 'users', + resourceDir: 'users', + loadToSpec: true, + }, + { + kind: 'AplVersion', + envDir, + jsonPathExpression: '$.versions', + pathGlob: `${envDir}/env/settings/versions.yaml`, + processAs: 'mapItem', + resourceGroup: 'platformSettings', + resourceDir: 'settings', + loadToSpec: true, + }, + { + kind: 'AplTeamCodeRepo', + envDir, + jsonPathExpression: '$.teamConfig.*.codeRepos[*]', + pathGlob: `${envDir}/env/teams/*/codeRepos/*.yaml`, + processAs: 'arrayItem', + resourceGroup: 'team', + resourceDir: 'codeRepos', + loadToSpec: false, + }, + { + kind: 'AplTeamBuild', + envDir, + jsonPathExpression: '$.teamConfig.*.builds[*]', + pathGlob: `${envDir}/env/teams/*/builds/*.yaml`, + processAs: 'arrayItem', + resourceGroup: 'team', + resourceDir: 'builds', + loadToSpec: true, + }, + { + kind: 'AplTeamWorkload', + envDir, + jsonPathExpression: '$.teamConfig.*.workloads[*]', + pathGlob: `${envDir}/env/teams/*/workloads/*.yaml`, + processAs: 'arrayItem', + resourceGroup: 'team', + resourceDir: 'workloads', + loadToSpec: true, + }, + { + kind: 'AplTeamWorkloadValues', + envDir, + jsonPathExpression: '$.teamConfig.*.workloadValues[*]', + pathGlob: `${envDir}/env/teams/*/workloadValues/*.yaml`, + processAs: 'arrayItem', + resourceGroup: 'team', + resourceDir: 'workloadValues', + loadToSpec: false, + }, + { + kind: 'AplTeamService', + envDir, + jsonPathExpression: '$.teamConfig.*.services[*]', + pathGlob: `${envDir}/env/teams/*/services/*.yaml`, + processAs: 'arrayItem', + resourceGroup: 'team', + resourceDir: 'services', + loadToSpec: true, + }, + { + kind: 'AplTeamSecret', + envDir, + jsonPathExpression: '$.teamConfig.*.sealedsecrets[*]', + pathGlob: `${envDir}/env/teams/*/sealedsecrets/*.yaml`, + processAs: 'arrayItem', + resourceGroup: 'team', + resourceDir: 'sealedsecrets', + loadToSpec: false, + }, + { + kind: 'AplTeamBackup', + envDir, + jsonPathExpression: '$.teamConfig.*.backups[*]', + pathGlob: `${envDir}/env/teams/*/backups/*.yaml`, + processAs: 'arrayItem', + resourceGroup: 'team', + resourceDir: 'backups', + loadToSpec: true, + }, + { + kind: 'AplTeamProject', + envDir, + jsonPathExpression: '$.teamConfig.*.projects[*]', + pathGlob: `${envDir}/env/teams/*/projects/*.yaml`, + processAs: 'arrayItem', + resourceGroup: 'team', + resourceDir: 'projects', + loadToSpec: true, + }, + { + kind: 'AplTeamNetworkControl', + envDir, + jsonPathExpression: '$.teamConfig.*.netpols[*]', + pathGlob: `${envDir}/env/teams/*/netpols/*.yaml`, + processAs: 'arrayItem', + resourceGroup: 'team', + resourceDir: 'netpols', + loadToSpec: true, + }, + { + kind: 'AplTeamSettingSet', + envDir, + jsonPathExpression: '$.teamConfig.*.settings', + pathGlob: `${envDir}/env/teams/*/*settings{.yaml,.yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'team', + resourceDir: '.', + loadToSpec: true, + }, + { + kind: 'AplTeamTool', + envDir, + jsonPathExpression: '$.teamConfig.*.apps', + pathGlob: `${envDir}/env/teams/*/*apps{.yaml,.yaml.dec}`, + processAs: 'mapItem', + resourceGroup: 'team', + resourceDir: '.', + loadToSpec: true, + }, + { + kind: 'AplTeamPolicy', + envDir, + jsonPathExpression: '$.teamConfig.*.policies', + pathGlob: `${envDir}/env/teams/*/policies.yaml`, + processAs: 'mapItem', + resourceGroup: 'team', + resourceDir: '.', + loadToSpec: true, + }, + ] + return maps +} - let secretDataRelativePath = `${inSecretRelativeFilePath}${this.secretFilePostfix}` +export function hasCorrespondingDecryptedFile(filePath: string, fileList: Array): boolean { + return fileList.includes(`${filePath}.dec`) +} - if (this.secretFilePostfix) { - const secretExists = await this.fileExists(inSecretRelativeFilePath) - // In case secret file does not exists, create new one and let sops to encrypt it in place - if (!secretExists) secretDataRelativePath = inSecretRelativeFilePath - } +export function getFileMap(kind: AplKind, envDir: string): FileMap { + const fileMaps = getFileMaps(envDir) + const fileMapFiltered = fileMaps.find((fileMap) => fileMap.kind === kind) + return fileMapFiltered! +} - await this.writeFile(secretDataRelativePath, secretData) - await this.writeFile(dataPath, plainData) +export function renderManifest(fileMap: FileMap, jsonPath: jsonpath.PathComponent[], data: Record) { + //TODO remove this custom workaround for workloadValues + const manifest = + fileMap.kind === 'AplTeamWorkloadValues' + ? omit(data, ['id', 'name', 'teamId']) + : { + kind: fileMap.kind, + metadata: { + name: getResourceName(fileMap, jsonPath, data), + labels: {}, + }, + spec: data, + } + if (fileMap.resourceGroup === 'team' && fileMap.kind !== 'AplTeamWorkloadValues') { + manifest.metadata.labels['apl.io/teamId'] = getTeamNameFromJsonPath(jsonPath) } - isRootClone(): boolean { - return this.path === env.GIT_LOCAL_PATH - } + return manifest +} - hasRemote(): boolean { - return !!env.GIT_REPO_URL +export function renderManifestForSecrets(fileMap: FileMap, data: Record) { + return { + kind: fileMap.kind, + spec: data, } +} - async initFromTestFolder(): Promise { - // we inflate GIT_LOCAL_PATH from the ./test folder - debug(`DEV mode: using local folder values`) - await copy(join(process.cwd(), 'test'), env.GIT_LOCAL_PATH, { - recursive: true, - overwrite: false, - }) - await this.init(false) - await this.git.checkoutLocalBranch(this.branch) - await this.git.add('.') - await this.addConfig() - await this.git.commit('initial commit', undefined, this.getOptions()) - } +export async function unsetValuesFile(envDir: string): Promise { + const valuesPath = path.join(envDir, 'values-repo.yaml') + await rm(valuesPath, { force: true }) + return valuesPath +} - async clone(): Promise { - debug(`Checking if local git repository exists at: ${this.path}`) - const isRepo = await this.git.checkIsRepo(CheckRepoActions.IS_REPO_ROOT) - // remote root url - this.url = getUrl(`${env.GIT_REPO_URL}`) - if (!isRepo) { - debug(`Initializing repo...`) - if (!this.hasRemote() && this.isRootClone()) return await this.initFromTestFolder() - else if (!this.isRootClone()) { - // child clone, point to remote root - this.urlAuth = getUrlAuth(this.url, env.GIT_USER, env.GIT_PASSWORD) - } - debug(`Cloning from '${this.url}' to '${this.path}'`) - await this.git.clone(this.urlAuth!, this.path) - await this.addConfig() - await this.git.checkout(this.branch) - } else if (this.url) { - debug('Repo already exists. Checking out correct branch.') - // Git fetch ensures that local git repository is synced with remote repository - await this.git.fetch({}) - await this.git.checkout(this.branch) - } - this.commitSha = await this.getCommitSha() - } +export function unsetValuesFileSync(envDir: string): string { + const valuesPath = path.join(envDir, 'values-repo.yaml') + rmSync(valuesPath, { force: true }) + return valuesPath +} - getOptions() { - const options = {} - if (env.isDev) options['--no-verify'] = null // only for dev do we have git hooks blocking direct commit - return options +export async function loadFileToSpec( + filePath: string, + fileMap: FileMap, + spec: Record, + deps = { loadYaml }, +): Promise { + const jsonPath = getJsonPath(fileMap, filePath) + const data = await deps.loadYaml(filePath) + if (fileMap.processAs === 'arrayItem') { + const ref: Record[] = get(spec, jsonPath) + //TODO remove this custom workaround for workloadValues as it has no spec + if (fileMap.kind === 'AplTeamWorkloadValues') { + const name = filePath.match(/\/([^/]+)\.yaml$/)?.[1] + ref.push({ ...data, name }) + } else if (fileMap.kind === 'AplTeamSecret') { + const name = filePath.match(/\/([^/]+)\.yaml$/)?.[1] + ref.push({ ...data?.spec, name }) + } else { + ref.push(data?.spec) + } + } else { + const ref: Record = get(spec, jsonPath) + // Decrypted secrets may need to be merged with plain text specs + const newRef = merge(cloneDeep(ref), data?.spec) + set(spec, jsonPath, newRef) } +} - async commit(editor: string): Promise { - await this.git.add('./*') - const summary = await this.git.commit(`otomi-api commit by ${editor}`, undefined, this.getOptions()) - debug(`Commit summary: ${JSON.stringify(summary)}`) - return summary +export function initSpec(fileMap: FileMap, jsonPath: string, spec: Record) { + if (fileMap.processAs === 'arrayItem') { + set(spec, jsonPath, []) + } else { + set(spec, jsonPath, {}) } +} - async pull(skipRequest = false, skipMsg = false): Promise { - // test root can't pull as it has no remote - if (!this.url) return - debug('Pulling') - try { - const summary = await this.git.pull(this.remote, this.branch, { '--rebase': 'true' }) - const summJson = JSON.stringify(summary) - debug(`Pull summary: ${summJson}`) - this.commitSha = await this.getCommitSha() - if (!skipRequest) await this.requestInitValues() - await this.initSops() - } catch (e) { - debug('Could not pull from remote. Upstream commits? Marked db as corrupt.', e) - this.corrupt = true - if (!skipMsg) { - const msg: DbMessage = { editor: 'system', state: 'corrupt', reason: 'conflict' } - getIo().emit('db', msg) - } - try { - // Remove local changes so that no conflict can happen - debug('Removing local changes.') - await this.git.reset(ResetMode.HARD) - debug(`Go to ${this.branch} branch`) - await this.git.checkout(this.branch) - debug('Removing local changes.') - await this.git.reset(ResetMode.HARD) - debug('Cleaning local values and directories.') - await this.git.clean(CleanOptions.FORCE, ['-d']) - debug('Get the latest branch from:', this.remote) - await this.git.fetch(this.remote, this.branch) - debug('Reconciling divergent branches.') - await this.git.merge([`${this.remote}/${this.branch}`, '--strategy-option=theirs']) - debug('Trying to remove upstream commits: ', this.remote) - await this.git.push([this.remote, this.branch, '--force']) - } catch (error) { - debug('Failed to remove upstream commits: ', error) - throw new GitPullError('Failed to remove upstream commits!') - } - debug('Removed upstream commits!') - const cleanMsg: DbMessage = { editor: 'system', state: 'clean', reason: 'restored' } - getIo().emit('db', cleanMsg) - this.corrupt = false - } +export async function loadToSpec( + spec: Record, + fileMap: FileMap, + deps = { loadFileToSpec }, +): Promise { + const globOptions = { + nodir: true, // Exclude directories + dot: false, } + const files: string[] = globSync(fileMap.pathGlob, globOptions).sort() + const promises: Promise[] = [] - async push(): Promise { - if (!this.url && this.isRootClone()) return - debug('Pushing') - const summary = await this.git.push(this.remote, this.branch) - debug('Pushed. Summary: ', summary) - return - } + files.forEach((filePath) => { + const jsonPath = getJsonPath(fileMap, filePath) + initSpec(fileMap, jsonPath, spec) + if (hasCorrespondingDecryptedFile(filePath, files)) return + promises.push(deps.loadFileToSpec(filePath, fileMap, spec)) + }) - async getCommitSha(): Promise { - return this.git.revparse('HEAD') - } + await Promise.all(promises) +} - async save(editor: string): Promise { - // prepare values first - try { - await this.requestPrepareValues() - } catch (e) { - debug(`ERROR: ${JSON.stringify(e)}`) - if (e.response) { - const { status } = e.response as AxiosResponse - if (status === 422) throw new ValidationError() - throw HttpError.fromCode(status) - } - throw new HttpError(500, `${e}`) - } - // all good? commit - await this.commit(editor) - try { - // we are in a unique developer branch, so we can pull, push, and merge - // with the remote root, which might have been modified by another developer - // since this is a child branch, we don't need to re-init - // retry up to 3 times to pull and push if there are conflicts - const retries = env.GIT_PUSH_RETRIES - for (let attempt = 1; attempt <= retries; attempt++) { - await this.git.pull(this.remote, this.branch, { '--rebase': 'true', '--depth': '5' }) - try { - await this.push() - break - } catch (error) { - if (attempt === retries) throw error - debug(`Attempt ${attempt} failed. Retrying...`) - await new Promise((resolve) => setTimeout(resolve, 50)) - } - } - } catch (e) { - debug(`${e.message.trim()} for command ${JSON.stringify(e.task?.commands)}`) - debug(`Merge error: ${JSON.stringify(e)}`) - throw new GitPullError() - } - } +export async function loadValues(envDir: string, deps = { loadToSpec }): Promise> { + //We need everything to load to spec for the API + const fileMaps = getFileMaps(envDir) + const spec = {} + + await Promise.all( + fileMaps.map(async (fileMap) => { + await deps.loadToSpec(spec, fileMap) + }), + ) + return spec } -export default async function getRepo( - path: string, - url: string, - user: string, - email: string, - password: string, - branch: string, - method: 'clone' | 'init' = 'clone', -): Promise { - await ensureDir(path, { mode: 0o744 }) - const urlNormalized = getUrl(url) - const urlAuth = getUrlAuth(urlNormalized, user, password) - const repo = new Repo(path, urlNormalized, user, email, urlAuth, branch) - await repo[method]() - return repo +export async function getKmsSettings(envDir: string, deps = { loadToSpec }): Promise> { + const kmsFiles = getFileMap('AplKms', envDir) + const spec = {} + await deps.loadToSpec(spec, kmsFiles) + return spec } diff --git a/src/services/RepoService.test.ts b/src/services/RepoService.test.ts new file mode 100644 index 000000000..89f30e566 --- /dev/null +++ b/src/services/RepoService.test.ts @@ -0,0 +1,188 @@ +import { App, Repo, User } from '../otomi-models' +import { RepoService } from './RepoService' +import { TeamConfigService } from './TeamConfigService' +import { AlreadyExists } from '../error' + +jest.mock('uuid', () => ({ + v4: jest.fn(() => 'mocked-uuid'), +})) + +describe('RepoService', () => { + let service: RepoService + let repo: Repo + + beforeEach(() => { + repo = { + apps: [], + users: [], + teamConfig: {}, + cluster: {}, + dns: {}, + ingress: {}, + otomi: { version: '1.0.0' }, + smtp: { smarthost: 'smtp.mailtrap.io' }, + platformBackups: {}, + alerts: {}, + databases: {}, + kms: {}, + obj: {}, + oidc: { issuer: 'https://issuer.com', clientID: 'client-id', clientSecret: 'client-secret' }, + versions: { version: '1.0.0' }, + } as Repo + service = new RepoService(repo) + }) + + describe('getTeamConfigService', () => { + test('should throw an error if team config does not exist', () => { + expect(() => service.getTeamConfigService('nonexistent-team')).toThrow( + 'TeamConfig for nonexistent-team does not exist.', + ) + }) + + test('should return an instance of TeamConfigService when team config exists', () => { + service.createTeamConfig('team1', { name: 'Team 1' }) + const teamConfigService = service.getTeamConfigService('team1') + + expect(teamConfigService).toBeInstanceOf(TeamConfigService) + expect(service.getTeamConfig('team1')).toBeDefined() + }) + }) + + describe('Users', () => { + const user: User = { email: 'user@test.com', firstName: 'user', lastName: 'test' } + + test('should create a user', () => { + const createdUser = service.createUser(user) + expect(createdUser).toEqual({ email: 'user@test.com', id: 'mocked-uuid', firstName: 'user', lastName: 'test' }) + expect(service.getUsers()).toHaveLength(1) + }) + + test('should throw an error if user already exists', () => { + service.createUser(user) + expect(() => service.createUser(user)).toThrow(AlreadyExists) + }) + + test('should retrieve a user by ID', () => { + const createdUser = service.createUser(user) + expect(service.getUser(createdUser.id!)).toEqual(createdUser) + }) + + test('should delete a user', () => { + const createdUser = service.createUser(user) + service.deleteUser(createdUser.email) + expect(service.getUsers()).toHaveLength(0) + }) + }) + + describe('Apps', () => { + const app: App = { id: 'app1', enabled: true } + + test('should retrieve all apps', () => { + service.getRepo().apps.push(app) + expect(service.getApps()).toContain(app) + }) + + test('should retrieve a specific app', () => { + service.getRepo().apps.push(app) + expect(service.getApp('app1')).toEqual(app) + }) + + test('should throw an error when retrieving a non-existent app', () => { + expect(() => service.getApp('nonexistent')).toThrow('App[nonexistent] does not exist.') + }) + + test('should update an app', () => { + service.getRepo().apps.push(app) + const updatedApp = service.updateApp('app1', { enabled: false }) + expect(updatedApp.enabled).toBe(false) + }) + + test('should delete an app', () => { + service.getRepo().apps.push(app) + service.deleteApp('app1') + expect(service.getApps()).toHaveLength(0) + }) + }) + + describe('Team Config', () => { + test('should create a team config', () => { + const teamConfig = service.createTeamConfig('team1', { name: 'Team 1' }) + expect(teamConfig.settings).toEqual({ name: 'Team 1', id: 'team1' }) + expect(service.getTeamConfig('team1')).toBeDefined() + }) + + test('should throw an error if team config already exists', () => { + service.createTeamConfig('team1', { name: 'Team 1' }) + expect(() => service.createTeamConfig('team1', { name: 'Duplicate Team' })).toThrow(AlreadyExists) + }) + + test('should delete a team config', () => { + service.createTeamConfig('team1', { name: 'Team 1' }) + service.deleteTeamConfig('team1') + expect(service.getTeamConfig('team1')).toBeUndefined() + }) + }) + + describe('Collection Functions', () => { + test('should retrieve a collection', () => { + service.getRepo().cluster = { name: 'Test Cluster' } + expect(service.getCollection('cluster')).toEqual({ name: 'Test Cluster' }) + }) + + test('should throw an error for non-existent collection', () => { + expect(() => service.getCollection('nonexistent')).toThrow( + 'Getting repo collection [nonexistent] does not exist.', + ) + }) + + test('should update an existing collection', () => { + service.getRepo().cluster = { name: 'Old Cluster' } + service.updateCollection('cluster', { name: 'Updated Cluster' }) + expect(service.getCollection('cluster')).toEqual({ name: 'Updated Cluster' }) + }) + + test('should throw an error when updating a non-existent collection', () => { + expect(() => service.updateCollection('nonexistent', { key: 'value' })).toThrow( + 'Updating repo collection [nonexistent] does not exist.', + ) + }) + }) + + describe('Settings', () => { + test('should retrieve settings', () => { + service.getRepo().cluster = { name: 'Cluster A' } + service.getRepo().dns = { provider: { linode: { apiToken: 'test' } } } + const settings = service.getSettings() + + expect(settings.cluster).toEqual({ name: 'Cluster A' }) + expect(settings.dns).toEqual({ provider: { linode: { apiToken: 'test' } } }) + }) + + test('should update settings', () => { + service.updateSettings({ cluster: { name: 'Updated Cluster' } }) + expect(service.getSettings().cluster).toEqual({ name: 'Updated Cluster' }) + }) + }) + + describe('Global Retrieval Functions', () => { + test('should return all users emails', () => { + service.createUser({ email: 'user1@test.com', firstName: 'user', lastName: 'test' }) + service.createUser({ email: 'user2@test.com', firstName: 'user', lastName: 'test' }) + expect(service.getUsersEmail()).toEqual(['user1@test.com', 'user2@test.com']) + }) + + test('should return all builds', () => { + service.createTeamConfig('team1', { name: 'Team 1' }) + + service.getTeamConfigService('team1').createBuild({ name: 'Build1' }) + expect(service.getAllBuilds()).toHaveLength(1) + }) + + test('should return all projects', () => { + service.createTeamConfig('team1', { name: 'Team 1' }) + + service.getTeamConfigService('team1').createProject({ name: 'Project1' }) + expect(service.getAllProjects()).toHaveLength(1) + }) + }) +}) diff --git a/src/services/RepoService.ts b/src/services/RepoService.ts new file mode 100644 index 000000000..11ebcb593 --- /dev/null +++ b/src/services/RepoService.ts @@ -0,0 +1,312 @@ +import { find, has, map, mapValues, merge, remove, set } from 'lodash' +import { v4 as uuidv4 } from 'uuid' +import { AlreadyExists } from '../error' +import { + Alerts, + App, + Backup, + Build, + Cluster, + CodeRepo, + Dns, + Ingress, + Kms, + Netpol, + Otomi, + Policies, + Project, + Repo, + SealedSecret, + Service, + Settings, + Smtp, + Team, + TeamConfig, + User, + Versions, + Workload, +} from '../otomi-models' +import { TeamConfigService } from './TeamConfigService' + +export class RepoService { + // We can create an LRU cache if needed with a lot of teams. + private teamConfigServiceCache = new Map() + + constructor(private repo: Repo) { + this.repo.apps ??= [] + this.repo.alerts ??= {} as Alerts + this.repo.cluster ??= {} as Cluster + this.repo.databases ??= {} + this.repo.dns ??= {} as Dns + this.repo.ingress ??= {} as Ingress + this.repo.kms ??= {} as Kms + this.repo.obj ??= {} + this.repo.otomi ??= {} as Otomi + this.repo.platformBackups ??= {} + this.repo.users ??= [] + this.repo.versions ??= {} as Versions + this.repo.teamConfig ??= {} + } + + public getTeamConfigService(teamId: string): TeamConfigService { + if (!this.repo.teamConfig[teamId]) { + throw new Error(`TeamConfig for ${teamId} does not exist.`) + } + + // Check if we already have an instance cached + if (!this.teamConfigServiceCache.has(teamId)) { + // If not, create a new one and store it in the cache + this.teamConfigServiceCache.set(teamId, new TeamConfigService(this.repo.teamConfig[teamId])) + } + + // Return the cached instance + return this.teamConfigServiceCache.get(teamId)! + } + + public getApp(id: string): App { + const app = find(this.repo.apps, { id }) + if (!app) { + throw new Error(`App[${id}] does not exist.`) + } + return app + } + + public getApps(): App[] { + return this.repo.apps ?? [] + } + + public updateApp(id: string, updates: Partial): App { + const app = find(this.repo.apps, { id }) + if (!app) { + throw new Error(`App[${id}] does not exist.`) + } + return merge(app, updates) + } + + public deleteApp(id: string): void { + remove(this.repo.apps, { id }) + } + + public createUser(user: User): User { + const newUser = { ...user, id: user.id ?? uuidv4() } + if (find(this.repo.users, { email: newUser.email })) { + throw new AlreadyExists(`User[${user.email}] already exists.`) + } + this.repo.users.push(newUser) + return newUser + } + + public getUser(id: string): User { + const user = find(this.repo.users, { id }) + if (!user) { + throw new Error(`User[${id}] does not exist.`) + } + return user + } + + public getUsers(): User[] { + return this.repo.users ?? [] + } + + public getUsersEmail(): string[] { + return map(this.repo.users, 'email') + } + + public updateUser(id: string, updates: Partial): User { + const user = find(this.repo.users, { id }) + if (!user) throw new Error(`User[${id}] does not exist.`) + return merge(user, updates) + } + + public deleteUser(email: string): void { + remove(this.repo.users, { email }) + } + + private getDefaultTeamConfig(): TeamConfig { + return { + builds: [], + codeRepos: [], + workloads: [], + services: [], + sealedsecrets: [], + backups: [], + projects: [], + netpols: [], + settings: {} as Team, + apps: [], + policies: {} as Policies, + workloadValues: [], + } + } + + public createTeamConfig(teamName: string, team: Team): TeamConfig { + if (has(this.repo.teamConfig, teamName)) { + throw new AlreadyExists(`TeamConfig[${teamName}] already exists.`) + } + const newTeam = this.getDefaultTeamConfig() + newTeam.settings = team + newTeam.settings.id = teamName + this.repo.teamConfig[teamName] = newTeam + return this.repo.teamConfig[teamName] + } + + public getTeamConfig(teamId: string): TeamConfig | undefined { + return this.repo.teamConfig[teamId] + } + + public deleteTeamConfig(teamId: string): void { + if (!has(this.repo.teamConfig, teamId)) { + throw new Error(`TeamConfig[${teamId}] does not exist.`) + } + delete this.repo.teamConfig[teamId] + } + + public getCluster(): Cluster { + return this.repo.cluster + } + + public getDns(): Dns { + return this.repo.dns + } + + public getIngress(): Ingress { + return this.repo.ingress + } + public getOtomi(): Otomi { + return this.repo.otomi + } + + public getSmtp(): Smtp { + return this.repo.smtp + } + + public getObj(): any | undefined { + return this.repo.obj + } + + public getPlatformBackups(): any | undefined { + return this.repo.platformBackups + } + + public getSettings(): Settings { + const settings: Settings = { + alerts: this.repo.alerts, + cluster: this.repo.cluster, + dns: this.repo.dns, + ingress: this.repo.ingress, + kms: this.repo.kms, + obj: this.repo.obj, + otomi: this.repo.otomi, + platformBackups: this.repo.platformBackups, + } + + if (this.repo.smtp) { + settings.smtp = this.repo.smtp + } + if (this.repo.oidc) { + settings.oidc = this.repo.oidc + } + + return settings + } + + public updateSettings(updates: Partial): void { + merge(this.repo, updates) + } + + public getRepo(): Repo { + return this.repo + } + + public setRepo(repo: Repo): void { + this.repo = repo + } + + public getAllTeamSettings(): Team[] { + return map(this.repo.teamConfig, 'settings').filter(Boolean) ?? [] + } + + public getAllNetpols(): Netpol[] { + return ( + Object.entries(this.repo.teamConfig) + .flatMap(([teamId, config]) => config.netpols?.map((netpol) => ({ ...netpol, teamId }))) + .filter(Boolean) ?? [] + ) + } + + public getAllProjects(): Project[] { + return ( + Object.entries(this.repo.teamConfig) + .flatMap(([teamId, config]) => config.projects?.map((project) => ({ ...project, teamId }))) + .filter(Boolean) ?? [] + ) + } + + public getAllBuilds(): Build[] { + return ( + Object.entries(this.repo.teamConfig) + .flatMap(([teamId, config]) => config.builds?.map((build) => ({ ...build, teamId }))) + .filter(Boolean) ?? [] + ) + } + + public getAllPolicies(): Record { + return mapValues(this.repo.teamConfig, 'policies') + } + + public getAllWorkloads(): Workload[] { + return ( + Object.entries(this.repo.teamConfig) + .flatMap(([teamId, config]) => config.workloads?.map((workload) => ({ ...workload, teamId }))) + .filter(Boolean) ?? [] + ) + } + + public getAllServices(): Service[] { + return ( + Object.entries(this.repo.teamConfig) + .flatMap(([teamId, config]) => config.services?.map((service) => ({ ...service, teamId }))) + .filter(Boolean) ?? [] + ) + } + + public getAllSealedSecrets(): SealedSecret[] { + return ( + Object.entries(this.repo.teamConfig) + .flatMap(([teamId, config]) => config.sealedsecrets?.map((secret) => ({ ...secret, teamId }))) + .filter(Boolean) ?? [] + ) + } + + public getAllBackups(): Backup[] { + return ( + Object.entries(this.repo.teamConfig) + .flatMap(([teamId, config]) => config.backups?.map((backup) => ({ ...backup, teamId }))) + .filter(Boolean) ?? [] + ) + } + + public getAllCodeRepos(): CodeRepo[] { + return ( + Object.entries(this.repo.teamConfig) + .flatMap(([teamId, config]) => config.codeRepos?.map((repo) => ({ ...repo, teamId }))) + .filter(Boolean) ?? [] + ) + } + + /** Retrieve a collection dynamically from the Repo */ + public getCollection(collectionId: string): any { + if (!has(this.repo, collectionId)) { + throw new Error(`Getting repo collection [${collectionId}] does not exist.`) + } + return this.repo[collectionId] + } + + /** Update a collection dynamically in the Repo */ + public updateCollection(collectionId: string, data: any): void { + if (!has(this.repo, collectionId)) { + throw new Error(`Updating repo collection [${collectionId}] does not exist.`) + } + set(this.repo, collectionId, data) + } +} diff --git a/src/services/TeamConfigService.test.ts b/src/services/TeamConfigService.test.ts new file mode 100644 index 000000000..c9ca8ac6c --- /dev/null +++ b/src/services/TeamConfigService.test.ts @@ -0,0 +1,377 @@ +// Mock UUID to generate predictable values +import { + App, + Backup, + Build, + Netpol, + Project, + SealedSecret, + Service, + TeamConfig, + Workload, + WorkloadValues, +} from '../otomi-models' +import { TeamConfigService } from './TeamConfigService' +import { AlreadyExists, NotExistError } from '../error' + +jest.mock('uuid', () => ({ + v4: jest.fn(() => 'mocked-uuid'), +})) + +describe('TeamConfigService', () => { + let service: TeamConfigService + let teamConfig: TeamConfig + + beforeEach(() => { + teamConfig = { + builds: [], + codeRepos: [], + workloads: [], + workloadValues: [], + services: [], + sealedsecrets: [], + backups: [], + projects: [], + netpols: [], + apps: [], + policies: {}, + settings: { name: 'team1' }, + } as TeamConfig + service = new TeamConfigService(teamConfig) + }) + + describe('Builds', () => { + const build: Build = { name: 'TestBuild' } + test('should create a build', () => { + const createdBuild = service.createBuild(build) + + expect(createdBuild).toEqual({ name: 'TestBuild', id: 'mocked-uuid' }) + expect(service.getBuilds()).toHaveLength(1) + }) + + test('should throw an error if creating a duplicate build', () => { + service.createBuild(build) + + expect(() => service.createBuild(build)).toThrow(AlreadyExists) + }) + + test('should retrieve a build by id', () => { + const createdBuild = service.createBuild(build) + + expect(service.getBuild(createdBuild.name)).toEqual(createdBuild) + }) + + test('should throw an error when retrieving a non-existent build', () => { + expect(() => service.getBuild('non-existent')).toThrow(NotExistError) + }) + + test('should update a build', () => { + const createdBuild = service.createBuild(build) + + const updatedBuild = service.updateBuild(createdBuild.name, { name: 'UpdatedBuild' }) + expect(updatedBuild.name).toBe('UpdatedBuild') + }) + + test('should delete a build', () => { + const createdBuild = service.createBuild(build) + + service.deleteBuild(createdBuild.name) + expect(service.getBuilds()).toHaveLength(0) + }) + }) + + describe('Workloads', () => { + const workload: Workload = { name: 'TestWorkload', url: 'http://test.com' } + test('should create a workload', () => { + const createdWorkload = service.createWorkload(workload) + + expect(createdWorkload).toEqual({ name: 'TestWorkload', id: 'mocked-uuid', url: 'http://test.com' }) + expect(service.getWorkloads()).toHaveLength(1) + }) + + test('should throw an error if creating a duplicate workload', () => { + service.createWorkload(workload) + + expect(() => service.createWorkload(workload)).toThrow(AlreadyExists) + }) + + test('should retrieve a workload by id', () => { + const createdWorkload = service.createWorkload(workload) + + expect(service.getWorkload(createdWorkload.name)).toEqual(createdWorkload) + }) + + test('should throw an error when retrieving a non-existent workload', () => { + expect(() => service.getWorkload('non-existent')).toThrow(NotExistError) + }) + + test('should delete a workload', () => { + const createdWorkload = service.createWorkload(workload) + + service.deleteWorkload(createdWorkload.name) + expect(service.getWorkloads()).toHaveLength(0) + }) + }) + + describe('Services', () => { + const serviceData: Service = { name: 'TestService', ingress: {} } + test('should create a service', () => { + const createdService = service.createService(serviceData) + + expect(createdService).toEqual({ name: 'TestService', id: 'mocked-uuid', ingress: {} }) + expect(service.getServices()).toHaveLength(1) + }) + + test('should throw an error if creating a duplicate service', () => { + service.createService(serviceData) + + expect(() => service.createService(serviceData)).toThrow(AlreadyExists) + }) + + test('should retrieve a service by id', () => { + const createdService = service.createService(serviceData) + + expect(service.getService(createdService.name)).toEqual(createdService) + }) + + test('should throw an error when retrieving a non-existent service', () => { + expect(() => service.getService('non-existent')).toThrow(NotExistError) + }) + + test('should delete a service', () => { + const createdService = service.createService(serviceData) + + service.deleteService(createdService.name) + expect(service.getServices()).toHaveLength(0) + }) + }) + + describe('SealedSecrets', () => { + const secret: SealedSecret = { + name: 'TestSecret', + type: 'kubernetes.io/opaque', + encryptedData: [{ key: 'key', value: 'value' }], + } + test('should create a sealed secret', () => { + const createdSecret = service.createSealedSecret(secret) + + expect(createdSecret).toEqual({ + name: 'TestSecret', + id: 'mocked-uuid', + type: 'kubernetes.io/opaque', + encryptedData: [{ key: 'key', value: 'value' }], + }) + expect(service.getSealedSecrets()).toHaveLength(1) + }) + + test('should throw an error if creating a duplicate sealed secret', () => { + service.createSealedSecret(secret) + + expect(() => service.createSealedSecret(secret)).toThrow(AlreadyExists) + }) + + test('should retrieve a sealed secret by id', () => { + const createdSecret = service.createSealedSecret(secret) + + expect(service.getSealedSecret(createdSecret.name)).toEqual(createdSecret) + }) + + test('should throw an error when retrieving a non-existent sealed secret', () => { + expect(() => service.getSealedSecret('non-existent')).toThrow(NotExistError) + }) + + test('should delete a sealed secret', () => { + const createdSecret = service.createSealedSecret(secret) + + service.deleteSealedSecret(createdSecret.name) + expect(service.getSealedSecrets()).toHaveLength(0) + }) + }) + + describe('WorkloadValues', () => { + const workloadValues: WorkloadValues = { name: 'TestWorkloadValues', values: { test: 'values' } } + + test('should create workload values', () => { + const created = service.createWorkloadValues(workloadValues) + expect(created).toEqual({ name: 'TestWorkloadValues', id: 'mocked-uuid', values: { test: 'values' } }) + expect(service.getWorkloadValues(created.name!)).toEqual(created) + }) + + test('should throw an error when creating duplicate workload values', () => { + service.createWorkloadValues(workloadValues) + expect(() => service.createWorkloadValues(workloadValues)).toThrow(AlreadyExists) + }) + + test('should delete workload values', () => { + const created = service.createWorkloadValues(workloadValues) + service.deleteWorkloadValues(created.name!) + expect(() => service.getWorkloadValues(created.name!)).toThrow(NotExistError) + }) + }) + + describe('Backups', () => { + const backup: Backup = { name: 'TestBackup', ttl: '1', schedule: '0 0 * * *' } + + test('should create a backup', () => { + const created = service.createBackup(backup) + expect(created).toEqual({ name: 'TestBackup', id: 'mocked-uuid', ttl: '1', schedule: '0 0 * * *' }) + expect(service.getBackup(created.name)).toEqual(created) + }) + + test('should throw an error when creating duplicate backup', () => { + service.createBackup(backup) + expect(() => service.createBackup(backup)).toThrow(AlreadyExists) + }) + + test('should delete a backup', () => { + const created = service.createBackup(backup) + service.deleteBackup(created.name) + expect(() => service.getBackup(created.name)).toThrow(NotExistError) + }) + }) + + describe('Projects', () => { + const project: Project = { name: 'TestProject' } + + test('should create a project', () => { + const created = service.createProject(project) + expect(created).toEqual({ name: 'TestProject', id: 'mocked-uuid' }) + expect(service.getProject(created.name)).toEqual(created) + }) + + test('should throw an error when creating duplicate project', () => { + service.createProject(project) + expect(() => service.createProject(project)).toThrow(AlreadyExists) + }) + + test('should delete a project', () => { + const created = service.createProject(project) + service.deleteProject(created.name) + expect(() => service.getProject(created.name)).toThrow(NotExistError) + }) + }) + + describe('Netpols', () => { + const netpol: Netpol = { name: 'TestNetpol' } + + test('should create a netpol', () => { + const created = service.createNetpol(netpol) + expect(created).toEqual({ name: 'TestNetpol', id: 'mocked-uuid' }) + expect(service.getNetpol(created.name)).toEqual(created) + }) + + test('should throw an error when creating duplicate netpol', () => { + service.createNetpol(netpol) + expect(() => service.createNetpol(netpol)).toThrow(AlreadyExists) + }) + + test('should delete a netpol', () => { + const created = service.createNetpol(netpol) + service.deleteNetpol(created.name) + expect(() => service.getNetpol(created.name)).toThrow(NotExistError) + }) + }) + + describe('Apps', () => { + const app: App = { id: 'app1' } + + test('should create an app', () => { + const created = service.createApp(app) + expect(created).toEqual({ id: 'app1' }) + expect(service.getApp(created.id)).toEqual(created) + }) + + test('should throw an error when creating duplicate app', () => { + service.createApp(app) + expect(() => service.createApp(app)).toThrow(AlreadyExists) + }) + }) + + describe('Policies', () => { + test('should retrieve policies', () => { + expect(service.getPolicies()).toEqual({}) + }) + + test('should update policies', () => { + service.updatePolicies({ 'require-limits': { action: 'Audit' } }) + expect(service.getPolicies()).toEqual({ 'require-limits': { action: 'Audit' } }) + }) + + test('should retrieve a single policy', () => { + service.updatePolicies({ 'require-limits': { action: 'Audit' } }) + expect(service.getPolicy('require-limits')).toEqual({ action: 'Audit' }) + }) + }) + + describe('Settings', () => { + test('should retrieve settings', () => { + expect(service.getSettings()).toEqual({ name: 'team1' }) + }) + + test('should update settings', () => { + const updated = service.updateSettings({ name: 'UpdatedTeam' }) + expect(updated).toEqual({ name: 'UpdatedTeam' }) + expect(service.getSettings().name).toBe('UpdatedTeam') + }) + }) + + describe('doesProjectNameExist', () => { + test('should return false when no projects exist', () => { + expect(service.doesProjectNameExist('NonExistentProject')).toBe(false) + }) + + test('should return true when a build with the given name exists', () => { + service.createBuild({ name: 'ExistingBuild' }) + expect(service.doesProjectNameExist('ExistingBuild')).toBe(true) + }) + + test('should return true when a workload with the given name exists', () => { + service.createWorkload({ name: 'ExistingWorkload', url: 'http://example.com' }) + expect(service.doesProjectNameExist('ExistingWorkload')).toBe(true) + }) + + test('should return true when a service with the given name exists', () => { + service.createService({ name: 'ExistingService', ingress: {} }) + expect(service.doesProjectNameExist('ExistingService')).toBe(true) + }) + + test('should return false when the name does not match any existing project', () => { + service.createBuild({ name: 'SomeBuild' }) + service.createWorkload({ name: 'SomeWorkload', url: 'http://example.com' }) + service.createService({ name: 'SomeService', ingress: {} }) + expect(service.doesProjectNameExist('NonExistentProject')).toBe(false) + }) + }) + + describe('getCollection', () => { + test('should retrieve an existing collection', () => { + service.createBuild({ name: 'TestBuild' }) + expect(service.getCollection('builds')).toEqual([{ name: 'TestBuild', id: expect.any(String) }]) + }) + + test('should throw an error when trying to retrieve a non-existent collection', () => { + expect(() => service.getCollection('nonExistentCollection')).toThrowError( + 'Getting TeamConfig collection [nonExistentCollection] does not exist.', + ) + }) + }) + + describe('updateCollection', () => { + test('should update an existing collection', () => { + service.createBuild({ name: 'Build1' }) + service.updateCollection('builds', [{ name: 'UpdatedBuild' }]) + expect(service.getCollection('builds')).toEqual([{ name: 'UpdatedBuild' }]) + }) + + test('should create a new collection if it does not exist', () => { + service.updateCollection('customCollection', [{ key: 'value' }]) + expect(service.getCollection('customCollection')).toEqual([{ key: 'value' }]) + }) + + test('should replace the collection with the new value', () => { + service.createBuild({ name: 'OldBuild' }) + service.updateCollection('builds', [{ name: 'NewBuild' }]) + expect(service.getCollection('builds')).toEqual([{ name: 'NewBuild' }]) + }) + }) +}) diff --git a/src/services/TeamConfigService.ts b/src/services/TeamConfigService.ts new file mode 100644 index 000000000..4b3cf4084 --- /dev/null +++ b/src/services/TeamConfigService.ts @@ -0,0 +1,480 @@ +import { find, has, merge, remove, set } from 'lodash' +import { v4 as uuidv4 } from 'uuid' +import { AlreadyExists, NotExistError } from '../error' +import { + App, + Backup, + Build, + CodeRepo, + Netpol, + Policies, + Policy, + Project, + SealedSecret, + Service, + Team, + TeamConfig, + Workload, + WorkloadValues, +} from '../otomi-models' + +export class TeamConfigService { + constructor(private teamConfig: TeamConfig) { + this.teamConfig.builds ??= [] + this.teamConfig.workloads ??= [] + this.teamConfig.workloadValues ??= [] + this.teamConfig.services ??= [] + this.teamConfig.sealedsecrets ??= [] + this.teamConfig.backups ??= [] + this.teamConfig.projects ??= [] + this.teamConfig.netpols ??= [] + this.teamConfig.apps ??= [] + this.teamConfig.policies ??= {} + } + + // ===================================== + // == BUILDS CRUD == + // ===================================== + + public createBuild(build: Build): Build { + this.teamConfig.builds ??= [] + const newBuild = { ...build, id: build.id ?? uuidv4() } + if (find(this.teamConfig.builds, { name: newBuild.name })) { + throw new AlreadyExists(`Build[${newBuild.name}] already exists.`) + } + this.teamConfig.builds.push(newBuild) + return newBuild + } + + public getBuild(name: string): Build { + const build = find(this.teamConfig.builds, { name }) + if (!build) { + throw new NotExistError(`Build[${name}] does not exist.`) + } + return build + } + + public getBuilds(): Build[] { + const teamId = this.teamConfig.settings?.id + return (this.teamConfig.builds ?? []).map((build) => ({ + ...build, + teamId, + })) + } + + public updateBuild(name: string, updates: Partial): Build { + const build = find(this.teamConfig.builds, { name }) + if (!build) throw new NotExistError(`Build[${name}] does not exist.`) + return merge(build, updates) + } + + public deleteBuild(name: string): void { + remove(this.teamConfig.builds, { name }) + } + + // ===================================== + // == CODEREPOS CRUD == + // ===================================== + + public createCodeRepo(codeRepo: CodeRepo): CodeRepo { + this.teamConfig.codeRepos ??= [] + if (find(this.teamConfig.codeRepos, { name: codeRepo.name })) { + throw new AlreadyExists(`CodeRepo[${codeRepo.name}] already exists.`) + } + this.teamConfig.codeRepos.push(codeRepo) + return codeRepo + } + + public getCodeRepo(name: string): CodeRepo { + const codeRepo = find(this.teamConfig.codeRepos, { name }) + if (!codeRepo) { + throw new NotExistError(`CodeRepo[${name}] does not exist.`) + } + return codeRepo + } + + public getCodeRepos(): CodeRepo[] { + const teamId = this.teamConfig.settings?.id + return (this.teamConfig.codeRepos ?? []).map((codeRepo) => ({ + ...codeRepo, + teamId, + })) + } + + public updateCodeRepo(name: string, updates: Partial): CodeRepo { + const codeRepo = find(this.teamConfig.codeRepos, { name }) + if (!codeRepo) throw new NotExistError(`CodeRepo[${name}] does not exist.`) + return merge(codeRepo, updates) + } + + public deleteCodeRepo(name: string): void { + remove(this.teamConfig.codeRepos, { name }) + } + + // ===================================== + // == WORKLOADS CRUD == + // ===================================== + + public createWorkload(workload: Workload): Workload { + this.teamConfig.workloads ??= [] + const newWorkload = { ...workload, id: workload.id ?? uuidv4() } + if (find(this.teamConfig.workloads, { name: newWorkload.name })) { + throw new AlreadyExists(`Workload[${newWorkload.name}] already exists.`) + } + + this.teamConfig.workloads.push(newWorkload) + return newWorkload + } + + public getWorkload(name: string): Workload { + const workload = find(this.teamConfig.workloads, { name }) + if (!workload) { + throw new NotExistError(`Workload[${name}] does not exist.`) + } + return workload + } + + public getWorkloads(): Workload[] { + const teamId = this.teamConfig.settings?.id + return (this.teamConfig.workloads ?? []).map((workloadValues) => ({ + ...workloadValues, + teamId, + })) + } + + public updateWorkload(name: string, updates: Partial): Workload { + const workload = find(this.teamConfig.workloads, { name }) + if (!workload) throw new NotExistError(`Workload[${name}] does not exist.`) + return merge(workload, updates) + } + + public deleteWorkload(name: string): void { + remove(this.teamConfig.workloads, { name }) + } + + // ===================================== + // == WORKLOADVALUES CRUD == + // ===================================== + + public createWorkloadValues(workloadValues: WorkloadValues): WorkloadValues { + this.teamConfig.workloadValues ??= [] + const newWorkloadValues = { ...workloadValues, id: workloadValues.id ?? uuidv4() } + if ( + find(this.teamConfig.workloadValues, { name: newWorkloadValues.name }) || + find(this.teamConfig.workloadValues, { id: newWorkloadValues.id }) + ) { + throw new AlreadyExists(`WorkloadValues[${newWorkloadValues.name}] already exists.`) + } + this.teamConfig.workloadValues.push(newWorkloadValues) + return newWorkloadValues + } + + public getWorkloadValues(name: string): WorkloadValues { + const workloadValues = find(this.teamConfig.workloadValues, { name }) + if (!workloadValues) { + throw new NotExistError(`WorkloadValues[${name}] does not exist.`) + } + return workloadValues + } + + public updateWorkloadValues(name: string, updates: Partial): WorkloadValues { + const workloadValues = find(this.teamConfig.workloadValues, { name }) + if (!workloadValues) throw new NotExistError(`WorkloadValues[${name}] does not exist.`) + return merge(workloadValues, updates) + } + + public deleteWorkloadValues(name: string): void { + remove(this.teamConfig.workloadValues, { name }) + } + + // ===================================== + // == SERVICES CRUD == + // ===================================== + + public createService(service: Service): Service { + this.teamConfig.services ??= [] + const newService = { ...service, id: service.id ?? uuidv4() } + if (find(this.teamConfig.services, { name: newService.name })) { + throw new AlreadyExists(`Service[${newService.name}] already exists.`) + } + this.teamConfig.services.push(newService) + return newService + } + + public getService(name: string): Service { + const service = find(this.teamConfig.services, { name }) + if (!service) { + throw new NotExistError(`Service[${name}] does not exist.`) + } + return service + } + + public getServices(): Service[] { + const teamId = this.teamConfig.settings?.id + return (this.teamConfig.services ?? []).map((service) => ({ + ...service, + teamId, + })) + } + + public updateService(name: string, updates: Partial): Service { + const service = find(this.teamConfig.services, { name }) + if (!service) throw new NotExistError(`Service[${name}] does not exist.`) + return merge(service, updates) + } + + public deleteService(name: string): void { + remove(this.teamConfig.services, { name }) + } + + // ===================================== + // == SEALED SECRETS CRUD == + // ===================================== + + public createSealedSecret(secret: SealedSecret): SealedSecret { + this.teamConfig.sealedsecrets ??= [] + const newSecret = { ...secret, id: secret.id ?? uuidv4() } + if (find(this.teamConfig.sealedsecrets, { name: newSecret.name })) { + throw new AlreadyExists(`SealedSecret[${newSecret.name}] already exists.`) + } + this.teamConfig.sealedsecrets.push(newSecret) + return newSecret + } + + public getSealedSecret(name: string): SealedSecret { + const sealedSecrets = find(this.teamConfig.sealedsecrets, { name }) + if (!sealedSecrets) { + throw new NotExistError(`SealedSecret[${name}] does not exist.`) + } + return sealedSecrets + } + + public getSealedSecrets(): SealedSecret[] { + const teamId = this.teamConfig.settings?.id + return (this.teamConfig.sealedsecrets ?? []).map((sealedSecret) => ({ + ...sealedSecret, + teamId, + })) + } + + public updateSealedSecret(name: string, updates: Partial): SealedSecret { + const secret = find(this.teamConfig.sealedsecrets, { name }) + if (!secret) throw new NotExistError(`SealedSecret[${name}] does not exist.`) + return merge(secret, updates) + } + + public deleteSealedSecret(name: string): void { + remove(this.teamConfig.sealedsecrets, { name }) + } + + // ===================================== + // == BACKUPS CRUD == + // ===================================== + + public createBackup(backup: Backup): Backup { + this.teamConfig.backups ??= [] + const newBackup = { ...backup, id: backup.id ?? uuidv4() } + if (find(this.teamConfig.backups, { name: newBackup.name })) { + throw new AlreadyExists(`Backup[${newBackup.name}] already exists.`) + } + this.teamConfig.backups.push(newBackup) + return newBackup + } + + public getBackup(name: string): Backup { + const backup = find(this.teamConfig.backups, { name }) + if (!backup) { + throw new NotExistError(`Backup[${name}] does not exist.`) + } + return backup + } + + public getBackups(): Backup[] { + const teamId = this.teamConfig.settings?.id + return (this.teamConfig.backups ?? []).map((backup) => ({ + ...backup, + teamId, + })) + } + + public updateBackup(name: string, updates: Partial): Backup { + const backup = find(this.teamConfig.backups, { name }) + if (!backup) throw new NotExistError(`Backup[${name}] does not exist.`) + return merge(backup, updates) + } + + public deleteBackup(name: string): void { + remove(this.teamConfig.backups, { name }) + } + + // ===================================== + // == PROJECTS CRUD == + // ===================================== + + public createProject(project: Project): Project { + this.teamConfig.projects ??= [] + const newProject = { ...project, id: project.id ?? uuidv4() } + if (find(this.teamConfig.projects, { name: newProject.name })) { + throw new AlreadyExists(`Project[${newProject.name}] already exists.`) + } + this.teamConfig.projects.push(newProject) + return newProject + } + + public getProject(name: string): Project { + const project = find(this.teamConfig.projects, { name }) + if (!project) { + throw new NotExistError(`Project[${name}] does not exist.`) + } + return project + } + + public getProjects(): Project[] { + const teamId = this.teamConfig.settings?.id + return (this.teamConfig.projects ?? []).map((project) => ({ + ...project, + teamId, + })) + } + + public updateProject(name: string, updates: Partial): Project { + const project = find(this.teamConfig.projects, { name }) + if (!project) throw new NotExistError(`Project[${name}] does not exist.`) + return merge(project, updates) + } + + public deleteProject(name: string): void { + remove(this.teamConfig.projects, { name }) + } + + // ===================================== + // == NETPOLS CRUD == + // ===================================== + + public createNetpol(netpol: Netpol): Netpol { + this.teamConfig.netpols ??= [] + const newNetpol = { ...netpol, id: netpol.id ?? uuidv4() } + if (find(this.teamConfig.netpols, { name: newNetpol.name })) { + throw new AlreadyExists(`Netpol[${newNetpol.name}] already exists.`) + } + this.teamConfig.netpols.push(newNetpol) + return newNetpol + } + + public getNetpol(name: string): Netpol { + const netpol = find(this.teamConfig.netpols, { name }) + if (!netpol) { + throw new NotExistError(`Netpol[${name}] does not exist.`) + } + return netpol + } + + public getNetpols(): Netpol[] { + const teamId = this.teamConfig.settings?.id + return (this.teamConfig.netpols ?? []).map((netpol) => ({ + ...netpol, + teamId, + })) + } + + public updateNetpol(name: string, updates: Partial): Netpol { + const netpol = find(this.teamConfig.netpols, { name }) + if (!netpol) { + throw new NotExistError(`Netpol[${name}] does not exist.`) + } + return merge(netpol, updates) + } + + public deleteNetpol(name: string): void { + remove(this.teamConfig.netpols, { name }) + } + + // ===================================== + // == SETTINGS CRUD == + // ===================================== + + public getSettings(): Team { + return this.teamConfig.settings + } + + public updateSettings(updates: Partial): Team { + if (!this.teamConfig.settings) { + this.teamConfig.settings = { name: updates.name || '' } + } + return merge(this.teamConfig.settings, updates) + } + + // ===================================== + // == APPS CRUD == + // ===================================== + + public createApp(app: App): App { + this.teamConfig.apps ??= [] + const newApp = { ...app, id: app.id ?? uuidv4() } + if (find(this.teamConfig.apps, { id: newApp.id })) { + throw new AlreadyExists(`App[${app.id}] already exists.`) + } + this.teamConfig.apps.push(newApp) + return newApp + } + + public getApp(id: string): App { + const app = find(this.teamConfig.apps, { id }) + if (!app) { + throw new NotExistError(`App[${id}] does not exist.`) + } + return app + } + + public getApps(): App[] { + const teamId = this.teamConfig.settings?.id + return (this.teamConfig.apps ?? []).map((app) => ({ + ...app, + teamId, + })) + } + + public setApps(apps: App[]) { + this.teamConfig.apps = apps + } + + // ===================================== + // == POLICIES CRUD == + // ===================================== + + public getPolicies(): Policies { + return this.teamConfig.policies + } + + public getPolicy(key: string): Policy { + return this.teamConfig.policies[key] + } + + public updatePolicies(updates: Partial): void { + if (!this.teamConfig.policies) { + this.teamConfig.policies = {} + } + merge(this.teamConfig.policies, updates) + } + + public doesProjectNameExist(name: string): boolean { + return ( + (this.teamConfig.builds && this.teamConfig.builds.some((build) => build.name === name)) || + (this.teamConfig.workloads && this.teamConfig.workloads.some((workload) => workload.name === name)) || + (this.teamConfig.services && this.teamConfig.services.some((service) => service.name === name)) + ) + } + + /** Retrieve a collection dynamically from the Teamconfig */ + public getCollection(collectionId: string): any { + if (!has(this.teamConfig, collectionId)) { + throw new Error(`Getting TeamConfig collection [${collectionId}] does not exist.`) + } + return this.teamConfig[collectionId] + } + + /** Update a collection dynamically in the Teamconfig */ + public updateCollection(collectionId: string, data: any): void { + set(this.teamConfig, collectionId, data) + } +} diff --git a/src/utils.ts b/src/utils.ts index ec0991c5f..e53f09e43 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,15 +1,16 @@ import axios from 'axios' import cleanDeep, { CleanOptions } from 'clean-deep' import { pathExists } from 'fs-extra' -import { readFile } from 'fs/promises' -import { isArray, memoize, mergeWith, omit } from 'lodash' +import { readdir, readFile } from 'fs/promises' +import { isArray, isEmpty, memoize, mergeWith, omit } from 'lodash' import cloneDeep from 'lodash/cloneDeep' import { Cluster, Dns } from 'src/otomi-models' -import { parse } from 'yaml' +import { parse, stringify } from 'yaml' import { BASEURL } from './constants' export function arrayToObject(array: [] = [], keyName = 'name', keyValue = 'value'): Record { const obj = {} + if (!Array.isArray(array)) return array array.forEach((item) => { const cloneItem = cloneDeep(item) obj[cloneItem[keyName]] = cloneItem[keyValue] @@ -165,3 +166,17 @@ export const argQuoteStrip = (s: string) => { // use lodash mergeWith to avoid merging arrays export const mergeData = (orig, extra) => mergeWith(orig, extra, (a, b) => (isArray(b) ? b : undefined)) + +export const getDirNames = async (dir: string, opts?: { skipHidden: boolean }): Promise => { + const dirs = await readdir(dir, { withFileTypes: true }) + const dirNames: Array = [] + dirs.map((dirOrFile) => { + if (opts?.skipHidden && dirOrFile.name.startsWith('.')) return + if (dirOrFile.isDirectory()) dirNames.push(dirOrFile.name) + }) + return dirNames +} + +export const objectToYaml = (obj: Record, indent = 4, lineWidth = 200): string => { + return isEmpty(obj) ? '' : stringify(obj, { indent, lineWidth }) +} diff --git a/src/utils/coderepoUtils.test.ts b/src/utils/codeRepoUtils.test.ts similarity index 98% rename from src/utils/coderepoUtils.test.ts rename to src/utils/codeRepoUtils.test.ts index 9b6cd9837..efe0c967b 100644 --- a/src/utils/coderepoUtils.test.ts +++ b/src/utils/codeRepoUtils.test.ts @@ -4,7 +4,7 @@ import { chmod, writeFile } from 'fs/promises' import simpleGit, { SimpleGit } from 'simple-git' import { OtomiError } from 'src/error' import { v4 as uuidv4 } from 'uuid' -import { getGiteaRepoUrls, normalizeRepoUrl, testPrivateRepoConnect, testPublicRepoConnect } from './coderepoUtils' +import { getGiteaRepoUrls, normalizeRepoUrl, testPrivateRepoConnect, testPublicRepoConnect } from './codeRepoUtils' jest.mock('simple-git', () => ({ __esModule: true, @@ -27,7 +27,7 @@ jest.mock('src/error', () => ({ }), })) -describe('coderepoUtils', () => { +describe('codeRepoUtils', () => { beforeEach(() => { jest.clearAllMocks() }) diff --git a/src/utils/coderepoUtils.ts b/src/utils/codeRepoUtils.ts similarity index 100% rename from src/utils/coderepoUtils.ts rename to src/utils/codeRepoUtils.ts diff --git a/src/utils/sealedSecretUtils.ts b/src/utils/sealedSecretUtils.ts index 4c16b080d..4b703494d 100644 --- a/src/utils/sealedSecretUtils.ts +++ b/src/utils/sealedSecretUtils.ts @@ -41,7 +41,10 @@ export function encryptSecretItem(certificate, secretName, ns, data, scope) { return out } -export type EncryptedDataRecord = Record +export interface EncryptedDataRecord { + key: string + value: string +} export interface SealedSecretManifestType { apiVersion: string @@ -54,7 +57,7 @@ export interface SealedSecretManifestType { labels?: Record } spec: { - encryptedData: EncryptedDataRecord + encryptedData: EncryptedDataRecord[] template: { type: | 'kubernetes.io/opaque' @@ -75,7 +78,7 @@ export interface SealedSecretManifestType { export function sealedSecretManifest( data: SealedSecret, - encryptedData: EncryptedDataRecord, + encryptedData: EncryptedDataRecord[], namespace: string, ): SealedSecretManifestType { const annotations = data.metadata?.annotations?.reduce((acc, item) => { diff --git a/src/utils/userUtils.ts b/src/utils/userUtils.ts index 18fe7468a..fc132afcb 100644 --- a/src/utils/userUtils.ts +++ b/src/utils/userUtils.ts @@ -34,7 +34,7 @@ export async function getKeycloakUsers( realm: string, username: string, password: string, -): Promise<{ email: string }[]> { +): Promise { try { const token = await getKeycloakToken(keycloakBaseUrl, realm, username, password) const url = `${keycloakBaseUrl}/admin/realms/${realm}/users` @@ -45,12 +45,10 @@ export async function getKeycloakUsers( }, }) - const users = [] as { email: string }[] + const users = [] as string[] for (const user of response.data) { if (user.username === env.ROOT_KEYCLOAK_USER) continue - users.push({ - email: user.email, - }) + users.push(user.email) } return users } catch (error) {