From ab98d05d545b0d0ef403df05f97c40a9c3277e59 Mon Sep 17 00:00:00 2001 From: CasLubbers Date: Wed, 12 Mar 2025 14:27:52 +0100 Subject: [PATCH 1/8] fix(git save): increase git pull retries and remove doRestore (#651) Git pull retries have increase to 10 and doRestore have been replaced with always cleaning the session Co-authored-by: Jehoszafat Zimnowoda <17126497+j-zimnowoda@users.noreply.github.com> (cherry picked from commit a8ae5ef815ca8494f5c0228677949cf90f450a2a) --- src/k8s_operations.ts | 6 ++++-- src/otomi-stack.ts | 19 +++++++++---------- src/repo.ts | 18 ++++++++++++++---- src/validators.ts | 6 +++++- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/k8s_operations.ts b/src/k8s_operations.ts index 3f9bb6630..b8ae5aa39 100644 --- a/src/k8s_operations.ts +++ b/src/k8s_operations.ts @@ -391,7 +391,7 @@ export async function getSecretValues(name: string, namespace: string): Promise< } return decodedData } catch (error) { - debug(`Failed to get secret values for ${name} in ${namespace}.`) + if (process.env.NODE_ENV !== 'development') debug(`Failed to get secret values for ${name} in ${namespace}.`) } } @@ -417,7 +417,9 @@ export async function getSealedSecretSyncedStatus(name: string, namespace: strin } return 'NotFound' } catch (error) { - debug(`Failed to get SealedSecret synced status for ${name} in ${namespace}.`) + if (process.env.NODE_ENV !== 'development') + debug(`Failed to get SealedSecret synced status for ${name} in ${namespace}.`) + return 'NotFound' } } diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index 188ae8cf2..27b4c77e0 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -3,15 +3,15 @@ import * as k8s from '@kubernetes/client-node' import { V1ObjectReference } from '@kubernetes/client-node' import Debug from 'debug' -import { ObjectStorageKeyRegions, getRegions } from '@linode/api-v4' +import { getRegions, ObjectStorageKeyRegions } from '@linode/api-v4' import { emptyDir, existsSync, pathExists, rmSync, unlink } from 'fs-extra' -import { readFile, readdir, writeFile } from 'fs/promises' +import { readdir, readFile, writeFile } from 'fs/promises' import { generate as generatePassword } from 'generate-password' import { cloneDeep, filter, get, isArray, isEmpty, map, omit, pick, set, unset } from 'lodash' import { getAppList, getAppSchema, getSpec } from 'src/app' import Db from 'src/db' -import { AlreadyExists, GitPullError, HttpError, OtomiError, PublicUrlExists, ValidationError } from 'src/error' -import { DbMessage, cleanAllSessions, cleanSession, getIo, getSessionStack } from 'src/middleware' +import { AlreadyExists, HttpError, OtomiError, PublicUrlExists, ValidationError } from 'src/error' +import { cleanAllSessions, cleanSession, DbMessage, getIo, getSessionStack } from 'src/middleware' import { App, Backup, @@ -42,6 +42,7 @@ import { import getRepo, { Repo } from 'src/repo' import { arrayToObject, getServiceUrl, getValuesSchema, objectToArray, removeBlankAttributes } from 'src/utils' import { + cleanEnv, CUSTOM_ROOT_CA, DEFAULT_PLATFORM_ADMIN_EMAIL, EDITOR_INACTIVITY_TIMEOUT, @@ -56,7 +57,6 @@ import { PREINSTALLED_EXCLUDED_APPS, TOOLS_HOST, VERSIONS, - cleanEnv, } from 'src/validators' import { v4 as uuidv4 } from 'uuid' import { parse as parseYaml, stringify as stringifyYaml } from 'yaml' @@ -83,7 +83,7 @@ import { getPolicies } from './utils/policiesUtils' import { EncryptedDataRecord, encryptSecretItem, sealedSecretManifest } from './utils/sealedSecretUtils' import { getKeycloakUsers, isValidUsername } from './utils/userUtils' import { ObjectStorageClient } from './utils/wizardUtils' -import { NewChartPayload, fetchWorkloadCatalog, sparseCloneChart } from './utils/workloadUtils' +import { fetchWorkloadCatalog, NewChartPayload, sparseCloneChart } from './utils/workloadUtils' interface ExcludedApp extends App { managed: boolean @@ -1437,16 +1437,15 @@ export default class OtomiStack { }) } debug(`Updated root stack values with ${this.sessionId} changes`) - // and remove editor from the session - await cleanSession(this.sessionId!) const sha = await rootStack.repo.getCommitSha() this.emitPipelineStatus(sha) } catch (e) { - // git conflict with upstream changes, clean up and restore the DB - if (e instanceof GitPullError) await this.doRestore() const msg: DbMessage = { editor: 'system', state: 'corrupt', reason: 'deploy' } getIo().emit('db', msg) throw e + } finally { + // Clean up the session + await cleanSession(this.sessionId!) } } diff --git a/src/repo.ts b/src/repo.ts index 6f8070989..e46ca0b3a 100644 --- a/src/repo.ts +++ b/src/repo.ts @@ -8,7 +8,16 @@ import stringifyJson from 'json-stable-stringify' import { cloneDeep, get, isEmpty, merge, set, unset } from 'lodash' import { basename, dirname, join } from 'path' import simpleGit, { CheckRepoActions, CleanOptions, CommitResult, ResetMode, SimpleGit } from 'simple-git' -import { GIT_BRANCH, GIT_LOCAL_PATH, GIT_PASSWORD, GIT_REPO_URL, GIT_USER, TOOLS_HOST, cleanEnv } from 'src/validators' +import { + cleanEnv, + GIT_BRANCH, + GIT_LOCAL_PATH, + GIT_PASSWORD, + GIT_PUSH_RETRIES, + GIT_REPO_URL, + GIT_USER, + TOOLS_HOST, +} from 'src/validators' import { parse as parseYaml, stringify as stringifyYaml } from 'yaml' import { BASEURL } from './constants' import { GitPullError, HttpError, ValidationError } from './error' @@ -24,6 +33,7 @@ const env = cleanEnv({ GIT_PASSWORD, GIT_REPO_URL, GIT_USER, + GIT_PUSH_RETRIES, TOOLS_HOST, }) @@ -372,16 +382,16 @@ export class Repo { // with the remote root, which might have been modified by another developer // since this is a child branch, we don't need to re-init // retry up to 3 times to pull and push if there are conflicts - const skipInit = true - const retries = 3 + const retries = env.GIT_PUSH_RETRIES for (let attempt = 1; attempt <= retries; attempt++) { + await this.git.pull(this.remote, this.branch, { '--rebase': 'true', '--depth': '5' }) try { - await this.pull(skipInit) await this.push() break } catch (error) { if (attempt === retries) throw error debug(`Attempt ${attempt} failed. Retrying...`) + await new Promise((resolve) => setTimeout(resolve, 50)) } } } catch (e) { diff --git a/src/validators.ts b/src/validators.ts index 7012fa788..ae7b58c85 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -1,4 +1,4 @@ -import { CleanOptions, CleanedEnvAccessors, ValidatorSpec, bool, cleanEnv as clean, json, num, str } from 'envalid' +import { bool, CleanedEnvAccessors, cleanEnv as clean, CleanOptions, json, num, str, ValidatorSpec } from 'envalid' export const AUTHZ_MOCK_IS_PLATFORM_ADMIN = bool({ desc: 'Indicate if a mocked user is a platform admin', @@ -75,6 +75,10 @@ export const EXPRESS_PAYLOAD_LIMIT = str({ desc: 'The express payload limit', default: '500kb', }) +export const GIT_PUSH_RETRIES = num({ + desc: 'Amount of retries we do to push and pull in the git save function', + default: 10, +}) const { env } = process export function cleanEnv( validators: { [K in keyof T]: ValidatorSpec }, From ab6d57cf328bf1a0bec6a40c1a81c0e15be6e51a Mon Sep 17 00:00:00 2001 From: CasLubbers Date: Wed, 12 Mar 2025 14:43:28 +0100 Subject: [PATCH 2/8] fix: add policies to api check (#652) * fix: add policies to api check (cherry picked from commit 4d384f2044d172f9398d9acde51ff757714ffa38) --- src/api.authz.test.ts | 38 ++++++++++++++++++++++++++++++++++++++ src/middleware/authz.ts | 14 +++++++++++--- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/api.authz.test.ts b/src/api.authz.test.ts index 903c6492a..327c16cf3 100644 --- a/src/api.authz.test.ts +++ b/src/api.authz.test.ts @@ -733,4 +733,42 @@ describe('API authz tests', () => { .expect(200) }) }) + + describe('Policy endpoint tests', () => { + const data = { action: 'Enforce', severity: 'high' } + + test('platform admin can get policies', async () => { + await agent + .get('/v1/teams/team1/policies') + .set('Authorization', `Bearer ${platformAdminToken}`) + .expect(200) + .expect('Content-Type', /json/) + }) + + test('platform admin can update policies', async () => { + await agent + .put('/v1/teams/team1/policies/disallow-selinux') + .send(data) + .set('Authorization', `Bearer ${platformAdminToken}`) + .expect(200) + .expect('Content-Type', /json/) + }) + + test('team member can get policies', async () => { + await agent + .get('/v1/teams/team1/policies') + .set('Authorization', `Bearer ${teamMemberToken}`) + .expect(200) + .expect('Content-Type', /json/) + }) + + test('team member can not update policies', async () => { + await agent + .put('/v1/teams/team1/policies/disallow-selinux') + .send(data) + .set('Authorization', `Bearer ${teamMemberToken}`) + .expect(403) + .expect('Content-Type', /json/) + }) + }) }) diff --git a/src/middleware/authz.ts b/src/middleware/authz.ts index 6375e93ea..d1ca3b930 100644 --- a/src/middleware/authz.ts +++ b/src/middleware/authz.ts @@ -58,8 +58,9 @@ export function authorize(req: OpenApiRequestExt, res, next, authz: Authz, db: D valid = authz.hasSelfService(teamId, 'access', 'downloadKubeConfig') else if (action === 'read' && schemaName === 'DockerConfig') valid = authz.hasSelfService(teamId, 'access', 'downloadDockerConfig') - else if (action === 'create' && schemaName === 'Cloudtty') - valid = authz.hasSelfService(body.teamId, 'access', 'shell') + else if (action === 'create' && schemaName === 'Cloudtty') valid = authz.hasSelfService(teamId, 'access', 'shell') + else if (action === 'update' && schemaName === 'Policy') + valid = authz.hasSelfService(teamId, 'policies', 'edit policies') else valid = authz.validateWithCasl(action, schemaName, teamId) const env = cleanEnv({}) // TODO: Debug purpose only for removal of license @@ -75,6 +76,7 @@ export function authorize(req: OpenApiRequestExt, res, next, authz: Authz, db: D Secret: 'secrets', Service: 'services', Team: 'teams', + Policy: 'policies', } const selector = renameKeys(req.params) @@ -87,7 +89,13 @@ export function authorize(req: OpenApiRequestExt, res, next, authz: Authz, db: D {}, ) - if (action === 'update') dataOrig = db.getItemReference(collection, selector, false) as Record + if (action === 'update') { + if (collection === 'policies') { + const policies = db.db.get(['policies']).value() + const id = req.params.policyId + dataOrig = policies[teamId][id] + } else dataOrig = db.getItemReference(collection, selector, false) as Record + } const violatedAttributes = authz.validateWithAbac(action, schemaName, teamId, req.body, dataOrig) if (violatedAttributes.length > 0) { return res.status(403).send({ From 8dd5f05020764565c5a3e5d5739d7e59767b258e Mon Sep 17 00:00:00 2001 From: Ferruh <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Mar 2025 15:34:44 +0100 Subject: [PATCH 3/8] feat: update fetch workload catalog (#655) (cherry picked from commit b5db0b441b0767862cf2d7aa8ee0bfd18453ab7a) --- src/utils/workloadUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index 9b09fd150..b1ffc6047 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -258,7 +258,7 @@ export async function fetchWorkloadCatalog(url: string, helmChartsDir: string, t if (!rbac[folder] || rbac[folder].includes(`team-${teamId}`) || teamId === 'admin') { const catalogItem = { name: folder, - values, + values: values || '{}', icon: chartMetadata?.icon, chartVersion: chartMetadata?.version, chartDescription: chartMetadata?.description, From 1e523ae71f8b45cc8b1211b96cfe21d6217ca35a Mon Sep 17 00:00:00 2001 From: Ferruh <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Mar 2025 17:31:47 +0100 Subject: [PATCH 4/8] feat: update add new helm chart (#653) * feat: update add new helm chart * feat: update support for gitlab * fix: normalizedUrl & code conventions * fix: gitlab regex * feat: add GIT_PROVIDER_URL_PATTERNS env --------- Co-authored-by: jeho <17126497+j-zimnowoda@users.noreply.github.com> (cherry picked from commit f7d6e3e63296ff2b922951c5f1748e42f99351ed) --- src/api/helmChartContent.ts | 20 + src/openapi/api.yaml | 22 ++ src/otomi-stack.ts | 26 +- src/utils/workloadUtils.test.ts | 653 ++++++++++++++++++++++++++++---- src/utils/workloadUtils.ts | 166 +++++--- src/validators.ts | 8 + 6 files changed, 767 insertions(+), 128 deletions(-) create mode 100644 src/api/helmChartContent.ts diff --git a/src/api/helmChartContent.ts b/src/api/helmChartContent.ts new file mode 100644 index 000000000..c0ae27677 --- /dev/null +++ b/src/api/helmChartContent.ts @@ -0,0 +1,20 @@ +import Debug from 'debug' +import { Operation, OperationHandlerArray } from 'express-openapi' +import { OpenApiRequestExt } from 'src/otomi-models' + +const debug = Debug('otomi:api:helmChartContent') + +export default function (): OperationHandlerArray { + const get: Operation = [ + async ({ otomi, query }: OpenApiRequestExt, res): Promise => { + debug(`gethelmChartContent ${query?.url}`) + const v = await otomi.getHelmChartContent(query?.url as string) + res.json(v) + return v + }, + ] + const api = { + get, + } + return api +} diff --git a/src/openapi/api.yaml b/src/openapi/api.yaml index 99dc8bb1f..0398179ff 100644 --- a/src/openapi/api.yaml +++ b/src/openapi/api.yaml @@ -1289,6 +1289,28 @@ paths: application/json: schema: type: object + '/helmChartContent': + get: + operationId: getHelmChartContent + parameters: + - name: url + in: query + description: URL of the helm chart + schema: + type: string + responses: + <<: *DefaultGetResponses + '200': + description: Successfully obtained helm chart content + content: + application/json: + schema: + type: object + properties: + values: + type: object + error: + type: string '/createWorkloadCatalog': post: operationId: createWorkloadCatalog diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index 27b4c77e0..9660eca1d 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -83,7 +83,7 @@ import { getPolicies } from './utils/policiesUtils' import { EncryptedDataRecord, encryptSecretItem, sealedSecretManifest } from './utils/sealedSecretUtils' import { getKeycloakUsers, isValidUsername } from './utils/userUtils' import { ObjectStorageClient } from './utils/wizardUtils' -import { fetchWorkloadCatalog, NewChartPayload, sparseCloneChart } from './utils/workloadUtils' +import { fetchChartYaml, fetchWorkloadCatalog, NewHelmChartValues, sparseCloneChart } from './utils/workloadUtils' interface ExcludedApp extends App { managed: boolean @@ -1236,34 +1236,36 @@ export default class OtomiStack { } } - async createWorkloadCatalog(body: NewChartPayload): Promise { - const { url, chartName, chartPath, chartIcon, revision, allowTeams } = body + async getHelmChartContent(url: string): Promise { + return await fetchChartYaml(url) + } + + async createWorkloadCatalog(body: NewHelmChartValues): Promise { + const { gitRepositoryUrl, chartTargetDirName, chartIcon, allowTeams } = body const uuid = uuidv4() - const helmChartsDir = `/tmp/otomi/charts/${uuid}` + const localHelmChartsDir = `/tmp/otomi/charts/${uuid}` const helmChartCatalogUrl = env.HELM_CHART_CATALOG const { user, email } = this.repo try { await sparseCloneChart( - url, + gitRepositoryUrl, + localHelmChartsDir, helmChartCatalogUrl, user, email, - chartName, - chartPath, - helmChartsDir, - revision, + chartTargetDirName, chartIcon, allowTeams, ) return true - } catch (err) { - debug(`error while parsing chart ${err.message}`) + } catch (error) { + debug('Error adding new Helm chart to catalog') return false } finally { // Clean up: if the temporary directory exists, remove it. - if (existsSync(helmChartsDir)) rmSync(helmChartsDir, { recursive: true, force: true }) + if (existsSync(localHelmChartsDir)) rmSync(localHelmChartsDir, { recursive: true, force: true }) } } diff --git a/src/utils/workloadUtils.test.ts b/src/utils/workloadUtils.test.ts index 2f1abf1a8..a03ae64ff 100644 --- a/src/utils/workloadUtils.test.ts +++ b/src/utils/workloadUtils.test.ts @@ -1,12 +1,20 @@ // workloadUtils.test.ts -import Debug from 'debug' +import axios from 'axios' import * as fsExtra from 'fs-extra' import * as fsPromises from 'fs/promises' -import simpleGit, { SimpleGit } from 'simple-git' +import simpleGit from 'simple-git' import YAML from 'yaml' -import { sparseCloneChart, updateChartIconInYaml, updateRbacForNewChart } from './workloadUtils' -const debug = Debug('otomi:api:workloadCatalog') +import { + detectGitProvider, + fetchChartYaml, + fetchWorkloadCatalog, + getGitCloneUrl, + sparseCloneChart, + updateChartIconInYaml, + updateRbacForNewChart, +} from './workloadUtils' +jest.mock('axios') jest.mock('fs/promises', () => ({ writeFile: jest.fn(), readdir: jest.fn(), @@ -33,6 +41,169 @@ jest.mock('simple-git', () => ({ })), })) +// Save the original environment variables +const originalEnv = process.env + +// ---------------------------------------------------------------- +// Tests for detectGitProvider +describe('detectGitProvider', () => { + test('returns null for undefined or non-string inputs', () => { + expect(detectGitProvider(undefined)).toBeNull() + expect(detectGitProvider(null)).toBeNull() + expect(detectGitProvider(123 as any)).toBeNull() + }) + + test('detects GitHub URLs correctly', () => { + const url = 'https://github.com/owner/repo/blob/main/path/to/file.yaml' + const result = detectGitProvider(url) + expect(result).toEqual({ + provider: 'github', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/file.yaml', + }) + }) + + test('detects GitLab URLs correctly', () => { + const url = 'https://gitlab.com/owner/charts/repo/-/blob/main/path/to/file.yaml' + const result = detectGitProvider(url) + expect(result).toEqual({ + provider: 'gitlab', + owner: 'owner', + repo: 'charts/repo', + branch: 'main', + filePath: 'path/to/file.yaml', + }) + }) + + test('detects Bitbucket URLs correctly', () => { + const url = 'https://bitbucket.org/owner/repo/src/main/path/to/file.yaml' + const result = detectGitProvider(url) + expect(result).toEqual({ + provider: 'bitbucket', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/file.yaml', + }) + }) + + test('returns null for unsupported URLs', () => { + const url = 'https://example.com/owner/repo/main/file.yaml' + expect(detectGitProvider(url)).toBeNull() + }) + + test('handles URLs with trailing slashes', () => { + const url = 'https://github.com/owner/repo/blob/main/path/to/file.yaml/' + const result = detectGitProvider(url) + expect(result).toEqual({ + provider: 'github', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/file.yaml', + }) + }) +}) + +// ---------------------------------------------------------------- +// Tests for getGitCloneUrl +describe('getGitCloneUrl', () => { + test('returns null for null input', () => { + expect(getGitCloneUrl(null)).toBeNull() + }) + + test('returns GitHub clone URL', () => { + const details = { + provider: 'github', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/Chart.yaml', + } + expect(getGitCloneUrl(details)).toBe('https://github.com/owner/repo.git') + }) + + test('returns GitLab clone URL', () => { + const details = { + provider: 'gitlab', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/Chart.yaml', + } + expect(getGitCloneUrl(details)).toBe('https://gitlab.com/owner/repo.git') + }) + + test('returns Bitbucket clone URL', () => { + const details = { + provider: 'bitbucket', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/Chart.yaml', + } + expect(getGitCloneUrl(details)).toBe('https://bitbucket.org/owner/repo.git') + }) + + test('returns null for unsupported provider', () => { + const details = { + provider: 'unsupported', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/Chart.yaml', + } + expect(getGitCloneUrl(details)).toBeNull() + }) +}) + +// ---------------------------------------------------------------- +// Tests for fetchChartYaml +describe('fetchChartYaml', () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + test('successfully fetches and parses Chart.yaml', async () => { + const url = 'https://github.com/owner/repo/blob/main/charts/mychart/Chart.yaml' + const mockChartData = 'name: mychart\nversion: 1.0.0\ndescription: Test Chart' + const expectedValues = { name: 'mychart', version: '1.0.0', description: 'Test Chart' } + + jest.spyOn(axios, 'get').mockResolvedValue({ data: mockChartData }) + + const result = await fetchChartYaml(url) + + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(axios.get).toHaveBeenCalledWith( + 'https://raw.githubusercontent.com/owner/repo/main/charts/mychart/Chart.yaml', + { responseType: 'text' }, + ) + expect(result).toEqual({ values: expectedValues, error: '' }) + }) + + test('returns error for unsupported Git provider', async () => { + const url = 'https://example.com/owner/repo/main/charts/mychart/Chart.yaml' + + const result = await fetchChartYaml(url) + + expect(result).toEqual({ values: {}, error: 'Unsupported Git provider or invalid URL format.' }) + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(axios.get).not.toHaveBeenCalled() + }) + + test('handles axios errors gracefully', async () => { + const url = 'https://github.com/owner/repo/blob/main/charts/mychart/Chart.yaml' + + jest.spyOn(axios, 'get').mockRejectedValue(new Error('Network error')) + + const result = await fetchChartYaml(url) + + expect(result).toEqual({ values: {}, error: 'Error fetching helm chart content.' }) + }) +}) + // ---------------------------------------------------------------- // Tests for updateChartIconInYaml describe('updateChartIconInYaml', () => { @@ -41,18 +212,60 @@ describe('updateChartIconInYaml', () => { }) test('updates the icon field when newIcon is provided', async () => { - const chartObject = { name: 'Test Chart', icon: '' } + const chartObject = { name: 'Test Chart', version: '1.0.0' } const fileContent = YAML.stringify(chartObject) ;(fsExtra.readFile as jest.Mock).mockResolvedValue(fileContent) const fakePath = '/tmp/test/Chart.yaml' const newIcon = 'https://example.com/new-icon.png' - const expectedObject = { name: 'Test Chart', icon: newIcon } + const expectedObject = { name: 'Test Chart', version: '1.0.0', icon: newIcon } const expectedContent = YAML.stringify(expectedObject) + await updateChartIconInYaml(fakePath, newIcon) expect(fsExtra.readFile).toHaveBeenCalledWith(fakePath, 'utf-8') expect(fsPromises.writeFile).toHaveBeenCalledWith(fakePath, expectedContent, 'utf-8') }) + + test('replaces existing icon when newIcon is provided', async () => { + const chartObject = { name: 'Test Chart', version: '1.0.0', icon: 'https://example.com/old-icon.png' } + const fileContent = YAML.stringify(chartObject) + ;(fsExtra.readFile as jest.Mock).mockResolvedValue(fileContent) + const fakePath = '/tmp/test/Chart.yaml' + const newIcon = 'https://example.com/new-icon.png' + const expectedObject = { name: 'Test Chart', version: '1.0.0', icon: newIcon } + const expectedContent = YAML.stringify(expectedObject) + + await updateChartIconInYaml(fakePath, newIcon) + + expect(fsPromises.writeFile).toHaveBeenCalledWith(fakePath, expectedContent, 'utf-8') + }) + + test('does not change icon when newIcon is empty', async () => { + const chartObject = { name: 'Test Chart', version: '1.0.0', icon: 'https://example.com/old-icon.png' } + const fileContent = YAML.stringify(chartObject) + ;(fsExtra.readFile as jest.Mock).mockResolvedValue(fileContent) + const fakePath = '/tmp/test/Chart.yaml' + const newIcon = '' + + await updateChartIconInYaml(fakePath, newIcon) + + // Verify writeFile was called, but the icon wasn't changed + expect(fsPromises.writeFile).toHaveBeenCalled() + const writeFileArgs = (fsPromises.writeFile as jest.Mock).mock.calls[0] + const writtenContent = writeFileArgs[1] + const parsedContent = YAML.parse(writtenContent) + expect(parsedContent.icon).toBe('https://example.com/old-icon.png') + }) + + test('handles errors gracefully', async () => { + ;(fsExtra.readFile as jest.Mock).mockRejectedValue(new Error('File not found')) + const fakePath = '/tmp/test/Chart.yaml' + const newIcon = 'https://example.com/new-icon.png' + + // Should not throw + await expect(updateChartIconInYaml(fakePath, newIcon)).resolves.not.toThrow() + expect(fsPromises.writeFile).not.toHaveBeenCalled() + }) }) // ---------------------------------------------------------------- @@ -87,90 +300,396 @@ describe('updateRbacForNewChart', () => { const expected = { rbac: { [chartKey]: [] }, betaCharts: [] } expect(fsPromises.writeFile).toHaveBeenCalledWith(`${fakeSparsePath}/rbac.yaml`, YAML.stringify(expected), 'utf-8') }) + + test('creates rbac.yaml when it does not exist', async () => { + ;(fsExtra.readFile as jest.Mock).mockRejectedValue(new Error('File not found')) + const fakeSparsePath = '/tmp/test' + const chartKey = 'quickstart-cassandra' + + await updateRbacForNewChart(fakeSparsePath, chartKey, true) + + const expected = { rbac: { [chartKey]: null }, betaCharts: [] } + expect(fsPromises.writeFile).toHaveBeenCalledWith(`${fakeSparsePath}/rbac.yaml`, YAML.stringify(expected), 'utf-8') + }) + + test('preserves existing rbac entries when adding new chart', async () => { + const rbacObject = { rbac: { 'existing-chart': ['team-1'] }, betaCharts: ['existing-chart'] } + const fileContent = YAML.stringify(rbacObject) + ;(fsExtra.readFile as jest.Mock).mockResolvedValue(fileContent) + const fakeSparsePath = '/tmp/test' + const chartKey = 'quickstart-cassandra' + + await updateRbacForNewChart(fakeSparsePath, chartKey, false) + + const expected = { + rbac: { + 'existing-chart': ['team-1'], + [chartKey]: [], + }, + betaCharts: ['existing-chart'], + } + expect(fsPromises.writeFile).toHaveBeenCalledWith(`${fakeSparsePath}/rbac.yaml`, YAML.stringify(expected), 'utf-8') + }) }) // ---------------------------------------------------------------- // Tests for sparseCloneChart describe('sparseCloneChart', () => { - const fakeSparsePath = '/tmp/test' - const fakeHelmCatalogUrl = 'https://gitea.example.com/otomi/charts.git' - const fakeUser = 'TestUser' - const fakeEmail = 'test@example.com' - const fakeChartName = 'cassandra' - const fakeChartPath = 'bitnami/cassandra' - const fakeRevision = 'main' - const fakeChartIcon = 'https://example.com/icon.png' + const gitRepositoryUrl = 'https://github.com/bitnami/charts/blob/main/bitnami/cassandra/Chart.yaml' + const localHelmChartsDir = '/tmp/otomi/charts/uuid' + const helmChartCatalogUrl = 'https://gitea.example.com/otomi/charts.git' + const user = 'test-user' + const email = 'test@example.com' + const chartTargetDirName = 'cassandra' + const chartIcon = 'https://example.com/icon.png' + const allowTeams = true beforeEach(() => { jest.clearAllMocks() + // Set up environment variables for tests + process.env = { ...originalEnv, GIT_USER: 'git-user', GIT_PASSWORD: 'git-password' } + // Mock necessary function responses + ;(fsExtra.existsSync as jest.Mock).mockReturnValue(false) }) - test('sparseCloneChart returns true on successful clone and push', async () => { - // Arrange: simulate a successful run - const mockGit: Partial = { - env: jest.fn(), - clone: jest.fn().mockResolvedValueOnce('success'), - cwd: jest.fn().mockResolvedValueOnce('success'), - raw: jest.fn().mockResolvedValueOnce('success'), - checkout: jest.fn().mockResolvedValueOnce('success'), - push: jest.fn().mockResolvedValueOnce('success'), - addConfig: jest.fn().mockResolvedValueOnce('success'), - add: jest.fn().mockResolvedValueOnce('success'), - pull: jest.fn().mockResolvedValueOnce('success'), - init: jest.fn().mockResolvedValueOnce('success'), - addRemote: jest.fn().mockResolvedValueOnce('success'), - commit: jest.fn().mockResolvedValueOnce('success'), - } + afterEach(() => { + // Restore original environment + process.env = originalEnv + }) + test('successfully clones and processes a chart repository', async () => { + // Setup mock git instance + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + cwd: jest.fn().mockResolvedValue(undefined), + raw: jest.fn().mockResolvedValue(undefined), + checkout: jest.fn().mockResolvedValue(undefined), + addConfig: jest.fn().mockResolvedValue(undefined), + add: jest.fn().mockResolvedValue(undefined), + commit: jest.fn().mockResolvedValue(undefined), + pull: jest.fn().mockResolvedValue(undefined), + push: jest.fn().mockResolvedValue(undefined), + } ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + const result = await sparseCloneChart( - 'https://github.com/bitnami/charts.git', - fakeHelmCatalogUrl, - fakeUser, - fakeEmail, - fakeChartName, - fakeChartPath, - fakeSparsePath, - fakeRevision, - fakeChartIcon, - true, + gitRepositoryUrl, + localHelmChartsDir, + helmChartCatalogUrl, + user, + email, + chartTargetDirName, + chartIcon, + allowTeams, ) expect(result).toBe(true) - // Assert that renameSync was called to move the chart folder. + expect(fsExtra.mkdirSync).toHaveBeenCalledWith(localHelmChartsDir, { recursive: true }) + expect(fsExtra.mkdirSync).toHaveBeenCalledWith(`${localHelmChartsDir}-newChart`, { recursive: true }) + expect(mockGit.clone).toHaveBeenCalledTimes(2) // Once for catalog repo, once for chart repo + expect(mockGit.raw).toHaveBeenCalledWith(['sparse-checkout', 'init', '--cone']) + expect(mockGit.raw).toHaveBeenCalledWith(['sparse-checkout', 'set', 'bitnami/cassandra/']) + expect(mockGit.checkout).toHaveBeenCalledWith('main') expect(fsExtra.renameSync).toHaveBeenCalled() - // And that the simpleGit clone and push methods were called. - expect(mockGit.clone).toHaveBeenCalled() - expect(mockGit.push).toHaveBeenCalled() + expect(fsExtra.rmSync).toHaveBeenCalled() + // Verify addConfig was called with correct user/email + expect(mockGit.addConfig).toHaveBeenCalledWith('user.name', user) + expect(mockGit.addConfig).toHaveBeenCalledWith('user.email', email) + // Verify commit and push were called + expect(mockGit.add).toHaveBeenCalledWith('.') + expect(mockGit.commit).toHaveBeenCalledWith(`Add ${chartTargetDirName} helm chart`) + expect(mockGit.pull).toHaveBeenCalledWith('origin', 'main', { '--rebase': null }) + expect(mockGit.push).toHaveBeenCalledWith('origin', 'main') + }) + + test('handles Gitea URLs by encoding credentials', async () => { + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + cwd: jest.fn().mockResolvedValue(undefined), + raw: jest.fn().mockResolvedValue(undefined), + checkout: jest.fn().mockResolvedValue(undefined), + addConfig: jest.fn().mockResolvedValue(undefined), + add: jest.fn().mockResolvedValue(undefined), + commit: jest.fn().mockResolvedValue(undefined), + pull: jest.fn().mockResolvedValue(undefined), + push: jest.fn().mockResolvedValue(undefined), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + await sparseCloneChart( + gitRepositoryUrl, + localHelmChartsDir, + helmChartCatalogUrl, + user, + email, + chartTargetDirName, + chartIcon, + allowTeams, + ) + + // Check that clone was called with encoded URL + const encodedUrl = `https://git-user:git-password@gitea.example.com/otomi/charts.git` + expect(mockGit.clone.mock.calls[0][0]).toBe(encodedUrl) + }) + + test('properly handles empty chartIcon parameter', async () => { + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + cwd: jest.fn().mockResolvedValue(undefined), + raw: jest.fn().mockResolvedValue(undefined), + checkout: jest.fn().mockResolvedValue(undefined), + addConfig: jest.fn().mockResolvedValue(undefined), + add: jest.fn().mockResolvedValue(undefined), + commit: jest.fn().mockResolvedValue(undefined), + pull: jest.fn().mockResolvedValue(undefined), + push: jest.fn().mockResolvedValue(undefined), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await sparseCloneChart( + gitRepositoryUrl, + localHelmChartsDir, + helmChartCatalogUrl, + user, + email, + chartTargetDirName, + '', // Empty chart icon + allowTeams, + ) + + expect(result).toBe(true) + // Should not attempt to update the chart icon + const chartYamlPath = `${localHelmChartsDir}/${chartTargetDirName}/Chart.yaml` + expect(fsExtra.readFile).not.toHaveBeenCalledWith(chartYamlPath, 'utf-8') + }) + + test('creates directory if it does not exist', async () => { + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + cwd: jest.fn().mockResolvedValue(undefined), + raw: jest.fn().mockResolvedValue(undefined), + checkout: jest.fn().mockResolvedValue(undefined), + addConfig: jest.fn().mockResolvedValue(undefined), + add: jest.fn().mockResolvedValue(undefined), + commit: jest.fn().mockResolvedValue(undefined), + pull: jest.fn().mockResolvedValue(undefined), + push: jest.fn().mockResolvedValue(undefined), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + ;(fsExtra.existsSync as jest.Mock).mockReturnValueOnce(false) + + await sparseCloneChart( + gitRepositoryUrl, + localHelmChartsDir, + helmChartCatalogUrl, + user, + email, + chartTargetDirName, + chartIcon, + allowTeams, + ) + + expect(fsExtra.mkdirSync).toHaveBeenCalledWith(localHelmChartsDir, { recursive: true }) }) +}) + +// ---------------------------------------------------------------- +// Tests for fetchWorkloadCatalog +describe('fetchWorkloadCatalog', () => { + const url = 'https://gitea.example.com/otomi/charts.git' + const helmChartsDir = '/tmp/otomi/charts/uuid' + + beforeEach(() => { + jest.clearAllMocks() + process.env = { ...originalEnv, GIT_USER: 'git-user', GIT_PASSWORD: 'git-password' } + ;(fsExtra.existsSync as jest.Mock).mockReturnValue(false) + + // Mock directory structure + const files = ['.git', 'chart1', 'chart2', 'README.md', 'rbac.yaml'] + ;(fsPromises.readdir as jest.Mock).mockResolvedValue(files) + + // Mock rbac.yaml content + const rbacContent = YAML.stringify({ + rbac: { + chart1: null, + chart2: ['team-2'], + }, + betaCharts: ['chart2'], + }) + ;(fsExtra.readFile as jest.Mock).mockImplementation((path) => { + if (path.endsWith('rbac.yaml')) return Promise.resolve(rbacContent) + + if (path.endsWith('chart1/README.md')) return Promise.resolve('# Chart 1 README') + + if (path.endsWith('chart1/values.yaml')) return Promise.resolve('key: value') + + if (path.endsWith('chart1/Chart.yaml')) { + return Promise.resolve( + YAML.stringify({ + name: 'chart1', + version: '1.0.0', + description: 'Test Chart 1', + icon: 'https://example.com/icon1.png', + }), + ) + } + if (path.endsWith('chart2/README.md')) return Promise.resolve('# Chart 2 README') - test('sparseCloneChart returns false when an error occurs', async () => { - // Arrange: simulate an error in the cloning process. - const mockGit: Partial = { - env: jest.fn(), - clone: jest.fn().mockRejectedValueOnce(new Error('Clone failed')), - push: jest.fn().mockResolvedValueOnce('success'), + if (path.endsWith('chart2/values.yaml')) return Promise.resolve('key: value') + + if (path.endsWith('chart2/Chart.yaml')) { + return Promise.resolve( + YAML.stringify({ + name: 'chart2', + version: '2.0.0', + description: 'Test Chart 2', + icon: 'https://example.com/icon2.png', + }), + ) + } + return Promise.reject(new Error(`File not found: ${path}`)) + }) + }) + + afterEach(() => { + process.env = originalEnv + }) + + test('clones repository and builds catalog for admin team', async () => { + // Setup mock git instance + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await fetchWorkloadCatalog(url, helmChartsDir, 'admin') + + expect(fsExtra.mkdirSync).toHaveBeenCalledWith(helmChartsDir, { recursive: true }) + expect(mockGit.clone).toHaveBeenCalledWith( + 'https://git-user:git-password@gitea.example.com/otomi/charts.git', + helmChartsDir, + ) + expect(result).toEqual({ + helmCharts: ['chart1', 'chart2'], + catalog: [ + { + name: 'chart1', + values: 'key: value', + icon: 'https://example.com/icon1.png', + chartVersion: '1.0.0', + chartDescription: 'Test Chart 1', + readme: '# Chart 1 README', + isBeta: false, + }, + { + name: 'chart2', + values: 'key: value', + icon: 'https://example.com/icon2.png', + chartVersion: '2.0.0', + chartDescription: 'Test Chart 2', + readme: '# Chart 2 README', + isBeta: true, + }, + ], + }) + }) + test('filters catalog based on team permissions', async () => { + // Setup mock git instance + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await fetchWorkloadCatalog(url, helmChartsDir, '1') + + // Only chart1 should be accessible to team-1 + expect(result).toEqual({ + helmCharts: ['chart1'], + catalog: [ + { + name: 'chart1', + values: 'key: value', + icon: 'https://example.com/icon1.png', + chartVersion: '1.0.0', + chartDescription: 'Test Chart 1', + readme: '# Chart 1 README', + isBeta: false, + }, + ], + }) + }) + + test('handles non-existent README files', async () => { + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + } ;(simpleGit as jest.Mock).mockReturnValue(mockGit) - let result: boolean | undefined - try { - result = await sparseCloneChart( - 'https://github.com/bitnami/charts.git', - fakeHelmCatalogUrl, - fakeUser, - fakeEmail, - fakeChartName, - fakeChartPath, - fakeSparsePath, - fakeRevision, - fakeChartIcon, - true, - ) - } catch (error) { - result = false + // Make the README.md file read fail + ;(fsExtra.readFile as jest.Mock).mockImplementation((path) => { + if (path.endsWith('chart1/README.md')) return Promise.reject(new Error('File not found')) + + if (path.endsWith('chart1/values.yaml')) return Promise.resolve('key: value') + + if (path.endsWith('chart1/Chart.yaml')) { + return Promise.resolve( + YAML.stringify({ + name: 'chart1', + version: '1.0.0', + description: 'Test Chart 1', + icon: 'https://example.com/icon1.png', + }), + ) + } + if (path.endsWith('rbac.yaml')) { + return Promise.resolve( + YAML.stringify({ + rbac: { chart1: null }, + betaCharts: [], + }), + ) + } + return Promise.reject(new Error(`File not found: ${path}`)) + }) + + const result = await fetchWorkloadCatalog(url, helmChartsDir, 'admin') + + // Should include chart1 with default README message + expect(result.catalog[0].readme).toBe('There is no `README` for this chart.') + }) + + test('handles missing rbac.yaml file', async () => { + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), } - expect(result).toBe(false) + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + // Make the rbac.yaml file read fail + ;(fsExtra.readFile as jest.Mock).mockImplementation((path) => { + if (path.endsWith('rbac.yaml')) return Promise.reject(new Error('File not found')) + + if (path.endsWith('chart1/README.md')) return Promise.resolve('# Chart 1 README') + + if (path.endsWith('chart1/values.yaml')) return Promise.resolve('key: value') + + if (path.endsWith('chart1/Chart.yaml')) { + return Promise.resolve( + YAML.stringify({ + name: 'chart1', + version: '1.0.0', + description: 'Test Chart 1', + icon: 'https://example.com/icon1.png', + }), + ) + } + return Promise.reject(new Error(`File not found: ${path}`)) + }) + + const result = await fetchWorkloadCatalog(url, helmChartsDir, 'admin') + + // Should include charts in the catalog + expect(result.helmCharts).toEqual(['chart1']) + expect(result.catalog).toHaveLength(1) }) }) diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index b1ffc6047..b743570fb 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -1,23 +1,100 @@ +import axios from 'axios' import Debug from 'debug' import { existsSync, mkdirSync, readFile, renameSync, rmSync } from 'fs-extra' import { readdir, writeFile } from 'fs/promises' import path from 'path' import simpleGit, { SimpleGit } from 'simple-git' +import { GIT_PROVIDER_URL_PATTERNS, cleanEnv } from 'src/validators' import YAML from 'yaml' const debug = Debug('apl:workloadUtils') -export interface NewChartValues { - url: string - chartName: string - chartIcon?: string - chartPath: string - revision: string - allowTeams: boolean +const env = cleanEnv({ + GIT_PROVIDER_URL_PATTERNS, +}) + +export function detectGitProvider(url) { + if (!url || typeof url !== 'string') return null + + const normalizedUrl = new URL(url).origin + new URL(url).pathname.replace(/\/*$/, '') + + const { github, gitlab, bitbucket } = env.GIT_PROVIDER_URL_PATTERNS as { + github: string + gitlab: string + bitbucket: string + } + const githubPattern = new RegExp(github || 'github\\.com\\/([^\\/]+)\\/([^\\/]+)\\/(?:blob|raw)\\/([^\\/]+)\\/(.+)') + const gitlabPattern = new RegExp( + gitlab || 'gitlab\\.com\\/([^\\/]+)\\/([^\\/]+)(?:\\/([^\\/]+))?\\/(?:\\-\\/(?:blob|raw))\\/([^\\/]+)\\/(.+)', + ) + const bitbucketPattern = new RegExp( + bitbucket || 'bitbucket\\.org\\/([^\\/]+)\\/([^\\/]+)\\/(?:src|raw)\\/([^\\/]+)\\/(.+)', + ) + + let match = normalizedUrl.match(githubPattern) + if (match) return { provider: 'github', owner: match[1], repo: match[2], branch: match[3], filePath: match[4] } + + match = normalizedUrl.match(gitlabPattern) + if (match) { + return { + provider: 'gitlab', + owner: match[1], + repo: match[3] ? `${match[2]}/${match[3]}` : match[2], // Handle optional subgroup + branch: match[4], + filePath: match[5], + } + } + + match = normalizedUrl.match(bitbucketPattern) + if (match) return { provider: 'bitbucket', owner: match[1], repo: match[2], branch: match[3], filePath: match[4] } + + return null +} + +function getGitRawUrl(details) { + if (!details) return null + + if (details.provider === 'github') + return `https://raw.githubusercontent.com/${details.owner}/${details.repo}/${details.branch}/${details.filePath}` + if (details.provider === 'gitlab') + return `https://gitlab.com/${details.owner}/${details.repo}/-/raw/${details.branch}/${details.filePath}` + if (details.provider === 'bitbucket') + return `https://bitbucket.org/${details.owner}/${details.repo}/raw/${details.branch}/${details.filePath}` + + return null } -export interface NewChartPayload extends NewChartValues { - teamId: string +export function getGitCloneUrl(details) { + if (!details) return null + + if (details.provider === 'github') return `https://github.com/${details.owner}/${details.repo}.git` + if (details.provider === 'gitlab') return `https://gitlab.com/${details.owner}/${details.repo}.git` + if (details.provider === 'bitbucket') return `https://bitbucket.org/${details.owner}/${details.repo}.git` + + return null +} + +export async function fetchChartYaml(url) { + try { + const details = detectGitProvider(url) + if (!details) return { values: {}, error: 'Unsupported Git provider or invalid URL format.' } + + const rawUrl = getGitRawUrl(details) + if (!rawUrl) return { values: {}, error: `Could not generate raw URL for provider: ${details.provider}` } + + const response = await axios.get(rawUrl, { responseType: 'text' }) + return { values: YAML.parse(response.data as string), error: '' } + } catch (error) { + console.error('Error fetching Chart.yaml:', error.message) + return { values: {}, error: 'Error fetching helm chart content.' } + } +} + +export interface NewHelmChartValues { + gitRepositoryUrl: string + chartTargetDirName: string + chartIcon?: string + allowTeams: boolean } function throwChartError(message: string) { @@ -27,7 +104,6 @@ function throwChartError(message: string) { } throw err } - function isGiteaURL(url: string) { let hostname = '' if (url) { @@ -41,7 +117,6 @@ function isGiteaURL(url: string) { const giteaPattern = /^gitea\..+/i return giteaPattern.test(hostname) } - /** * Reads the Chart.yaml file at the given path, updates (or sets) its icon field, * and writes the updated content back to disk. @@ -54,14 +129,12 @@ export async function updateChartIconInYaml(chartYamlPath: string, newIcon: stri const fileContent = await readFile(chartYamlPath, 'utf-8') const chartObject = YAML.parse(fileContent) if (newIcon && newIcon.trim() !== '') chartObject.icon = newIcon - const newContent = YAML.stringify(chartObject) await writeFile(chartYamlPath, newContent, 'utf-8') } catch (error) { debug(`Error updating chart icon in ${chartYamlPath}:`, error) } } - /** * Updates the rbac.yaml file in the specified folder by adding a new chart key. * @@ -83,21 +156,17 @@ export async function updateRbacForNewChart(sparsePath: string, chartKey: string // Create a default structure if the file doesn't exist. rbacData = { rbac: {}, betaCharts: [] } } - // Ensure the "rbac" section exists. if (!rbacData.rbac) rbacData.rbac = {} - // Add the new chart entry if it doesn't exist. // If allowTeams is false, set the value to an empty array ([]), // otherwise (if true) set it to null. if (!(chartKey in rbacData.rbac)) rbacData.rbac[chartKey] = allowTeams ? null : [] - // Stringify the updated YAML content and write it back. const newContent = YAML.stringify(rbacData) await writeFile(rbacFilePath, newContent, 'utf-8') debug(`Updated rbac.yaml: added ${chartKey}: ${allowTeams ? 'null' : '[]'}`) } - class chartRepo { localPath: string chartRepoUrl: string @@ -125,45 +194,47 @@ class chartRepo { await this.git.push('origin', 'main') } } - /** * Clones a repository using sparse checkout, checks out a specific revision, * and moves the contents of the desired subdirectory (sparsePath) to the root of the target folder. * - * @param url - The base Git repository URL (e.g. "https://github.com/nats-io/k8s.git") - * @param chartName - The target folder name for the clone (will be the final chart folder, e.g. "nats") - * @param chartPath - The path in github where the chart is located - * @param sparsePath - The subdirectory to sparse checkout (e.g. "helm/charts/nats") - * @param revision - The branch or commit to checkout (e.g. "main") + * @param gitRepositoryUrl - The base Git repository URL (e.g. "https://github.com/nats-io/k8s.git") + * @param localHelmChartsDir - The subdirectory to sparse checkout (e.g. "/tmp/otomi/charts/uuid") + * @param helmChartCatalogUrl - The URL of the (Gitea) Helm Chart Catalog (e.g. "https://gitea./otomi/charts.git") + * @param user - The Git username (e.g. "otomi-admin") + * @param email - The Git email (e.g. "not@us.ed") + * @param chartTargetDirName - The target folder name for the clone (will be the final chart folder, e.g. "nats") * @param chartIcon - the icon URL path (e.g https://myimage.com/imageurl) * @param allowTeams - Boolean indicating if teams are allowed to use the chart. * If false, the key is set to []. * If true, the key is set to null. */ export async function sparseCloneChart( - url: string, + gitRepositoryUrl: string, + localHelmChartsDir: string, helmChartCatalogUrl: string, user: string, email: string, - chartName: string, - chartPath: string, - sparsePath: string, // e.g. "/tmp/otomi/charts/uuid" - revision: string, + chartTargetDirName: string, chartIcon?: string, allowTeams?: boolean, ): Promise { - const temporaryCloneDir = `${sparsePath}-new` // Temporary clone directory - const checkoutPath = `${sparsePath}/${chartName}` // Final destination + const details = detectGitProvider(gitRepositoryUrl) + const gitCloneUrl = getGitCloneUrl(details) as string + const chartPath = details?.filePath.replace('Chart.yaml', '') as string + const revision = details?.branch as string + const temporaryCloneDir = `${localHelmChartsDir}-newChart` + const finalDestinationPath = `${localHelmChartsDir}/${chartTargetDirName}` - if (!existsSync(sparsePath)) mkdirSync(sparsePath, { recursive: true }) + if (!existsSync(localHelmChartsDir)) mkdirSync(localHelmChartsDir, { recursive: true }) let gitUrl = helmChartCatalogUrl - if (isGiteaURL(url)) { - const [protocol, bareUrl] = url.split('://') + if (isGiteaURL(helmChartCatalogUrl)) { + const [protocol, bareUrl] = helmChartCatalogUrl.split('://') const encodedUser = encodeURIComponent(process.env.GIT_USER as string) const encodedPassword = encodeURIComponent(process.env.GIT_PASSWORD as string) gitUrl = `${protocol}://${encodedUser}:${encodedPassword}@${bareUrl}` } - const gitRepo = new chartRepo(sparsePath, gitUrl, user, email) + const gitRepo = new chartRepo(localHelmChartsDir, gitUrl, user, email) await gitRepo.clone() if (!existsSync(temporaryCloneDir)) mkdirSync(temporaryCloneDir, { recursive: true }) @@ -174,43 +245,40 @@ export async function sparseCloneChart( const git = simpleGit() - // Clone the repository into the folder named checkoutPath. - debug(`Cloning repository: ${url} into ${checkoutPath}`) - await git.clone(url, temporaryCloneDir, ['--filter=blob:none', '--no-checkout']) + debug(`Cloning repository: ${gitCloneUrl} into ${temporaryCloneDir}`) + await git.clone(gitCloneUrl, temporaryCloneDir, ['--filter=blob:none', '--no-checkout']) - // Initialize sparse checkout in cone mode within checkoutPath. - debug(`Initializing sparse checkout in cone mode at ${checkoutPath}`) + debug(`Initializing sparse checkout in cone mode at ${temporaryCloneDir}`) await git.cwd(temporaryCloneDir) await git.raw(['sparse-checkout', 'init', '--cone']) - // Set the sparse checkout to only include the specified chartPath. debug(`Setting sparse checkout path to ${chartPath}`) await git.raw(['sparse-checkout', 'set', chartPath]) - // Checkout the desired revision (branch or commit) within checkoutPath. - debug(`Checking out revision: ${revision}`) + debug(`Checking out the desired revision (branch or commit): ${revision}`) await git.checkout(revision) - // Move the contents of the sparse folder (chartPath) to the repository root. - // This moves files from "checkoutPath/chartPath/*" to "checkoutPath/" - renameSync(path.join(temporaryCloneDir, chartPath), checkoutPath) + // Move files from "temporaryCloneDir/chartPath/*" to "finalDestinationPath/" + renameSync(path.join(temporaryCloneDir, chartPath), finalDestinationPath) + + // Remove the .git directory from the final destination. + rmSync(`${finalDestinationPath}/.git`, { recursive: true, force: true }) // Remove the leftover temporary clone directory. - // For chartPath "bitnami/cassandra", the top-level folder is "bitnami". rmSync(temporaryCloneDir, { recursive: true, force: true }) // Update Chart.yaml with the new icon if one is provided. if (chartIcon && chartIcon.trim() !== '') { - const chartYamlPath = `${checkoutPath}/Chart.yaml` + const chartYamlPath = `${finalDestinationPath}/Chart.yaml` await updateChartIconInYaml(chartYamlPath, chartIcon) } // update rbac file - await updateRbacForNewChart(sparsePath, chartName, allowTeams as boolean) + await updateRbacForNewChart(localHelmChartsDir, chartTargetDirName, allowTeams as boolean) // pull&push new chart changes await gitRepo.addConfig() - await gitRepo.commitAndPush(chartName) + await gitRepo.commitAndPush(chartTargetDirName) return true } diff --git a/src/validators.ts b/src/validators.ts index ae7b58c85..7df90ca11 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -47,6 +47,14 @@ export const HELM_CHART_CATALOG = str({ desc: 'The helm chart catalog', devDefault: 'https://github.com/linode/apl-charts.git', }) +export const GIT_PROVIDER_URL_PATTERNS = json({ + desc: 'Regular expressions to match and extract information from URLs of supported git providers (GitHub, GitLab, Bitbucket) for cloning Helm charts.', + default: { + github: 'github\\.com\\/([^\\/]+)\\/([^\\/]+)\\/(?:blob|raw)\\/([^\\/]+)\\/(.+)', + gitlab: 'gitlab\\.com\\/([^\\/]+)\\/([^\\/]+)(?:\\/([^\\/]+))?\\/(?:\\-\\/(?:blob|raw))\\/([^\\/]+)\\/(.+)', + bitbucket: 'bitbucket\\.org\\/([^\\/]+)\\/([^\\/]+)\\/(?:src|raw)\\/([^\\/]+)\\/(.+)', + }, +}) export const OIDC_ENDPOINT = str() export const REGION = str({ desc: 'The cloud region' }) export const ROARR_LOG = bool({ desc: 'To enable Lightship logs', default: false }) From ae7509ebcc3470c3c0342fe5a137f0ea637c58a0 Mon Sep 17 00:00:00 2001 From: Ferruh <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 13 Mar 2025 09:45:37 +0100 Subject: [PATCH 5/8] fix: create service & form validation nullable error (#656) (cherry picked from commit b52712d39d029bbcbeae39d7e7a478270374f2cb) --- src/openapi/service.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/openapi/service.yaml b/src/openapi/service.yaml index 368a86473..46f840bea 100644 --- a/src/openapi/service.yaml +++ b/src/openapi/service.yaml @@ -199,5 +199,6 @@ IngressPublic: - public default: public nullable: true + type: object description: Will only accept traffic coming from an external loadbalancer. title: External From 498f57092b2ac4f93697be026854ee8ae49e589f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 13 Mar 2025 16:10:32 +0100 Subject: [PATCH 6/8] chore(release): 3.7.1 --- CHANGELOG.md | 30 ++++++++++++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebddfce3e..533c2b47c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,36 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [3.7.1](https://github.com/redkubes/otomi-api/compare/v3.5.0...v3.7.1) (2025-03-13) + + +### Features + +* add code repository schema & endpoints ([#628](https://github.com/redkubes/otomi-api/issues/628)) ([a8cb958](https://github.com/redkubes/otomi-api/commit/a8cb9581951b598b5d7e4a6ae5673e8f5d7815bc)) +* add username blacklist and validation ([#619](https://github.com/redkubes/otomi-api/issues/619)) ([34142f9](https://github.com/redkubes/otomi-api/commit/34142f91dfb0c25187c9f9c4a7b9daebaeb4da7e)) +* add self-service option for admins to add external helm charts to the catalog ([#642](https://github.com/redkubes/otomi-api/issues/642)) ([eb2e268](https://github.com/redkubes/otomi-api/commit/eb2e268a69bd6aaa910f1e8ee1203ce376c91665)) +* change all tests to Jest ([#625](https://github.com/redkubes/otomi-api/issues/625)) ([4cfaab9](https://github.com/redkubes/otomi-api/commit/4cfaab9a3202ac29ca95d70c413ab586795548bb)) +* execute initSops before we loadValues to prevent data being shown as encrypted ([#610](https://github.com/redkubes/otomi-api/issues/610)) ([9a61589](https://github.com/redkubes/otomi-api/commit/9a61589df24919debd9d57ee159dd95aa13f0a1b)) +* remove deploy changes button ([#608](https://github.com/redkubes/otomi-api/issues/608)) ([6e045c9](https://github.com/redkubes/otomi-api/commit/6e045c984689df81c5b7e8dfad0e3a958856940d)) +* remove locking mechanism & update api flow ([#611](https://github.com/redkubes/otomi-api/issues/611)) ([7829668](https://github.com/redkubes/otomi-api/commit/782966887068cc6a10aeb53e3adca4b960a390e7)) +* Save and deploy team secrets as SealedSecrets manifests ([#623](https://github.com/redkubes/otomi-api/issues/623)) ([79d61ce](https://github.com/redkubes/otomi-api/commit/79d61ceeb4eb8e4eb917acd23062913feb17ce9a)) +* update add new helm chart ([#653](https://github.com/redkubes/otomi-api/issues/653)) ([1e523ae](https://github.com/redkubes/otomi-api/commit/1e523ae71f8b45cc8b1211b96cfe21d6217ca35a)) +* update fetch workload catalog ([#655](https://github.com/redkubes/otomi-api/issues/655)) ([8dd5f05](https://github.com/redkubes/otomi-api/commit/8dd5f05020764565c5a3e5d5739d7e59767b258e)) + + +### Bug Fixes + +* add policies to api check ([#652](https://github.com/redkubes/otomi-api/issues/652)) ([ab6d57c](https://github.com/redkubes/otomi-api/commit/ab6d57cf328bf1a0bec6a40c1a81c0e15be6e51a)) +* align name field titles in the resource forms ([#605](https://github.com/redkubes/otomi-api/issues/605)) ([2f0db13](https://github.com/redkubes/otomi-api/commit/2f0db133254e993a7807105efcf30cf2e2917265)) +* create project ([#624](https://github.com/redkubes/otomi-api/issues/624)) ([794689a](https://github.com/redkubes/otomi-api/commit/794689a3d147430d1d5ca923eeaa3366ea622c9c)) +* create service & form validation nullable error ([#656](https://github.com/redkubes/otomi-api/issues/656)) ([ae7509e](https://github.com/redkubes/otomi-api/commit/ae7509ebcc3470c3c0342fe5a137f0ea637c58a0)) +* createObjWizard 500 errors ([#636](https://github.com/redkubes/otomi-api/issues/636)) ([c2ca7f7](https://github.com/redkubes/otomi-api/commit/c2ca7f7466da03a975138c4f4768b978e3655211)) +* enable trivy for isPreInstalled=true ([#603](https://github.com/redkubes/otomi-api/issues/603)) ([810e6f4](https://github.com/redkubes/otomi-api/commit/810e6f40b76c954412f7109d0ae97dc1a4933d85)) +* generate special password ([#632](https://github.com/redkubes/otomi-api/issues/632)) ([72e68ba](https://github.com/redkubes/otomi-api/commit/72e68bacd0e95719c53e8456f8cde10aa916e491)) +* **git save:** increase git pull retries and remove doRestore ([#651](https://github.com/redkubes/otomi-api/issues/651)) ([ab98d05](https://github.com/redkubes/otomi-api/commit/ab98d05d545b0d0ef403df05f97c40a9c3277e59)) +* main module to be placed in dist/src ([#633](https://github.com/redkubes/otomi-api/issues/633)) ([ef9b438](https://github.com/redkubes/otomi-api/commit/ef9b4380c85bfc57edb59d679c66feefadfd8d58)) +* update fetchWorkloadCatalog and align with the new feature ([#643](https://github.com/redkubes/otomi-api/issues/643)) ([a52ab12](https://github.com/redkubes/otomi-api/commit/a52ab12d5fae5a85cbc4a7fc8ed46eccca58eed0)) + ## [3.7.0](https://github.com/redkubes/otomi-api/compare/v3.5.0...v3.7.0) (2025-03-07) diff --git a/package-lock.json b/package-lock.json index 4479c85bb..a6b565a8d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@redkubes/otomi-api", - "version": "3.7.0", + "version": "3.7.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@redkubes/otomi-api", - "version": "3.7.0", + "version": "3.7.1", "hasInstallScript": true, "license": "ISC", "dependencies": { diff --git a/package.json b/package.json index 2eef1e6ac..22683d7d9 100644 --- a/package.json +++ b/package.json @@ -162,7 +162,7 @@ "tag": true } }, - "version": "3.7.0", + "version": "3.7.1", "watch": { "build:models": { "patterns": [ From 5723fb7b2be2c0e07adf707e49a8972e75f22a97 Mon Sep 17 00:00:00 2001 From: Ferruh <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 14 Mar 2025 08:36:47 +0100 Subject: [PATCH 7/8] fix: create policies with the new team creation (#657) * fix: create policies with the new team creation * fix: create policies for teams (cherry picked from commit f4e556e92dfe48798f89d6d5980f2a06791f1559) --- src/otomi-stack.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index 9660eca1d..b59ec2273 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -620,14 +620,15 @@ export default class OtomiStack { if (id !== 'admin' && (isShared || inTeamApps)) this.db.createItem('apps', {}, { teamId: id, id: appId }, appId) }) - if (!data.id) { - const policies = getPolicies() - this.db.db.set(`policies[${data.name}]`, policies).write() - } if (deploy) { + if (!data.id) { + const policies = getPolicies() + this.db.db.set(`policies[${data.name}]`, policies).write() + await this.saveTeamPolicies(data.name) + } const secretPaths = this.getSecretPaths() await this.saveTeams(secretPaths) - await this.doDeployment(['teams']) + await this.doDeployment(['teams', 'policies']) } return team } From 9eb1bfcb57ef020ecfa36058e4ba04ca2717cdec Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 14 Mar 2025 11:22:28 +0100 Subject: [PATCH 8/8] chore(release): 3.7.2 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 533c2b47c..7ae4326be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [3.7.2](https://github.com/redkubes/otomi-api/compare/v3.7.1...v3.7.2) (2025-03-14) + + +### Bug Fixes + +* create policies with the new team creation ([#657](https://github.com/redkubes/otomi-api/issues/657)) ([5723fb7](https://github.com/redkubes/otomi-api/commit/5723fb7b2be2c0e07adf707e49a8972e75f22a97)) + ### [3.7.1](https://github.com/redkubes/otomi-api/compare/v3.5.0...v3.7.1) (2025-03-13) diff --git a/package-lock.json b/package-lock.json index a6b565a8d..95f9ab86d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@redkubes/otomi-api", - "version": "3.7.1", + "version": "3.7.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@redkubes/otomi-api", - "version": "3.7.1", + "version": "3.7.2", "hasInstallScript": true, "license": "ISC", "dependencies": { diff --git a/package.json b/package.json index 22683d7d9..d20c27533 100644 --- a/package.json +++ b/package.json @@ -162,7 +162,7 @@ "tag": true } }, - "version": "3.7.1", + "version": "3.7.2", "watch": { "build:models": { "patterns": [