From 15118a6630171f7c310dce64ac7153fb2bf279e4 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Mar 2025 01:42:51 +0100 Subject: [PATCH 1/5] feat: update add new helm chart --- src/api/helmChartContent.ts | 20 + src/openapi/api.yaml | 22 ++ src/otomi-stack.ts | 26 +- src/utils/workloadUtils.test.ts | 653 ++++++++++++++++++++++++++++---- src/utils/workloadUtils.ts | 142 ++++--- 5 files changed, 735 insertions(+), 128 deletions(-) create mode 100644 src/api/helmChartContent.ts diff --git a/src/api/helmChartContent.ts b/src/api/helmChartContent.ts new file mode 100644 index 000000000..c0ae27677 --- /dev/null +++ b/src/api/helmChartContent.ts @@ -0,0 +1,20 @@ +import Debug from 'debug' +import { Operation, OperationHandlerArray } from 'express-openapi' +import { OpenApiRequestExt } from 'src/otomi-models' + +const debug = Debug('otomi:api:helmChartContent') + +export default function (): OperationHandlerArray { + const get: Operation = [ + async ({ otomi, query }: OpenApiRequestExt, res): Promise => { + debug(`gethelmChartContent ${query?.url}`) + const v = await otomi.getHelmChartContent(query?.url as string) + res.json(v) + return v + }, + ] + const api = { + get, + } + return api +} diff --git a/src/openapi/api.yaml b/src/openapi/api.yaml index 99dc8bb1f..0398179ff 100644 --- a/src/openapi/api.yaml +++ b/src/openapi/api.yaml @@ -1289,6 +1289,28 @@ paths: application/json: schema: type: object + '/helmChartContent': + get: + operationId: getHelmChartContent + parameters: + - name: url + in: query + description: URL of the helm chart + schema: + type: string + responses: + <<: *DefaultGetResponses + '200': + description: Successfully obtained helm chart content + content: + application/json: + schema: + type: object + properties: + values: + type: object + error: + type: string '/createWorkloadCatalog': post: operationId: createWorkloadCatalog diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index 188ae8cf2..650f7d8bf 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -83,7 +83,7 @@ import { getPolicies } from './utils/policiesUtils' import { EncryptedDataRecord, encryptSecretItem, sealedSecretManifest } from './utils/sealedSecretUtils' import { getKeycloakUsers, isValidUsername } from './utils/userUtils' import { ObjectStorageClient } from './utils/wizardUtils' -import { NewChartPayload, fetchWorkloadCatalog, sparseCloneChart } from './utils/workloadUtils' +import { NewHelmChartValues, fetchChartYaml, fetchWorkloadCatalog, sparseCloneChart } from './utils/workloadUtils' interface ExcludedApp extends App { managed: boolean @@ -1236,34 +1236,36 @@ export default class OtomiStack { } } - async createWorkloadCatalog(body: NewChartPayload): Promise { - const { url, chartName, chartPath, chartIcon, revision, allowTeams } = body + async getHelmChartContent(url: string): Promise { + return await fetchChartYaml(url) + } + + async createWorkloadCatalog(body: NewHelmChartValues): Promise { + const { gitRepositoryUrl, chartTargetDirName, chartIcon, allowTeams } = body const uuid = uuidv4() - const helmChartsDir = `/tmp/otomi/charts/${uuid}` + const localHelmChartsDir = `/tmp/otomi/charts/${uuid}` const helmChartCatalogUrl = env.HELM_CHART_CATALOG const { user, email } = this.repo try { await sparseCloneChart( - url, + gitRepositoryUrl, + localHelmChartsDir, helmChartCatalogUrl, user, email, - chartName, - chartPath, - helmChartsDir, - revision, + chartTargetDirName, chartIcon, allowTeams, ) return true - } catch (err) { - debug(`error while parsing chart ${err.message}`) + } catch (error) { + debug('Error adding new Helm chart to catalog') return false } finally { // Clean up: if the temporary directory exists, remove it. - if (existsSync(helmChartsDir)) rmSync(helmChartsDir, { recursive: true, force: true }) + if (existsSync(localHelmChartsDir)) rmSync(localHelmChartsDir, { recursive: true, force: true }) } } diff --git a/src/utils/workloadUtils.test.ts b/src/utils/workloadUtils.test.ts index 2f1abf1a8..c488605c4 100644 --- a/src/utils/workloadUtils.test.ts +++ b/src/utils/workloadUtils.test.ts @@ -1,12 +1,20 @@ // workloadUtils.test.ts -import Debug from 'debug' +import axios from 'axios' import * as fsExtra from 'fs-extra' import * as fsPromises from 'fs/promises' -import simpleGit, { SimpleGit } from 'simple-git' +import simpleGit from 'simple-git' import YAML from 'yaml' -import { sparseCloneChart, updateChartIconInYaml, updateRbacForNewChart } from './workloadUtils' -const debug = Debug('otomi:api:workloadCatalog') +import { + detectGitProvider, + fetchChartYaml, + fetchWorkloadCatalog, + getGitCloneUrl, + sparseCloneChart, + updateChartIconInYaml, + updateRbacForNewChart, +} from './workloadUtils' +jest.mock('axios') jest.mock('fs/promises', () => ({ writeFile: jest.fn(), readdir: jest.fn(), @@ -33,6 +41,169 @@ jest.mock('simple-git', () => ({ })), })) +// Save the original environment variables +const originalEnv = process.env + +// ---------------------------------------------------------------- +// Tests for detectGitProvider +describe('detectGitProvider', () => { + test('returns null for undefined or non-string inputs', () => { + expect(detectGitProvider(undefined)).toBeNull() + expect(detectGitProvider(null)).toBeNull() + expect(detectGitProvider(123 as any)).toBeNull() + }) + + test('detects GitHub URLs correctly', () => { + const url = 'https://github.com/owner/repo/blob/main/path/to/file.yaml' + const result = detectGitProvider(url) + expect(result).toEqual({ + provider: 'github', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/file.yaml', + }) + }) + + test('detects GitLab URLs correctly', () => { + const url = 'https://gitlab.com/owner/repo/-/blob/main/path/to/file.yaml' + const result = detectGitProvider(url) + expect(result).toEqual({ + provider: 'gitlab', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/file.yaml', + }) + }) + + test('detects Bitbucket URLs correctly', () => { + const url = 'https://bitbucket.org/owner/repo/src/main/path/to/file.yaml' + const result = detectGitProvider(url) + expect(result).toEqual({ + provider: 'bitbucket', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/file.yaml', + }) + }) + + test('returns null for unsupported URLs', () => { + const url = 'https://example.com/owner/repo/main/file.yaml' + expect(detectGitProvider(url)).toBeNull() + }) + + test('handles URLs with trailing slashes', () => { + const url = 'https://github.com/owner/repo/blob/main/path/to/file.yaml/' + const result = detectGitProvider(url) + expect(result).toEqual({ + provider: 'github', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/file.yaml', + }) + }) +}) + +// ---------------------------------------------------------------- +// Tests for getGitCloneUrl +describe('getGitCloneUrl', () => { + test('returns null for null input', () => { + expect(getGitCloneUrl(null)).toBeNull() + }) + + test('returns GitHub clone URL', () => { + const details = { + provider: 'github', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/Chart.yaml', + } + expect(getGitCloneUrl(details)).toBe('https://github.com/owner/repo.git') + }) + + test('returns GitLab clone URL', () => { + const details = { + provider: 'gitlab', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/Chart.yaml', + } + expect(getGitCloneUrl(details)).toBe('https://gitlab.com/owner/repo.git') + }) + + test('returns Bitbucket clone URL', () => { + const details = { + provider: 'bitbucket', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/Chart.yaml', + } + expect(getGitCloneUrl(details)).toBe('https://bitbucket.org/owner/repo.git') + }) + + test('returns null for unsupported provider', () => { + const details = { + provider: 'unsupported', + owner: 'owner', + repo: 'repo', + branch: 'main', + filePath: 'path/to/Chart.yaml', + } + expect(getGitCloneUrl(details)).toBeNull() + }) +}) + +// ---------------------------------------------------------------- +// Tests for fetchChartYaml +describe('fetchChartYaml', () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + test('successfully fetches and parses Chart.yaml', async () => { + const url = 'https://github.com/owner/repo/blob/main/charts/mychart/Chart.yaml' + const mockChartData = 'name: mychart\nversion: 1.0.0\ndescription: Test Chart' + const expectedValues = { name: 'mychart', version: '1.0.0', description: 'Test Chart' } + + jest.spyOn(axios, 'get').mockResolvedValue({ data: mockChartData }) + + const result = await fetchChartYaml(url) + + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(axios.get).toHaveBeenCalledWith( + 'https://raw.githubusercontent.com/owner/repo/main/charts/mychart/Chart.yaml', + { responseType: 'text' }, + ) + expect(result).toEqual({ values: expectedValues, error: '' }) + }) + + test('returns error for unsupported Git provider', async () => { + const url = 'https://example.com/owner/repo/main/charts/mychart/Chart.yaml' + + const result = await fetchChartYaml(url) + + expect(result).toEqual({ values: {}, error: 'Unsupported Git provider or invalid URL format.' }) + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(axios.get).not.toHaveBeenCalled() + }) + + test('handles axios errors gracefully', async () => { + const url = 'https://github.com/owner/repo/blob/main/charts/mychart/Chart.yaml' + + jest.spyOn(axios, 'get').mockRejectedValue(new Error('Network error')) + + const result = await fetchChartYaml(url) + + expect(result).toEqual({ values: {}, error: 'Error fetching helm chart content.' }) + }) +}) + // ---------------------------------------------------------------- // Tests for updateChartIconInYaml describe('updateChartIconInYaml', () => { @@ -41,18 +212,60 @@ describe('updateChartIconInYaml', () => { }) test('updates the icon field when newIcon is provided', async () => { - const chartObject = { name: 'Test Chart', icon: '' } + const chartObject = { name: 'Test Chart', version: '1.0.0' } const fileContent = YAML.stringify(chartObject) ;(fsExtra.readFile as jest.Mock).mockResolvedValue(fileContent) const fakePath = '/tmp/test/Chart.yaml' const newIcon = 'https://example.com/new-icon.png' - const expectedObject = { name: 'Test Chart', icon: newIcon } + const expectedObject = { name: 'Test Chart', version: '1.0.0', icon: newIcon } const expectedContent = YAML.stringify(expectedObject) + await updateChartIconInYaml(fakePath, newIcon) expect(fsExtra.readFile).toHaveBeenCalledWith(fakePath, 'utf-8') expect(fsPromises.writeFile).toHaveBeenCalledWith(fakePath, expectedContent, 'utf-8') }) + + test('replaces existing icon when newIcon is provided', async () => { + const chartObject = { name: 'Test Chart', version: '1.0.0', icon: 'https://example.com/old-icon.png' } + const fileContent = YAML.stringify(chartObject) + ;(fsExtra.readFile as jest.Mock).mockResolvedValue(fileContent) + const fakePath = '/tmp/test/Chart.yaml' + const newIcon = 'https://example.com/new-icon.png' + const expectedObject = { name: 'Test Chart', version: '1.0.0', icon: newIcon } + const expectedContent = YAML.stringify(expectedObject) + + await updateChartIconInYaml(fakePath, newIcon) + + expect(fsPromises.writeFile).toHaveBeenCalledWith(fakePath, expectedContent, 'utf-8') + }) + + test('does not change icon when newIcon is empty', async () => { + const chartObject = { name: 'Test Chart', version: '1.0.0', icon: 'https://example.com/old-icon.png' } + const fileContent = YAML.stringify(chartObject) + ;(fsExtra.readFile as jest.Mock).mockResolvedValue(fileContent) + const fakePath = '/tmp/test/Chart.yaml' + const newIcon = '' + + await updateChartIconInYaml(fakePath, newIcon) + + // Verify writeFile was called, but the icon wasn't changed + expect(fsPromises.writeFile).toHaveBeenCalled() + const writeFileArgs = (fsPromises.writeFile as jest.Mock).mock.calls[0] + const writtenContent = writeFileArgs[1] + const parsedContent = YAML.parse(writtenContent) + expect(parsedContent.icon).toBe('https://example.com/old-icon.png') + }) + + test('handles errors gracefully', async () => { + ;(fsExtra.readFile as jest.Mock).mockRejectedValue(new Error('File not found')) + const fakePath = '/tmp/test/Chart.yaml' + const newIcon = 'https://example.com/new-icon.png' + + // Should not throw + await expect(updateChartIconInYaml(fakePath, newIcon)).resolves.not.toThrow() + expect(fsPromises.writeFile).not.toHaveBeenCalled() + }) }) // ---------------------------------------------------------------- @@ -87,90 +300,396 @@ describe('updateRbacForNewChart', () => { const expected = { rbac: { [chartKey]: [] }, betaCharts: [] } expect(fsPromises.writeFile).toHaveBeenCalledWith(`${fakeSparsePath}/rbac.yaml`, YAML.stringify(expected), 'utf-8') }) + + test('creates rbac.yaml when it does not exist', async () => { + ;(fsExtra.readFile as jest.Mock).mockRejectedValue(new Error('File not found')) + const fakeSparsePath = '/tmp/test' + const chartKey = 'quickstart-cassandra' + + await updateRbacForNewChart(fakeSparsePath, chartKey, true) + + const expected = { rbac: { [chartKey]: null }, betaCharts: [] } + expect(fsPromises.writeFile).toHaveBeenCalledWith(`${fakeSparsePath}/rbac.yaml`, YAML.stringify(expected), 'utf-8') + }) + + test('preserves existing rbac entries when adding new chart', async () => { + const rbacObject = { rbac: { 'existing-chart': ['team-1'] }, betaCharts: ['existing-chart'] } + const fileContent = YAML.stringify(rbacObject) + ;(fsExtra.readFile as jest.Mock).mockResolvedValue(fileContent) + const fakeSparsePath = '/tmp/test' + const chartKey = 'quickstart-cassandra' + + await updateRbacForNewChart(fakeSparsePath, chartKey, false) + + const expected = { + rbac: { + 'existing-chart': ['team-1'], + [chartKey]: [], + }, + betaCharts: ['existing-chart'], + } + expect(fsPromises.writeFile).toHaveBeenCalledWith(`${fakeSparsePath}/rbac.yaml`, YAML.stringify(expected), 'utf-8') + }) }) // ---------------------------------------------------------------- // Tests for sparseCloneChart describe('sparseCloneChart', () => { - const fakeSparsePath = '/tmp/test' - const fakeHelmCatalogUrl = 'https://gitea.example.com/otomi/charts.git' - const fakeUser = 'TestUser' - const fakeEmail = 'test@example.com' - const fakeChartName = 'cassandra' - const fakeChartPath = 'bitnami/cassandra' - const fakeRevision = 'main' - const fakeChartIcon = 'https://example.com/icon.png' + const gitRepositoryUrl = 'https://github.com/bitnami/charts/blob/main/bitnami/cassandra/Chart.yaml' + const localHelmChartsDir = '/tmp/otomi/charts/uuid' + const helmChartCatalogUrl = 'https://gitea.example.com/otomi/charts.git' + const user = 'test-user' + const email = 'test@example.com' + const chartTargetDirName = 'cassandra' + const chartIcon = 'https://example.com/icon.png' + const allowTeams = true beforeEach(() => { jest.clearAllMocks() + // Set up environment variables for tests + process.env = { ...originalEnv, GIT_USER: 'git-user', GIT_PASSWORD: 'git-password' } + // Mock necessary function responses + ;(fsExtra.existsSync as jest.Mock).mockReturnValue(false) }) - test('sparseCloneChart returns true on successful clone and push', async () => { - // Arrange: simulate a successful run - const mockGit: Partial = { - env: jest.fn(), - clone: jest.fn().mockResolvedValueOnce('success'), - cwd: jest.fn().mockResolvedValueOnce('success'), - raw: jest.fn().mockResolvedValueOnce('success'), - checkout: jest.fn().mockResolvedValueOnce('success'), - push: jest.fn().mockResolvedValueOnce('success'), - addConfig: jest.fn().mockResolvedValueOnce('success'), - add: jest.fn().mockResolvedValueOnce('success'), - pull: jest.fn().mockResolvedValueOnce('success'), - init: jest.fn().mockResolvedValueOnce('success'), - addRemote: jest.fn().mockResolvedValueOnce('success'), - commit: jest.fn().mockResolvedValueOnce('success'), - } + afterEach(() => { + // Restore original environment + process.env = originalEnv + }) + test('successfully clones and processes a chart repository', async () => { + // Setup mock git instance + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + cwd: jest.fn().mockResolvedValue(undefined), + raw: jest.fn().mockResolvedValue(undefined), + checkout: jest.fn().mockResolvedValue(undefined), + addConfig: jest.fn().mockResolvedValue(undefined), + add: jest.fn().mockResolvedValue(undefined), + commit: jest.fn().mockResolvedValue(undefined), + pull: jest.fn().mockResolvedValue(undefined), + push: jest.fn().mockResolvedValue(undefined), + } ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + const result = await sparseCloneChart( - 'https://github.com/bitnami/charts.git', - fakeHelmCatalogUrl, - fakeUser, - fakeEmail, - fakeChartName, - fakeChartPath, - fakeSparsePath, - fakeRevision, - fakeChartIcon, - true, + gitRepositoryUrl, + localHelmChartsDir, + helmChartCatalogUrl, + user, + email, + chartTargetDirName, + chartIcon, + allowTeams, ) expect(result).toBe(true) - // Assert that renameSync was called to move the chart folder. + expect(fsExtra.mkdirSync).toHaveBeenCalledWith(localHelmChartsDir, { recursive: true }) + expect(fsExtra.mkdirSync).toHaveBeenCalledWith(`${localHelmChartsDir}-newChart`, { recursive: true }) + expect(mockGit.clone).toHaveBeenCalledTimes(2) // Once for catalog repo, once for chart repo + expect(mockGit.raw).toHaveBeenCalledWith(['sparse-checkout', 'init', '--cone']) + expect(mockGit.raw).toHaveBeenCalledWith(['sparse-checkout', 'set', 'bitnami/cassandra']) + expect(mockGit.checkout).toHaveBeenCalledWith('main') expect(fsExtra.renameSync).toHaveBeenCalled() - // And that the simpleGit clone and push methods were called. - expect(mockGit.clone).toHaveBeenCalled() - expect(mockGit.push).toHaveBeenCalled() + expect(fsExtra.rmSync).toHaveBeenCalled() + // Verify addConfig was called with correct user/email + expect(mockGit.addConfig).toHaveBeenCalledWith('user.name', user) + expect(mockGit.addConfig).toHaveBeenCalledWith('user.email', email) + // Verify commit and push were called + expect(mockGit.add).toHaveBeenCalledWith('.') + expect(mockGit.commit).toHaveBeenCalledWith(`Add ${chartTargetDirName} helm chart`) + expect(mockGit.pull).toHaveBeenCalledWith('origin', 'main', { '--rebase': null }) + expect(mockGit.push).toHaveBeenCalledWith('origin', 'main') + }) + + test('handles Gitea URLs by encoding credentials', async () => { + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + cwd: jest.fn().mockResolvedValue(undefined), + raw: jest.fn().mockResolvedValue(undefined), + checkout: jest.fn().mockResolvedValue(undefined), + addConfig: jest.fn().mockResolvedValue(undefined), + add: jest.fn().mockResolvedValue(undefined), + commit: jest.fn().mockResolvedValue(undefined), + pull: jest.fn().mockResolvedValue(undefined), + push: jest.fn().mockResolvedValue(undefined), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + await sparseCloneChart( + gitRepositoryUrl, + localHelmChartsDir, + helmChartCatalogUrl, + user, + email, + chartTargetDirName, + chartIcon, + allowTeams, + ) + + // Check that clone was called with encoded URL + const encodedUrl = `https://git-user:git-password@gitea.example.com/otomi/charts.git` + expect(mockGit.clone.mock.calls[0][0]).toBe(encodedUrl) + }) + + test('properly handles empty chartIcon parameter', async () => { + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + cwd: jest.fn().mockResolvedValue(undefined), + raw: jest.fn().mockResolvedValue(undefined), + checkout: jest.fn().mockResolvedValue(undefined), + addConfig: jest.fn().mockResolvedValue(undefined), + add: jest.fn().mockResolvedValue(undefined), + commit: jest.fn().mockResolvedValue(undefined), + pull: jest.fn().mockResolvedValue(undefined), + push: jest.fn().mockResolvedValue(undefined), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await sparseCloneChart( + gitRepositoryUrl, + localHelmChartsDir, + helmChartCatalogUrl, + user, + email, + chartTargetDirName, + '', // Empty chart icon + allowTeams, + ) + + expect(result).toBe(true) + // Should not attempt to update the chart icon + const chartYamlPath = `${localHelmChartsDir}/${chartTargetDirName}/Chart.yaml` + expect(fsExtra.readFile).not.toHaveBeenCalledWith(chartYamlPath, 'utf-8') + }) + + test('creates directory if it does not exist', async () => { + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + cwd: jest.fn().mockResolvedValue(undefined), + raw: jest.fn().mockResolvedValue(undefined), + checkout: jest.fn().mockResolvedValue(undefined), + addConfig: jest.fn().mockResolvedValue(undefined), + add: jest.fn().mockResolvedValue(undefined), + commit: jest.fn().mockResolvedValue(undefined), + pull: jest.fn().mockResolvedValue(undefined), + push: jest.fn().mockResolvedValue(undefined), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + ;(fsExtra.existsSync as jest.Mock).mockReturnValueOnce(false) + + await sparseCloneChart( + gitRepositoryUrl, + localHelmChartsDir, + helmChartCatalogUrl, + user, + email, + chartTargetDirName, + chartIcon, + allowTeams, + ) + + expect(fsExtra.mkdirSync).toHaveBeenCalledWith(localHelmChartsDir, { recursive: true }) }) +}) + +// ---------------------------------------------------------------- +// Tests for fetchWorkloadCatalog +describe('fetchWorkloadCatalog', () => { + const url = 'https://gitea.example.com/otomi/charts.git' + const helmChartsDir = '/tmp/otomi/charts/uuid' + + beforeEach(() => { + jest.clearAllMocks() + process.env = { ...originalEnv, GIT_USER: 'git-user', GIT_PASSWORD: 'git-password' } + ;(fsExtra.existsSync as jest.Mock).mockReturnValue(false) + + // Mock directory structure + const files = ['.git', 'chart1', 'chart2', 'README.md', 'rbac.yaml'] + ;(fsPromises.readdir as jest.Mock).mockResolvedValue(files) + + // Mock rbac.yaml content + const rbacContent = YAML.stringify({ + rbac: { + chart1: null, + chart2: ['team-2'], + }, + betaCharts: ['chart2'], + }) + ;(fsExtra.readFile as jest.Mock).mockImplementation((path) => { + if (path.endsWith('rbac.yaml')) return Promise.resolve(rbacContent) + + if (path.endsWith('chart1/README.md')) return Promise.resolve('# Chart 1 README') + + if (path.endsWith('chart1/values.yaml')) return Promise.resolve('key: value') + + if (path.endsWith('chart1/Chart.yaml')) { + return Promise.resolve( + YAML.stringify({ + name: 'chart1', + version: '1.0.0', + description: 'Test Chart 1', + icon: 'https://example.com/icon1.png', + }), + ) + } + if (path.endsWith('chart2/README.md')) return Promise.resolve('# Chart 2 README') - test('sparseCloneChart returns false when an error occurs', async () => { - // Arrange: simulate an error in the cloning process. - const mockGit: Partial = { - env: jest.fn(), - clone: jest.fn().mockRejectedValueOnce(new Error('Clone failed')), - push: jest.fn().mockResolvedValueOnce('success'), + if (path.endsWith('chart2/values.yaml')) return Promise.resolve('key: value') + + if (path.endsWith('chart2/Chart.yaml')) { + return Promise.resolve( + YAML.stringify({ + name: 'chart2', + version: '2.0.0', + description: 'Test Chart 2', + icon: 'https://example.com/icon2.png', + }), + ) + } + return Promise.reject(new Error(`File not found: ${path}`)) + }) + }) + + afterEach(() => { + process.env = originalEnv + }) + + test('clones repository and builds catalog for admin team', async () => { + // Setup mock git instance + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await fetchWorkloadCatalog(url, helmChartsDir, 'admin') + + expect(fsExtra.mkdirSync).toHaveBeenCalledWith(helmChartsDir, { recursive: true }) + expect(mockGit.clone).toHaveBeenCalledWith( + 'https://git-user:git-password@gitea.example.com/otomi/charts.git', + helmChartsDir, + ) + expect(result).toEqual({ + helmCharts: ['chart1', 'chart2'], + catalog: [ + { + name: 'chart1', + values: 'key: value', + icon: 'https://example.com/icon1.png', + chartVersion: '1.0.0', + chartDescription: 'Test Chart 1', + readme: '# Chart 1 README', + isBeta: false, + }, + { + name: 'chart2', + values: 'key: value', + icon: 'https://example.com/icon2.png', + chartVersion: '2.0.0', + chartDescription: 'Test Chart 2', + readme: '# Chart 2 README', + isBeta: true, + }, + ], + }) + }) + test('filters catalog based on team permissions', async () => { + // Setup mock git instance + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await fetchWorkloadCatalog(url, helmChartsDir, '1') + + // Only chart1 should be accessible to team-1 + expect(result).toEqual({ + helmCharts: ['chart1'], + catalog: [ + { + name: 'chart1', + values: 'key: value', + icon: 'https://example.com/icon1.png', + chartVersion: '1.0.0', + chartDescription: 'Test Chart 1', + readme: '# Chart 1 README', + isBeta: false, + }, + ], + }) + }) + + test('handles non-existent README files', async () => { + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), + } ;(simpleGit as jest.Mock).mockReturnValue(mockGit) - let result: boolean | undefined - try { - result = await sparseCloneChart( - 'https://github.com/bitnami/charts.git', - fakeHelmCatalogUrl, - fakeUser, - fakeEmail, - fakeChartName, - fakeChartPath, - fakeSparsePath, - fakeRevision, - fakeChartIcon, - true, - ) - } catch (error) { - result = false + // Make the README.md file read fail + ;(fsExtra.readFile as jest.Mock).mockImplementation((path) => { + if (path.endsWith('chart1/README.md')) return Promise.reject(new Error('File not found')) + + if (path.endsWith('chart1/values.yaml')) return Promise.resolve('key: value') + + if (path.endsWith('chart1/Chart.yaml')) { + return Promise.resolve( + YAML.stringify({ + name: 'chart1', + version: '1.0.0', + description: 'Test Chart 1', + icon: 'https://example.com/icon1.png', + }), + ) + } + if (path.endsWith('rbac.yaml')) { + return Promise.resolve( + YAML.stringify({ + rbac: { chart1: null }, + betaCharts: [], + }), + ) + } + return Promise.reject(new Error(`File not found: ${path}`)) + }) + + const result = await fetchWorkloadCatalog(url, helmChartsDir, 'admin') + + // Should include chart1 with default README message + expect(result.catalog[0].readme).toBe('There is no `README` for this chart.') + }) + + test('handles missing rbac.yaml file', async () => { + const mockGit = { + clone: jest.fn().mockResolvedValue(undefined), } - expect(result).toBe(false) + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + // Make the rbac.yaml file read fail + ;(fsExtra.readFile as jest.Mock).mockImplementation((path) => { + if (path.endsWith('rbac.yaml')) return Promise.reject(new Error('File not found')) + + if (path.endsWith('chart1/README.md')) return Promise.resolve('# Chart 1 README') + + if (path.endsWith('chart1/values.yaml')) return Promise.resolve('key: value') + + if (path.endsWith('chart1/Chart.yaml')) { + return Promise.resolve( + YAML.stringify({ + name: 'chart1', + version: '1.0.0', + description: 'Test Chart 1', + icon: 'https://example.com/icon1.png', + }), + ) + } + return Promise.reject(new Error(`File not found: ${path}`)) + }) + + const result = await fetchWorkloadCatalog(url, helmChartsDir, 'admin') + + // Should include charts in the catalog + expect(result.helmCharts).toEqual(['chart1']) + expect(result.catalog).toHaveLength(1) }) }) diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index 9b09fd150..f798fcec5 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -1,3 +1,5 @@ +/* eslint-disable no-useless-escape */ +import axios from 'axios' import Debug from 'debug' import { existsSync, mkdirSync, readFile, renameSync, rmSync } from 'fs-extra' import { readdir, writeFile } from 'fs/promises' @@ -7,17 +9,71 @@ import YAML from 'yaml' const debug = Debug('apl:workloadUtils') -export interface NewChartValues { - url: string - chartName: string - chartIcon?: string - chartPath: string - revision: string - allowTeams: boolean +export const detectGitProvider = (url) => { + if (!url || typeof url !== 'string') return null + + const normalizedUrl = url.replace(/\/*$/, '') + + const githubPattern = /github\.com\/([^\/]+)\/([^\/]+)(?:\/(?:blob|raw))?\/([^\/]+)\/(.+)/ + const gitlabPattern = /gitlab\.com\/([^\/]+)\/([^\/]+)\/(?:\-\/(?:blob|raw))\/([^\/]+)\/(.+)/ + const bitbucketPattern = /bitbucket\.org\/([^\/]+)\/([^\/]+)\/(?:src|raw)\/([^\/]+)\/(.+)/ + + let match = normalizedUrl.match(githubPattern) + if (match) return { provider: 'github', owner: match[1], repo: match[2], branch: match[3], filePath: match[4] } + + match = normalizedUrl.match(gitlabPattern) + if (match) return { provider: 'gitlab', owner: match[1], repo: match[2], branch: match[3], filePath: match[4] } + + match = normalizedUrl.match(bitbucketPattern) + if (match) return { provider: 'bitbucket', owner: match[1], repo: match[2], branch: match[3], filePath: match[4] } + + return null +} + +const getGitRawUrl = (details) => { + if (!details) return null + + if (details.provider === 'github') + return `https://raw.githubusercontent.com/${details.owner}/${details.repo}/${details.branch}/${details.filePath}` + if (details.provider === 'gitlab') + return `https://gitlab.com/${details.owner}/${details.repo}/-/raw/${details.branch}/${details.filePath}` + if (details.provider === 'bitbucket') + return `https://bitbucket.org/${details.owner}/${details.repo}/raw/${details.branch}/${details.filePath}` + + return null +} + +export const getGitCloneUrl = (details) => { + if (!details) return null + + if (details.provider === 'github') return `https://github.com/${details.owner}/${details.repo}.git` + if (details.provider === 'gitlab') return `https://gitlab.com/${details.owner}/${details.repo}.git` + if (details.provider === 'bitbucket') return `https://bitbucket.org/${details.owner}/${details.repo}.git` + + return null +} + +export const fetchChartYaml = async (url) => { + try { + const details = detectGitProvider(url) + if (!details) return { values: {}, error: 'Unsupported Git provider or invalid URL format.' } + + const rawUrl = getGitRawUrl(details) + if (!rawUrl) return { values: {}, error: `Could not generate raw URL for provider: ${details.provider}` } + + const response = await axios.get(rawUrl, { responseType: 'text' }) + return { values: YAML.parse(response.data as string), error: '' } + } catch (error) { + console.error('Error fetching Chart.yaml:', error.message) + return { values: {}, error: 'Error fetching helm chart content.' } + } } -export interface NewChartPayload extends NewChartValues { - teamId: string +export interface NewHelmChartValues { + gitRepositoryUrl: string + chartTargetDirName: string + chartIcon?: string + allowTeams: boolean } function throwChartError(message: string) { @@ -27,7 +83,6 @@ function throwChartError(message: string) { } throw err } - function isGiteaURL(url: string) { let hostname = '' if (url) { @@ -41,7 +96,6 @@ function isGiteaURL(url: string) { const giteaPattern = /^gitea\..+/i return giteaPattern.test(hostname) } - /** * Reads the Chart.yaml file at the given path, updates (or sets) its icon field, * and writes the updated content back to disk. @@ -54,14 +108,12 @@ export async function updateChartIconInYaml(chartYamlPath: string, newIcon: stri const fileContent = await readFile(chartYamlPath, 'utf-8') const chartObject = YAML.parse(fileContent) if (newIcon && newIcon.trim() !== '') chartObject.icon = newIcon - const newContent = YAML.stringify(chartObject) await writeFile(chartYamlPath, newContent, 'utf-8') } catch (error) { debug(`Error updating chart icon in ${chartYamlPath}:`, error) } } - /** * Updates the rbac.yaml file in the specified folder by adding a new chart key. * @@ -83,21 +135,17 @@ export async function updateRbacForNewChart(sparsePath: string, chartKey: string // Create a default structure if the file doesn't exist. rbacData = { rbac: {}, betaCharts: [] } } - // Ensure the "rbac" section exists. if (!rbacData.rbac) rbacData.rbac = {} - // Add the new chart entry if it doesn't exist. // If allowTeams is false, set the value to an empty array ([]), // otherwise (if true) set it to null. if (!(chartKey in rbacData.rbac)) rbacData.rbac[chartKey] = allowTeams ? null : [] - // Stringify the updated YAML content and write it back. const newContent = YAML.stringify(rbacData) await writeFile(rbacFilePath, newContent, 'utf-8') debug(`Updated rbac.yaml: added ${chartKey}: ${allowTeams ? 'null' : '[]'}`) } - class chartRepo { localPath: string chartRepoUrl: string @@ -125,45 +173,47 @@ class chartRepo { await this.git.push('origin', 'main') } } - /** * Clones a repository using sparse checkout, checks out a specific revision, * and moves the contents of the desired subdirectory (sparsePath) to the root of the target folder. * - * @param url - The base Git repository URL (e.g. "https://github.com/nats-io/k8s.git") - * @param chartName - The target folder name for the clone (will be the final chart folder, e.g. "nats") - * @param chartPath - The path in github where the chart is located - * @param sparsePath - The subdirectory to sparse checkout (e.g. "helm/charts/nats") - * @param revision - The branch or commit to checkout (e.g. "main") + * @param gitRepositoryUrl - The base Git repository URL (e.g. "https://github.com/nats-io/k8s.git") + * @param localHelmChartsDir - The subdirectory to sparse checkout (e.g. "/tmp/otomi/charts/uuid") + * @param helmChartCatalogUrl - The URL of the (Gitea) Helm Chart Catalog (e.g. "https://gitea./otomi/charts.git") + * @param user - The Git username (e.g. "otomi-admin") + * @param email - The Git email (e.g. "not@us.ed") + * @param chartTargetDirName - The target folder name for the clone (will be the final chart folder, e.g. "nats") * @param chartIcon - the icon URL path (e.g https://myimage.com/imageurl) * @param allowTeams - Boolean indicating if teams are allowed to use the chart. * If false, the key is set to []. * If true, the key is set to null. */ export async function sparseCloneChart( - url: string, + gitRepositoryUrl: string, + localHelmChartsDir: string, helmChartCatalogUrl: string, user: string, email: string, - chartName: string, - chartPath: string, - sparsePath: string, // e.g. "/tmp/otomi/charts/uuid" - revision: string, + chartTargetDirName: string, chartIcon?: string, allowTeams?: boolean, ): Promise { - const temporaryCloneDir = `${sparsePath}-new` // Temporary clone directory - const checkoutPath = `${sparsePath}/${chartName}` // Final destination + const details = detectGitProvider(gitRepositoryUrl) + const gitCloneUrl = getGitCloneUrl(details) as string + const chartPath = details?.filePath.replace('/Chart.yaml', '') as string + const revision = details?.branch as string + const temporaryCloneDir = `${localHelmChartsDir}-newChart` + const finalDestinationPath = `${localHelmChartsDir}/${chartTargetDirName}` - if (!existsSync(sparsePath)) mkdirSync(sparsePath, { recursive: true }) + if (!existsSync(localHelmChartsDir)) mkdirSync(localHelmChartsDir, { recursive: true }) let gitUrl = helmChartCatalogUrl - if (isGiteaURL(url)) { - const [protocol, bareUrl] = url.split('://') + if (isGiteaURL(helmChartCatalogUrl)) { + const [protocol, bareUrl] = helmChartCatalogUrl.split('://') const encodedUser = encodeURIComponent(process.env.GIT_USER as string) const encodedPassword = encodeURIComponent(process.env.GIT_PASSWORD as string) gitUrl = `${protocol}://${encodedUser}:${encodedPassword}@${bareUrl}` } - const gitRepo = new chartRepo(sparsePath, gitUrl, user, email) + const gitRepo = new chartRepo(localHelmChartsDir, gitUrl, user, email) await gitRepo.clone() if (!existsSync(temporaryCloneDir)) mkdirSync(temporaryCloneDir, { recursive: true }) @@ -174,43 +224,37 @@ export async function sparseCloneChart( const git = simpleGit() - // Clone the repository into the folder named checkoutPath. - debug(`Cloning repository: ${url} into ${checkoutPath}`) - await git.clone(url, temporaryCloneDir, ['--filter=blob:none', '--no-checkout']) + debug(`Cloning repository: ${gitCloneUrl} into ${temporaryCloneDir}`) + await git.clone(gitCloneUrl, temporaryCloneDir, ['--filter=blob:none', '--no-checkout']) - // Initialize sparse checkout in cone mode within checkoutPath. - debug(`Initializing sparse checkout in cone mode at ${checkoutPath}`) + debug(`Initializing sparse checkout in cone mode at ${temporaryCloneDir}`) await git.cwd(temporaryCloneDir) await git.raw(['sparse-checkout', 'init', '--cone']) - // Set the sparse checkout to only include the specified chartPath. debug(`Setting sparse checkout path to ${chartPath}`) await git.raw(['sparse-checkout', 'set', chartPath]) - // Checkout the desired revision (branch or commit) within checkoutPath. - debug(`Checking out revision: ${revision}`) + debug(`Checking out the desired revision (branch or commit): ${revision}`) await git.checkout(revision) - // Move the contents of the sparse folder (chartPath) to the repository root. - // This moves files from "checkoutPath/chartPath/*" to "checkoutPath/" - renameSync(path.join(temporaryCloneDir, chartPath), checkoutPath) + // Move files from "temporaryCloneDir/chartPath/*" to "finalDestinationPath/" + renameSync(path.join(temporaryCloneDir, chartPath), finalDestinationPath) // Remove the leftover temporary clone directory. - // For chartPath "bitnami/cassandra", the top-level folder is "bitnami". rmSync(temporaryCloneDir, { recursive: true, force: true }) // Update Chart.yaml with the new icon if one is provided. if (chartIcon && chartIcon.trim() !== '') { - const chartYamlPath = `${checkoutPath}/Chart.yaml` + const chartYamlPath = `${finalDestinationPath}/Chart.yaml` await updateChartIconInYaml(chartYamlPath, chartIcon) } // update rbac file - await updateRbacForNewChart(sparsePath, chartName, allowTeams as boolean) + await updateRbacForNewChart(localHelmChartsDir, chartTargetDirName, allowTeams as boolean) // pull&push new chart changes await gitRepo.addConfig() - await gitRepo.commitAndPush(chartName) + await gitRepo.commitAndPush(chartTargetDirName) return true } From 41844b661b305796d939d95dca2248682cd414cc Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Mar 2025 11:20:44 +0100 Subject: [PATCH 2/5] feat: update support for gitlab --- src/utils/workloadUtils.test.ts | 6 +++--- src/utils/workloadUtils.ts | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/utils/workloadUtils.test.ts b/src/utils/workloadUtils.test.ts index c488605c4..a03ae64ff 100644 --- a/src/utils/workloadUtils.test.ts +++ b/src/utils/workloadUtils.test.ts @@ -66,12 +66,12 @@ describe('detectGitProvider', () => { }) test('detects GitLab URLs correctly', () => { - const url = 'https://gitlab.com/owner/repo/-/blob/main/path/to/file.yaml' + const url = 'https://gitlab.com/owner/charts/repo/-/blob/main/path/to/file.yaml' const result = detectGitProvider(url) expect(result).toEqual({ provider: 'gitlab', owner: 'owner', - repo: 'repo', + repo: 'charts/repo', branch: 'main', filePath: 'path/to/file.yaml', }) @@ -388,7 +388,7 @@ describe('sparseCloneChart', () => { expect(fsExtra.mkdirSync).toHaveBeenCalledWith(`${localHelmChartsDir}-newChart`, { recursive: true }) expect(mockGit.clone).toHaveBeenCalledTimes(2) // Once for catalog repo, once for chart repo expect(mockGit.raw).toHaveBeenCalledWith(['sparse-checkout', 'init', '--cone']) - expect(mockGit.raw).toHaveBeenCalledWith(['sparse-checkout', 'set', 'bitnami/cassandra']) + expect(mockGit.raw).toHaveBeenCalledWith(['sparse-checkout', 'set', 'bitnami/cassandra/']) expect(mockGit.checkout).toHaveBeenCalledWith('main') expect(fsExtra.renameSync).toHaveBeenCalled() expect(fsExtra.rmSync).toHaveBeenCalled() diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index f798fcec5..aaca8d2d1 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -15,14 +15,22 @@ export const detectGitProvider = (url) => { const normalizedUrl = url.replace(/\/*$/, '') const githubPattern = /github\.com\/([^\/]+)\/([^\/]+)(?:\/(?:blob|raw))?\/([^\/]+)\/(.+)/ - const gitlabPattern = /gitlab\.com\/([^\/]+)\/([^\/]+)\/(?:\-\/(?:blob|raw))\/([^\/]+)\/(.+)/ + const gitlabPattern = /gitlab\.com\/([^\/]+)\/([^\/]+)\/([^\/]+)\/(?:\-\/(?:blob|raw))\/([^\/]+)\/(.+)/ const bitbucketPattern = /bitbucket\.org\/([^\/]+)\/([^\/]+)\/(?:src|raw)\/([^\/]+)\/(.+)/ let match = normalizedUrl.match(githubPattern) if (match) return { provider: 'github', owner: match[1], repo: match[2], branch: match[3], filePath: match[4] } match = normalizedUrl.match(gitlabPattern) - if (match) return { provider: 'gitlab', owner: match[1], repo: match[2], branch: match[3], filePath: match[4] } + if (match) { + return { + provider: 'gitlab', + owner: match[1], + repo: `${match[2]}/${match[3]}`, + branch: match[4], + filePath: match[5], + } + } match = normalizedUrl.match(bitbucketPattern) if (match) return { provider: 'bitbucket', owner: match[1], repo: match[2], branch: match[3], filePath: match[4] } @@ -200,7 +208,7 @@ export async function sparseCloneChart( ): Promise { const details = detectGitProvider(gitRepositoryUrl) const gitCloneUrl = getGitCloneUrl(details) as string - const chartPath = details?.filePath.replace('/Chart.yaml', '') as string + const chartPath = details?.filePath.replace('Chart.yaml', '') as string const revision = details?.branch as string const temporaryCloneDir = `${localHelmChartsDir}-newChart` const finalDestinationPath = `${localHelmChartsDir}/${chartTargetDirName}` @@ -240,6 +248,9 @@ export async function sparseCloneChart( // Move files from "temporaryCloneDir/chartPath/*" to "finalDestinationPath/" renameSync(path.join(temporaryCloneDir, chartPath), finalDestinationPath) + // Remove the .git directory from the final destination. + rmSync(`${finalDestinationPath}/.git`, { recursive: true, force: true }) + // Remove the leftover temporary clone directory. rmSync(temporaryCloneDir, { recursive: true, force: true }) From b7951d49412174efb4af13de6cee1875d9432712 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Mar 2025 12:02:41 +0100 Subject: [PATCH 3/5] fix: normalizedUrl & code conventions --- src/utils/workloadUtils.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index aaca8d2d1..e23b776b0 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -9,10 +9,10 @@ import YAML from 'yaml' const debug = Debug('apl:workloadUtils') -export const detectGitProvider = (url) => { +export function detectGitProvider(url) { if (!url || typeof url !== 'string') return null - const normalizedUrl = url.replace(/\/*$/, '') + const normalizedUrl = new URL(url).origin + new URL(url).pathname.replace(/\/*$/, '') const githubPattern = /github\.com\/([^\/]+)\/([^\/]+)(?:\/(?:blob|raw))?\/([^\/]+)\/(.+)/ const gitlabPattern = /gitlab\.com\/([^\/]+)\/([^\/]+)\/([^\/]+)\/(?:\-\/(?:blob|raw))\/([^\/]+)\/(.+)/ @@ -38,7 +38,7 @@ export const detectGitProvider = (url) => { return null } -const getGitRawUrl = (details) => { +function getGitRawUrl(details) { if (!details) return null if (details.provider === 'github') @@ -51,7 +51,7 @@ const getGitRawUrl = (details) => { return null } -export const getGitCloneUrl = (details) => { +export function getGitCloneUrl(details) { if (!details) return null if (details.provider === 'github') return `https://github.com/${details.owner}/${details.repo}.git` @@ -61,7 +61,7 @@ export const getGitCloneUrl = (details) => { return null } -export const fetchChartYaml = async (url) => { +export async function fetchChartYaml(url) { try { const details = detectGitProvider(url) if (!details) return { values: {}, error: 'Unsupported Git provider or invalid URL format.' } From 9762ac565d045d5c7ecce72d442bb494aa7e77d6 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Mar 2025 12:32:01 +0100 Subject: [PATCH 4/5] fix: gitlab regex --- src/utils/workloadUtils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index e23b776b0..ba8739d3c 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -14,8 +14,8 @@ export function detectGitProvider(url) { const normalizedUrl = new URL(url).origin + new URL(url).pathname.replace(/\/*$/, '') - const githubPattern = /github\.com\/([^\/]+)\/([^\/]+)(?:\/(?:blob|raw))?\/([^\/]+)\/(.+)/ - const gitlabPattern = /gitlab\.com\/([^\/]+)\/([^\/]+)\/([^\/]+)\/(?:\-\/(?:blob|raw))\/([^\/]+)\/(.+)/ + const githubPattern = /github\.com\/([^\/]+)\/([^\/]+)\/(?:blob|raw)\/([^\/]+)\/(.+)/ + const gitlabPattern = /gitlab\.com\/([^\/]+)\/([^\/]+)(?:\/([^\/]+))?\/(?:\-\/(?:blob|raw))\/([^\/]+)\/(.+)/ const bitbucketPattern = /bitbucket\.org\/([^\/]+)\/([^\/]+)\/(?:src|raw)\/([^\/]+)\/(.+)/ let match = normalizedUrl.match(githubPattern) @@ -26,7 +26,7 @@ export function detectGitProvider(url) { return { provider: 'gitlab', owner: match[1], - repo: `${match[2]}/${match[3]}`, + repo: match[3] ? `${match[2]}/${match[3]}` : match[2], // Handle optional subgroup branch: match[4], filePath: match[5], } From 2ab85270be055cd3b883064f04445abc9fc8bf67 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 12 Mar 2025 12:55:35 +0100 Subject: [PATCH 5/5] feat: add GIT_PROVIDER_URL_PATTERNS env --- src/utils/workloadUtils.ts | 21 +++++++++++++++++---- src/validators.ts | 8 ++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/utils/workloadUtils.ts b/src/utils/workloadUtils.ts index ba8739d3c..3a31b51d7 100644 --- a/src/utils/workloadUtils.ts +++ b/src/utils/workloadUtils.ts @@ -1,22 +1,35 @@ -/* eslint-disable no-useless-escape */ import axios from 'axios' import Debug from 'debug' import { existsSync, mkdirSync, readFile, renameSync, rmSync } from 'fs-extra' import { readdir, writeFile } from 'fs/promises' import path from 'path' import simpleGit, { SimpleGit } from 'simple-git' +import { GIT_PROVIDER_URL_PATTERNS, cleanEnv } from 'src/validators' import YAML from 'yaml' const debug = Debug('apl:workloadUtils') +const env = cleanEnv({ + GIT_PROVIDER_URL_PATTERNS, +}) + export function detectGitProvider(url) { if (!url || typeof url !== 'string') return null const normalizedUrl = new URL(url).origin + new URL(url).pathname.replace(/\/*$/, '') - const githubPattern = /github\.com\/([^\/]+)\/([^\/]+)\/(?:blob|raw)\/([^\/]+)\/(.+)/ - const gitlabPattern = /gitlab\.com\/([^\/]+)\/([^\/]+)(?:\/([^\/]+))?\/(?:\-\/(?:blob|raw))\/([^\/]+)\/(.+)/ - const bitbucketPattern = /bitbucket\.org\/([^\/]+)\/([^\/]+)\/(?:src|raw)\/([^\/]+)\/(.+)/ + const { github, gitlab, bitbucket } = env.GIT_PROVIDER_URL_PATTERNS as { + github: string + gitlab: string + bitbucket: string + } + const githubPattern = new RegExp(github || 'github\\.com\\/([^\\/]+)\\/([^\\/]+)\\/(?:blob|raw)\\/([^\\/]+)\\/(.+)') + const gitlabPattern = new RegExp( + gitlab || 'gitlab\\.com\\/([^\\/]+)\\/([^\\/]+)(?:\\/([^\\/]+))?\\/(?:\\-\\/(?:blob|raw))\\/([^\\/]+)\\/(.+)', + ) + const bitbucketPattern = new RegExp( + bitbucket || 'bitbucket\\.org\\/([^\\/]+)\\/([^\\/]+)\\/(?:src|raw)\\/([^\\/]+)\\/(.+)', + ) let match = normalizedUrl.match(githubPattern) if (match) return { provider: 'github', owner: match[1], repo: match[2], branch: match[3], filePath: match[4] } diff --git a/src/validators.ts b/src/validators.ts index 7012fa788..59ffd53dc 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -47,6 +47,14 @@ export const HELM_CHART_CATALOG = str({ desc: 'The helm chart catalog', devDefault: 'https://github.com/linode/apl-charts.git', }) +export const GIT_PROVIDER_URL_PATTERNS = json({ + desc: 'Regular expressions to match and extract information from URLs of supported git providers (GitHub, GitLab, Bitbucket) for cloning Helm charts.', + default: { + github: 'github\\.com\\/([^\\/]+)\\/([^\\/]+)\\/(?:blob|raw)\\/([^\\/]+)\\/(.+)', + gitlab: 'gitlab\\.com\\/([^\\/]+)\\/([^\\/]+)(?:\\/([^\\/]+))?\\/(?:\\-\\/(?:blob|raw))\\/([^\\/]+)\\/(.+)', + bitbucket: 'bitbucket\\.org\\/([^\\/]+)\\/([^\\/]+)\\/(?:src|raw)\\/([^\\/]+)\\/(.+)', + }, +}) export const OIDC_ENDPOINT = str() export const REGION = str({ desc: 'The cloud region' }) export const ROARR_LOG = bool({ desc: 'To enable Lightship logs', default: false })