diff --git a/migrations/extract/0002_even_marvel_boy.sql b/migrations/extract/0002_even_marvel_boy.sql new file mode 100644 index 000000000..4a5e93762 --- /dev/null +++ b/migrations/extract/0002_even_marvel_boy.sql @@ -0,0 +1 @@ +ALTER TABLE repositories ADD `name` text NOT NULL; \ No newline at end of file diff --git a/migrations/extract/meta/0002_snapshot.json b/migrations/extract/meta/0002_snapshot.json new file mode 100644 index 000000000..6c2a24d79 --- /dev/null +++ b/migrations/extract/meta/0002_snapshot.json @@ -0,0 +1,399 @@ +{ + "version": "5", + "dialect": "sqlite", + "id": "fe237ba1-c2a4-4363-8541-1a1857fd7a97", + "prevId": "a89d4156-ecbd-4e33-b823-372c918503f0", + "tables": { + "members": { + "name": "members", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "external_id": { + "name": "external_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "members_external_id_idx": { + "name": "members_external_id_idx", + "columns": [ + "external_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "merge_request_commits": { + "name": "merge_request_commits", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "merge_request_id": { + "name": "merge_request_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "authored_date": { + "name": "authored_date", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "committed_date": { + "name": "committed_date", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "author_name": { + "name": "author_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "author_email": { + "name": "author_email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "committer_name": { + "name": "committer_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "committer_email": { + "name": "committer_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "merge_request_commits_external_id_idx": { + "name": "merge_request_commits_external_id_idx", + "columns": [ + "external_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "merge_request_diffs": { + "name": "merge_request_diffs", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "merge_request_id": { + "name": "merge_request_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "new_path": { + "name": "new_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "old_path": { + "name": "old_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "a_mode": { + "name": "a_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "b_mode": { + "name": "b_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "new_file": { + "name": "new_file", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "renamed_file": { + "name": "renamed_file", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_file": { + "name": "deleted_file", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "diff": { + "name": "diff", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "diffs_merge_request_id_newPath_idx": { + "name": "diffs_merge_request_id_newPath_idx", + "columns": [ + "merge_request_id", + "new_path" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "merge_requests": { + "name": "merge_requests", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "external_id": { + "name": "external_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "merge_request_id": { + "name": "merge_request_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "merge_requests_external_id_idx": { + "name": "merge_requests_external_id_idx", + "columns": [ + "external_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "namespaces": { + "name": "namespaces", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "external_id": { + "name": "external_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "namespaces_external_id_idx": { + "name": "namespaces_external_id_idx", + "columns": [ + "external_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "repositories": { + "name": "repositories", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "external_id": { + "name": "external_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "repositories_external_id_idx": { + "name": "repositories_external_id_idx", + "columns": [ + "external_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "repositories_to_members": { + "name": "repositories_to_members", + "columns": { + "repository_id": { + "name": "repository_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "member_id": { + "name": "member_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "repositories_to_members_repository_id_member_id_pk": { + "columns": [ + "member_id", + "repository_id" + ] + } + }, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + } +} \ No newline at end of file diff --git a/migrations/extract/meta/_journal.json b/migrations/extract/meta/_journal.json index e1f0de7dd..5b22aaebe 100644 --- a/migrations/extract/meta/_journal.json +++ b/migrations/extract/meta/_journal.json @@ -15,6 +15,13 @@ "when": 1690531910887, "tag": "0001_wealthy_medusa", "breakpoints": true + }, + { + "idx": 2, + "version": "5", + "when": 1690804131705, + "tag": "0002_even_marvel_boy", + "breakpoints": true } ] } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index edad033dc..40cda8bf0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2707,6 +2707,151 @@ "node": ">= 8" } }, + "node_modules/@octokit/auth-token": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", + "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/core": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.0.0.tgz", + "integrity": "sha512-YbAtMWIrbZ9FCXbLwT9wWB8TyLjq9mxpKdgB3dUNxQcIVTf9hJ70gRPwAcqGZdY6WdJPZ0I7jLaaNDCiloGN2A==", + "dependencies": { + "@octokit/auth-token": "^4.0.0", + "@octokit/graphql": "^7.0.0", + "@octokit/request": "^8.0.2", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/endpoint": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.0.tgz", + "integrity": "sha512-szrQhiqJ88gghWY2Htt8MqUDO6++E/EIXqJ2ZEp5ma3uGS46o7LZAzSLt49myB7rT+Hfw5Y6gO3LmOxGzHijAQ==", + "dependencies": { + "@octokit/types": "^11.0.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.1.tgz", + "integrity": "sha512-T5S3oZ1JOE58gom6MIcrgwZXzTaxRnxBso58xhozxHpOqSTgDS6YNeEUvZ/kRvXgPrRz/KHnZhtb7jUMRi9E6w==", + "dependencies": { + "@octokit/request": "^8.0.1", + "@octokit/types": "^11.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==" + }, + "node_modules/@octokit/plugin-paginate-rest": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-8.0.0.tgz", + "integrity": "sha512-2xZ+baZWUg+qudVXnnvXz7qfrTmDeYPCzangBVq/1gXxii/OiS//4shJp9dnCCvj1x+JAm9ji1Egwm1BA47lPQ==", + "dependencies": { + "@octokit/types": "^11.0.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": ">=5" + } + }, + "node_modules/@octokit/plugin-request-log": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-4.0.0.tgz", + "integrity": "sha512-2uJI1COtYCq8Z4yNSnM231TgH50bRkheQ9+aH8TnZanB6QilOnx8RMD2qsnamSOXtDj0ilxvevf5fGsBhBBzKA==", + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": ">=5" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-9.0.0.tgz", + "integrity": "sha512-KquMF/VB1IkKNiVnzJKspY5mFgGyLd7HzdJfVEGTJFzqu9BRFNWt+nwTCMuUiWc72gLQhRWYubTwOkQj+w/1PA==", + "dependencies": { + "@octokit/types": "^11.0.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": ">=5" + } + }, + "node_modules/@octokit/request": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.1.tgz", + "integrity": "sha512-8N+tdUz4aCqQmXl8FpHYfKG9GelDFd7XGVzyN8rc6WxVlYcfpHECnuRkgquzz+WzvHTK62co5di8gSXnzASZPQ==", + "dependencies": { + "@octokit/endpoint": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.1.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "dependencies": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/rest": { + "version": "20.0.1", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-20.0.1.tgz", + "integrity": "sha512-wROV21RwHQIMNb2Dgd4+pY+dVy1Dwmp85pBrgr6YRRDYRBu9Gb+D73f4Bl2EukZSj5hInq2Tui9o7gAQpc2k2Q==", + "dependencies": { + "@octokit/core": "^5.0.0", + "@octokit/plugin-paginate-rest": "^8.0.0", + "@octokit/plugin-request-log": "^4.0.0", + "@octokit/plugin-rest-endpoint-methods": "^9.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "dependencies": { + "@octokit/openapi-types": "^18.0.0" + } + }, "node_modules/@peculiar/asn1-schema": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/@peculiar/asn1-schema/-/asn1-schema-2.3.6.tgz", @@ -3627,6 +3772,11 @@ "form-data": "^3.0.0" } }, + "node_modules/@types/parse-link-header": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/parse-link-header/-/parse-link-header-2.0.1.tgz", + "integrity": "sha512-BrKNSrRTqn3UkMXvdVtr/znJch0PMBpEvEP8oBkxDx7eEGntuFLI+WpA5HGsNHK4SlqyhaMa+Ks0ViwyixQB5w==" + }, "node_modules/@types/prettier": { "version": "2.7.3", "license": "MIT" @@ -4395,6 +4545,11 @@ } ] }, + "node_modules/before-after-hook": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", + "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" + }, "node_modules/better-sqlite3": { "version": "8.5.0", "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-8.5.0.tgz", @@ -5074,6 +5229,11 @@ "node": ">=0.4.0" } }, + "node_modules/deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" + }, "node_modules/dequal": { "version": "2.0.3", "license": "MIT", @@ -6999,6 +7159,14 @@ "node": ">=8" } }, + "node_modules/is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-promise": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz", @@ -9853,6 +10021,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parse-link-header": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/parse-link-header/-/parse-link-header-2.0.0.tgz", + "integrity": "sha512-xjU87V0VyHZybn2RrCX5TIFGxTVZE6zqqZWMPlIKiSKuWh/X5WZdt+w1Ki1nXB+8L/KtL+nZ4iq+sfI6MrhhMw==", + "dependencies": { + "xtend": "~4.0.1" + } + }, "node_modules/path-exists": { "version": "4.0.0", "license": "MIT", @@ -11787,6 +11963,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/universal-user-agent": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", + "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" + }, "node_modules/universalify": { "version": "0.1.2", "license": "MIT", @@ -12085,6 +12266,14 @@ "resolved": "https://registry.npmjs.org/xcase/-/xcase-2.0.1.tgz", "integrity": "sha512-UmFXIPU+9Eg3E9m/728Bii0lAIuoc+6nbrNUKaRPJOFp91ih44qqGlWtxMB6kXFrRD6po+86ksHM5XHCfk6iPw==" }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -12240,7 +12429,10 @@ "dependencies": { "@acme/extract-schema": "^1.0.0", "@gitbeaker/core": "^39.8.0", - "@gitbeaker/rest": "^39.8.0" + "@gitbeaker/rest": "^39.8.0", + "@octokit/rest": "^20.0.1", + "@types/parse-link-header": "^2.0.1", + "parse-link-header": "^2.0.0" } }, "packages/schemas/extract": { diff --git a/packages/functions/extract/src/get-merge-requests.test.ts b/packages/functions/extract/src/get-merge-requests.test.ts index 27f3cd2f3..9f93ea3be 100644 --- a/packages/functions/extract/src/get-merge-requests.test.ts +++ b/packages/functions/extract/src/get-merge-requests.test.ts @@ -24,7 +24,7 @@ beforeAll(() => { migrate(db, { migrationsFolder: "../../../migrations/extract" }); - fetchMergeRequests = jest.fn((externalRepositoryId: number, page?: number, perPage?: number) => { + fetchMergeRequests = jest.fn((externalRepositoryId: number, namespaceName:string, repositoryName: string, page?: number, perPage?: number) => { switch (externalRepositoryId) { case 1000: @@ -70,7 +70,7 @@ afterAll(async () => { describe('get-merge-request:', () => { describe('getMergeRequests', () => { test('should create insert merge request data in the database', async () => { - const { mergeRequests, paginationInfo } = await getMergeRequests({ externalRepositoryId: 1000 }, context); + const { mergeRequests, paginationInfo } = await getMergeRequests({ externalRepositoryId: 1000, namespaceName: '', repositoryName: ''}, context); expect(mergeRequests).toBeDefined(); expect(paginationInfo).toBeDefined(); diff --git a/packages/functions/extract/src/get-merge-requests.ts b/packages/functions/extract/src/get-merge-requests.ts index e36b2ef04..87e20d22c 100644 --- a/packages/functions/extract/src/get-merge-requests.ts +++ b/packages/functions/extract/src/get-merge-requests.ts @@ -4,6 +4,8 @@ import type { Pagination, SourceControl } from "@acme/source-control"; export type GetMergeRequestsInput = { externalRepositoryId: number; + namespaceName: string; + repositoryName: string; page?: number; perPage?: number; }; @@ -19,10 +21,10 @@ export type GetMergeRequestsEntities = Pick; export type GetMergeRequestsFunction = ExtractFunction; export const getMergeRequests: GetMergeRequestsFunction = async ( - { externalRepositoryId }, + { externalRepositoryId, namespaceName, repositoryName }, { integrations, db, entities } ) => { - const { mergeRequests, pagination } = await integrations.sourceControl.fetchMergeRequests(externalRepositoryId); + const { mergeRequests, pagination } = await integrations.sourceControl.fetchMergeRequests(externalRepositoryId, namespaceName, repositoryName); await db.insert(entities.mergeRequests).values(mergeRequests) .onConflictDoNothing({ target: entities.mergeRequests.externalId }) diff --git a/packages/functions/extract/src/get-repository.test.ts b/packages/functions/extract/src/get-repository.test.ts index 8e765a1d0..633354da2 100644 --- a/packages/functions/extract/src/get-repository.test.ts +++ b/packages/functions/extract/src/get-repository.test.ts @@ -16,7 +16,7 @@ let db: ReturnType; let context: Context; let fetchRepository: jest.Mock> const databaseName = 'get-repository.db'; @@ -30,7 +30,7 @@ beforeAll(() => { switch (externalRepositoryId) { case 1000: return Promise.resolve({ - repository: { externalId: 1000 }, + repository: { externalId: 1000, name: 'repo' }, namespace: { externalId: 2000, name: 'gengar' } }); default: @@ -58,7 +58,7 @@ afterAll(async () => { describe('get-repository', () => { describe('getRepository', () => { test('should insert values into db', async () => { - const { namespace, repository } = await getRepository({ externalRepositoryId: 1000 }, context); + const { namespace, repository } = await getRepository({ externalRepositoryId: 1000, namespaceName: '', repositoryName: '' }, context); expect(namespace).not.toBeNull(); expect(repository).toBeDefined(); diff --git a/packages/functions/extract/src/get-repository.ts b/packages/functions/extract/src/get-repository.ts index cd67b6e6c..135890e0e 100644 --- a/packages/functions/extract/src/get-repository.ts +++ b/packages/functions/extract/src/get-repository.ts @@ -4,6 +4,8 @@ import type { SourceControl } from "@acme/source-control"; export type GetRepositoryInput = { externalRepositoryId: number; + namespaceName: string; + repositoryName: string; }; export type GetRepositoryOutput = { @@ -17,11 +19,11 @@ export type GetRepositoryEntities = Pick; export const getRepository: GetRepositoryFunction = async ( - { externalRepositoryId }, + { externalRepositoryId, namespaceName, repositoryName }, { integrations, db, entities } ) => { - - const { repository, namespace } = await integrations.sourceControl.fetchRepository(externalRepositoryId); + + const { repository, namespace } = await integrations.sourceControl.fetchRepository(externalRepositoryId, namespaceName, repositoryName); await db.insert(entities.repositories).values(repository) .onConflictDoNothing({ target: entities.repositories.externalId }) diff --git a/packages/integrations/source-control/package.json b/packages/integrations/source-control/package.json index 031a22dc7..e74d91ec9 100644 --- a/packages/integrations/source-control/package.json +++ b/packages/integrations/source-control/package.json @@ -13,6 +13,9 @@ "dependencies": { "@acme/extract-schema": "^1.0.0", "@gitbeaker/core": "^39.8.0", - "@gitbeaker/rest": "^39.8.0" + "@gitbeaker/rest": "^39.8.0", + "@octokit/rest": "^20.0.1", + "@types/parse-link-header": "^2.0.1", + "parse-link-header": "^2.0.0" } } diff --git a/packages/integrations/source-control/src/github/index.ts b/packages/integrations/source-control/src/github/index.ts new file mode 100644 index 000000000..ecccc1255 --- /dev/null +++ b/packages/integrations/source-control/src/github/index.ts @@ -0,0 +1,66 @@ +import type { SourceControl } from '..'; +import { Octokit } from '@octokit/rest'; +import parseLinkHeader from "parse-link-header"; + +import type { NewRepository, NewNamespace, NewMergeRequest } from "@acme/extract-schema"; +import type { Pagination } from '../source-control'; + +export class GitHubSourceControl implements SourceControl { + + private api: Octokit; + + constructor(auth?: string | object) { + this.api = new Octokit({ + auth, // TODO: Need to look into https://github.com/octokit/authentication-strategies.js + }) + } + + async fetchRepository(externalRepositoryId: number, namespaceName: string, repositoryName: string): Promise<{ repository: NewRepository; namespace?: NewNamespace }> { + const result = await this.api.repos.get({ + owner: namespaceName, + repo: repositoryName + }); + + return { + repository: { + externalId: result.data.id, + name: result.data.name, + }, + namespace: { + externalId: result.data.owner.id, + name: result.data.owner.login + } + } + } + + async fetchMergeRequests(externalRepositoryId: number, namespaceName: string, repositoryName: string, page?: number, perPage?: number): Promise<{ mergeRequests: NewMergeRequest[]; pagination: Pagination; }> { + page = page || 1; + perPage = perPage || 30; + + const result = await this.api.pulls.list({ + owner: namespaceName, + repo: repositoryName, + page: page, + per_page: perPage, + state: "all" + }); + + const linkHeader = parseLinkHeader(result.headers.link) || { next: { per_page: perPage } }; + + const pagination = { + page, + perPage: ('next' in linkHeader) ? Number(linkHeader.next?.per_page) : Number(linkHeader.prev?.per_page), + totalPages: (!('last' in linkHeader)) ? page : Number(linkHeader.last?.page) + } satisfies Pagination; + + return { + mergeRequests: result.data.map(mergeRequest => ({ + externalId: mergeRequest.id, + mergeRequestId: mergeRequest.number, + repositoryId: externalRepositoryId, // todo: wait until drizzle fixes returning clause + })), + pagination + } + } + +} \ No newline at end of file diff --git a/packages/integrations/source-control/src/gitlab/index.ts b/packages/integrations/source-control/src/gitlab/index.ts index 29dbb7e00..1861ce142 100644 --- a/packages/integrations/source-control/src/gitlab/index.ts +++ b/packages/integrations/source-control/src/gitlab/index.ts @@ -13,13 +13,15 @@ export class GitlabSourceControl implements SourceControl { }); } - async fetchRepository(externalRepositoryId: number): Promise<{ repository: NewRepository, namespace?: NewNamespace }> { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async fetchRepository(externalRepositoryId: number, namespaceName: string, repositoryName: string): Promise<{ repository: NewRepository, namespace?: NewNamespace }> { const project = await this.api.Projects.show(externalRepositoryId); const namespace = project.namespace; return { repository: { externalId: project.id, + name: project.name } satisfies NewRepository, namespace: { externalId: namespace.id, @@ -29,7 +31,7 @@ export class GitlabSourceControl implements SourceControl { } // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await - async fetchMergeRequests(externalRepositoryId: number, page?: number, perPage?: number): Promise<{ mergeRequests: NewMergeRequest[], pagination: Pagination }> { + async fetchMergeRequests(externalRepositoryId: number, namespaceName: string, repositoryName: string, page?: number, perPage?: number): Promise<{ mergeRequests: NewMergeRequest[], pagination: Pagination }> { throw new Error('Method not implemented.'); } } diff --git a/packages/integrations/source-control/src/index.ts b/packages/integrations/source-control/src/index.ts index 03b8ee901..91095ccb0 100644 --- a/packages/integrations/source-control/src/index.ts +++ b/packages/integrations/source-control/src/index.ts @@ -1,3 +1,3 @@ export type { Pagination, SourceControl } from './source-control'; export { GitlabSourceControl } from './gitlab'; - +export { GitHubSourceControl } from './github'; diff --git a/packages/integrations/source-control/src/source-control.ts b/packages/integrations/source-control/src/source-control.ts index 17c27288d..63efac57b 100644 --- a/packages/integrations/source-control/src/source-control.ts +++ b/packages/integrations/source-control/src/source-control.ts @@ -7,6 +7,6 @@ export type Pagination = { }; export interface SourceControl { - fetchRepository(externalRepositoryId: number): Promise<{ repository: NewRepository, namespace?: NewNamespace }>; - fetchMergeRequests(externalRepositoryId: number, page?: number, perPage?: number): Promise<{ mergeRequests: NewMergeRequest[], pagination: Pagination }>; + fetchRepository(externalRepositoryId: number, namespaceName: string, repositoryName: string): Promise<{ repository: NewRepository, namespace?: NewNamespace }>; + fetchMergeRequests(externalRepositoryId: number, namespaceName: string, repositoryName: string, page?: number, perPage?: number): Promise<{ mergeRequests: NewMergeRequest[], pagination: Pagination }>; } diff --git a/packages/schemas/extract/src/merge-requests.ts b/packages/schemas/extract/src/merge-requests.ts index 56ad1b2aa..d63a3b368 100644 --- a/packages/schemas/extract/src/merge-requests.ts +++ b/packages/schemas/extract/src/merge-requests.ts @@ -3,8 +3,9 @@ import { sqliteTable, integer, uniqueIndex } from 'drizzle-orm/sqlite-core'; export const mergeRequests = sqliteTable('merge_requests', { id: integer('id').primaryKey(), + /* Gitlab -> id */ externalId: integer('external_id').notNull(), - /* Gitlab -> iid */ + /* Gitlab -> iid, GitHub -> number */ mergeRequestId: integer('merge_request_id').notNull(), repositoryId: integer('repository_id').notNull(), }, (mergeRequests) => ({ diff --git a/packages/schemas/extract/src/repositories.ts b/packages/schemas/extract/src/repositories.ts index 47f0f159e..dd0a55139 100644 --- a/packages/schemas/extract/src/repositories.ts +++ b/packages/schemas/extract/src/repositories.ts @@ -1,9 +1,10 @@ import type { InferModel } from 'drizzle-orm'; -import { sqliteTable, integer, uniqueIndex } from 'drizzle-orm/sqlite-core'; +import { sqliteTable, integer,text, uniqueIndex } from 'drizzle-orm/sqlite-core'; export const repositories = sqliteTable('repositories', { id: integer('id').primaryKey(), externalId: integer('external_id').notNull(), + name: text('name').notNull(), }, (projects) => ({ uniqueExternalId: uniqueIndex('repositories_external_id_idx').on(projects.externalId), }));