From 6b33f5d5302b1df2efd7fe230db936756752e6ce Mon Sep 17 00:00:00 2001
From: bkellam <bshizzle1234@gmail.com>
Date: Wed, 20 Nov 2024 13:04:38 -0800
Subject: [PATCH 1/2] Add parameter  to the gitlab config schema to allow for
 indexing all projects visible to a given token (if any)

---
 packages/backend/src/gitlab.ts     | 17 ++++++++++++++++-
 packages/backend/src/schemas/v2.ts |  4 ++++
 schemas/v2/index.json              |  5 +++++
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts
index 5795c32f6..58793c751 100644
--- a/packages/backend/src/gitlab.ts
+++ b/packages/backend/src/gitlab.ts
@@ -7,6 +7,7 @@ import path from 'path';
 import micromatch from "micromatch";
 
 const logger = createLogger("GitLab");
+const GITLAB_CLOUD_HOSTNAME = "gitlab.com";
 
 export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppContext) => {
     const token = config.token ? getTokenFromConfig(config.token, ctx) : undefined;
@@ -18,9 +19,24 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
             host: config.url,
         } : {}),
     });
+    const hostname = config.url ? new URL(config.url).hostname : GITLAB_CLOUD_HOSTNAME;
+
 
     let allProjects: ProjectSchema[] = [];
 
+    if (config.all === true) {
+        if (hostname !== GITLAB_CLOUD_HOSTNAME) {
+            logger.debug(`Fetching all projects visible in ${config.url}...`);
+            const { durationMs, data: _projects } = await measure(() => api.Projects.all({
+                perPage: 100,
+            }));
+            logger.debug(`Found ${_projects.length} projects in ${durationMs}ms.`);
+            allProjects = allProjects.concat(_projects);
+        } else {
+            logger.warn(`Ignoring option all:true in ${ctx.configPath} : host is ${GITLAB_CLOUD_HOSTNAME}`);
+        }
+    }
+
     if (config.groups) {
         const _projects = (await Promise.all(config.groups.map(async (group) => {
             logger.debug(`Fetching project info for group ${group}...`);
@@ -62,7 +78,6 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
 
     let repos: GitRepository[] = allProjects
         .map((project) => {
-            const hostname = config.url ? new URL(config.url).hostname : "gitlab.com";
             const repoId = `${hostname}/${project.path_with_namespace}`;
             const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`))
             const isFork = project.forked_from_project !== undefined;
diff --git a/packages/backend/src/schemas/v2.ts b/packages/backend/src/schemas/v2.ts
index 644f564ef..2de8bf1bc 100644
--- a/packages/backend/src/schemas/v2.ts
+++ b/packages/backend/src/schemas/v2.ts
@@ -93,6 +93,10 @@ export interface GitLabConfig {
    * The URL of the GitLab host. Defaults to https://gitlab.com
    */
   url?: string;
+  /**
+   * Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com .
+   */
+  all?: boolean;
   /**
    * List of users to sync with. All projects owned by the user and visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property.
    */
diff --git a/schemas/v2/index.json b/schemas/v2/index.json
index 0f7f79c3b..4c2c3275c 100644
--- a/schemas/v2/index.json
+++ b/schemas/v2/index.json
@@ -189,6 +189,11 @@
                     ],
                     "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$"
                 },
+                "all": {
+                    "type": "boolean",
+                    "default": false,
+                    "description": "Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com ."
+                },
                 "users": {
                     "type": "array",
                     "items": {

From 67d565f5ee6b11ce3684aafff37b1768c8833bef Mon Sep 17 00:00:00 2001
From: bkellam <bshizzle1234@gmail.com>
Date: Wed, 20 Nov 2024 13:19:06 -0800
Subject: [PATCH 2/2] example & changelog

---
 CHANGELOG.md       | 1 +
 configs/basic.json | 9 +++++++++
 2 files changed, 10 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 413f15912..59478cf8f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Added `DOMAIN_SUB_PATH` environment variable to allow overriding the default domain subpath. ([#74](https://github.com/sourcebot-dev/sourcebot/pull/74))
+- Added option `all` to the GitLab index schema, allowing for indexing all projects in a self-hosted GitLab instance. ([#84](https://github.com/sourcebot-dev/sourcebot/pull/84)) 
 
 ## [2.4.3] - 2024-11-18
 
diff --git a/configs/basic.json b/configs/basic.json
index 78b8588e0..b57de3a48 100644
--- a/configs/basic.json
+++ b/configs/basic.json
@@ -59,6 +59,15 @@
         {
             "type": "local",
             "path": "/path/to/local/repo"
+        },
+        // Index all projects in a self-hosted GitLab instance
+        // that are visible to the provided token.
+        // Note: this does not work with gitlab.com
+        {
+            "type": "gitlab",
+            "url": "https://gitlab.example.com",
+            "token": "my-token",
+            "all": true
         }
     ]
 }
\ No newline at end of file