feat(webserver, db): refactor gitlab and github integrations into unified integration / third party repository services

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "crates/llama-cpp-bindings/llama.cpp"]
+[submodule "crates/llama-cpp-server/llama.cpp"]
 	path = crates/llama-cpp-server/llama.cpp
 	url = https://github.com/ggerganov/llama.cpp

ee/tabby-db-macros/src/lib.rs

@@ -109,7 +109,7 @@ pub fn query_paged_as(input: TokenStream) -> TokenStream {
     let backwards = input.backwards;
     quote! {
         sqlx::query_as(&crate::make_pagination_query_with_condition({
-            let _ = sqlx::query_as!(#typ, "SELECT " + #columns + " FROM " + #table_name);
+            let _ = sqlx::query_as!(#typ, "SELECT " + #columns + " FROM (SELECT * FROM " + #table_name + ")");
             &#table_name
         }, &[ #(#column_args),* ], #limit, #skip_id, #backwards, #condition))
     }

ee/tabby-db/migrations/0029_merged-provider-tables.up.sql

@@ -2,7 +2,7 @@ CREATE TABLE integration_access_tokens(
     id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
     kind TEXT NOT NULL,
     display_name TEXT NOT NULL,
-    access_token TEXT,
+    access_token TEXT NOT NULL,
     error TEXT,
     created_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now')),
     updated_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now'))
@@ -20,3 +20,9 @@ CREATE TABLE provided_repositories(
     FOREIGN KEY (integration_access_token_id) REFERENCES integration_access_tokens(id) ON DELETE CASCADE,
     CONSTRAINT idx_unique_provider_id_vendor_id UNIQUE (integration_access_token_id, vendor_id)
 );
+
+INSERT INTO integration_access_tokens(kind, display_name, access_token)
+    SELECT 'github', display_name, access_token FROM github_repository_provider;
+
+INSERT INTO integration_access_tokens(kind, display_name, access_token)
+    SELECT 'gitlab', display_name, access_token FROM gitlab_repository_provider;

ee/tabby-db/schema/schema.sql

@@ -176,7 +176,7 @@ CREATE TABLE integration_access_tokens(
   id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
   kind TEXT NOT NULL,
   display_name TEXT NOT NULL,
-  access_token TEXT,
+  access_token TEXT NOT NULL,
   error TEXT,
   created_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')),
   updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now'))

ee/tabby-db/src/integration_access_tokens.rs

@@ -10,17 +10,17 @@
     pub kind: String,
     pub error: Option<String>,
     pub display_name: String,
-    pub access_token: Option<String>,
+    pub access_token: String,
     pub created_at: DateTimeUtc,
     pub updated_at: DateTimeUtc,
 }
 
 impl DbConn {
     pub async fn create_integration_access_token(
         &self,
-        kind: &str,
-        name: &str,
-        access_token: &str,
+        kind: String,
+        name: String,
+        access_token: String,
     ) -> Result<i64> {
         let res = query!(
             "INSERT INTO integration_access_tokens(kind, display_name, access_token) VALUES (?, ?, ?);",
@@ -69,7 +69,7 @@
         access_token: Option<String>,
     ) -> Result<()> {
         let access_token = match access_token {
-            Some(access_token) => Some(access_token),
+            Some(access_token) => access_token,
             None => self.get_integration_access_token(id).await?.access_token,
         };
 
@@ -126,7 +126,7 @@
         });
         conditions.extend(id_condition);
 
-        let kind_condition = kind.map(|kind| format!("kind = {kind}"));
+        let kind_condition = kind.map(|kind| format!("kind = '{kind}'"));
         conditions.extend(kind_condition);
 
         let condition = (!conditions.is_empty()).then(|| conditions.join(" AND "));

ee/tabby-db/src/lib.rs

@@ -12,9 +12,11 @@ use chrono::{DateTime, Duration, NaiveDateTime, Utc};
 pub use email_setting::EmailSettingDAO;
 pub use github_repository_provider::{GithubProvidedRepositoryDAO, GithubRepositoryProviderDAO};
 pub use gitlab_repository_provider::{GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO};
+pub use integration_access_tokens::IntegrationAccessTokenDAO;
 pub use invitations::InvitationDAO;
 pub use job_runs::JobRunDAO;
 pub use oauth_credential::OAuthCredentialDAO;
+pub use provided_repositories::ProvidedRepositoryDAO;
 pub use repositories::RepositoryDAO;
 pub use server_setting::ServerSettingDAO;
 use sqlx::{

ee/tabby-db/src/provided_repositories.rs

@@ -61,7 +61,7 @@
 
     pub async fn list_provided_repositories(
         &self,
-        provider_ids: Vec<i64>,
+        integration_ids: Vec<i64>,
         kind: Option<String>,
         active: Option<bool>,
         limit: Option<usize>,
@@ -70,35 +70,37 @@
     ) -> Result<Vec<ProvidedRepositoryDAO>> {
         let mut conditions = vec![];
 
-        let provider_ids = provider_ids
+        let integration_ids = integration_ids
             .into_iter()
             .map(|id| id.to_string())
             .collect::<Vec<_>>()
             .join(", ");
-        if !provider_ids.is_empty() {
-            conditions.push(format!("access_token_provider_id IN ({provider_ids})"));
+        if !integration_ids.is_empty() {
+            conditions.push(format!(
+                "integration_access_token_id IN ({integration_ids})"
+            ));
         }
 
         let active_filter = active.map(|active| format!("active = {active}"));
         conditions.extend(active_filter);
 
-        let kind_filter = kind.map(|kind| format!("kind = {kind}"));
+        let kind_filter = kind.map(|kind| format!("kind = '{kind}'"));
         conditions.extend(kind_filter);
 
         let condition = (!conditions.is_empty()).then(|| conditions.join(" AND "));
 
         let repos = query_paged_as!(
             ProvidedRepositoryDAO,
-            "provided_repositories",
+            "provided_repositories JOIN integration_access_tokens ON integration_access_token_id = integration_access_tokens.id",
             [
                 "id",
                 "vendor_id",
                 "name",
                 "git_url",
                 "active",
                 "integration_access_token_id",
-                "created_at" as "created_at: DateTimeUtc",
-                "updated_at" as "updated_at: DateTimeUtc"
+                "created_at",
+                "updated_at"
             ],
             limit,
             skip_id,
@@ -128,3 +130,16 @@
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::DbConn;
+
+    #[tokio::test]
+    async fn test_list_provided_repositories() {
+        let db = DbConn::new_in_memory().await.unwrap();
+        db.list_github_provided_repositories(vec![], None, None, None, false)
+            .await
+            .unwrap();
+    }
+}

ee/tabby-schema/graphql/schema.graphql

@@ -376,9 +376,9 @@ type Query {
   users(after: String, before: String, first: Int, last: Int): UserConnection!
   invitations(after: String, before: String, first: Int, last: Int): InvitationConnection!
   githubRepositoryProviders(ids: [ID!], after: String, before: String, first: Int, last: Int): GithubRepositoryProviderConnection!
-  githubRepositories(providerIds: [ID!]!, active: Boolean, after: String, before: String, first: Int, last: Int): GithubProvidedRepositoryConnection!
+  githubRepositories(providerIds: [ID!], active: Boolean, after: String, before: String, first: Int, last: Int): GithubProvidedRepositoryConnection!
   gitlabRepositoryProviders(ids: [ID!], after: String, before: String, first: Int, last: Int): GitlabRepositoryProviderConnection!
-  gitlabRepositories(providerIds: [ID!]!, active: Boolean, after: String, before: String, first: Int, last: Int): GitlabProvidedRepositoryConnection!
+  gitlabRepositories(providerIds: [ID!], active: Boolean, after: String, before: String, first: Int, last: Int): GitlabProvidedRepositoryConnection!
   jobRuns(ids: [ID!], jobs: [String!], after: String, before: String, first: Int, last: Int): JobRunConnection!
   jobRunStats(jobs: [String!]): JobStats!
   emailSetting: EmailSetting

ee/tabby-schema/src/dao.rs

@@ -3,21 +3,26 @@
 use lazy_static::lazy_static;
 use tabby_db::{
     EmailSettingDAO, GithubProvidedRepositoryDAO, GithubRepositoryProviderDAO,
-    GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO, InvitationDAO, JobRunDAO,
-    OAuthCredentialDAO, RepositoryDAO, ServerSettingDAO, UserDAO, UserEventDAO,
+    GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO, IntegrationAccessTokenDAO,
+    InvitationDAO, JobRunDAO, OAuthCredentialDAO, ProvidedRepositoryDAO, RepositoryDAO,
+    ServerSettingDAO, UserDAO, UserEventDAO,
 };
 
-use crate::schema::{
-    auth::{self, OAuthCredential, OAuthProvider},
-    email::{AuthMethod, EmailSetting, Encryption},
-    job,
-    repository::{
-        GitRepository, GithubProvidedRepository, GithubRepositoryProvider,
-        GitlabProvidedRepository, GitlabRepositoryProvider, RepositoryProviderStatus,
+use crate::{
+    integration::{IntegrationAccessToken, IntegrationKind, IntegrationStatus},
+    repository::ProvidedRepository,
+    schema::{
+        auth::{self, OAuthCredential, OAuthProvider},
+        email::{AuthMethod, EmailSetting, Encryption},
+        job,
+        repository::{
+            GitRepository, GithubProvidedRepository, GithubRepositoryProvider,
+            GitlabProvidedRepository, GitlabRepositoryProvider, RepositoryProviderStatus,
+        },
+        setting::{NetworkSetting, SecuritySetting},
+        user_event::{EventKind, UserEvent},
+        CoreError,
     },
-    setting::{NetworkSetting, SecuritySetting},
-    user_event::{EventKind, UserEvent},
-    CoreError,
 };
 
 impl From<InvitationDAO> for auth::Invitation {
@@ -125,6 +130,44 @@
     }
 }
 
+impl TryFrom<ProvidedRepositoryDAO> for ProvidedRepository {
+    type Error = anyhow::Error;
+    fn try_from(value: ProvidedRepositoryDAO) -> Result<Self, Self::Error> {
+        Ok(Self {
+            id: value.id.as_id(),
+            integration_id: value.integration_access_token_id.as_id(),
+            active: value.active,
+            display_name: value.name,
+            git_url: value.git_url,
+            vendor_id: value.vendor_id,
+            created_at: *value.created_at,
+            updated_at: *value.updated_at,
+        })
+    }
+}
+
+impl TryFrom<IntegrationAccessTokenDAO> for IntegrationAccessToken {
+    type Error = anyhow::Error;
+    fn try_from(value: IntegrationAccessTokenDAO) -> anyhow::Result<Self> {
+        let status = if *value.created_at == *value.updated_at {
+            IntegrationStatus::Pending
+        } else if value.error.is_some() {
+            IntegrationStatus::Failed
+        } else {
+            IntegrationStatus::Ready
+        };
+        Ok(Self {
+            id: value.id.as_id(),
+            kind: IntegrationKind::from_enum_str(&value.kind)?,
+            display_name: value.display_name,
+            access_token: value.access_token,
+            created_at: *value.created_at,
+            updated_at: *value.updated_at,
+            status,
+        })
+    }
+}
+
 impl From<GithubRepositoryProviderDAO> for GithubRepositoryProvider {
     fn from(value: GithubRepositoryProviderDAO) -> Self {
         Self {
@@ -139,6 +182,64 @@
     }
 }
 
+impl From<ProvidedRepository> for GithubProvidedRepository {
+    fn from(value: ProvidedRepository) -> Self {
+        Self {
+            id: value.id,
+            vendor_id: value.vendor_id,
+            github_repository_provider_id: value.integration_id,
+            name: value.display_name,
+            git_url: value.git_url,
+            active: value.active,
+        }
+    }
+}
+
+impl From<IntegrationAccessToken> for GithubRepositoryProvider {
+    fn from(value: IntegrationAccessToken) -> Self {
+        Self {
+            id: value.id,
+            display_name: value.display_name,
+            status: value.status.into(),
+            access_token: Some(value.access_token),
+        }
+    }
+}
+
+impl From<IntegrationAccessToken> for GitlabRepositoryProvider {
+    fn from(value: IntegrationAccessToken) -> Self {
+        Self {
+            id: value.id,
+            display_name: value.display_name,
+            status: value.status.into(),
+            access_token: Some(value.access_token),
+        }
+    }
+}
+
+impl From<IntegrationStatus> for RepositoryProviderStatus {
+    fn from(value: IntegrationStatus) -> Self {
+        match value {
+            IntegrationStatus::Ready => Self::Ready,
+            IntegrationStatus::Pending => Self::Pending,
+            IntegrationStatus::Failed => Self::Failed,
+        }
+    }
+}
+
+impl From<ProvidedRepository> for GitlabProvidedRepository {
+    fn from(value: ProvidedRepository) -> Self {
+        Self {
+            id: value.id,
+            vendor_id: value.vendor_id,
+            gitlab_repository_provider_id: value.integration_id,
+            name: value.display_name,
+            git_url: value.git_url,
+            active: value.active,
+        }
+    }
+}
+
 impl From<GithubProvidedRepositoryDAO> for GithubProvidedRepository {
     fn from(value: GithubProvidedRepositoryDAO) -> Self {
         Self {
@@ -255,6 +356,23 @@
     }
 }
 
+impl DbEnum for IntegrationKind {
+    fn as_enum_str(&self) -> &'static str {
+        match self {
+            IntegrationKind::Github => "github",
+            IntegrationKind::Gitlab => "gitlab",
+        }
+    }
+
+    fn from_enum_str(s: &str) -> anyhow::Result<Self> {
+        match s {
+            "github" => Ok(IntegrationKind::Github),
+            "gitlab" => Ok(IntegrationKind::Gitlab),
+            _ => bail!("{s} is not a valid value for ProviderKind"),
+        }
+    }
+}
+
 impl DbEnum for Encryption {
     fn as_enum_str(&self) -> &'static str {
         match self {