Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(webserver, db): refactor gitlab and github integrations into unified integration / third party repository services #2088

Merged
merged 24 commits into from
May 17, 2024
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitmodules
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
[submodule "crates/llama-cpp-bindings/llama.cpp"]
[submodule "crates/llama-cpp-server/llama.cpp"]
path = crates/llama-cpp-server/llama.cpp
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems to be a bug introduced by a previous PR? Tests were failing until making this change

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did rebase against main

url = https://github.com/ggerganov/llama.cpp
2 changes: 1 addition & 1 deletion ee/tabby-db-macros/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ pub fn query_paged_as(input: TokenStream) -> TokenStream {
let backwards = input.backwards;
quote! {
sqlx::query_as(&crate::make_pagination_query_with_condition({
let _ = sqlx::query_as!(#typ, "SELECT " + #columns + " FROM " + #table_name);
let _ = sqlx::query_as!(#typ, "SELECT " + #columns + " FROM (SELECT * FROM " + #table_name + ")");
&#table_name
}, &[ #(#column_args),* ], #limit, #skip_id, #backwards, #condition))
}
Expand Down
8 changes: 7 additions & 1 deletion ee/tabby-db/migrations/0029_merged-provider-tables.up.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ CREATE TABLE integration_access_tokens(
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
wsxiaoys marked this conversation as resolved.
Show resolved Hide resolved
kind TEXT NOT NULL,
display_name TEXT NOT NULL,
access_token TEXT,
access_token TEXT NOT NULL,
error TEXT,
created_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now')),
updated_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now'))
Expand All @@ -20,3 +20,9 @@ CREATE TABLE provided_repositories(
FOREIGN KEY (integration_access_token_id) REFERENCES integration_access_tokens(id) ON DELETE CASCADE,
CONSTRAINT idx_unique_provider_id_vendor_id UNIQUE (integration_access_token_id, vendor_id)
);

INSERT INTO integration_access_tokens(kind, display_name, access_token)
SELECT 'github', display_name, access_token FROM github_repository_provider;
boxbeam marked this conversation as resolved.
Show resolved Hide resolved

INSERT INTO integration_access_tokens(kind, display_name, access_token)
SELECT 'gitlab', display_name, access_token FROM gitlab_repository_provider;
Binary file modified ee/tabby-db/schema.sqlite
Binary file not shown.
2 changes: 1 addition & 1 deletion ee/tabby-db/schema/schema.sql
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ CREATE TABLE integration_access_tokens(
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL,
display_name TEXT NOT NULL,
access_token TEXT,
access_token TEXT NOT NULL,
error TEXT,
created_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')),
updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now'))
Expand Down
12 changes: 6 additions & 6 deletions ee/tabby-db/src/integration_access_tokens.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,17 @@
pub kind: String,
pub error: Option<String>,
pub display_name: String,
pub access_token: Option<String>,
pub access_token: String,
pub created_at: DateTimeUtc,
pub updated_at: DateTimeUtc,
}

impl DbConn {
pub async fn create_integration_access_token(
&self,
kind: &str,
name: &str,
access_token: &str,
kind: String,
name: String,
access_token: String,

Check warning on line 23 in ee/tabby-db/src/integration_access_tokens.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-db/src/integration_access_tokens.rs#L21-L23

Added lines #L21 - L23 were not covered by tests
) -> Result<i64> {
let res = query!(
"INSERT INTO integration_access_tokens(kind, display_name, access_token) VALUES (?, ?, ?);",
Expand Down Expand Up @@ -69,7 +69,7 @@
access_token: Option<String>,
) -> Result<()> {
let access_token = match access_token {
Some(access_token) => Some(access_token),
Some(access_token) => access_token,

Check warning on line 72 in ee/tabby-db/src/integration_access_tokens.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-db/src/integration_access_tokens.rs#L72

Added line #L72 was not covered by tests
None => self.get_integration_access_token(id).await?.access_token,
};

Expand Down Expand Up @@ -126,7 +126,7 @@
});
conditions.extend(id_condition);

let kind_condition = kind.map(|kind| format!("kind = {kind}"));
let kind_condition = kind.map(|kind| format!("kind = '{kind}'"));
conditions.extend(kind_condition);

let condition = (!conditions.is_empty()).then(|| conditions.join(" AND "));
Expand Down
2 changes: 2 additions & 0 deletions ee/tabby-db/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@ use chrono::{DateTime, Duration, NaiveDateTime, Utc};
pub use email_setting::EmailSettingDAO;
pub use github_repository_provider::{GithubProvidedRepositoryDAO, GithubRepositoryProviderDAO};
pub use gitlab_repository_provider::{GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO};
pub use integration_access_tokens::IntegrationAccessTokenDAO;
pub use invitations::InvitationDAO;
pub use job_runs::JobRunDAO;
pub use oauth_credential::OAuthCredentialDAO;
pub use provided_repositories::ProvidedRepositoryDAO;
pub use repositories::RepositoryDAO;
pub use server_setting::ServerSettingDAO;
use sqlx::{
Expand Down
31 changes: 23 additions & 8 deletions ee/tabby-db/src/provided_repositories.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@

pub async fn list_provided_repositories(
&self,
provider_ids: Vec<i64>,
integration_ids: Vec<i64>,

Check warning on line 64 in ee/tabby-db/src/provided_repositories.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-db/src/provided_repositories.rs#L64

Added line #L64 was not covered by tests
kind: Option<String>,
active: Option<bool>,
limit: Option<usize>,
Expand All @@ -70,35 +70,37 @@
) -> Result<Vec<ProvidedRepositoryDAO>> {
let mut conditions = vec![];

let provider_ids = provider_ids
let integration_ids = integration_ids

Check warning on line 73 in ee/tabby-db/src/provided_repositories.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-db/src/provided_repositories.rs#L73

Added line #L73 was not covered by tests
.into_iter()
.map(|id| id.to_string())
.collect::<Vec<_>>()
.join(", ");
if !provider_ids.is_empty() {
conditions.push(format!("access_token_provider_id IN ({provider_ids})"));
if !integration_ids.is_empty() {
conditions.push(format!(
"integration_access_token_id IN ({integration_ids})"
));

Check warning on line 81 in ee/tabby-db/src/provided_repositories.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-db/src/provided_repositories.rs#L78-L81

Added lines #L78 - L81 were not covered by tests
}

let active_filter = active.map(|active| format!("active = {active}"));
conditions.extend(active_filter);

let kind_filter = kind.map(|kind| format!("kind = {kind}"));
let kind_filter = kind.map(|kind| format!("kind = '{kind}'"));

Check warning on line 87 in ee/tabby-db/src/provided_repositories.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-db/src/provided_repositories.rs#L87

Added line #L87 was not covered by tests
conditions.extend(kind_filter);

let condition = (!conditions.is_empty()).then(|| conditions.join(" AND "));

let repos = query_paged_as!(
ProvidedRepositoryDAO,
"provided_repositories",
"provided_repositories JOIN integration_access_tokens ON integration_access_token_id = integration_access_tokens.id",
[
"id",
"vendor_id",
"name",
"git_url",
"active",
"integration_access_token_id",
"created_at" as "created_at: DateTimeUtc",
"updated_at" as "updated_at: DateTimeUtc"
"created_at",
"updated_at"
],
limit,
skip_id,
Expand Down Expand Up @@ -128,3 +130,16 @@
Ok(())
}
}

#[cfg(test)]
mod tests {
use crate::DbConn;

#[tokio::test]
async fn test_list_provided_repositories() {
let db = DbConn::new_in_memory().await.unwrap();
db.list_github_provided_repositories(vec![], None, None, None, false)
.await
.unwrap();
}
}
4 changes: 2 additions & 2 deletions ee/tabby-schema/graphql/schema.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -376,9 +376,9 @@ type Query {
users(after: String, before: String, first: Int, last: Int): UserConnection!
invitations(after: String, before: String, first: Int, last: Int): InvitationConnection!
githubRepositoryProviders(ids: [ID!], after: String, before: String, first: Int, last: Int): GithubRepositoryProviderConnection!
githubRepositories(providerIds: [ID!]!, active: Boolean, after: String, before: String, first: Int, last: Int): GithubProvidedRepositoryConnection!
githubRepositories(providerIds: [ID!], active: Boolean, after: String, before: String, first: Int, last: Int): GithubProvidedRepositoryConnection!
gitlabRepositoryProviders(ids: [ID!], after: String, before: String, first: Int, last: Int): GitlabRepositoryProviderConnection!
gitlabRepositories(providerIds: [ID!]!, active: Boolean, after: String, before: String, first: Int, last: Int): GitlabProvidedRepositoryConnection!
gitlabRepositories(providerIds: [ID!], active: Boolean, after: String, before: String, first: Int, last: Int): GitlabProvidedRepositoryConnection!
boxbeam marked this conversation as resolved.
Show resolved Hide resolved
jobRuns(ids: [ID!], jobs: [String!], after: String, before: String, first: Int, last: Int): JobRunConnection!
jobRunStats(jobs: [String!]): JobStats!
emailSetting: EmailSetting
Expand Down
142 changes: 130 additions & 12 deletions ee/tabby-schema/src/dao.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,26 @@
use lazy_static::lazy_static;
use tabby_db::{
EmailSettingDAO, GithubProvidedRepositoryDAO, GithubRepositoryProviderDAO,
GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO, InvitationDAO, JobRunDAO,
OAuthCredentialDAO, RepositoryDAO, ServerSettingDAO, UserDAO, UserEventDAO,
GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO, IntegrationAccessTokenDAO,
InvitationDAO, JobRunDAO, OAuthCredentialDAO, ProvidedRepositoryDAO, RepositoryDAO,
ServerSettingDAO, UserDAO, UserEventDAO,
};

use crate::schema::{
auth::{self, OAuthCredential, OAuthProvider},
email::{AuthMethod, EmailSetting, Encryption},
job,
repository::{
GitRepository, GithubProvidedRepository, GithubRepositoryProvider,
GitlabProvidedRepository, GitlabRepositoryProvider, RepositoryProviderStatus,
use crate::{
integration::{IntegrationAccessToken, IntegrationKind, IntegrationStatus},
repository::ProvidedRepository,
schema::{
auth::{self, OAuthCredential, OAuthProvider},
email::{AuthMethod, EmailSetting, Encryption},
job,
repository::{
GitRepository, GithubProvidedRepository, GithubRepositoryProvider,
GitlabProvidedRepository, GitlabRepositoryProvider, RepositoryProviderStatus,
},
setting::{NetworkSetting, SecuritySetting},
user_event::{EventKind, UserEvent},
CoreError,
},
setting::{NetworkSetting, SecuritySetting},
user_event::{EventKind, UserEvent},
CoreError,
};

impl From<InvitationDAO> for auth::Invitation {
Expand Down Expand Up @@ -125,6 +130,44 @@
}
}

impl TryFrom<ProvidedRepositoryDAO> for ProvidedRepository {
type Error = anyhow::Error;
fn try_from(value: ProvidedRepositoryDAO) -> Result<Self, Self::Error> {
Ok(Self {
id: value.id.as_id(),
integration_id: value.integration_access_token_id.as_id(),
active: value.active,
display_name: value.name,
git_url: value.git_url,
vendor_id: value.vendor_id,
created_at: *value.created_at,
updated_at: *value.updated_at,
})
}

Check warning on line 146 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L135-L146

Added lines #L135 - L146 were not covered by tests
}

impl TryFrom<IntegrationAccessTokenDAO> for IntegrationAccessToken {
type Error = anyhow::Error;
fn try_from(value: IntegrationAccessTokenDAO) -> anyhow::Result<Self> {
let status = if *value.created_at == *value.updated_at {
IntegrationStatus::Pending
} else if value.error.is_some() {
IntegrationStatus::Failed

Check warning on line 155 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L151-L155

Added lines #L151 - L155 were not covered by tests
boxbeam marked this conversation as resolved.
Show resolved Hide resolved
} else {
IntegrationStatus::Ready

Check warning on line 157 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L157

Added line #L157 was not covered by tests
};
Ok(Self {
id: value.id.as_id(),
kind: IntegrationKind::from_enum_str(&value.kind)?,
display_name: value.display_name,
access_token: value.access_token,
created_at: *value.created_at,
updated_at: *value.updated_at,
status,

Check warning on line 166 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L160-L166

Added lines #L160 - L166 were not covered by tests
})
}

Check warning on line 168 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L168

Added line #L168 was not covered by tests
}

impl From<GithubRepositoryProviderDAO> for GithubRepositoryProvider {
fn from(value: GithubRepositoryProviderDAO) -> Self {
Self {
Expand All @@ -139,6 +182,64 @@
}
}

impl From<ProvidedRepository> for GithubProvidedRepository {
fn from(value: ProvidedRepository) -> Self {
Self {
id: value.id,
vendor_id: value.vendor_id,
github_repository_provider_id: value.integration_id,
name: value.display_name,
git_url: value.git_url,
active: value.active,
}
}

Check warning on line 195 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L186-L195

Added lines #L186 - L195 were not covered by tests
}

impl From<IntegrationAccessToken> for GithubRepositoryProvider {
fn from(value: IntegrationAccessToken) -> Self {
Self {
id: value.id,
display_name: value.display_name,
status: value.status.into(),
access_token: Some(value.access_token),
}
}

Check warning on line 206 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L199-L206

Added lines #L199 - L206 were not covered by tests
}

impl From<IntegrationAccessToken> for GitlabRepositoryProvider {
fn from(value: IntegrationAccessToken) -> Self {
Self {
id: value.id,
display_name: value.display_name,
status: value.status.into(),
access_token: Some(value.access_token),
}
}

Check warning on line 217 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L210-L217

Added lines #L210 - L217 were not covered by tests
}

impl From<IntegrationStatus> for RepositoryProviderStatus {
fn from(value: IntegrationStatus) -> Self {
match value {
IntegrationStatus::Ready => Self::Ready,
IntegrationStatus::Pending => Self::Pending,
IntegrationStatus::Failed => Self::Failed,

Check warning on line 225 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L221-L225

Added lines #L221 - L225 were not covered by tests
}
}

Check warning on line 227 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L227

Added line #L227 was not covered by tests
}

impl From<ProvidedRepository> for GitlabProvidedRepository {
fn from(value: ProvidedRepository) -> Self {
Self {
id: value.id,
vendor_id: value.vendor_id,
gitlab_repository_provider_id: value.integration_id,
name: value.display_name,
git_url: value.git_url,
active: value.active,
}
}

Check warning on line 240 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L231-L240

Added lines #L231 - L240 were not covered by tests
}

impl From<GithubProvidedRepositoryDAO> for GithubProvidedRepository {
fn from(value: GithubProvidedRepositoryDAO) -> Self {
Self {
Expand Down Expand Up @@ -255,6 +356,23 @@
}
}

impl DbEnum for IntegrationKind {
fn as_enum_str(&self) -> &'static str {
match self {
IntegrationKind::Github => "github",
IntegrationKind::Gitlab => "gitlab",

Check warning on line 363 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L360-L363

Added lines #L360 - L363 were not covered by tests
}
}

Check warning on line 365 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L365

Added line #L365 was not covered by tests

fn from_enum_str(s: &str) -> anyhow::Result<Self> {
match s {
"github" => Ok(IntegrationKind::Github),
"gitlab" => Ok(IntegrationKind::Gitlab),
_ => bail!("{s} is not a valid value for ProviderKind"),

Check warning on line 371 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L367-L371

Added lines #L367 - L371 were not covered by tests
}
}

Check warning on line 373 in ee/tabby-schema/src/dao.rs

View check run for this annotation

Codecov / codecov/patch

ee/tabby-schema/src/dao.rs#L373

Added line #L373 was not covered by tests
}

impl DbEnum for Encryption {
fn as_enum_str(&self) -> &'static str {
match self {
Expand Down
Loading
Loading