Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

qe: use one db per test on cockroachdb #3597

Closed
wants to merge 13 commits into from
8 changes: 6 additions & 2 deletions .github/workflows/query-engine.yml
Expand Up @@ -10,6 +10,7 @@ on:
- 'LICENSE'
- 'CODEOWNERS'
- 'renovate.json'
- 'introspection-engine/**'
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is an interesting addition - avoiding running the QE tests on IE only changes I assume?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, not reflected but it seems like a small win.


jobs:
rust-vitess-tests:
Expand All @@ -35,7 +36,10 @@ jobs:
single_threaded: false
connector: "sqlserver"
version: "2022"

- name: "cockroach_22_1_0"
single_threaded: false
connector: "cockroachdb"
version: "22.1.0"
env:
LOG_LEVEL: "info"
LOG_QUERIES: "y"
Expand All @@ -49,7 +53,7 @@ jobs:
TEST_CONNECTOR: ${{ matrix.database.connector }}
TEST_CONNECTOR_VERSION: ${{ matrix.database.version }}

runs-on: buildjet-16vcpu-ubuntu-2004
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

Expand Down
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion docker-compose.yml
Expand Up @@ -3,11 +3,13 @@ services:
cockroach_22_1_0:
image: prismagraphql/cockroachdb-custom:22.1.0
restart: always
command: start-single-node --insecure
command: start-single-node --insecure --store=type=mem,size=3GiB
ports:
- "26257:26257"
networks:
- databases
tmpfs: /cockroach/cockroach-data


cockroach_21_2_0_patched:
image: prismagraphql/cockroachdb-custom:21.2.0-patched
Expand Down
2 changes: 2 additions & 0 deletions migration-engine/qe-setup/Cargo.toml
Expand Up @@ -16,3 +16,5 @@ mongodb = "2.3.0"
tempfile = "3.3.0"
url = "2"
quaint.workspace = true
once_cell = "1.17.0"
crossbeam-channel = "0.5.6"
75 changes: 75 additions & 0 deletions migration-engine/qe-setup/src/cockroachdb.rs
@@ -0,0 +1,75 @@
use migration_core::migration_connector::{ConnectorError, ConnectorResult};
use quaint::{prelude::*, single::Quaint};
use url::Url;

pub(crate) async fn cockroach_setup(url: String, prisma_schema: &str) -> ConnectorResult<()> {
let mut url = Url::parse(&url).map_err(ConnectorError::url_parse_error)?;
let quaint_url = quaint::connector::PostgresUrl::new(url.clone()).unwrap();
let db_name = quaint_url.dbname();
let conn = create_admin_conn(&mut url).await?;

let query = format!(
r#"
DROP DATABASE IF EXISTS "{db_name}";
CREATE DATABASE "{db_name}";
"#
);

conn.raw_cmd(&query).await.unwrap();
crate::diff_and_apply(prisma_schema).await;

drop_db_when_thread_exits(url, db_name);

Ok(())
}

async fn create_admin_conn(url: &mut Url) -> ConnectorResult<Quaint> {
url.set_path("/postgres");
Ok(Quaint::new(url.as_ref()).await.unwrap())
}

fn drop_db_when_thread_exits(admin_url: Url, db_name: &str) {
use crossbeam_channel::*;
use once_cell::sync::OnceCell;
use std::{cell::RefCell, thread};
use test_setup::runtime::run_with_thread_local_runtime as tok;

// === Dramatis Personæ ===

// DB_DROP_THREAD: A thread that drops databases.
static DB_DROP_THREAD: OnceCell<Sender<String>> = OnceCell::new();

let sender = DB_DROP_THREAD.get_or_init(|| {
let (sender, receiver) = unbounded::<String>();

thread::spawn(move || {
let mut admin_url = admin_url;
let conn = tok(create_admin_conn(&mut admin_url)).unwrap();
loop {
// Receive new databases to drop.
for msg in receiver.iter() {
tok(conn.raw_cmd(&msg)).unwrap();
}
}
});

sender
});

// NOTIFIER: a thread local that notifies DB_DROP_THREAD when dropped.
struct Notifier(String, Sender<String>);

impl Drop for Notifier {
fn drop(&mut self) {
self.1.send(std::mem::take(&mut self.0)).unwrap()
}
}

thread_local! {
static NOTIFIER: RefCell<Option<Notifier>> = RefCell::new(None);
}

NOTIFIER.with(move |cell| {
*cell.borrow_mut() = Some(Notifier(format!("DROP DATABASE \"{db_name}\""), sender.clone()));
});
}
11 changes: 8 additions & 3 deletions migration-engine/qe-setup/src/lib.rs
@@ -1,13 +1,16 @@
//! Query Engine test setup.

#![allow(clippy::await_holding_lock)]

mod cockroachdb;
mod mongodb;
mod mssql;
mod mysql;
mod postgres;

pub use migration_core::migration_connector::ConnectorError;

use self::{mongodb::*, mssql::*, mysql::*, postgres::*};
use self::{cockroachdb::*, mongodb::*, mssql::*, mysql::*, postgres::*};
use enumflags2::BitFlags;
use migration_core::{
json_rpc::types::*,
Expand Down Expand Up @@ -40,9 +43,10 @@ pub async fn setup(prisma_schema: &str, db_schemas: &[&str]) -> ConnectorResult<
let (source, url, _preview_features) = parse_configuration(prisma_schema)?;

match &source.active_provider {
provider if [POSTGRES.provider_name(), COCKROACH.provider_name()].contains(provider) => {
provider if [POSTGRES.provider_name()].contains(provider) => {
postgres_setup(url, prisma_schema, db_schemas).await?
}
provider if COCKROACH.is_provider(provider) => cockroach_setup(url, prisma_schema).await?,
provider if MSSQL.is_provider(provider) => mssql_setup(url, prisma_schema, db_schemas).await?,
provider if MYSQL.is_provider(provider) => {
mysql_reset(&url).await?;
Expand All @@ -66,7 +70,7 @@ pub async fn teardown(prisma_schema: &str, db_schemas: &[&str]) -> ConnectorResu
let (source, url, _) = parse_configuration(prisma_schema)?;

match &source.active_provider {
provider if [POSTGRES.provider_name(), COCKROACH.provider_name()].contains(provider) => {
provider if [POSTGRES.provider_name()].contains(provider) => {
postgres_teardown(&url, db_schemas).await?;
}

Expand All @@ -76,6 +80,7 @@ pub async fn teardown(prisma_schema: &str, db_schemas: &[&str]) -> ConnectorResu
MSSQL.provider_name(),
MYSQL.provider_name(),
MONGODB.provider_name(),
COCKROACH.provider_name(),
]
.contains(provider) => {}

Expand Down
Expand Up @@ -30,6 +30,7 @@ mod metrics {
Sqlite => assert_eq!(total_queries, 9),
SqlServer(_) => assert_eq!(total_queries, 15),
MongoDb(_) => assert_eq!(total_queries, 5),
ConnectorVersion::CockroachDb => assert_eq!(total_queries, 10),
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I haven't dug to the bottom of this, but it would make sense for it to be from DROP DATABASE + CREATE DATABASE on test initialization becoming 1 query (instead of 2).

_ => assert_eq!(total_queries, 11),
}
assert_eq!(total_operations, 2);
Expand Down