diff --git a/airbyte-integrations/connectors/source-mssql/README.md b/airbyte-integrations/connectors/source-mssql/README.md new file mode 100644 index 0000000000000..4b239ea23d197 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/README.md @@ -0,0 +1,14 @@ +# MsSQL (SQL Server) Source + +## Performance Test + +### Use MsSQL script to populate the benchmark database + +In order to create a database with a certain number of tables, and a certain number of records in each of them, +you need to follow a few simple steps. + +1. Create a new database. +2. Follow the TODOs in **mssql-script.sql** to change the number of tables, and the number of records of different sizes. +3. Execute the script with your changes for the new database. +You can run the script use the MySQL command line client: - **mysql -h hostname -u user database < path/to/script/mssql-script.sql** +After the script finishes its work, you will receive the number of tables specified in the script, with names starting with **test_0** and ending with **test_(the number of tables minus 1)**. diff --git a/airbyte-integrations/connectors/source-mssql/mssql-script.sql b/airbyte-integrations/connectors/source-mssql/mssql-script.sql new file mode 100644 index 0000000000000..270ce92df65f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/mssql-script.sql @@ -0,0 +1,139 @@ +create procedure table_copy(@tablecount int) +as +begin +set nocount on; + +DECLARE @v_max_table int; +DECLARE @v_counter_table int; +DECLARE @tnamee VARCHAR(255); +set @v_max_table = @tablecount; +set @v_counter_table = 1; + +while @v_counter_table < @v_max_table begin +set @tnamee = concat('SELECT * INTO test_', @v_counter_table, ' FROM test;'); +EXEC (@tnamee); +set @v_counter_table = @v_counter_table + 1; +end; + +end; +go -- + +create procedure insert_rows( @allrows int, @insertcount int, @value NVARCHAR(max)) +as +begin +set nocount on; + +DECLARE @dummyIpsum varchar(255) +DECLARE @fieldText NVARCHAR(max) +set @fieldText = @value +DECLARE @vmax int; +DECLARE @vmaxx int; +DECLARE @vmaxoneinsert int; +DECLARE @counter int; +DECLARE @lastinsertcounter int; +DECLARE @lastinsert int; +DECLARE @fullloop int; +DECLARE @fullloopcounter int; +set @dummyIpsum = '''dummy_ipsum''' +set @vmax = @allrows; +set @vmaxx = @allrows; +set @vmaxoneinsert = @insertcount; +set @counter = 1; +set @lastinsertcounter = 1; +set @lastinsert = 0; +set @fullloop = 0; +set @fullloopcounter = 0; + +while @vmaxx <= @vmaxoneinsert begin + set @vmaxoneinsert = @vmaxx; + set @fullloop = @fullloop + 1; + set @vmaxx = @vmaxx + 1; +end; + +while @vmax > @vmaxoneinsert begin + set @fullloop = @fullloop + 1; + set @vmax = @vmax - @vmaxoneinsert; + set @lastinsert = @vmax; +end; + +DECLARE @insertTable NVARCHAR(MAX) +set @insertTable = CONVERT(NVARCHAR(max), 'insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values ('); +while @counter < @vmaxoneinsert begin + set @insertTable = CONVERT(NVARCHAR(max), concat(@insertTable, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP), (')); + set @counter = @counter + 1; +end; +set @insertTable = CONVERT(NVARCHAR(max), concat(@insertTable, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP);')); + +while @vmax < 1 begin + set @fullloop = 0 + set @vmax = 1 +end; + +while @fullloopcounter < @fullloop begin + EXEC (@insertTable); + set @fullloopcounter = @fullloopcounter + 1; +end; + +DECLARE @insertTableLasted NVARCHAR(max); +set @insertTableLasted = CONVERT(NVARCHAR(max), 'insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values ('); +while @lastinsertcounter < @lastinsert begin + set @insertTableLasted = CONVERT(NVARCHAR(max), concat(@insertTableLasted, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP), (')); + set @lastinsertcounter = @lastinsertcounter + 1; +end; + +set @insertTableLasted = CONVERT(NVARCHAR(max), concat(@insertTableLasted, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP);')); + +while @lastinsert > 0 begin + EXEC (@insertTableLasted); + set @lastinsert = 0; +end; + +end; +go -- + +create procedure table_create(@val int) +as +begin +set nocount on; + +-- SQLINES LICENSE FOR EVALUATION USE ONLY +create table test +( +id int check (id > 0) not null identity primary key, +varchar1 varchar(255), +varchar2 varchar(255), +varchar3 varchar(255), +varchar4 varchar(255), +varchar5 varchar(255), +longblobfield nvarchar(max), +timestampfield datetime2(0) +); + +DECLARE @extraSmallText NVARCHAR(max); +DECLARE @smallText NVARCHAR(max); +DECLARE @regularText NVARCHAR(max); +DECLARE @largeText NVARCHAR(max); +set @extraSmallText = '''test weight 50b - 50b text, 50b text, 50b text''' +set @smallText = CONCAT('''test weight 500b - ', REPLICATE('some text, some text, ', 20), '''') +set @regularText = CONCAT('''test weight 10kb - ', REPLICATE('some text, some text, some text, some text, ', 295), 'some text''') +set @largeText = CONCAT('''test weight 100kb - ', REPLICATE('some text, some text, some text, some text, ', 2225), 'some text''') +-- TODO: change the following @allrows to control the number of records with different sizes +-- number of 50B records +EXEC insert_rows @allrows = 0, @insertcount = 998, @value = @extraSmallText +-- number of 500B records +EXEC insert_rows @allrows = 0, @insertcount = 998, @value = @smallText +-- number of 10KB records +EXEC insert_rows @allrows = 0, @insertcount = 998, @value = @regularText +-- number of 100KB records +EXEC insert_rows @allrows = 0, @insertcount = 98, @value = @largeText +end; +go -- + +EXEC table_create @val = 0 +drop procedure if exists insert_rows; +drop procedure if exists table_create; + +-- TODO: change the value to control the number of tables +EXEC table_copy @tablecount = 1; +drop procedure if exists table_copy; +exec sp_rename 'test', 'test_0'; diff --git a/airbyte-integrations/connectors/source-mysql/README.md b/airbyte-integrations/connectors/source-mysql/README.md index f533c02074736..1f15ee12662e9 100644 --- a/airbyte-integrations/connectors/source-mysql/README.md +++ b/airbyte-integrations/connectors/source-mysql/README.md @@ -36,4 +36,17 @@ To run acceptance and custom integration tests: To run performance tests: ``` ./gradlew :airbyte-integrations:connectors:source-mysql:performanceTest -``` \ No newline at end of file +``` + +### Use MySQL script to populate the benchmark database + +In order to create a database with a certain number of tables, and a certain number of records in each of them, +you need to follow a few simple steps. + +1. Create a new database. +2. Follow the TODOs in **mssql-script.sql** to change the number of tables, and the number of records of different sizes. +3. Execute the script with your changes for the new database. + You can run the script use the MsSQL command line client: - **sqlcmd -S Serverinstance -E -i path/to/script/mssql-script.sql** + After the script finishes its work, you will receive the number of tables specified in the script, with names starting with **test_0** and ending with **test_(the number of tables minus 1)**. + + diff --git a/airbyte-integrations/connectors/source-mysql/mysql-script.sql b/airbyte-integrations/connectors/source-mysql/mysql-script.sql new file mode 100644 index 0000000000000..f127c057b4c1c --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/mysql-script.sql @@ -0,0 +1,134 @@ +delimiter # +create procedure table_copy(in tablecount int) +begin + +set @v_max_table = tablecount; +set @v_counter_table = 1; + +while @v_counter_table < @v_max_table do +set @tnamee = concat('create table IF NOT EXISTS test_', @v_counter_table, ' SELECT * FROM test;'); +PREPARE stmt from @tnamee; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; +set @v_counter_table = @v_counter_table + 1; +end while; +commit; + +end # + +delimiter ; + +delimiter # +create procedure insert_rows(in allrows int, in insertcount int, in value longblob) +begin + +set @dummyIpsum = '\'dummy_ipsum\''; +set @fieldText = value; +set @vmax = allrows; +set @vmaxx = allrows; +set @vmaxoneinsert = insertcount; +set @counter = 1; +set @lastinsertcounter = 1; +set @lastinsert = 0; +set @fullloop = 0; +set @fullloopcounter = 0; + +while @vmaxx <= @vmaxoneinsert do + set @vmaxoneinsert = @vmaxx; + set @fullloop = @fullloop + 1; + set @vmaxx = @vmaxx + 1; +end while; +commit; + +while @vmax > @vmaxoneinsert do + set @fullloop = @fullloop + 1; + set @vmax = @vmax - @vmaxoneinsert; + set @lastinsert = @vmax; +end while; +commit; + +set @insertTable = concat('insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values ('); +while @counter < @vmaxoneinsert do + set @insertTable = concat(@insertTable, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP), ('); + set @counter = @counter + 1; +end while; +commit; +set @insertTable = concat(@insertTable, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP);'); + +while @vmax < 1 do + set @fullloop = 0; + set @vmax = 1; +end while; +commit; + +while @fullloopcounter < @fullloop do + PREPARE runinsert from @insertTable; + EXECUTE runinsert; + DEALLOCATE PREPARE runinsert; + set @fullloopcounter = @fullloopcounter + 1; +end while; +commit; + +set @insertTableLasted = concat('insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values ('); +while @lastinsertcounter < @lastinsert do + set @insertTableLasted = concat(@insertTableLasted, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP), ('); + set @lastinsertcounter = @lastinsertcounter + 1; +end while; +commit; +set @insertTableLasted = concat(@insertTableLasted, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP);'); + +while @lastinsert > 0 do + PREPARE runinsert from @insertTableLasted; + EXECUTE runinsert; + DEALLOCATE PREPARE runinsert; + set @lastinsert = 0; +end while; +commit; + +end # + +delimiter ; + +delimiter # +create procedure table_create() +begin + +create table test +( +id int unsigned not null auto_increment primary key, +varchar1 varchar(255), +varchar2 varchar(255), +varchar3 varchar(255), +varchar4 varchar(255), +varchar5 varchar(255), +longblobfield longblob, +timestampfield timestamp +) +engine=innodb; + +set @extraSmallText = '\'test weight 50b - some text, some text, some text\''; +set @smallText = CONCAT('\'test weight 500b - ', REPEAT('some text, some text, ', 20), '\''); +set @regularText = CONCAT('\'test weight 10kb - ', REPEAT('some text, some text, ', 590), '\''); +set @largeText = CONCAT('\'test weight 100kb - ', REPEAT('some text, some text, ', 4450), '\''); + +-- TODO: change the following @allrows to control the number of records with different sizes +-- number of 50B records +call insert_rows(0, 5000000, @extraSmallText); +-- number of 500B records +call insert_rows(0, 50000, @smallText); +-- number of 10KB records +call insert_rows(0, 5000, @regularText); +-- number of 100KB records +call insert_rows(0, 50, @largeText); +end # + +delimiter ; + +call table_create(); +drop procedure if exists table_create; +drop procedure if exists insert_rows; + +-- TODO: change the value to control the number of tables +call table_copy(1); +drop procedure if exists table_copy; +ALTER TABLE test RENAME test_0; diff --git a/airbyte-integrations/connectors/source-postgres/1-create-copy-tables-procedure.sql b/airbyte-integrations/connectors/source-postgres/1-create-copy-tables-procedure.sql new file mode 100644 index 0000000000000..3d339e5ff0fa2 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/1-create-copy-tables-procedure.sql @@ -0,0 +1,14 @@ +create or replace procedure copy_table(tablecount int) +language plpgsql +as $$ +declare v_max_table int; v_counter_table int; v_tnamee VARCHAR(255); +begin + v_max_table := tablecount; + v_counter_table := 1; + while v_counter_table < v_max_table loop + EXECUTE format('create table test_%s as (select * from test t)', v_counter_table); + v_counter_table := v_counter_table + 1; + end loop; + commit; +end;$$ + diff --git a/airbyte-integrations/connectors/source-postgres/2-create-insert-rows-to-table-procedure.sql b/airbyte-integrations/connectors/source-postgres/2-create-insert-rows-to-table-procedure.sql new file mode 100644 index 0000000000000..af00b332233df --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/2-create-insert-rows-to-table-procedure.sql @@ -0,0 +1,68 @@ +create or replace procedure insert_rows(allrows int, insertcount int, value text) +language plpgsql +as $$ +declare dummyIpsum varchar(255); fieldText text; vmax int; vmaxx int; vmaxoneinsert int; counter int; +declare lastinsertcounter int; lastinsert int; fullloop int; fullloopcounter int; insertTable text; insertTableLasted text; + +begin + fieldText := value; + dummyIpsum = '''dummy_ipsum'''; + vmax = allrows; + vmaxx = allrows; + vmaxoneinsert = insertcount; + counter = 1; + lastinsertcounter = 1; + lastinsert = 0; + fullloop = 0; + fullloopcounter = 0; + + while vmaxx <= vmaxoneinsert loop + vmaxoneinsert := vmaxx; + fullloop := fullloop + 1; + vmaxx := vmaxx + 1; + end loop; + commit; + + while vmax > vmaxoneinsert loop + fullloop := fullloop + 1; + vmax := vmax - vmaxoneinsert; + lastinsert := vmax; + end loop; + commit; + + insertTable := 'insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values ('; + while counter < vmaxoneinsert loop + insertTable := concat(insertTable, dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', fieldText, ', CURRENT_TIMESTAMP), ('); + counter := counter + 1; + end loop; + commit; + insertTable := concat(insertTable, dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', fieldText, ', CURRENT_TIMESTAMP);'); + + while vmax < 1 loop + fullloop := 0; + vmax := 1; + end loop; + commit; + + while fullloopcounter < fullloop loop + EXECUTE insertTable; + fullloopcounter := fullloopcounter + 1; + end loop; + commit; + + insertTableLasted := 'insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values ('; + while lastinsertcounter < lastinsert loop + insertTableLasted := concat(insertTableLasted, dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', fieldText, ', CURRENT_TIMESTAMP), ('); + lastinsertcounter := lastinsertcounter + 1; + end loop; + commit; + insertTableLasted := concat(insertTableLasted, dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', fieldText, ', CURRENT_TIMESTAMP);'); + + while lastinsert > 0 loop + EXECUTE insertTableLasted; + lastinsert := 0; + end loop; + commit; +end;$$ + + diff --git a/airbyte-integrations/connectors/source-postgres/3-run-script.sql b/airbyte-integrations/connectors/source-postgres/3-run-script.sql new file mode 100644 index 0000000000000..a9dfb20a533e7 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/3-run-script.sql @@ -0,0 +1,26 @@ +create sequence test_seq; + +create table test( +id int check (id > 0) not null default nextval ('test_seq') primary key, +varchar1 varchar(255), +varchar2 varchar(255), +varchar3 varchar(255), +varchar4 varchar(255), +varchar5 varchar(255), +longblobfield bytea, +timestampfield timestamp(0)); + +-- TODO: change the following @allrows to control the number of records with different sizes +-- number of 50B records +call insert_rows(0, 500000, '''test weight 50b - some text, some text, some text'''); +-- number of 500B records +call insert_rows(0, 50000, CONCAT('''test weight 500b - ', repeat('some text, some text, ', 20), 'some text''')); +-- number of 10KB records +call insert_rows(0, 5000, CONCAT('''test weight 10kb - ', repeat('some text, some text, some text, some text, ', 295), 'some text''')); +-- number of 100KB records +call insert_rows(0, 50, CONCAT('''test weight 100kb - ', repeat('some text, some text, ', 4450), 'some text''')); +-- TODO: change the value to control the number of tables +call copy_table(0); +ALTER TABLE test RENAME TO test_0; + + diff --git a/airbyte-integrations/connectors/source-postgres/README.md b/airbyte-integrations/connectors/source-postgres/README.md new file mode 100644 index 0000000000000..87d2d41f7f032 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/README.md @@ -0,0 +1,17 @@ +# Postgres Source + +## Performance Test + +### Use Postgres script to populate the benchmark database + +In order to create a database with a certain number of tables, and a certain number of records in each of them, +you need to follow a few simple steps. + +1. Create a new database. +2. On the new database, run script **1-create-copy-tables-procedure.sql** to create the table copy procedure. + You can run the script use the Postgres command line client: - **psql -h host -d userstoreis -U admin -p port -a -q -f /path/to/script/1-create-copy-tables-procedure.sql** +3. Run script **2-create-insert-rows-to-table-procedure.sql** to create a procedure for creating a table with the specified number of records. +4. Follow the TODOs in **3-run-script.sql** to change the number of tables, and the number of records of different sizes. +5. Execute the **3-run-script.sql** script with your changes for the new database. After the script finishes its work, you will receive the number of tables specified in the script, with names starting with **test_0** and ending with **test_(the number of tables minus 1)**. + +