Added benchmarks scripts with small instruction (#8529)

* updated source-mongodb-v2 performance * updated code style * fixed remarks * fixed remarks * fixed remarks * updated strict encrypt source mongodb version * updated source mongodb work with empty collections * updated source mongodb timestamp cursor * updated mongodb source perfomance * fix code style * fix code style * updated tests and documentation * updated tests and documentation * updated tests and documentation * added vudangngoc changes * updated code style * updated code style * added benchmarks scripts with small instruction * fixed remarks * updated ci performance test sh script
airbytehq · Dec 12, 2021 · 7591883 · 7591883
1 parent 70d2f46
commit 7591883
Show file tree

Hide file tree

Showing 8 changed files with 426 additions and 1 deletion.
diff --git a/airbyte-integrations/connectors/source-mssql/README.md b/airbyte-integrations/connectors/source-mssql/README.md
@@ -0,0 +1,14 @@
+# MsSQL (SQL Server) Source
+
+## Performance Test
+
+### Use MsSQL script to populate the benchmark database
+
+In order to create a database with a certain number of tables, and a certain number of records in each of them, 
+you need to follow a few simple steps.
+
+1. Create a new database.
+2. Follow the TODOs in **mssql-script.sql** to change the number of tables, and the number of records of different sizes.
+3. Execute the script with your changes for the new database. 
+You can run the script use the MySQL command line client: - **mysql -h hostname -u user database < path/to/script/mssql-script.sql**
+After the script finishes its work, you will receive the number of tables specified in the script, with names starting with **test_0** and ending with **test_(the number of tables minus 1)**.
diff --git a/airbyte-integrations/connectors/source-mssql/mssql-script.sql b/airbyte-integrations/connectors/source-mssql/mssql-script.sql
@@ -0,0 +1,139 @@
+create procedure table_copy(@tablecount int)
+as
+begin
+set nocount on;
+
+DECLARE @v_max_table int;
+DECLARE @v_counter_table int;
+DECLARE @tnamee VARCHAR(255);
+set @v_max_table = @tablecount;
+set @v_counter_table = 1;
+
+while @v_counter_table < @v_max_table begin
+set @tnamee = concat('SELECT * INTO test_', @v_counter_table, ' FROM test;');
+EXEC (@tnamee);
+set @v_counter_table = @v_counter_table + 1;
+end;
+
+end;
+go --
+
+create procedure insert_rows( @allrows int, @insertcount int, @value NVARCHAR(max))
+as
+begin
+set nocount on;
+
+DECLARE @dummyIpsum varchar(255)
+DECLARE @fieldText NVARCHAR(max)
+set @fieldText = @value
+DECLARE @vmax int;
+DECLARE @vmaxx int;
+DECLARE @vmaxoneinsert int;
+DECLARE @counter int;
+DECLARE @lastinsertcounter int;
+DECLARE @lastinsert int;
+DECLARE @fullloop int;
+DECLARE @fullloopcounter int;
+set @dummyIpsum = '''dummy_ipsum'''
+set @vmax = @allrows;
+set @vmaxx = @allrows;
+set @vmaxoneinsert = @insertcount;
+set @counter = 1;
+set @lastinsertcounter = 1;
+set @lastinsert = 0;
+set @fullloop = 0;
+set @fullloopcounter = 0;
+
+while @vmaxx <= @vmaxoneinsert begin
+	set @vmaxoneinsert = @vmaxx;
+	set @fullloop = @fullloop + 1;
+	set @vmaxx = @vmaxx + 1;
+end;
+
+while @vmax > @vmaxoneinsert begin
+	set @fullloop = @fullloop + 1;
+	set @vmax = @vmax - @vmaxoneinsert;
+	set @lastinsert = @vmax;
+end;
+
+DECLARE @insertTable NVARCHAR(MAX)
+set @insertTable = CONVERT(NVARCHAR(max), 'insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values (');
+while @counter < @vmaxoneinsert begin
+	set @insertTable = CONVERT(NVARCHAR(max), concat(@insertTable, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP), ('));
+	set @counter = @counter + 1;
+end;
+set @insertTable = CONVERT(NVARCHAR(max), concat(@insertTable, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP);'));
+
+while @vmax < 1 begin
+	set @fullloop = 0
+	set @vmax = 1
+end;
+
+while @fullloopcounter < @fullloop begin
+	EXEC (@insertTable);
+	set @fullloopcounter = @fullloopcounter + 1;
+end;
+
+DECLARE @insertTableLasted NVARCHAR(max);
+set @insertTableLasted = CONVERT(NVARCHAR(max), 'insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values (');
+while @lastinsertcounter < @lastinsert begin
+	set @insertTableLasted = CONVERT(NVARCHAR(max), concat(@insertTableLasted, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP), ('));
+	set @lastinsertcounter = @lastinsertcounter + 1;
+end;
+
+set @insertTableLasted = CONVERT(NVARCHAR(max), concat(@insertTableLasted, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP);'));
+
+while @lastinsert > 0 begin
+	EXEC (@insertTableLasted);
+	set @lastinsert = 0;
+end;
+
+end;
+go --
+
+create procedure table_create(@val int)
+as
+begin
+set nocount on;
+
+-- SQLINES LICENSE FOR EVALUATION USE ONLY
+create table test
+(
+id int check (id > 0) not null identity primary key,
+varchar1 varchar(255),
+varchar2 varchar(255),
+varchar3 varchar(255),
+varchar4 varchar(255),
+varchar5 varchar(255),
+longblobfield nvarchar(max),
+timestampfield datetime2(0)
+);
+
+DECLARE @extraSmallText NVARCHAR(max);
+DECLARE @smallText NVARCHAR(max);
+DECLARE @regularText NVARCHAR(max);
+DECLARE @largeText NVARCHAR(max);
+set @extraSmallText = '''test weight 50b - 50b text, 50b text, 50b text'''
+set @smallText = CONCAT('''test weight 500b - ', REPLICATE('some text, some text, ', 20), '''')
+set @regularText = CONCAT('''test weight 10kb - ', REPLICATE('some text, some text, some text, some text, ', 295), 'some text''')
+set @largeText = CONCAT('''test weight 100kb - ', REPLICATE('some text, some text, some text, some text, ', 2225), 'some text''')
+-- TODO: change the following @allrows to control the number of records with different sizes
+-- number of 50B records
+EXEC insert_rows @allrows = 0, @insertcount = 998, @value = @extraSmallText
+-- number of 500B records
+EXEC insert_rows @allrows = 0, @insertcount = 998, @value = @smallText
+-- number of 10KB records
+EXEC insert_rows @allrows = 0, @insertcount = 998, @value = @regularText
+-- number of 100KB records
+EXEC insert_rows @allrows = 0, @insertcount = 98, @value = @largeText
+end;
+go --
+
+EXEC table_create @val = 0
+drop procedure if exists insert_rows;
+drop procedure if exists table_create;
+
+-- TODO: change the value to control the number of tables
+EXEC table_copy @tablecount = 1;
+drop procedure if exists table_copy;
+exec sp_rename 'test', 'test_0';
diff --git a/airbyte-integrations/connectors/source-mysql/README.md b/airbyte-integrations/connectors/source-mysql/README.md
@@ -36,4 +36,17 @@ To run acceptance and custom integration tests:
 To run performance tests:
 ```
 ./gradlew :airbyte-integrations:connectors:source-mysql:performanceTest
-```
+```
+
+### Use MySQL script to populate the benchmark database
+
+In order to create a database with a certain number of tables, and a certain number of records in each of them,
+you need to follow a few simple steps.
+
+1. Create a new database.
+2. Follow the TODOs in **mssql-script.sql** to change the number of tables, and the number of records of different sizes.
+3. Execute the script with your changes for the new database.
+   You can run the script use the MsSQL command line client: - **sqlcmd -S Serverinstance -E -i path/to/script/mssql-script.sql**
+   After the script finishes its work, you will receive the number of tables specified in the script, with names starting with **test_0** and ending with **test_(the number of tables minus 1)**.
+
+
diff --git a/airbyte-integrations/connectors/source-mysql/mysql-script.sql b/airbyte-integrations/connectors/source-mysql/mysql-script.sql
@@ -0,0 +1,134 @@
+delimiter #
+create procedure table_copy(in tablecount int)
+begin
+
+set @v_max_table = tablecount;
+set @v_counter_table = 1;
+
+while @v_counter_table < @v_max_table do
+set @tnamee = concat('create table IF NOT EXISTS test_', @v_counter_table, ' SELECT * FROM test;');
+PREPARE stmt from @tnamee;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
+set @v_counter_table = @v_counter_table + 1;
+end while;
+commit;
+
+end #
+
+delimiter ;
+
+delimiter #
+create procedure insert_rows(in allrows int, in insertcount int, in value longblob)
+begin
+
+set @dummyIpsum = '\'dummy_ipsum\'';
+set @fieldText = value;
+set @vmax = allrows;
+set @vmaxx = allrows;
+set @vmaxoneinsert = insertcount;
+set @counter = 1;
+set @lastinsertcounter = 1;
+set @lastinsert = 0;
+set @fullloop = 0;
+set @fullloopcounter = 0;
+
+while @vmaxx <= @vmaxoneinsert do
+	set @vmaxoneinsert = @vmaxx;
+	set @fullloop = @fullloop + 1;
+	set @vmaxx = @vmaxx + 1;
+end while;
+commit;
+
+while @vmax > @vmaxoneinsert do
+	set @fullloop = @fullloop + 1;
+	set @vmax = @vmax - @vmaxoneinsert;
+	set @lastinsert = @vmax;
+end while;
+commit;
+
+set @insertTable = concat('insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values (');
+while @counter < @vmaxoneinsert do
+	set @insertTable = concat(@insertTable, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP), (');
+	set @counter = @counter + 1;
+end while;
+commit;
+set @insertTable = concat(@insertTable, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP);');
+
+while @vmax < 1 do
+	set @fullloop = 0;
+	set @vmax = 1;
+end while;
+commit;
+
+while @fullloopcounter < @fullloop do
+	PREPARE runinsert from @insertTable;
+	EXECUTE runinsert;
+	DEALLOCATE PREPARE runinsert;
+	set @fullloopcounter = @fullloopcounter + 1;
+end while;
+commit;
+
+set @insertTableLasted = concat('insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values (');
+while @lastinsertcounter < @lastinsert do
+	set @insertTableLasted = concat(@insertTableLasted, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP), (');
+	set @lastinsertcounter = @lastinsertcounter + 1;
+end while;
+commit;
+set @insertTableLasted = concat(@insertTableLasted, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP);');
+
+while @lastinsert > 0 do
+	PREPARE runinsert from @insertTableLasted;
+	EXECUTE runinsert;
+	DEALLOCATE PREPARE runinsert;
+	set @lastinsert = 0;
+end while;
+commit;
+
+end #
+
+delimiter ;
+
+delimiter #
+create procedure table_create()
+begin
+
+create table test
+(
+id int unsigned not null auto_increment primary key,
+varchar1 varchar(255),
+varchar2 varchar(255),
+varchar3 varchar(255),
+varchar4 varchar(255),
+varchar5 varchar(255),
+longblobfield longblob,
+timestampfield timestamp
+)
+engine=innodb;
+
+set @extraSmallText = '\'test weight 50b - some text, some text, some text\'';
+set @smallText = CONCAT('\'test weight 500b - ', REPEAT('some text, some text, ', 20), '\'');
+set @regularText = CONCAT('\'test weight 10kb - ', REPEAT('some text, some text, ', 590), '\'');
+set @largeText = CONCAT('\'test weight 100kb - ', REPEAT('some text, some text, ', 4450), '\'');
+
+-- TODO: change the following @allrows to control the number of records with different sizes
+-- number of 50B records
+call insert_rows(0, 5000000, @extraSmallText);
+-- number of 500B records
+call insert_rows(0, 50000, @smallText);
+-- number of 10KB records
+call insert_rows(0, 5000, @regularText);
+-- number of 100KB records
+call insert_rows(0, 50, @largeText);
+end #
+
+delimiter ;
+
+call table_create();
+drop procedure if exists table_create;
+drop procedure if exists insert_rows;
+
+-- TODO: change the value to control the number of tables
+call table_copy(1);
+drop procedure if exists table_copy;
+ALTER TABLE test RENAME test_0;
diff --git a/airbyte-integrations/connectors/source-postgres/1-create-copy-tables-procedure.sql b/airbyte-integrations/connectors/source-postgres/1-create-copy-tables-procedure.sql
@@ -0,0 +1,14 @@
+create or replace procedure copy_table(tablecount int)
+language plpgsql
+as $$
+declare v_max_table int; v_counter_table int; v_tnamee VARCHAR(255);
+begin
+	v_max_table := tablecount;
+	v_counter_table := 1;
+    while v_counter_table < v_max_table loop
+      EXECUTE format('create table test_%s as (select * from test t)', v_counter_table);
+	  v_counter_table := v_counter_table + 1;
+   end loop;
+    commit;
+end;$$
+
diff --git a/airbyte-integrations/connectors/source-postgres/2-create-insert-rows-to-table-procedure.sql b/airbyte-integrations/connectors/source-postgres/2-create-insert-rows-to-table-procedure.sql
@@ -0,0 +1,68 @@
+create or replace procedure insert_rows(allrows int, insertcount int, value text)
+language plpgsql
+as $$
+declare dummyIpsum varchar(255); fieldText text; vmax int; vmaxx int; vmaxoneinsert int; counter int;
+declare lastinsertcounter int; lastinsert int; fullloop int; fullloopcounter int; insertTable text; insertTableLasted text;
+
+begin
+	fieldText := value;
+	dummyIpsum = '''dummy_ipsum''';
+	vmax = allrows;
+	vmaxx = allrows;
+	vmaxoneinsert = insertcount;
+	counter = 1;
+	lastinsertcounter = 1;
+	lastinsert = 0;
+	fullloop = 0;
+	fullloopcounter = 0;
+
+	while vmaxx <= vmaxoneinsert loop
+      vmaxoneinsert := vmaxx;
+	  fullloop := fullloop + 1;
+	  vmaxx := vmaxx + 1;
+   	end loop;
+    commit;
+
+   	while vmax > vmaxoneinsert loop
+      fullloop := fullloop + 1;
+	  vmax := vmax - vmaxoneinsert;
+	 lastinsert := vmax;
+   	end loop;
+    commit;
+
+   insertTable := 'insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values (';
+	while counter < vmaxoneinsert loop
+      insertTable := concat(insertTable, dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', fieldText, ', CURRENT_TIMESTAMP), (');
+	  counter := counter + 1;
+   	end loop;
+    commit;
+	insertTable := concat(insertTable, dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', fieldText, ', CURRENT_TIMESTAMP);');
+
+	while vmax < 1 loop
+      fullloop := 0;
+	  vmax := 1;
+   	end loop;
+    commit;
+
+   while fullloopcounter < fullloop loop
+      EXECUTE insertTable;
+	  fullloopcounter := fullloopcounter + 1;
+   	end loop;
+    commit;
+
+   insertTableLasted := 'insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values (';
+	while lastinsertcounter < lastinsert loop
+      insertTableLasted := concat(insertTableLasted, dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', fieldText, ', CURRENT_TIMESTAMP), (');
+      lastinsertcounter := lastinsertcounter + 1;
+   	end loop;
+    commit;
+	insertTableLasted := concat(insertTableLasted, dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', fieldText, ', CURRENT_TIMESTAMP);');
+
+	while lastinsert > 0 loop
+      EXECUTE insertTableLasted;
+	  lastinsert := 0;
+   	end loop;
+    commit;
+end;$$
+
+