From 058cb8b32b9e246f603493414453f12e20597876 Mon Sep 17 00:00:00 2001 From: Teun van Schagen Date: Wed, 3 Jul 2019 11:33:17 +0200 Subject: [PATCH] docs(): cleaned up code documentation --- .../Connectors/SqlServerConnector.cs | 12 +-- .../Evaluators/MySqlClosureTableEvaluator .cs | 13 ++- .../Evaluators/MySqlEvaluator.cs | 51 +++++++++-- .../Generators/MySqlClosureTableGenerator.cs | 63 +++++--------- .../Generators/MySqlGenerator.cs | 61 ++++++------- .../Generators/Neo4jGenerator.cs | 87 ++++++++++++------- DummyDataGenerator/Program.cs | 2 +- DummyDataGenerator/mysql_generate_schema.sql | 4 +- .../mysql_generate_schema_ct.sql | 4 +- 9 files changed, 171 insertions(+), 126 deletions(-) diff --git a/DummyDataGenerator/Connectors/SqlServerConnector.cs b/DummyDataGenerator/Connectors/SqlServerConnector.cs index d7c2a4b..a5419be 100644 --- a/DummyDataGenerator/Connectors/SqlServerConnector.cs +++ b/DummyDataGenerator/Connectors/SqlServerConnector.cs @@ -30,16 +30,16 @@ private SqlServerConnector() if (Connection == null) { SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder(); - builder.DataSource = "10.0.10.136"; - builder.UserID = "SA"; - builder.Password = "SQLServer2019"; - builder.InitialCatalog = "master"; + builder.DataSource = Env.GetString("SQLSERVER_HOST"); + builder.UserID = Env.GetString("SQLSERVER_USER"); + builder.Password = Env.GetString("SQLSERVER_PW"); + builder.InitialCatalog = Env.GetString("SQLSERVER_DB"); try { _connection = new SqlConnection(builder.ConnectionString); _connection.Open(); - Logger.Info("Created connection to SQL Server @ " + "10.0.10.136"); + Logger.Info("Created connection to SQL Server @ " + Env.GetString("SQLSERVER_HOST")); Logger.Debug("State: " + _connection.State.ToString()); } catch (SqlException e) @@ -54,7 +54,7 @@ public void Close() try { _connection?.Close(); - Logger.Info("Closed connection to SQL Server @ " + "10.0.10.136"); + Logger.Info("Closed connection to SQL Server @ " + Env.GetString("SQLSERVER_HOST")); } catch (SqlException e) { diff --git a/DummyDataGenerator/Evaluators/MySqlClosureTableEvaluator .cs b/DummyDataGenerator/Evaluators/MySqlClosureTableEvaluator .cs index ee93846..040c7a3 100644 --- a/DummyDataGenerator/Evaluators/MySqlClosureTableEvaluator .cs +++ b/DummyDataGenerator/Evaluators/MySqlClosureTableEvaluator .cs @@ -12,17 +12,16 @@ namespace DummyDataGenerator.Evaluators { class MySqlClosureTableEvaluator : MySqlEvaluator { + // add the databases you want to evaluate here private void AddDatabases() { - databases.Add("scm_ct_breadth2_depth2"); - databases.Add("scm_ct_breadth2_depth4"); - databases.Add("scm_ct_breadth2_depth6"); - databases.Add("scm_ct_breadth2_depth8"); - databases.Add("scm_ct_breadth2_depth10"); - databases.Add("scm_ct_breadth2_depth12"); - databases.Add("scm_ct_breadth2_depth14"); + databases.Add("scm_ct_b2_d10"); } + /// + /// Reads all .sql files from the directory and adds them to the list + /// + #warning due to limitations (of query length) of the MySQL performance schema, q must be contained somewhere in the query in order to recognize it as this particular query private void AddQueries() { string path = @"../../../Queries/SQL/Closure Table/"; diff --git a/DummyDataGenerator/Evaluators/MySqlEvaluator.cs b/DummyDataGenerator/Evaluators/MySqlEvaluator.cs index e49879d..f0b7fec 100644 --- a/DummyDataGenerator/Evaluators/MySqlEvaluator.cs +++ b/DummyDataGenerator/Evaluators/MySqlEvaluator.cs @@ -22,6 +22,9 @@ class MySqlEvaluator protected List databases = new List(); protected List queries = new List(); + /// + /// Runs the performance evaluator for all added databases and queries, and outputs a file with the results + /// public void Execute() { AddDatabases(); @@ -43,17 +46,16 @@ public void Execute() WriteOutputToFile("mysql_eval.csv"); } + // add databases you want to evaluate here private void AddDatabases() { - databases.Add("v2_scm_al_breadth2_depth2"); - databases.Add("v2_scm_al_breadth2_depth4"); - databases.Add("v2_scm_al_breadth2_depth6"); - databases.Add("v2_scm_al_breadth2_depth8"); - databases.Add("v2_scm_al_breadth2_depth10"); - databases.Add("v2_scm_al_breadth2_depth12"); - databases.Add("v2_scm_al_breadth2_depth14"); + databases.Add("scm_al_b2_d10"); } + /// + /// Reads all .sql files from the directory and adds them to the list + /// + #warning due to limitations (of query length) of the MySQL performance schema, q must be contained somewhere in the query in order to recognize it as this particular query private void AddQueries() { string path = @"../../../Queries/SQL/Adjacency List/"; @@ -72,6 +74,10 @@ private void AddQueries() } } + /// + /// Returns the size of the specified database in megabytes + /// + /// The name of the database private void CaptureDatabaseSize(string database) { string statement = string.Format("USE {0}; SELECT sum(round(((data_length + index_length) / 1024 / 1024), 2)) as 'size_mb' FROM information_schema.TABLES WHERE table_schema = '{0}'", database); @@ -80,6 +86,10 @@ private void CaptureDatabaseSize(string database) output += database + " (" + size + " MB) "; } + /// + /// Enabled the performance schema on MySQL in order to capture the query response times + /// + /// The name of the database private void EnablePerformanceSchema(string database) { Logger.Debug("Enabling performance schema"); @@ -100,6 +110,13 @@ private void EnablePerformanceSchema(string database) command.ExecuteNonQuery(); } + /// + /// Executes a query on the database a specified number of times + /// + /// Number of the query + /// The actual SQL code to be executed + /// Number of times the query is executed + /// private void ExecuteQuery(int queryNumber, string query, int repetitions, string database) { Logger.Info("Executing query: " + queryNumber + " " + NUMBER_OF_REPETITIONS + " times on database: " + database); @@ -111,6 +128,13 @@ private void ExecuteQuery(int queryNumber, string query, int repetitions, string } } + /// + /// Gets the result from the enabled performance schema for the specified query, using 'q' and the number of the query (example 'q1') + /// + /// + /// + /// + #warning due to limitations (of query length) of the MySQL performance schema, q must be contained somewhere in the query in order to recognize it as this particular query private void RetrieveResult(int queryNumber, string query, int repetitions) { Logger.Info("Retrieving results for query: " + (queryNumber)); @@ -148,6 +172,8 @@ private void RetrieveResult(int queryNumber, string query, int repetitions) Logger.Info("Average execution time: " + totaExecutionTime / repetitions); + // gets individual response times + /*foreach (int i in eventIds) { string statement2 = string.Format("SELECT event_name AS Stage, TRUNCATE(TIMER_WAIT/1000000000000,6) AS Duration FROM performance_schema.events_stages_history_long WHERE NESTING_EVENT_ID = {0}", i); @@ -184,6 +210,10 @@ private void CloseConnection() connector.Close(); } + /// + /// Disables the performance schema for the specified database + /// + /// Name of the database private void DisablePerformanceSchema(string database) { Logger.Debug("Disabling performance schema"); @@ -197,6 +227,9 @@ private void DisablePerformanceSchema(string database) command.ExecuteNonQuery(); } + /// + /// Adds headers to the output file based on the no. of repetitions + /// private void AddHeadersToOutput() { Logger.Debug("Adding headers to file.."); @@ -208,6 +241,10 @@ private void AddHeadersToOutput() output += "\n"; } + /// + /// Writes the results to a file in csv format + /// + /// private void WriteOutputToFile(string fileName) { Logger.Debug("Writing output to file.."); diff --git a/DummyDataGenerator/Generators/MySqlClosureTableGenerator.cs b/DummyDataGenerator/Generators/MySqlClosureTableGenerator.cs index 4168730..ccdecc4 100644 --- a/DummyDataGenerator/Generators/MySqlClosureTableGenerator.cs +++ b/DummyDataGenerator/Generators/MySqlClosureTableGenerator.cs @@ -3,8 +3,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Threading; namespace DummyDataGenerator.Generators { @@ -29,6 +27,9 @@ public override void GenerateData(Configuration config, bool allowMultipleThread Console.WriteLine("Program done, press enter to continue"); } + /// + /// Drops and creates the database, as to start off with a clean sheet + /// private new void RefreshDatabaseSchema() { var watch = System.Diagnostics.Stopwatch.StartNew(); @@ -55,33 +56,14 @@ private new void RefreshDatabaseSchema() com.ExecuteNonQuery(); } - struct ProductTreeData - { - public ProductTreeData(int org, int productsPerSupplier, int depth, int breadthPerLevel, int orgIter, int prodIter) - { - this.org = org; - this.productsPerSupplier = productsPerSupplier; - this.depth = depth; - this.breadthPerLevel = breadthPerLevel; - this.orgIter = orgIter; - this.prodIter = prodIter; - } - - public int org { get; } - public int productsPerSupplier { get; } - public int depth { get; } - public int breadthPerLevel { get; } - public int orgIter; - public int prodIter; - } - /// - /// + /// Generates all of the product trees necessary (amount of top level suppliers * amount of products) + /// starts with the top level products, and then proceeds with all branches in the tree /// - /// - /// - /// - /// + /// The id's of the top level organizations + /// The number of products each supplier should get + /// The depth of each chain (in how many branches does each product split up) + /// The breadth of each chain per level; the number of products to generate for the product above private void GenerateProductTrees(int[] organizationIdentifiers, int productsPerSupplier, int depth, int breadthPerLevel) { var watchTotal = System.Diagnostics.Stopwatch.StartNew(); @@ -148,7 +130,6 @@ private void GenerateProductTrees(int[] organizationIdentifiers, int productsPer Console.WriteLine("Took " + watchTotal.ElapsedMilliseconds + "ms " + "(" + (watchTotal.ElapsedMilliseconds / 1000) + "s)" + "(" + (watchTotal.ElapsedMilliseconds / 1000 / 60) + "m) in total"); } - /// /// Adds a single row to a single product hierarchy /// @@ -190,7 +171,18 @@ private List GenerateProductRowAndRelations(List allParentProductIdent // first, retrieve the parent products and their path lengths List ids = new List(); List pathlengths = new List(); - string stmt = string.Format("WITH RECURSIVE cte ( ppid, cpid, pl ) AS ( SELECT parent.id, child.id, path_length FROM consists_of JOIN product AS parent ON consists_of.parent_product_id = parent.id JOIN product AS child ON consists_of.child_product_id = child.id WHERE child.id = {0} UNION ALL SELECT parent.id, child.id, path_length FROM consists_of JOIN product AS parent ON consists_of.parent_product_id = parent.id JOIN product AS child ON consists_of.child_product_id = child.id JOIN cte ON cte.ppid = child.id WHERE consists_of.child_product_id <> consists_of.parent_product_id ) SELECT ppid, MAX(pl) AS pl FROM cte GROUP BY ppid ORDER BY ppid DESC", childProductId); + string stmt = string.Format(@"WITH RECURSIVE cte ( ppid, cpid, pl ) AS ( + SELECT parent.id, child.id, path_length FROM consists_of + JOIN product AS parent ON consists_of.parent_product_id = parent.id + JOIN product AS child ON consists_of.child_product_id = child.id + WHERE child.id = {0} + UNION ALL + SELECT parent.id, child.id, path_length FROM consists_of + JOIN product AS parent ON consists_of.parent_product_id = parent.id + JOIN product AS child ON consists_of.child_product_id = child.id + JOIN cte ON cte.ppid = child.id WHERE consists_of.child_product_id <> consists_of.parent_product_id ) + SELECT ppid, MAX(pl) AS pl FROM cte GROUP BY ppid ORDER BY ppid DESC" + , childProductId); MySqlCommand cmd = new MySqlCommand(stmt, connector.Connection); MySqlDataReader reader = cmd.ExecuteReader(); while (reader.Read()) @@ -202,24 +194,11 @@ private List GenerateProductRowAndRelations(List allParentProductIdent } reader.Close(); - // select all of our parent products for the created child product - /*string statement4 = string.Format("WITH RECURSIVE cte ( ppid, cpid, pl ) AS ( SELECT parent.id, child.id, path_length FROM consists_of JOIN product AS parent ON consists_of.parent_product_id = parent.id JOIN product AS child ON consists_of.child_product_id = child.id WHERE child.id = {0} UNION ALL SELECT parent.id, child.id, path_length FROM consists_of JOIN product AS parent ON consists_of.parent_product_id = parent.id JOIN product AS child ON consists_of.child_product_id = child.id JOIN cte ON cte.ppid = child.id WHERE consists_of.child_product_id <> consists_of.parent_product_id ) SELECT GROUP_CONCAT(DISTINCT concat(ppid, '; ', pl) ORDER BY ppid DESC SEPARATOR ',') AS ids FROM cte", childProductId); - MySqlCommand com4 = new MySqlCommand(statement4, c); - string list = (string) com4.ExecuteScalar(); - List result = list.Split(',').ToList(); - foreach (string r in result) - { - ids.Add(Int32.Parse(r.Split(";").ToArray()[0])); - pathlengths.Add(Int32.Parse(r.Split(";").ToArray()[1])); - }*/ - // skip the first two id's, because they the first one is referencing themselves, and the second one was already inserted before (initial parent > child relation) for (int k = 2; k < ids.Count; k++) { // inserto into hierarchy string statement5 = string.Format("INSERT INTO consists_of(parent_product_id, child_product_id, path_length) VALUES(" + ids[k] + "," + childProductId + ", " + (pathlengths[k] + 1) +");"); - //Console.WriteLine("Depth: " + k); - // Console.WriteLine("Inserting: " + productIds[k]); MySqlCommand com5 = new MySqlCommand(statement5, c); com5.ExecuteNonQuery(); } diff --git a/DummyDataGenerator/Generators/MySqlGenerator.cs b/DummyDataGenerator/Generators/MySqlGenerator.cs index 02ba94a..2564e0b 100644 --- a/DummyDataGenerator/Generators/MySqlGenerator.cs +++ b/DummyDataGenerator/Generators/MySqlGenerator.cs @@ -88,6 +88,7 @@ protected int[] GenerateTopLevelOrganizations(int organizations) { MySqlCommand com = InsertOrganization(true, i); com.ExecuteNonQuery(); + // add the inserted primary keys to a list result.Add((int)com.LastInsertedId); } watch.Stop(); @@ -96,18 +97,19 @@ protected int[] GenerateTopLevelOrganizations(int organizations) } /// - /// Generatea a set number of organizations to be used during the tree generation + /// Generate a set number of locations to be used during the tree generation /// - /// The number of organiations - /// a list of generated organization ids - protected int[] GenerateOrganizations(int organizations) + /// The number of locations + /// a list of generated locations ids + protected int[] GenerateOrganizations(int numberOfOrganizations) { List result = new List(); var watch = System.Diagnostics.Stopwatch.StartNew(); - for (int i = 0; i < organizations; i++) + for (int i = 0; i < numberOfOrganizations; i++) { MySqlCommand com = InsertOrganization(false, i); com.ExecuteNonQuery(); + // add the inserted primary keys to a list result.Add((int)com.LastInsertedId); } watch.Stop(); @@ -120,14 +122,15 @@ protected int[] GenerateOrganizations(int organizations) /// /// The number of activities to be generated /// a list of generated activity ids - protected int[] GenerateActivities(int activites) + protected int[] GenerateActivities(int numberOfActivites) { List result = new List(); var watch = System.Diagnostics.Stopwatch.StartNew(); - for (int i = 0; i < activites; i++) + for (int i = 0; i < numberOfActivites; i++) { MySqlCommand com = InsertActivity(i); com.ExecuteNonQuery(); + // add the inserted primary keys to a list result.Add((int)com.LastInsertedId); } watch.Stop(); @@ -148,6 +151,7 @@ protected int[] GenerateLocations(int locations) { MySqlCommand com = InsertLocation(i); com.ExecuteNonQuery(); + // add the inserted primary keys to a list result.Add((int) com.LastInsertedId); } watch.Stop(); @@ -156,15 +160,17 @@ protected int[] GenerateLocations(int locations) } /// - /// Generates the product trees for specified organizations + /// Generates all of the product trees necessary (amount of top level suppliers * amount of products) + /// starts with the top level products, and then proceeds with all branches in the tree /// /// The id's of the top level organizations /// The number of products each supplier should get - /// The depth of each chain + /// The depth of each chain (in how many branches does each product split up) /// The breadth of each chain per level; the number of products to generate for the product above private void GenerateProductTrees(int[] organizationIdentifiers, int productsPerSupplier, int depth, int breadthPerLevel) { var watchTotal = System.Diagnostics.Stopwatch.StartNew(); + // for each top level supplier for (int i = 0; i < organizationIdentifiers.Length; i++) { @@ -176,32 +182,32 @@ private void GenerateProductTrees(int[] organizationIdentifiers, int productsPer var watch = System.Diagnostics.Stopwatch.StartNew(); // generate the top level product - - //string statement = "INSERT INTO product(name) VALUES(" + "'Top Level Product #o" + (i + 1) + "-p" + (j + 1) + "');"; MySqlCommand com = InsertProduct(true, i * productsPerSupplier + j); com.ExecuteNonQuery(); + // and get the primary key back int topLevelId = (int) com.LastInsertedId; - // generate supplies relation + // generate the supplies relation for the top level supplier string statement = "INSERT INTO supplies(organization_id, product_id, activity_id, location_id) VALUES(" + organizationIdentifiers[i] + ", " + topLevelId + ", " + (i+1) + ", " + (i+1) +")"; MySqlCommand com2 = new MySqlCommand(statement, connector.Connection); com2.ExecuteNonQuery(); List previousResults = new List(); - // for the depth of the chain + + // for each level in the depth of the chain for (int k = 0; k < depth; k++) { // in the first pass, generate the products for our top level product if (k == 0) { previousResults.Add(topLevelId); - previousResults = GenerateProductRowAndRelations(i, k + 1, previousResults, breadthPerLevel); + previousResults = GenerateProductRowAndRelations(previousResults, breadthPerLevel); } // in subsequent passes, take the previous row of products and generate a new underlying row for all of them else { - previousResults = GenerateProductRowAndRelations(i, k + 1, previousResults, breadthPerLevel); + previousResults = GenerateProductRowAndRelations(previousResults, breadthPerLevel); } } @@ -215,13 +221,12 @@ private void GenerateProductTrees(int[] organizationIdentifiers, int productsPer } /// - /// Adds a single row to a single product hierarchy + /// Generates a single row of products for the tree, given the list of parent product ids /// - /// an identifier for a chain, to more clearly identify the products associated to this chain in the database /// the list of parent products for which a number of child products has to be generated /// the number of child products that have to be generated per parent product /// a list of the id's of the child products that have been created - private List GenerateProductRowAndRelations(int chainId, int depth, List parentProductIdentifiers, int breadthPerLevel) + private List GenerateProductRowAndRelations(List parentProductIdentifiers, int breadthPerLevel) { List childProductsCreated = new List(); int i = 0; @@ -230,8 +235,7 @@ private List GenerateProductRowAndRelations(int chainId, int depth, List GenerateProductRowAndRelations(int chainId, int depth, List /// the total number of organizations specified - /// the total number of activities specified - /// the total number of products specified + /// the total number of activities specified + /// the total number of products specified + /// the chain depth specified + /// the chain breadth specified protected void AddOrganizationsAndActivitiesToProductTree(int numberOfOrganizations, int numberOfTopLevelOrganizations, int numberOfActivities, int chainDepth, int chainBreadth) { var watch = System.Diagnostics.Stopwatch.StartNew(); @@ -275,6 +281,7 @@ protected void AddOrganizationsAndActivitiesToProductTree(int numberOfOrganizati { // first, check if the product inserted isn't a top level product by calculating the number of products in each individual product chain double skip = Math.Pow(chainBreadth, chainDepth + 1) - 1; + // if it's the first one, skip the product if (i % skip == 0) { Logger.Debug("Skipping product: " + (i + 1)); @@ -284,7 +291,7 @@ protected void AddOrganizationsAndActivitiesToProductTree(int numberOfOrganizati // if not, insert the product into the supplies table string statement2 = string.Format("INSERT INTO supplies(organization_id, activity_id, product_id, location_id) VALUES({0}, {1}, {2}, {3});", // add an offset of the toplevel suppliers, which are added first to the database, then modulo if the n.o. products outnumbers the n.o. organizations - (i % numberOfOrganizations) + numberOfTopLevelOrganizations + 1, // numberOfOrganizations + 1 + numberOfTopLevelOrganizations, + (i % numberOfOrganizations) + numberOfTopLevelOrganizations + 1, // modulo of the n.o.products outnumbers the n.o. activities i % numberOfActivities + 1, // use the iterator as the product id @@ -295,16 +302,10 @@ protected void AddOrganizationsAndActivitiesToProductTree(int numberOfOrganizati com2.ExecuteNonQuery(); } } - watch.Stop(); Logger.Info("Took " + watch.ElapsedMilliseconds + "ms " + "(" + (watch.ElapsedMilliseconds / 1000) + "s) to add organizations and activities to the product tree"); } - protected void AddLocations() - { - - } - /// /// Adds metadata to a separate table in the database /// @@ -334,7 +335,7 @@ protected void AddMetaData(Configuration config) com2.ExecuteNonQuery(); Logger.Debug("Added metadata to the database"); - // commit changes + // finally, commit the changes to write them to disk in the MySQL database string statement3 = "COMMIT;"; MySqlCommand com3 = new MySqlCommand(statement3, connector.Connection); com3.ExecuteNonQuery(); diff --git a/DummyDataGenerator/Generators/Neo4jGenerator.cs b/DummyDataGenerator/Generators/Neo4jGenerator.cs index b76e851..6fe7504 100644 --- a/DummyDataGenerator/Generators/Neo4jGenerator.cs +++ b/DummyDataGenerator/Generators/Neo4jGenerator.cs @@ -14,16 +14,19 @@ class Neo4jGenerator : Database Neo4jConnector connector; - public override void InitializeConnection() + /// + /// Initializes the singleton connection + /// + public override void InitializeConnection() { connector = Neo4jConnector.Instance(); } /// - /// + /// Generates the data with the specified parameters of the configuration /// - /// - public override void GenerateData(Configuration config, bool allowMultipleThreads) + /// The configuration that holds the paremeters for the data + public override void GenerateData(Configuration config, bool allowMultipleThreads) { RefreshSchema(); int[] topLevelOrganizations = GenerateTopLevelOrganizations(config.NumberOfTopLevelSuppliers); @@ -44,8 +47,9 @@ public override void GenerateData(Configuration config, bool allowMultipleThread } /// - /// + /// Deletes all nodes and relationships from the connected graph /// + #warning There is no option for dropping a database in Neo4j, when the graph is very large, this way of deleting can take a long time private void RefreshSchema() { var watch = System.Diagnostics.Stopwatch.StartNew(); @@ -59,10 +63,10 @@ private void RefreshSchema() } /// - /// + /// Generates a set number of top level organizations; organizations that do not supply any others /// - /// - /// + /// The number of organiations + /// a list of generated organization ids private int[] GenerateTopLevelOrganizations(int organizations) { var watch = System.Diagnostics.Stopwatch.StartNew(); @@ -84,10 +88,10 @@ private int[] GenerateTopLevelOrganizations(int organizations) } /// - /// + /// Generates a set number of organizations; they can be located anwywhere in the chain /// - /// - /// + /// The number of organiations + /// a list of generated organization ids private int[] GenerateOrganizations(int organizations) { List result = new List(); @@ -108,6 +112,11 @@ private int[] GenerateOrganizations(int organizations) return result.ToArray(); } + /// + /// Generate a set number of locations to be used during the tree generation + /// + /// The number of locations + /// a list of generated locations ids private int[] GenerateLocations(int locations) { List result = new List(); @@ -128,6 +137,9 @@ private int[] GenerateLocations(int locations) return result.ToArray(); } + /// + /// Used to pass data along to a thread, when in multi threaded mode + /// struct ProductTreeData { public ProductTreeData(int org, int productsPerSupplier, int depth, int breadthPerLevel, int orgIter, int prodIter, int locationId) @@ -150,6 +162,15 @@ public ProductTreeData(int org, int productsPerSupplier, int depth, int breadthP public int locationId; } + /// + /// Multi threaded way to generate all of the product trees necessary (amount of top level suppliers * amount of products) + /// starts with the top level products, and then proceeds with all branches in the tree + /// + /// The id's of the top level organizations + /// The number of products each supplier should get + /// The depth of each chain (in how many branches does each product split up) + /// The breadth of each chain per level; the number of products to generate for the product above + /// The location ids to be used private void mt_GenerateProductTrees(int[] organizationIds, int productsPerSupplier, int depth, int breadthPerLevel, int[] locations) { var watch = System.Diagnostics.Stopwatch.StartNew(); @@ -180,6 +201,10 @@ private void mt_GenerateProductTrees(int[] organizationIds, int productsPerSuppl } + /// + /// Generates one product tree, to be used with a single thread of the multi-threaded way of generating product trees + /// + /// ProductTree data private void mt_GenerateProductTree(object data) { ProductTreeData d = (ProductTreeData) data; @@ -218,25 +243,27 @@ private void mt_GenerateProductTree(object data) if (k == 0) { previousResults.Add(topLevelId); - previousResults = GenerateProductRowAndRelations(d.orgIter, previousResults.ToArray(), d.breadthPerLevel); + previousResults = GenerateProductRowAndRelations(previousResults.ToArray(), d.breadthPerLevel); } // in subsequent passes, take the previous row of products and generate a new underlying row for all of them else { - previousResults = GenerateProductRowAndRelations(d.orgIter, previousResults.ToArray(), d.breadthPerLevel); + previousResults = GenerateProductRowAndRelations(previousResults.ToArray(), d.breadthPerLevel); } } } /// - /// + /// Generates all of the product trees necessary (amount of top level suppliers * amount of products) + /// starts with the top level products, and then proceeds with all branches in the tree /// - /// - /// - /// - /// + /// The id's of the top level organizations + /// The number of products each supplier should get + /// The depth of each chain (in how many branches does each product split up) + /// The breadth of each chain per level; the number of products to generate for the product above + /// The location ids to be used private void GenerateProductTrees(int[] organizationIdentifiers, int productsPerSupplier, int depth, int breadthPerLevel, int[] locationIdentifiers) { var watchTotal = System.Diagnostics.Stopwatch.StartNew(); @@ -285,13 +312,13 @@ private void GenerateProductTrees(int[] organizationIdentifiers, int productsPer if (k == 0) { previousResults.Add(topLevelId); - previousResults = GenerateProductRowAndRelations(i, previousResults.ToArray(), breadthPerLevel); + previousResults = GenerateProductRowAndRelations(previousResults.ToArray(), breadthPerLevel); } // in subsequent passes, take the previous row of products and generate a new underlying row for all of them else { - previousResults = GenerateProductRowAndRelations(i, previousResults.ToArray(), breadthPerLevel); + previousResults = GenerateProductRowAndRelations(previousResults.ToArray(), breadthPerLevel); } } @@ -305,13 +332,12 @@ private void GenerateProductTrees(int[] organizationIdentifiers, int productsPer } /// - /// + /// Generates a single row of products for the tree, given the list of parent product ids /// - /// - /// - /// - /// - private List GenerateProductRowAndRelations(int chainId, int[] parentProductIdentifiers, int breadthPerLevel) + /// the list of parent products for which a number of child products has to be generated + /// the number of child products that have to be generated per parent product + /// a list of the id's of the child products that have been created + private List GenerateProductRowAndRelations(int[] parentProductIdentifiers, int breadthPerLevel) { List childProductsCreated = new List(); @@ -347,10 +373,13 @@ private List GenerateProductRowAndRelations(int chainId, int[] parentProduc } /// - /// + /// Adds organizations and activities to all the products in the database /// - /// - /// + /// the total number of organizations specified + /// the total number of activities specified + /// the total number of products specified + /// the chain depth specified + /// the chain breadth specified private void AddOrganizationsAndActivitiesToProductTree(int[] organizationIds, int[] locationIds) { var watch = System.Diagnostics.Stopwatch.StartNew(); diff --git a/DummyDataGenerator/Program.cs b/DummyDataGenerator/Program.cs index 607f237..500397c 100644 --- a/DummyDataGenerator/Program.cs +++ b/DummyDataGenerator/Program.cs @@ -35,7 +35,7 @@ static void Main(string[] args) using (FileStream fs = File.Create(envPath)) { // adds the keys to the file - Byte[] info = new UTF8Encoding(true).GetBytes("MYSQL_HOST =\nMYSQL_DB = \nMYSQL_USER = \nMYSQL_PW = \nNEO4J_HOST = \nNEO4J_USER = \nNEO4J_PW ="); + Byte[] info = new UTF8Encoding(true).GetBytes("MYSQL_HOST =\nMYSQL_DB = \nMYSQL_USER = \nMYSQL_PW = \nNEO4J_HOST = \nNEO4J_USER = \nNEO4J_PW = \nSQLSERVER_HOST = \nSQLSERVER_USER = \nSQLSERVER_DB = \nSQLSERVER_PW = "); fs.Write(info, 0, info.Length); Logger.Info("Created .env file, please specify connection properties in the file and restart the program"); } diff --git a/DummyDataGenerator/mysql_generate_schema.sql b/DummyDataGenerator/mysql_generate_schema.sql index 173cd5e..2dc846b 100644 --- a/DummyDataGenerator/mysql_generate_schema.sql +++ b/DummyDataGenerator/mysql_generate_schema.sql @@ -1,5 +1,5 @@ -CREATE DATABASE IF NOT EXISTS `test_al_b2_d8` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */; -USE `test_al_b2_d8`; +CREATE DATABASE IF NOT EXISTS `scm_al_b2_d10` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */; +USE `scm_al_b2_d10`; -- MySQL dump 10.13 Distrib 8.0.16, for Win64 (x86_64) -- -- Host: localhost Database: scm_test_dummy_generated_v2 diff --git a/DummyDataGenerator/mysql_generate_schema_ct.sql b/DummyDataGenerator/mysql_generate_schema_ct.sql index 4c6f2ae..3ce815c 100644 --- a/DummyDataGenerator/mysql_generate_schema_ct.sql +++ b/DummyDataGenerator/mysql_generate_schema_ct.sql @@ -1,5 +1,5 @@ -CREATE DATABASE IF NOT EXISTS `test_ct_b2_d8` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */; -USE `test_ct_b2_d8`; +CREATE DATABASE IF NOT EXISTS `scm_ct_b2_d10` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */; +USE `scm_ct_b2_d10`; -- MySQL dump 10.13 Distrib 8.0.16, for Win64 (x86_64) -- -- Host: localhost Database: scm_test_dummy_generated_v2