Merge pull request #89 from dsietz/development

Close Issue #88

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "test-data-generation"
-version = "0.2.0"
+version = "0.2.1"
 edition = "2018"
 authors = ["dsietz <davidsietz@yahoo.com>"]
 repository = "https://github.com/dsietz/test-data-generation.git"

README.md

@@ -6,7 +6,7 @@
 Linux: [![Build Status](https://travis-ci.org/dsietz/test-data-generation.svg?branch=master)](https://travis-ci.org/dsietz/test-data-generation)
 Windows: [![Build status](https://ci.appveyor.com/api/projects/status/uw58v5t8ynwj8s8o/branch/master?svg=true)](https://ci.appveyor.com/project/dsietz/test-data-generation/branch/master)
 
-## [We've overhauled this crate and have made drastic improvements!](#head1234)
+## [Fast test data generation!](#head1234)
 
 ### Description
 For software development teams who need realistic test data for testing their software, this Test Data Generation library is a light-weight module that implements Markov decision process machine learning to quickly and easily profile sample data, create an algorithm, and produce representative test data without the need for persistent data sources, data cleaning, or remote services. Unlike other solutions, this open source solution can be integrated into your test source code, or wrapped into a web service or stand-alone utility.   
@@ -27,26 +27,23 @@ or production environment (option #1 above)
 
 ### Table of Contents
 - [Test Data Generation](#test-data-generation)
-- [We've overhauled this crate and have made drastic improvements!](#weve-overhauled-this-crate-and-have-made-drastic-improvements)
+- [Fast test data generation!](#fast-test-data-generation)
   - [Description](#description)
   - [Table of Contents](#table-of-contents)
 - [What's New](#whats-new)
 - [About](#about)
 - [Usage](#usage)
   - [Profile](#profile)
   - [Data Sample Parser](#data-sample-parser)
+- [Examples](#examples)
 - [How to Contribute](#how-to-contribute)
 - [License](#license)
 
 ## What's New
 
-Here's whats new in 0.2.0:
-<h3><a name="announcement"></a>We've overhauled this crate and have made drastic improvements!</h3> 
-
-+ Upgrade to Rust Edition 2018
-+ Support updated versions of dependent crates
-+ Simplified the crate structure and made it more intuitive
-+ Improved performance of parsing sample data. ( <b>It's extremely fast!</b> ) 
+Here's whats new in 0.2.1:
++ [Fix for issue #88](https://github.com/dsietz/test-data-generation/issues/88)
++ Added a new Demo 03 `cargo run --example 03_demo`
 
 ## About
 
@@ -164,6 +161,16 @@ fn main() {
 }
 ```
 
+## Examples
+
+This library comes with the following examples. To run the examples.
++ [Demo 1](https://github.com/dsietz/test-data-generation/blob/master/examples/01_demo.rs) : Demonstrates the basic feature of the library to generate dates and people's names from the built-in demo data sets. 
+   > `cargo run --example 01_demo`
++ [Demo 2](https://github.com/dsietz/test-data-generation/blob/master/examples/02_demo.rs) : Demonstrates the basic feature of the library to generate dates and people's names from a CSV file.
+   > `cargo run --example 02_demo`
++ [Demo 3](https://github.com/dsietz/test-data-generation/blob/master/examples/03_demo.rs) : Demonstrates the ability to conitnuously add new analyzed data to an existing profile. 
+   > `cargo run --example 03_demo`
+
 ## How to Contribute
 
 Details on how to contribute can be found in the [CONTRIBUTING](./CONTRIBUTING.md) file.

examples/01_demo.rs

@@ -4,7 +4,7 @@ use test_data_generation::data_sample_parser::DataSampleParser;
 
 fn main() {
 
-		// This example demonstrates the basic feature of the library to generate dates and people's names
+		// This example demonstrates the basic feature of the library to generate dates and people's names from the built-in demo data sets
 		// using demo sample that is included in the library. 
 
 		// initalize a new DataSampelParser

examples/02_demo.rs

@@ -4,7 +4,7 @@ use test_data_generation::data_sample_parser::DataSampleParser;
 
 fn main() {
 
-		// This example demonstrates the basic feature of the library to generate dates and people's names
+		// This example demonstrates the basic feature of the library to generate dates and people's names from a CSV file
 		// using sample data that is included in the library. 
 
 		// initalize a new DataSampelParser

examples/03_demo.rs

@@ -0,0 +1,52 @@
+extern crate test_data_generation;
+
+use std::path::Path;
+use test_data_generation::{Profile};
+
+fn main() {
+        // This example demonstrates the ability to conitnuously add new analyzed data to an existing profile. 
+        let profile_file = "./tests/samples/demo-03";
+		let mut profile = match Path::new(profile_file).exists() {
+          true => {
+            // use existing file
+            Profile::from_file(profile_file)
+          },
+          false => {
+            // create new file
+            println!("Creating new profile: {}", profile_file);
+            Profile::new_with_id("demo-03".to_string())
+          },
+        };
+
+        // analyze the first data set and save the profile.
+        profile.analyze("Jonny");
+        profile.analyze("Jon");
+        profile.analyze("Johnathon");
+        profile.analyze("John");
+        profile.analyze("Jonathon");
+        profile.pre_generate();
+        profile.save(&profile_file).unwrap();
+        println!("My new name is {}", profile.generate().to_string());
+
+        // analyze the second data set and add it to the saved profile.
+        let mut profile2 = Profile::from_file(profile_file);
+        profile2.analyze("Chris");
+        profile2.analyze("Kris");
+        profile2.analyze("Christopher");
+        profile2.analyze("Christian");
+        profile2.analyze("Krissy");
+        profile2.pre_generate();
+        profile2.save(&profile_file).unwrap();
+        println!("My new name is {}", profile2.generate().to_string());
+
+        // analyze the third data set and add it to the saved profile.
+        let mut profile3 = Profile::from_file(profile_file);
+        profile3.analyze("Dan");
+        profile3.analyze("Danny");
+        profile3.analyze("Danyl");
+        profile3.analyze("Dannie");
+        profile3.analyze("Danathon");
+        profile3.pre_generate();
+        profile3.save(&profile_file).unwrap();
+        println!("My new name is {}", profile3.generate().to_string());
+}

src/lib.rs

@@ -475,6 +475,9 @@ impl Profile {
 		// -> {"CcvccpSCvcc": 14.285714285714285, "CvccvccpSCvccvc": 14.285714285714285, "CvccvccpSCvccvv": 28.57142857142857, "CvcvcccpSCcvcv": 14.285714285714285, "CvcvpSCvccc": 14.285714285714285, "V~CcvvcpSCvccc": 14.285714285714285}
 		let n = self.patterns.len();
 
+		// see issue: https://github.com/dsietz/test-data-generation/issues/88
+		self.pattern_percentages.clear();
+
 		for m in 0..n {
 			self.pattern_percentages.push((self.pattern_keys[m].clone(), (self.pattern_vals[m] as f64 / self.pattern_total as f64) * 100.0));
 		}
@@ -487,6 +490,9 @@ impl Profile {
 		// -> [("CvccvccpSCvccvv", 28.57142857142857), ("CcvccpSCvcc", 42.857142857142854), ("CvccvccpSCvccvc", 57.14285714285714), ("CvcvcccpSCcvcv", 71.42857142857142), ("CvcvpSCvccc", 85.7142857142857), ("V~CcvvcpSCvccc", 99.99999999999997)]
 		let mut rank: f64 = 0.00;
 
+		// see issue: https://github.com/dsietz/test-data-generation/issues/88
+		self.pattern_ranks.clear();
+
 		for pttrn in self.pattern_percentages.iter() {
 			let tmp = pttrn.1 + rank;
 			self.pattern_ranks.push((pttrn.0.clone(),tmp));

tests/samples/demo-03.json

@@ -0,0 +1 @@
+{"id":"demo-03","patterns":{"Cccvc":1,"Cccvccvccvc":1,"Cccvccvvc":1,"Ccvc":1,"Ccvccc":1,"Cvc":2,"Cvcc":1,"Cvccc":3,"Cvccvccvc":1,"Cvccvv":1,"Cvcvccvc":2},"pattern_total":15,"pattern_keys":["Cccvc","Cccvccvccvc","Cccvccvvc","Ccvc","Ccvccc","Cvc","Cvcc","Cvccc","Cvccvccvc","Cvccvv","Cvcvccvc"],"pattern_vals":[1,1,1,1,1,2,1,3,1,1,2],"pattern_percentages":[["Cvccc",20.0],["Cvc",13.333333333333334],["Cvcvccvc",13.333333333333334],["Cccvc",6.666666666666667],["Cccvccvccvc",6.666666666666667],["Cccvccvvc",6.666666666666667],["Ccvc",6.666666666666667],["Ccvccc",6.666666666666667],["Cvcc",6.666666666666667],["Cvccvccvc",6.666666666666667],["Cvccvv",6.666666666666667]],"pattern_ranks":[["Cvccc",20.0],["Cvc",33.333333333333336],["Cvcvccvc",46.66666666666667],["Cccvc",53.333333333333336],["Cccvccvccvc",60.0],["Cccvccvvc",66.66666666666667],["Ccvc",73.33333333333334],["Ccvccc",80.00000000000001],["Cvcc",86.66666666666669],["Cvccvccvc",93.33333333333336],["Cvccvv",100.00000000000003]],"sizes":{"3":2,"4":2,"5":4,"6":2,"8":2,"9":2,"11":1},"size_total":15,"size_ranks":[[5,26.666666666666668],[3,40.0],[4,53.333333333333336],[6,66.66666666666667],[8,80.0],[9,93.33333333333333],[11,100.0]],"processors":4,"facts":[[{"key":"J","prior_key":null,"next_key":"o","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"y","prior_key":"n","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":4},{"key":"J","prior_key":null,"next_key":"o","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"J","prior_key":null,"next_key":"o","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"a","prior_key":"n","next_key":"t","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":4},{"key":"n","prior_key":"o","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":8},{"key":"J","prior_key":null,"next_key":"o","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"J","prior_key":null,"next_key":"o","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"t","prior_key":"a","next_key":"h","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":4},{"key":"C","prior_key":null,"next_key":"h","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"s","prior_key":"i","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":4},{"key":"K","prior_key":null,"next_key":"r","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"C","prior_key":null,"next_key":"h","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"s","prior_key":"i","next_key":"t","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":4},{"key":"h","prior_key":"p","next_key":"e","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":8},{"key":"C","prior_key":null,"next_key":"h","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"s","prior_key":"i","next_key":"t","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":4},{"key":"n","prior_key":"a","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":8},{"key":"K","prior_key":null,"next_key":"r","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"s","prior_key":"s","next_key":"y","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":4},{"key":"D","prior_key":null,"next_key":"a","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"D","prior_key":null,"next_key":"a","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"y","prior_key":"n","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":4},{"key":"D","prior_key":null,"next_key":"a","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"l","prior_key":"y","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":4},{"key":"D","prior_key":null,"next_key":"a","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"i","prior_key":"n","next_key":"e","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":4},{"key":"D","prior_key":null,"next_key":"a","pattern_placeholder":"C","starts_with":1,"ends_with":0,"index_offset":0},{"key":"t","prior_key":"a","next_key":"h","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":4}],[{"key":"o","prior_key":"J","next_key":"n","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":1},{"key":"o","prior_key":"J","next_key":"n","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":1},{"key":"o","prior_key":"J","next_key":"h","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":1},{"key":"t","prior_key":"a","next_key":"h","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":5},{"key":"o","prior_key":"J","next_key":"h","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":1},{"key":"o","prior_key":"J","next_key":"n","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":1},{"key":"h","prior_key":"t","next_key":"o","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":5},{"key":"h","prior_key":"C","next_key":"r","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":1},{"key":"r","prior_key":"K","next_key":"i","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":1},{"key":"h","prior_key":"C","next_key":"r","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":1},{"key":"t","prior_key":"s","next_key":"o","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":5},{"key":"e","prior_key":"h","next_key":"r","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":9},{"key":"h","prior_key":"C","next_key":"r","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":1},{"key":"t","prior_key":"s","next_key":"i","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":5},{"key":"r","prior_key":"K","next_key":"i","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":1},{"key":"y","prior_key":"s","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":5},{"key":"a","prior_key":"D","next_key":"n","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":1},{"key":"a","prior_key":"D","next_key":"n","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":1},{"key":"a","prior_key":"D","next_key":"n","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":1},{"key":"a","prior_key":"D","next_key":"n","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":1},{"key":"e","prior_key":"i","next_key":null,"pattern_placeholder":"v","starts_with":0,"ends_with":1,"index_offset":5},{"key":"a","prior_key":"D","next_key":"n","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":1},{"key":"h","prior_key":"t","next_key":"o","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":5}],[{"key":"n","prior_key":"o","next_key":"n","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":2},{"key":"n","prior_key":"o","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":2},{"key":"h","prior_key":"o","next_key":"n","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":2},{"key":"h","prior_key":"t","next_key":"o","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":6},{"key":"h","prior_key":"o","next_key":"n","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":2},{"key":"n","prior_key":"o","next_key":"a","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":2},{"key":"o","prior_key":"h","next_key":"n","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":6},{"key":"r","prior_key":"h","next_key":"i","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":2},{"key":"i","prior_key":"r","next_key":"s","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":2},{"key":"r","prior_key":"h","next_key":"i","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":2},{"key":"o","prior_key":"t","next_key":"p","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":6},{"key":"r","prior_key":"e","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":10},{"key":"r","prior_key":"h","next_key":"i","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":2},{"key":"i","prior_key":"t","next_key":"a","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":6},{"key":"i","prior_key":"r","next_key":"s","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":2},{"key":"n","prior_key":"a","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":2},{"key":"n","prior_key":"a","next_key":"n","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":2},{"key":"n","prior_key":"a","next_key":"y","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":2},{"key":"n","prior_key":"a","next_key":"n","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":2},{"key":"n","prior_key":"a","next_key":"a","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":2},{"key":"o","prior_key":"h","next_key":"n","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":6}],[{"key":"n","prior_key":"n","next_key":"y","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":3},{"key":"n","prior_key":"h","next_key":"a","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":3},{"key":"o","prior_key":"h","next_key":"n","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":7},{"key":"n","prior_key":"h","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":3},{"key":"a","prior_key":"n","next_key":"t","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":3},{"key":"n","prior_key":"o","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":7},{"key":"i","prior_key":"r","next_key":"s","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":3},{"key":"s","prior_key":"i","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":3},{"key":"i","prior_key":"r","next_key":"s","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":3},{"key":"p","prior_key":"o","next_key":"h","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":7},{"key":"i","prior_key":"r","next_key":"s","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":3},{"key":"a","prior_key":"i","next_key":"n","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":7},{"key":"s","prior_key":"i","next_key":"s","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":3},{"key":"n","prior_key":"n","next_key":"y","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":3},{"key":"y","prior_key":"n","next_key":"l","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":3},{"key":"n","prior_key":"n","next_key":"i","pattern_placeholder":"c","starts_with":0,"ends_with":0,"index_offset":3},{"key":"a","prior_key":"n","next_key":"t","pattern_placeholder":"v","starts_with":0,"ends_with":0,"index_offset":3},{"key":"n","prior_key":"o","next_key":null,"pattern_placeholder":"c","starts_with":0,"ends_with":1,"index_offset":7}]]}