Test fix

.circleci/config.yml

@@ -17,6 +17,7 @@ jobs:
           name: Run Tests
           command: |
             make test
+          no_output_timeout: 30m
       - store_test_results:
           path: /home/circleci/project/tests/test-reports
       - store_artifacts:

tests/.gitignore

@@ -1 +1,2 @@
 *.pyc
+results/

tests/large/README.md

@@ -0,0 +1,12 @@
+Convention:
+* Test conf files are located at `conf` folder;
+* Sample files are located at `sample` folder;
+* Other utils related tools are located at `utils` folder;
+* `fileName` in `conf` settings is relative which will write results to folder `tests/large/results`;
+
+How to add a new functional test:
+* Add eventgen conf file in `conf` folder;(`sampleDir = tests/large/sample` should be in the conf)
+* Add sample file defined in above eventgen conf in folder `sample`;
+* Add a new functional test `py` file and add test case;
+* Use `eventgen_test_helper` fixture to create a helper instance and use `get_events()` to get events generated;
+* Pass `timeout=60` if you want to stop eventgen instance after 60s;

tests/large/conf/eventgen_replay.conf

@@ -0,0 +1,14 @@
+[replay]
+sampleDir = tests/large/sample
+mode = replay
+sampletype = raw
+outputMode = stdout
+end = 1
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = replaytimestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_replay_backfill.conf

@@ -0,0 +1,15 @@
+[replay]
+sampleDir = tests/large/sample
+backfill = -5s
+sampletype = raw
+outputMode = stdout
+mode = replay
+end = 2
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = replaytimestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_replay_csv.conf

@@ -0,0 +1,10 @@
+[timeorder]
+sampleDir = tests/large/sample
+mode = replay
+sampletype = csv
+timeField = _time
+outputMode = stdout
+
+token.0.token = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}
+token.0.replacementType = replaytimestamp
+token.0.replacement = %Y-%m-%dT%H:%M:%S

tests/large/conf/eventgen_replay_end_1.conf

@@ -0,0 +1,15 @@
+[replay]
+sampleDir = tests/large/sample
+mode = replay
+earliest = -5s
+sampletype = raw
+outputMode = stdout
+end = 2
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = replaytimestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_replay_end_2.conf

@@ -0,0 +1,15 @@
+[replay]
+sampleDir = tests/large/sample
+mode = replay
+earliest = -5s
+sampletype = raw
+outputMode = stdout
+end = -1
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = replaytimestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_replay_timeMultiple.conf

@@ -0,0 +1,14 @@
+[replay]
+sampleDir = tests/large/sample
+mode = replay
+sampletype = raw
+outputMode = stdout
+timeMultiple = 0.5
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = replaytimestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_sample.conf

@@ -0,0 +1,15 @@
+[sample]
+sampleDir = tests/large/sample
+mode = sample
+earliest = -15s
+sampletype = raw
+outputMode = stdout
+end = 1
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_sample_backfill.conf

@@ -0,0 +1,15 @@
+[sample]
+sampleDir = tests/large/sample
+mode = sample
+backfill = -15s
+sampletype = raw
+outputMode = stdout
+end = 1
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_sample_breaker.conf

@@ -0,0 +1,14 @@
+[breakersample]
+sampleDir = tests/large/sample
+outputMode = file
+fileName = tests/large/results/eventgen_sample_breaker.result
+count = 3
+earliest = -3s
+latest = now
+interval = 3
+breaker = ^\d{14}\.\d{6}
+end = 1
+
+token.0.token = ^(\d{14})\.\d{6}
+token.0.replacementType = timestamp
+token.0.replacement = %Y%m%d%H%M%S

tests/large/conf/eventgen_sample_count.conf

@@ -0,0 +1,16 @@
+[sample]
+sampleDir = tests/large/sample
+mode = sample
+earliest = -15s
+sampletype = raw
+outputMode = stdout
+end = 1
+count = 5
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_sample_csv.conf

@@ -0,0 +1,14 @@
+[timeorder]
+sampleDir = tests/large/sample
+mode = sample
+sampletype = csv
+outputMode = stdout
+end = 1
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_sample_earliest.conf

@@ -0,0 +1,16 @@
+[sample]
+sampleDir = tests/large/sample
+mode = sample
+earliest = -15s
+sampletype = raw
+outputMode = file
+fileName = tests/large/results/eventgen_sample_earliest.result
+end = 1
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_sample_end.conf

@@ -0,0 +1,16 @@
+[sample]
+sampleDir = tests/large/sample
+mode = sample
+earliest = -15s
+sampletype = raw
+outputMode = file
+fileName = tests/large/results/eventgen_sample_end.result
+end = 1
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_sample_interval.conf

@@ -0,0 +1,15 @@
+[sample]
+sampleDir = tests/large/sample
+mode = sample
+sampletype = raw
+outputMode = file
+fileName = tests/large/results/eventgen_sample_interval.result
+interval = 10
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_sample_latest.conf

@@ -0,0 +1,16 @@
+[sample]
+sampleDir = tests/large/sample
+mode = sample
+latest = +15s
+sampletype = raw
+outputMode = file
+fileName = tests/large/results/eventgen_sample_latest.result
+end = 1
+
+token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
+token.0.replacementType = timestamp
+token.0.replacement = %Y-%m-%d %H:%M:%S
+
+token.1.token = @@integer
+token.1.replacementType = random
+token.1.replacement = integer[0:10]

tests/large/conf/eventgen_token_replacement.conf

@@ -0,0 +1,62 @@
+[tokenreplacement.sample]
+sampleDir = tests/large/sample
+mode = sample
+sampletype = raw
+outputMode = stdout
+end = 1
+
+token.0.token = "start":"(\d+)
+token.0.replacementType = timestamp
+token.0.replacement = %s
+
+token.1.token = "cp":"([\d\.]+)
+token.1.replacementType = file
+token.1.replacement = tests/large/sample/cp.csv
+
+token.2.token = "country":"(\w+)
+token.2.replacementType = mvfile
+token.2.replacement = tests/large/sample/city.csv:1
+
+token.3.token = "city":"(\w+)
+token.3.replacementType = file
+token.3.replacement = tests/large/sample/city.csv:2
+
+token.4.token = "lat":"(-?\d+.\d+)
+token.4.replacementType = file
+token.4.replacement = tests/large/sample/city.csv:4
+
+token.5.token = "long":"(-?\d+.\d+)
+token.5.replacementType = file
+token.5.replacement = tests/large/sample/city.csv:5
+
+token.6.token = "id":"([\w\-]+)
+token.6.replacementType = file
+token.6.replacement = tests/large/sample/id.csv
+
+token.7.token = "bytes":"(\d+)
+token.7.replacementType = random
+token.7.replacement = integer[40:5000]
+
+token.8.token = "cliIP":"(\d+.\d+.\d+.\d+)
+token.8.replacementType = file
+token.8.replacement = tests/large/sample/ip.csv
+
+token.9.token = "lastByte":"(\d+)
+token.9.replacementType = static
+token.9.replacement = 0
+
+token.10.token = "receiver_id":"(\d*)
+token.10.replacementType = integerid
+token.10.replacement = 1
+
+token.11.token = "Ak_IP":"(\d+.\d+.\d+.\d+)
+token.11.replacementType = random
+token.11.replacement = ipv4
+
+token.12.token = "forward-origin-ip":"(\d+.\d+.\d+.\d+)
+token.12.replacementType = random
+token.12.replacement = ipv6
+
+token.13.token = "end-user-ip":"(\d+.\d+.\d+.\d+)
+token.13.replacementType = random
+token.13.replacement = mac

tests/large/conftest.py

@@ -0,0 +1,18 @@
+import pytest
+
+from utils.eventgen_test_helper import EventgenTestHelper
+
+
+@pytest.fixture
+def eventgen_test_helper():
+    """Returns a function to create EventgenTestHelper instance based on config file"""
+    created_instances = []
+
+    def _create_eventgen_test_helper_instance(conf, timeout=None):
+        instance = EventgenTestHelper(conf, timeout)
+        created_instances.append(instance)
+        return instance
+    yield _create_eventgen_test_helper_instance
+
+    for instance in created_instances:
+        instance.tear_down()

tests/large/sample/breakersample

@@ -0,0 +1,9 @@
+20110414114247.083068
+PercentProcessorTime=@@proc_time
+PercentUserTime=37
+wmi_type=CPUTime
+
+20110414114247.083068
+PercentProcessorTime=100
+PercentUserTime=37
+wmi_type=CPUTime