From a4cbe0bc38e4b9235dfda6fb9324565f0b7255ca Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Mon, 27 Jan 2020 16:55:03 +0100 Subject: [PATCH 1/6] Fix typo in travis ci --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 38aca0d..6b04d29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,7 @@ script: - pip install . # Generate parsed rules - python rules2yml.py -d rules + - python rules2yml.py -d rules # Check if rule syntax in osi is correct # - python test_cases.py From bfedd0f181fa058109cb253ae9f9d553172489f5 Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Wed, 29 Jan 2020 09:23:10 +0100 Subject: [PATCH 2/6] Install PyYaml and update osi --- open-simulation-interface | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/open-simulation-interface b/open-simulation-interface index 66501d4..c853a1c 160000 --- a/open-simulation-interface +++ b/open-simulation-interface @@ -1 +1 @@ -Subproject commit 66501d4094ed5706ab83c704eddf135e01e6af0c +Subproject commit c853a1ca2686598b3c1b8c088b9bb5557b722f4a diff --git a/setup.py b/setup.py index 418f9ea..0857df5 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ install_requires=[ 'iso3166', 'ruamel.yaml', + 'PyYaml', 'asteval', 'sphinx_rtd_theme', 'recommonmark', From 4794c0ada29cfd83ad03e0047cf2273e59425751 Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Tue, 4 Feb 2020 15:14:08 +0100 Subject: [PATCH 3/6] Add test for the txt2osi convert in ci --- .travis.yml | 38 +++++++++++++++++++++----------------- data/small_test.osi.lzma | 3 --- 2 files changed, 21 insertions(+), 20 deletions(-) delete mode 100644 data/small_test.osi.lzma diff --git a/.travis.yml b/.travis.yml index 6b04d29..586fdd5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,6 +52,26 @@ script: # Show validator usage - osivalidator -h + # Convert decompress small_test.txt and convert to small_test.osi + - lzma -d data/small_test.txt.lzma + - python open-simulation-interface/format/txt2osi.py -d data/small_test.txt + + # Run the validator on decompressed data + - osivalidator data/small_test.osi + - osivalidator data/small_test.osi -p + - osivalidator data/small_test.txt -f separated + - osivalidator data/small_test.txt -f separated -p + + # Run the validator on decompressed data with parsed rules + - osivalidator data/small_test.osi -r rules + - osivalidator data/small_test.osi -p -r rules + - osivalidator data/small_test.txt -f separated -r rules + - osivalidator data/small_test.txt -f separated -p -r rules + + # Compress *.osi and *.txt with lzma + - lzma -z data/small_test.txt + - lzma -z data/small_test.osi + # Run validator on a small test with already existing rules - osivalidator data/small_test.osi.lzma - osivalidator data/small_test.osi.lzma -p @@ -62,20 +82,4 @@ script: - osivalidator data/small_test.osi.lzma -r rules - osivalidator data/small_test.osi.lzma -p -r rules - osivalidator data/small_test.txt.lzma -f separated -r rules - - osivalidator data/small_test.txt.lzma -f separated -p -r rules - - # Decompress both traces - - lzma -d data/small_test.osi.lzma - - lzma -d data/small_test.txt.lzma - - # Run the validator on decompressed data - - osivalidator data/small_test.osi - - osivalidator data/small_test.osi -p - - osivalidator data/small_test.txt -f separated - - osivalidator data/small_test.txt -f separated -p - - # Run the validator on decompressed data with parsed rules - - osivalidator data/small_test.osi -r rules - - osivalidator data/small_test.osi -p -r rules - - osivalidator data/small_test.txt -f separated -r rules - - osivalidator data/small_test.txt -f separated -p -r rules + - osivalidator data/small_test.txt.lzma -f separated -p -r rules \ No newline at end of file diff --git a/data/small_test.osi.lzma b/data/small_test.osi.lzma deleted file mode 100644 index 083d55b..0000000 --- a/data/small_test.osi.lzma +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6aa71cd856b6dbfeebee0c9b78a81e339bd3c958a745b3c325c67bbfd370f7d2 -size 130011 From 167ae1ee24c994fa7783c4eeeb10a738bb8cdd93 Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Tue, 4 Feb 2020 15:20:20 +0100 Subject: [PATCH 4/6] Fixed indentaion --- .travis.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 586fdd5..c232aae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,17 +56,17 @@ script: - lzma -d data/small_test.txt.lzma - python open-simulation-interface/format/txt2osi.py -d data/small_test.txt - # Run the validator on decompressed data - - osivalidator data/small_test.osi - - osivalidator data/small_test.osi -p - - osivalidator data/small_test.txt -f separated - - osivalidator data/small_test.txt -f separated -p - - # Run the validator on decompressed data with parsed rules - - osivalidator data/small_test.osi -r rules - - osivalidator data/small_test.osi -p -r rules - - osivalidator data/small_test.txt -f separated -r rules - - osivalidator data/small_test.txt -f separated -p -r rules + # Run the validator on decompressed data + - osivalidator data/small_test.osi + - osivalidator data/small_test.osi -p + - osivalidator data/small_test.txt -f separated + - osivalidator data/small_test.txt -f separated -p + + # Run the validator on decompressed data with parsed rules + - osivalidator data/small_test.osi -r rules + - osivalidator data/small_test.osi -p -r rules + - osivalidator data/small_test.txt -f separated -r rules + - osivalidator data/small_test.txt -f separated -p -r rules # Compress *.osi and *.txt with lzma - lzma -z data/small_test.txt From 1a3c1599fae6be163b31d5278c6ac0deb6fc0eda Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Tue, 4 Feb 2020 16:17:43 +0100 Subject: [PATCH 5/6] Move unit test below trace compression --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c232aae..f6efef9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,9 +46,6 @@ script: # Check if rule syntax in osi is correct # - python test_cases.py - # Check rule correctness with unittests - - python -m unittest discover tests - # Show validator usage - osivalidator -h @@ -72,6 +69,9 @@ script: - lzma -z data/small_test.txt - lzma -z data/small_test.osi + # Check rule correctness with unittests + - python -m unittest discover tests + # Run validator on a small test with already existing rules - osivalidator data/small_test.osi.lzma - osivalidator data/small_test.osi.lzma -p From c3608d327f9035097c96e739dd006e03792e51fc Mon Sep 17 00:00:00 2001 From: Viktor Kreschenski Date: Thu, 6 Feb 2020 15:18:20 +0100 Subject: [PATCH 6/6] Added closing file and eof if clause --- osivalidator/osi_general_validator.py | 6 ++---- osivalidator/osi_trace.py | 15 +++++---------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/osivalidator/osi_general_validator.py b/osivalidator/osi_general_validator.py index 16aa91e..69a8b55 100755 --- a/osivalidator/osi_general_validator.py +++ b/osivalidator/osi_general_validator.py @@ -167,17 +167,15 @@ def main(): except Exception as e: print(str(e)) - LOGGER.flush(LOGS) + LOGGER.flush(LOGS) MESSAGE_CACHE.clear() BAR.finish() - - # Grab major OSI version + DATA.trace_file.close() # Synthetize LOGGER.synthetize_results_from_sqlite() - def close_pool(pool): """Cleanly close a pool to free the memory""" pool.close() diff --git a/osivalidator/osi_trace.py b/osivalidator/osi_trace.py index eee1209..ff9002e 100644 --- a/osivalidator/osi_trace.py +++ b/osivalidator/osi_trace.py @@ -53,9 +53,6 @@ def __init__(self, buffer_size, show_progress=True, path=None, type_name="Sensor self.show_progress = show_progress self.retrieved_trace_size = 0 - if path is not None and type_name is not None: - self.from_file(path) - # Open and Read text file def from_file(self, path, type_name="SensorView", max_index=-1, format_type=None): """Import a trace from a file""" @@ -101,7 +98,7 @@ def retrieve_message(self): counter = 0 # Counter is needed to enable correct buffer parsing of serialized messages # Check if user decided to use buffer - if self.buffer_size != 0 and type(self.buffer_size)==int: + if self.buffer_size != 0 and type(self.buffer_size) == int: # Run while the end of file is not reached while not eof and message_offset < trace_size: @@ -114,8 +111,7 @@ def retrieve_message(self): message_length = struct.unpack(" self.buffer_size * (counter + 1) - # Check if the last INT (length=4) is found and then exit - if after_tell - before_tell == self._int_length: + # Check if reached end of file + if self.trace_file.tell() == trace_size: self.retrieved_trace_size = self.message_offsets[-1] - self.message_offsets.pop() # Remove the last element since after that there is no message coming - self.trace_file.seek(trace_size) # Set the cursor to the end of the file + self.message_offsets.pop() # Remove the last element since after that there is no message coming break while eof: