diff --git a/project/scripts/compile.py b/project/scripts/compile.py index e19fb252..a70341b1 100755 --- a/project/scripts/compile.py +++ b/project/scripts/compile.py @@ -23,7 +23,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -import os + import sys import time import argparse @@ -39,35 +39,22 @@ class Compiler(object): def __init__(self, path_to_files, skips_file_name, need_to_prepare_c_code=True): self.need_to_prepare_c_code = need_to_prepare_c_code - self.skips = settings.get_skips(skips_file_name) if skips_file_name else {} + self.skips_file_name = skips_file_name self.path_to_tests = path_to_files self.path_to_c2eo_build = settings.get_setting('path_to_c2eo_build') self.transpilation_units = [] def compile(self): start_time = time.time() - self.transpilation_units = Transpiler(self.path_to_tests, '', self.need_to_prepare_c_code).transpile() - skips = self.remove_skip_files() + self.transpilation_units, skip_result = Transpiler(self.path_to_tests, self.skips_file_name, + self.need_to_prepare_c_code).transpile() if self.transpilation_units: EOBuilder().build() passes = set(unit['unique_name'] for unit in self.transpilation_units) - passes -= set(file for value in skips.values() for file in value.keys()) - result = {tools.PASS: passes, tools.SKIP: skips} - tools.pprint_result('COMPILE', len(self.transpilation_units), int(time.time() - start_time), result, 0) - return self.transpilation_units - - def remove_skip_files(self): - skips = {} - for unit in self.transpilation_units: - for _filter, comment in self.skips.items(): - if _filter in unit['name']: - if comment not in skips: - skips[comment] = {} - if os.path.exists(unit['src_eo_file']): - os.remove(unit['src_eo_file']) - skips[comment][unit['unique_name']] = set() - break - return skips + result = {tools.PASS: passes, tools.SKIP: skip_result} + tests_count = len(self.transpilation_units) + sum(map(len, skip_result.values())) + tools.pprint_result('COMPILE', tests_count, int(time.time() - start_time), result, 0) + return self.transpilation_units, skip_result def create_parser(): diff --git a/project/scripts/test.py b/project/scripts/test.py index f319b14c..08bf87f3 100755 --- a/project/scripts/test.py +++ b/project/scripts/test.py @@ -41,7 +41,7 @@ class Tests(object): def __init__(self, path_to_tests, skips_file_name): - self.skips = settings.get_skips(skips_file_name) if skips_file_name else {} + self.skips_file_name = skips_file_name self.path_to_tests = path_to_tests self.path_to_c2eo_build = settings.get_setting('path_to_c2eo_build') self.path_to_eo_src = settings.get_setting('path_to_eo_src') @@ -53,15 +53,16 @@ def __init__(self, path_to_tests, skips_file_name): def test(self): start_time = time.time() - self.transpilation_units = Compiler(self.path_to_tests, '').compile() + self.transpilation_units, skip_result = Compiler(self.path_to_tests, self.skips_file_name).compile() if self.transpilation_units: self.get_result_for_tests() with tools.thread_pool() as threads: - results = threads.map(self.compare_test_results, self.transpilation_units) + results = threads.map(compare_test_results, self.transpilation_units) result = group_comparison_results(results) + result[tools.SKIP] = skip_result + tests_count = len(self.transpilation_units) + sum(map(len, skip_result.values())) _is_failed = len(result[tools.ERROR]) + len(result[tools.EXCEPTION]) - tools.pprint_result('TEST', len(self.transpilation_units), int(time.time() - start_time), result, - _is_failed) + tools.pprint_result('TEST', tests_count, int(time.time() - start_time), result, _is_failed) return _is_failed def get_result_for_tests(self): @@ -117,20 +118,6 @@ def get_result_for_eo_file(self, unit): self.test_handled_count += 1 tools.print_progress_bar(self.test_handled_count, len(self.transpilation_units)) - def compare_test_results(self, unit): - for _filter, comment in self.skips.items(): - if _filter in unit['name']: - return unit, True, False, False, comment - - with open(unit['result_c_file'], 'r') as f: - c_data = f.readlines() - with open(unit['result_eo_file'], 'r') as f: - eo_data = f.readlines() - is_except, is_equal, log_data = compare_files(c_data, eo_data) - with open(os.path.join(unit['result_path'], f'{unit["name"]}.log'), 'w') as f: - f.writelines(log_data) - return unit, False, is_except, is_equal, log_data - def compare_files(c_data, eo_data): if is_exception(c_data): @@ -178,14 +165,10 @@ def compare_lines(c_data, eo_data): def group_comparison_results(results): - result = {tools.PASS: [], tools.ERROR: [], tools.EXCEPTION: {}, tools.SKIP: {}} + result = {tools.PASS: [], tools.ERROR: [], tools.EXCEPTION: {}} tools.pprint('Getting results\n', slowly=True) - for unit, is_skip, is_except, is_equal, log_data in results: - if is_skip: - if log_data not in result[tools.SKIP]: - result[tools.SKIP][log_data] = {} - result[tools.SKIP][log_data][unit['unique_name']] = set() - elif is_except: + for unit, is_except, is_equal, log_data in results: + if is_except: log_data = ''.join(log_data) if log_data not in result[tools.EXCEPTION]: result[tools.EXCEPTION][log_data] = {} @@ -197,6 +180,17 @@ def group_comparison_results(results): return result +def compare_test_results(unit): + with open(unit['result_c_file'], 'r') as f: + c_data = f.readlines() + with open(unit['result_eo_file'], 'r') as f: + eo_data = f.readlines() + is_except, is_equal, log_data = compare_files(c_data, eo_data) + with open(os.path.join(unit['result_path'], f'{unit["name"]}.log'), 'w') as f: + f.writelines(log_data) + return unit, is_except, is_equal, log_data + + def create_parser(): _parser = argparse.ArgumentParser(description='the script for testing the correctness of the execution of ' 'translated files from C to EO') diff --git a/project/scripts/transpile.py b/project/scripts/transpile.py index c13364a4..6660d5d3 100755 --- a/project/scripts/transpile.py +++ b/project/scripts/transpile.py @@ -63,7 +63,6 @@ def __init__(self, path_to_c_files, skips_file_name, need_to_prepare_c_code=True self.transpilation_units = [] self.replaced_path = f'{os.path.split(self.path_to_c_files[:-1])[0]}{os.sep}' self.files_handled_count = 0 - self.files_count = 0 self.ignored_transpilation_warnings = settings.get_setting('ignored_transpilation_warnings') if not self.ignored_transpilation_warnings: self.ignored_transpilation_warnings = [] @@ -75,18 +74,21 @@ def transpile(self): clean_before_transpilation.main(self.path_to_c_files) c_files = tools.search_files_by_patterns(self.path_to_c_files, ['*.c'], filters=self.filters, recursive=True, print_files=True) - self.files_count = len(c_files) + with tools.thread_pool() as threads: + self.transpilation_units = list(threads.map(self.make_unit, c_files)) + generate_unique_names_for_units(self.transpilation_units) + skip_result = self.check_skips() original_path = os.getcwd() os.chdir(self.path_to_c2eo_transpiler) tools.pprint('\nTranspile files:\n', slowly=True) - tools.print_progress_bar(0, self.files_count) + tools.print_progress_bar(0, len(self.transpilation_units)) with tools.thread_pool() as threads: - self.transpilation_units = list(threads.map(self.start_transpilation, c_files)) - generate_unique_names_for_units(self.transpilation_units, 2) + threads.map(self.start_transpilation, self.transpilation_units) result = self.group_transpilation_results() + result[tools.SKIP] = skip_result + tests_count = len(self.transpilation_units) + sum(map(len, skip_result.values())) is_failed = sum(map(len, result[tools.EXCEPTION].values())) - tools.pprint_result('TRANSPILE', len(self.transpilation_units), int(time.time() - start_time), result, - is_failed) + tools.pprint_result('TRANSPILE', tests_count, int(time.time() - start_time), result, is_failed) if is_failed: exit(f'transpilation failed') @@ -97,7 +99,7 @@ def transpile(self): self.generate_run_sh(self.transpilation_units[0]['full_name']) tools.pprint('\nTranspilation done\n') os.chdir(original_path) - return self.transpilation_units + return self.transpilation_units, skip_result def build_c2eo(self): if self.need_to_generate_codecov: @@ -107,19 +109,36 @@ def build_c2eo(self): else: build_c2eo.main(self.path_to_c2eo_build) - def start_transpilation(self, c_file): + def make_unit(self, c_file): path, name, _ = tools.split_path(c_file, with_end_sep=True) rel_c_path = path.replace(self.replaced_path, '') full_name = f'{tools.make_name_from_path(rel_c_path)}.{name}' prepared_c_file, result_path = self.prepare_c_file(path, name, c_file) - eo_file = f'{full_name}.eo' - transpile_cmd = f'{self.codecov_arg} ./c2eo {prepared_c_file} {eo_file}' + return {'c_file': c_file, 'rel_c_path': rel_c_path, 'full_name': full_name, 'prepared_c_file': prepared_c_file, + 'result_path': result_path, 'name': name, 'unique_name': name} + + def check_skips(self): + skip_units = [] + skips = {} + for unit in self.transpilation_units: + for _filter, comment in self.skips.items(): + if _filter in unit['name']: + if comment not in skips: + skips[comment] = {} + skips[comment][unit['unique_name']] = set() + skip_units.append(unit) + self.transpilation_units = list(filter(lambda x: x not in skip_units, self.transpilation_units)) + return skips + + def start_transpilation(self, unit): + eo_file = f'{unit["full_name"]}.eo' + transpile_cmd = f'{self.codecov_arg} ./c2eo {unit["prepared_c_file"]} {eo_file}' result = subprocess.run(transpile_cmd, shell=True, capture_output=True, text=True) self.files_handled_count += 1 - tools.print_progress_bar(self.files_handled_count, self.files_count) - return {'c_file': c_file, 'rel_c_path': rel_c_path, 'full_name': full_name, 'transpilation_result': result, - 'eo_file': os.path.abspath(eo_file), 'rel_eo_file': os.path.join(rel_c_path, f'{name}.eo'), - 'name': name, 'result_path': result_path, 'prepared_c_file': prepared_c_file, 'unique_name': name} + tools.print_progress_bar(self.files_handled_count, len(self.transpilation_units)) + unit['transpilation_result'] = result + unit['eo_file'] = os.path.abspath(eo_file) + unit['rel_eo_file'] = os.path.join(unit["rel_c_path"], f'{unit["name"]}.eo') def prepare_c_file(self, path, file_name, c_file): with open(f'{c_file}', 'r', encoding='ISO-8859-1') as f: @@ -153,17 +172,10 @@ def create_plug_file(self, unit, message): def group_transpilation_results(self): result = {tools.PASS: set([unit['unique_name'] for unit in self.transpilation_units]), tools.NOTE: {}, - tools.WARNING: {}, tools.ERROR: {}, tools.EXCEPTION: {}, tools.SKIP: {}} + tools.WARNING: {}, tools.ERROR: {}, tools.EXCEPTION: {}} tools.pprint('\nGetting results\n', slowly=True, on_the_next_line=True) for unit in self.transpilation_units: - skip_message = self.check_unit_skip(unit) exception_message = check_unit_exception(unit) - if skip_message: - if skip_message not in result[tools.SKIP]: - result[tools.SKIP][skip_message] = {} - result[tools.SKIP][skip_message][unit['unique_name']] = set() - continue - if exception_message: if exception_message not in [tools.EXCEPTION]: result[tools.EXCEPTION][exception_message] = {} @@ -186,15 +198,10 @@ def group_transpilation_results(self): if unit['unique_name'] in place: result[status][message][unit['unique_name']].add(place.split(':', 1)[1][:-2]) - for status in [tools.EXCEPTION, tools.SKIP]: + for status in [tools.EXCEPTION]: result[tools.PASS] -= set(file for value in result[status].values() for file in value.keys()) return result - def check_unit_skip(self, unit): - for _filter, comment in self.skips.items(): - if _filter in unit['name']: - return comment - def move_transpiled_files(self): difference = [] for unit in self.transpilation_units: @@ -232,7 +239,7 @@ def generate_run_sh(self, full_name): f.write(code) -def generate_unique_names_for_units(units, i): +def generate_unique_names_for_units(units, words_in_name=2): names = {} collision_names = {} for unit in units: @@ -246,9 +253,9 @@ def generate_unique_names_for_units(units, i): for name, _units in collision_names.items(): units.extend(_units) for unit in _units: - unit['unique_name'] = f'{os.sep.join(unit["rel_c_path"].split(os.sep)[-i:])}{unit["name"]}' + unit['unique_name'] = f'{os.sep.join(unit["rel_c_path"].split(os.sep)[-words_in_name:])}{unit["name"]}' if len(collision_names) > 0: - generate_unique_names_for_units(units, i + 1) + generate_unique_names_for_units(units, words_in_name + 1) def check_unit_exception(unit):