From 246164a5c01f2bd68ac0a2a3e7348febb99c67df Mon Sep 17 00:00:00 2001 From: vhristov5555 Date: Tue, 3 Jul 2018 16:11:42 +0300 Subject: [PATCH] Add create, add platform and build performance test. --- core/tns/tns.py | 40 ++-- tests/perf/cli_operations_time.py | 254 ++++++++++++++++++++++ tests/perf/cli_operations_time_values.csv | 25 +++ 3 files changed, 304 insertions(+), 15 deletions(-) create mode 100644 tests/perf/cli_operations_time.py create mode 100644 tests/perf/cli_operations_time_values.csv diff --git a/core/tns/tns.py b/core/tns/tns.py index ee0fa60c..7976c397 100644 --- a/core/tns/tns.py +++ b/core/tns/tns.py @@ -51,7 +51,7 @@ def __get_final_package_name(app_name, platform=Platform.NONE): elif platform is Platform.IOS: return app_name.replace(" ", "").replace("-", "").replace("_", "").replace("\"", "") else: - raise "Invalid platform!" + raise Exception("Invalid platform!") @staticmethod def __get_app_name_from_attributes(attributes={}): @@ -99,7 +99,8 @@ def get_app_id(app_name, platform=Platform.NONE): return json.get('nativescript').get('id') @staticmethod - def run_tns_command(command, tns_path=None, attributes={}, log_trace=False, timeout=COMMAND_TIMEOUT, wait=True): + def run_tns_command(command, tns_path=None, attributes={}, log_trace=False, timeout=COMMAND_TIMEOUT, wait=True, + measureTime=False): cmd = TNS_PATH + " " + command if tns_path is not None: cmd = tns_path + " " + command @@ -108,6 +109,9 @@ def run_tns_command(command, tns_path=None, attributes={}, log_trace=False, time cmd += " " + k + " " + v if log_trace: cmd += " --log trace" + + if measureTime: + cmd = "TIME " + cmd print cmd output = run(command=cmd, timeout=timeout, wait=wait) return output @@ -205,7 +209,7 @@ def ensure_app_resources(path): @staticmethod def create_app(app_name, attributes={}, log_trace=False, assert_success=True, update_modules=True, - force_clean=True): + force_clean=True, measureTime=False): if force_clean: if File.exists(app_name): @@ -225,9 +229,10 @@ def create_app(app_name, attributes={}, log_trace=False, assert_success=True, up attr = {"--template": "tns-template-hello-world"} attr.update(attributes) if app_name is None: - output = Tns.run_tns_command("create ", attributes=attr, log_trace=log_trace) + output = Tns.run_tns_command("create ", attributes=attr, log_trace=log_trace, measureTime=measureTime) else: - output = Tns.run_tns_command("create \"" + app_name + "\"", attributes=attr, log_trace=log_trace) + output = Tns.run_tns_command("create \"" + app_name + "\"", attributes=attr, log_trace=log_trace, + measureTime=measureTime) if assert_success: TnsAsserts.created(app_name=app_name, output=output) if update_modules: @@ -292,7 +297,7 @@ def create_app_ng(app_name, attributes={}, log_trace=False, assert_success=True, @staticmethod def platform_add(platform=Platform.NONE, version=None, attributes={}, assert_success=True, log_trace=False, - tns_path=None): + tns_path=None, measureTime=False): platform_string = Tns.__get_platform_string(platform) @@ -300,7 +305,7 @@ def platform_add(platform=Platform.NONE, version=None, attributes={}, assert_suc platform_string = platform_string + "@" + version output = Tns.run_tns_command("platform add " + platform_string, attributes=attributes, log_trace=log_trace, - tns_path=tns_path) + tns_path=tns_path, measureTime=measureTime) # Verify platforms added app_name = Tns.__get_app_name_from_attributes(attributes) @@ -354,17 +359,21 @@ def platform_update(platform=Platform.NONE, version=None, attributes={}, assert_ return output @staticmethod - def platform_add_android(version=None, attributes={}, assert_success=True, log_trace=False, tns_path=None): + def platform_add_android(version=None, attributes={}, assert_success=True, log_trace=False, tns_path=None, + measureTime=False): return Tns.platform_add(platform=Platform.ANDROID, version=version, attributes=attributes, assert_success=assert_success, log_trace=log_trace, - tns_path=tns_path) + tns_path=tns_path, + measureTime=measureTime) @staticmethod - def platform_add_ios(version=None, attributes={}, assert_success=True, log_trace=False, tns_path=None): + def platform_add_ios(version=None, attributes={}, assert_success=True, log_trace=False, tns_path=None, + measureTime=False): return Tns.platform_add(Platform.IOS, version=version, attributes=attributes, assert_success=assert_success, log_trace=log_trace, - tns_path=tns_path) + tns_path=tns_path, + measureTime=measureTime) @staticmethod def platform_list(attributes={}, log_trace=False, tns_path=None): @@ -415,8 +424,9 @@ def prepare_ios(attributes={}, assert_success=True, log_trace=False, tns_path=No return output @staticmethod - def build_android(attributes={}, assert_success=True, tns_path=None, log_trace=False): - output = Tns.run_tns_command("build android", attributes=attributes, tns_path=tns_path, log_trace=log_trace) + def build_android(attributes={}, assert_success=True, tns_path=None, log_trace=False, measureTime=False): + output = Tns.run_tns_command("build android", attributes=attributes, tns_path=tns_path, log_trace=log_trace, + measureTime=measureTime) if assert_success: # Verify output of build command assert "Project successfully built" in output, "Build failed!" + os.linesep + output @@ -468,7 +478,7 @@ def build_android(attributes={}, assert_success=True, tns_path=None, log_trace=F return output @staticmethod - def build_ios(attributes={}, assert_success=True, tns_path=None, log_trace=False): + def build_ios(attributes={}, assert_success=True, tns_path=None, log_trace=False, measureTime=False): if "--teamId" not in attributes.keys() \ and "--team-id" not in attributes.keys() \ @@ -476,7 +486,7 @@ def build_ios(attributes={}, assert_success=True, tns_path=None, log_trace=False attr = {"--provision": PROVISIONING} attributes.update(attr) - output = Tns.run_tns_command("build ios", attributes=attributes, tns_path=tns_path, log_trace=log_trace) + output = Tns.run_tns_command("build ios", attributes=attributes, tns_path=tns_path, log_trace=log_trace, measureTime=measureTime) app_name = Tns.__get_app_name_from_attributes(attributes=attributes) app_name = app_name.replace("\"", "") # Handle projects with space diff --git a/tests/perf/cli_operations_time.py b/tests/perf/cli_operations_time.py new file mode 100644 index 00000000..246707ae --- /dev/null +++ b/tests/perf/cli_operations_time.py @@ -0,0 +1,254 @@ +import csv +import os +from time import sleep + +from nose_parameterized import parameterized + +from core.base_class.BaseClass import BaseClass +from core.device.emulator import Emulator +from core.device.simulator import Simulator +from core.git.git import Git +from core.npm.npm import Npm +from core.osutils.command import run +from core.osutils.file import File +from core.osutils.folder import Folder +from core.settings.settings import ANDROID_PACKAGE, TYPESCRIPT_PACKAGE, WEBPACK_PACKAGE, ANDROID_KEYSTORE_PATH, \ + ANDROID_KEYSTORE_PASS, ANDROID_KEYSTORE_ALIAS_PASS, ANDROID_KEYSTORE_ALIAS, TEST_RUN_HOME, PROVISIONING, IOS_PACKAGE +from core.tns.tns import Tns +from core.tns.tns_platform_type import Platform +from core.tns.tns_verifications import TnsAsserts +import re + + +def read_data(platform, app_name=None): + csv_file_path = os.path.join(TEST_RUN_HOME, 'tests', 'perf', 'cli_operations_time_values.csv') + csv_list = tuple(csv.reader(open(csv_file_path, 'rb'), delimiter=',')) + if app_name is None: + return [tuple(l) for l in csv_list if l[2].startswith(platform)] + else: + return [tuple(l) for l in csv_list if (l[2].startswith(platform) and l[0] == app_name)] + + +class PerfBuildTests(BaseClass): + platform = os.getenv('PLATFORM', None) + APP_NAME = os.getenv('APP_NAME', None) + DATA = read_data(platform, APP_NAME) + + @staticmethod + def get_time(log): + m = re.search('\d+\.\d+ real', log) + time = m.group(0).split(".")[0] + return float(time) + + @staticmethod + def assert_time(expected, actual, tolerance=20, error_message="Startup time is not expected.", + verification_errors=[]): + print "Actual startup: " + str(actual) + print "Expected startup: " + str(expected) + x = int(expected) + y = int(actual) + if actual >= 0: + diff = abs(x - y) * 1.00 + try: + assert diff <= x * tolerance * 0.01, error_message + except AssertionError, e: + verification_errors.append(str(e)) + return verification_errors + + @staticmethod + def report_add_column_titles(): + + with open('perfResults.csv', 'a+') as csvfile: + fieldnames = ['app_name', 'configuration', 'platform', 'action', 'expected_first_start', + 'actual_first_start', + 'expected_second_start', 'actual_second_start'] + + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + + writer.writeheader() + + @staticmethod + def report_add_data(demo, config, platform, expected_tns_create_time, actual_tns_create_time, + expected_tns_platform_add_time, + actual_tns_platform_add_time, expected_tns_build_time, actual_tns_build_time): + with open('perfResults.csv', 'a+') as csvfile: + fieldnames = ['app_name', 'configuration', 'platform', 'expected_tns_create_time', + 'actual_tns_create_time', + 'expected_tns_platform_add_time', 'actual_tns_platform_add_time', + 'expected_tns_build_time', 'actual_tns_build_time'] + + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + + writer.writerow({'app_name': demo.split('/')[-1], 'configuration': config, 'platform': platform, + 'expected_tns_create_time': str(expected_tns_create_time), + 'actual_tns_create_time': str(actual_tns_create_time), + 'expected_tns_platform_add_time': str(expected_tns_platform_add_time), + 'actual_tns_platform_add_time': str(actual_tns_platform_add_time), + 'expected_tns_build_time': str(expected_tns_build_time), + 'actual_tns_build_time': str(actual_tns_build_time) + }) + + @classmethod + def setUpClass(cls): + BaseClass.setUpClass(cls.__name__) + Npm.cache_clean() + Emulator.stop() + Simulator.stop() + PerfBuildTests.report_add_column_titles() + + def setUp(self): + Tns.kill() + BaseClass.setUp(self) + + def tearDown(self): + BaseClass.tearDown(self) + Tns.kill() + + @classmethod + def tearDownClass(cls): + BaseClass.tearDownClass() + + @parameterized.expand(DATA) + def test_tns_commands_time(self, demo, config, platform, expected_tns_create_time, + expected_tns_platform_add_time, expected_tns_build_time): + + perf_loop = int(os.getenv('RUN_TIMES', '3')) + tns_create_log = '' + tns_platform_add_log = '' + tns_build_log = '' + actual_tns_create_time = 0.0 + actual_tns_platform_add_time = 0.0 + actual_tns_build_time = 0.0 + for x in range(0, perf_loop): + print "Number of run " + str((x + 1)) + if x > 0: + sleep(60) + if "ios" in platform: + attributes = {"--path": self.app_name, + "--provision": PROVISIONING, + "--release": ""} + else: + attributes = {"--path": self.app_name, + "--keyStorePath": ANDROID_KEYSTORE_PATH, + "--keyStorePassword": ANDROID_KEYSTORE_PASS, + "--keyStoreAlias": ANDROID_KEYSTORE_ALIAS, + "--keyStoreAliasPassword": ANDROID_KEYSTORE_ALIAS_PASS, + "--release": ""} + + if "webpack" in config: + attr = {"--bundle": "", "--env.uglify": "", "--env.aot": ""} + attributes.update(attr) + + if "snapshot" in config: + attr = {"--env.snapshot": ""} + attributes.update(attr) + + if "template" in demo: + Npm.cache_clean() + sleep(10) + tns_create_log = Tns.create_app(self.app_name, attributes={"--template": "https://github.com/" + demo}, + measureTime=True) + Npm.cache_clean() + sleep(20) + if "android" in platform: + tns_platform_add_log = Tns.platform_add_android( + attributes={"--path": self.app_name, "--frameworkPath": ANDROID_PACKAGE}, + measureTime=True) + else: + tns_platform_add_log = Tns.platform_add_ios( + attributes={"--path": self.app_name, "--frameworkPath": IOS_PACKAGE}, + measureTime=True) + sleep(10) + if "-ng" in demo: + Tns.update_angular(self.app_name) + if "-ng" in demo or "-ts" in demo: + Npm.uninstall(package="nativescript-dev-typescript", option='--save-dev', folder=self.app_name) + Npm.install(package=TYPESCRIPT_PACKAGE, option='--save-dev', folder=self.app_name) + if "vue" not in demo: + Tns.install_npm(package=WEBPACK_PACKAGE, option='--save-dev', folder=self.app_name) + sleep(10) + else: + Npm.cache_clean() + sleep(10) + Folder.cleanup(self.app_name) + Git.clone_repo(repo_url="https://github.com/" + demo, local_folder=self.app_name) + if "android" in platform: + Tns.platform_remove(platform=Platform.ANDROID, attributes={"--path": self.app_name}, + assert_success=False) + sleep(5) + Npm.cache_clean() + sleep(10) + tns_platform_add_log = Tns.platform_add_android( + attributes={"--path": self.app_name, "--frameworkPath": ANDROID_PACKAGE}, + measureTime=True) + sleep(20) + else: + Tns.platform_remove(platform=Platform.IOS, attributes={"--path": self.app_name}, + assert_success=False) + sleep(5) + Npm.cache_clean() + sleep(10) + tns_platform_add_log = Tns.platform_add_ios( + attributes={"--path": self.app_name, "--frameworkPath": IOS_PACKAGE}, + measureTime=True) + sleep(20) + Tns.update_modules(self.app_name) + Npm.uninstall(package="nativescript-dev-webpack", option='--save-dev', folder=self.app_name) + Tns.install_npm(package=WEBPACK_PACKAGE, option='--save-dev', folder=self.app_name) + + json = str(TnsAsserts.get_package_json(self.app_name)) + if "angular" in json: + Tns.update_angular(self.app_name) + if "typescript" in json: + Npm.uninstall(package="nativescript-dev-typescript", option='--save-dev', folder=self.app_name) + Npm.install(package=TYPESCRIPT_PACKAGE, option='--save-dev', folder=self.app_name) + sleep(10) + if "timeline" in config: + package_json = os.path.join(self.app_name, 'app', 'package.json') + File.replace(package_json, "\"main\": \"main.js\"", "\"main\": \"main.js\",\"profiling\": \"timeline\"") + + if "android" in platform: + tns_build_log = Tns.build_android(attributes=attributes, measureTime=True) + else: + tns_build_log = Tns.build_ios(attributes=attributes, measureTime=True) + + actual_tns_create_time += PerfBuildTests.get_time(tns_create_log) + actual_tns_platform_add_time += PerfBuildTests.get_time(tns_platform_add_log) + actual_tns_build_time += PerfBuildTests.get_time(tns_build_log) + run('rm -rf {0}'.format(self.app_name)) + + actual_tns_create_time = actual_tns_create_time / perf_loop + actual_tns_platform_add_time = actual_tns_platform_add_time / perf_loop + actual_tns_build_time = actual_tns_build_time / perf_loop + verification_errors = [] + PerfBuildTests.report_add_data(demo, config, platform, expected_tns_create_time, actual_tns_create_time, + expected_tns_platform_add_time, + actual_tns_platform_add_time, expected_tns_build_time, actual_tns_build_time) + + message = "Tns create project command for platform {1} for {0} with {4} configuration is {3} s. " \ + "The expected time is {2} s.".format(demo, + platform, expected_tns_create_time, + actual_tns_create_time, config) + + verification_errors = PerfBuildTests.assert_time(expected=expected_tns_create_time, + actual=actual_tns_create_time, + tolerance=20, + error_message=message, verification_errors=verification_errors) + + message = "Tns platform add command for platform {1} for {0} with {4} configuration is {3} s. " \ + "The expected time is {2} s.".format(demo, platform, expected_tns_platform_add_time, + actual_tns_platform_add_time, config) + + verification_errors = PerfBuildTests.assert_time(expected=expected_tns_platform_add_time, + actual=actual_tns_platform_add_time, + tolerance=20, + error_message=message, verification_errors=verification_errors) + message = "Tns build command for platform {1} for {0} with {4} configuration is {3} s. " \ + "The expected time is {2} s.".format(demo, platform, expected_tns_build_time, + actual_tns_build_time, config) + + verification_errors = PerfBuildTests.assert_time(expected=expected_tns_build_time, actual=actual_tns_build_time, + tolerance=20, + error_message=message, verification_errors=verification_errors) + + self.assertEqual([], verification_errors) diff --git a/tests/perf/cli_operations_time_values.csv b/tests/perf/cli_operations_time_values.csv new file mode 100644 index 00000000..f24cb21d --- /dev/null +++ b/tests/perf/cli_operations_time_values.csv @@ -0,0 +1,25 @@ +Repo,Config,Platform,FirstStart,SecondStart +NativeScript/template-hello-world,default,android,45,24,105 +NativeScript/template-hello-world,webpack,android,38,25,76 +NativeScript/template-hello-world,webpack-snapshot,android,36,25,130 +NativeScript/template-hello-world-ng,default,android,1,1,1 +NativeScript/template-hello-world-ng,webpack,android,1,1,1 +NativeScript/template-hello-world-ng,webpack-snapshot,android,1,1,1 +nativescript-vue/nativescript-vue-template,default,android,1,1,1 +NativeScript/nativescript-sdk-examples-ng,default,android,1,1,1 +NativeScript/nativescript-sdk-examples-ng,webpack,android,1,1,1 +NativeScript/nativescript-sdk-examples-ng,webpack-snapshot,android,1,1,1 +NativeScript/sample-Groceries,default,android,1,1,1 +NativeScript/sample-Groceries,webpack,android,1,1,1 +NativeScript/sample-Groceries,webpack-snapshot,android,1,1,1 +NativeScript/sample-Groceries,timeline,android,1,1,1 +NativeScript/template-hello-world,default,ios,50,13,57 +NativeScript/template-hello-world,webpack,ios,51,12,85 +NativeScript/template-hello-world-ng,default,ios,1,1,1 +NativeScript/template-hello-world-ng,webpack,ios,1,1,1 +nativescript-vue/nativescript-vue-template,default,ios,1,1,1 +NativeScript/nativescript-sdk-examples-ng,default,ios,1,1,1 +NativeScript/nativescript-sdk-examples-ng,webpack,ios,1,1,1 +NativeScript/sample-Groceries,default,ios,1,1,1 +NativeScript/sample-Groceries,webpack,ios,1,1,1 +NativeScript/sample-Groceries,timeline,ios,1,1,1 \ No newline at end of file