From 6ebc2fe0470b91785f1f6e4a33c32404db36face Mon Sep 17 00:00:00 2001 From: Kyriakos Psarakis Date: Wed, 26 May 2021 01:12:28 +0300 Subject: [PATCH 1/7] Create .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fdadc78 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea +__pycache__ +consistency-test/tmp \ No newline at end of file From 0524e2cf9cf4d561dcfdeaed1fd7338b60d22877 Mon Sep 17 00:00:00 2001 From: Kyriakos Psarakis Date: Wed, 26 May 2021 01:13:04 +0300 Subject: [PATCH 2/7] update stress test to the new specs --- locustfile.py => stress-test/locustfile.py | 256 +++++++++++---------- 1 file changed, 137 insertions(+), 119 deletions(-) rename locustfile.py => stress-test/locustfile.py (52%) diff --git a/locustfile.py b/stress-test/locustfile.py similarity index 52% rename from locustfile.py rename to stress-test/locustfile.py index b014b5b..e24aa8d 100644 --- a/locustfile.py +++ b/stress-test/locustfile.py @@ -1,91 +1,113 @@ +import os.path import random +import json from typing import List -from locust import HttpLocust, TaskSet, TaskSequence, seq_task, between +from locust import HttpUser, SequentialTaskSet, between, task # replace the example urls and ports with the appropriate ones -ORDER_URL = "http://127.0.0.1:5000" -PAYMENT_URL = "http://127.0.0.1:5001" -STOCK_URL = "http://127.0.0.1:5002" -USER_URL = "http://127.0.0.1:5003" - - -def create_item(self): - price = random.randint(1, 10) - response = self.client.post(f"{STOCK_URL}/stock/item/create/{price}", name="/stock/item/create/[price]") - self.item_ids.append(response.json()['item_id']) +with open(os.path.join('..', 'urls.json')) as f: + urls = json.load(f) + ORDER_URL = urls['ORDER_URL'] + PAYMENT_URL = urls['PAYMENT_URL'] + STOCK_URL = urls['STOCK_URL'] + + +def create_item(session): + price = random.uniform(1.0, 10.0) + with session.client.post(f"{STOCK_URL}/stock/item/create/{price}", name="/stock/item/create/[price]", + catch_response=True) as response: + try: + item_id = response.json()['item_id'] + except json.JSONDecodeError: + response.failure("SERVER ERROR") + else: + session.item_ids.append(item_id) -def add_stock(self, item_idx: int): +def add_stock(session, item_idx: int): stock_to_add = random.randint(100, 1000) - self.client.post(f"{STOCK_URL}/stock/add/{self.item_ids[item_idx]}/{stock_to_add}", - name="/stock/add/[item_id]/[number]") + session.client.post(f"{STOCK_URL}/stock/add/{session.item_ids[item_idx]}/{stock_to_add}", + name="/stock/add/[item_id]/[number]") -def create_user(self): - response = self.client.post(f"{USER_URL}/users/create", name="/users/create/") - self.user_id = response.json()['user_id'] +def create_user(session): + with session.client.post(f"{PAYMENT_URL}/payment/create_user", name="/payment/create_user", + catch_response=True) as response: + try: + session.user_id = response.json()['user_id'] + except json.JSONDecodeError: + response.failure("SERVER ERROR") -def add_balance_to_user(self): - balance_to_add = random.randint(10000, 100000) - self.client.post(f"{USER_URL}/users/credit/add/{self.user_id}/{balance_to_add}", - name="/users/credit/add/[user_id]/[amount]") +def add_balance_to_user(session): + balance_to_add: float = random.uniform(10000.0, 100000.0) + session.client.post(f"{PAYMENT_URL}/payment/add_funds/{session.user_id}/{balance_to_add}", + name="/payment/add_funds/[user_id]/[amount]") -def create_order(self): - response = self.client.post(f"{ORDER_URL}/orders/create/{self.user_id}", name="/orders/create/[user_id]") - self.order_id = response.json()['order_id'] +def create_order(session): + with session.client.post(f"{ORDER_URL}/orders/create/{session.user_id}", name="/orders/create/[user_id]", + catch_response=True) as response: + try: + session.order_id = response.json()['order_id'] + except json.JSONDecodeError: + response.failure("SERVER ERROR") -def add_item_to_order(self, item_idx: int): - response = self.client.post(f"{ORDER_URL}/orders/addItem/{self.order_id}/{self.item_ids[item_idx]}", - name="/orders/addItem/[order_id]/[item_id]", catch_response=True) - if 400 <= response.status_code < 500: - response.failure(response.text) - else: - response.success() - - -def remove_item_from_order(self, item_idx: int): - response = self.client.delete(f"{ORDER_URL}/orders/removeItem/{self.order_id}/{self.item_ids[item_idx]}", - name="/orders/removeItem/[order_id]/[item_id]", catch_response=True) - if 400 <= response.status_code < 500: - response.failure(response.text) - else: - response.success() +def add_item_to_order(session, item_idx: int): + with session.client.post(f"{ORDER_URL}/orders/addItem/{session.order_id}/{session.item_ids[item_idx]}", + name="/orders/addItem/[order_id]/[item_id]", catch_response=True) as response: + if 400 <= response.status_code < 500: + response.failure(response.text) + else: + response.success() -def checkout_order(self): - response = self.client.post(f"{ORDER_URL}/orders/checkout/{self.order_id}", name="/orders/checkout/[order_id]", - catch_response=True) - if 400 <= response.status_code < 500: - response.failure(response.text) - else: - response.success() +def remove_item_from_order(session, item_idx: int): + with session.client.delete(f"{ORDER_URL}/orders/removeItem/{session.order_id}/{session.item_ids[item_idx]}", + name="/orders/removeItem/[order_id]/[item_id]", catch_response=True) as response: + if 400 <= response.status_code < 500: + response.failure(response.text) + else: + response.success() -def checkout_order_that_is_supposed_to_fail(self, reason: int): - response = self.client.post(f"{ORDER_URL}/orders/checkout/{self.order_id}", name="/orders/checkout/[order_id]", - catch_response=True) - if 400 <= response.status_code < 500: - response.success() - else: - if reason == 0: - response.failure("This was supposed to fail: Not enough stock") +def checkout_order(session): + with session.client.post(f"{ORDER_URL}/orders/checkout/{session.order_id}", name="/orders/checkout/[order_id]", + catch_response=True) as response: + if 400 <= response.status_code < 500: + response.failure(response.text) else: - response.failure("This was supposed to fail: Not enough credit") + response.success() -def make_items_stock_zero(self, item_idx: int): - stock_to_subtract = self.client.get(f"{STOCK_URL}/stock/find/{self.item_ids[item_idx]}", - name="/stock/find/[item_id]").json()['stock'] - self.client.post(f"{STOCK_URL}/stock/subtract/{self.item_ids[item_idx]}/{stock_to_subtract}", - name="/stock/add/[item_id]/[number]") +def checkout_order_that_is_supposed_to_fail(session, reason: int): + with session.client.post(f"{ORDER_URL}/orders/checkout/{session.order_id}", name="/orders/checkout/[order_id]", + catch_response=True) as response: + if 400 <= response.status_code < 500: + response.success() + else: + if reason == 0: + response.failure("This was supposed to fail: Not enough stock") + else: + response.failure("This was supposed to fail: Not enough credit") + + +def make_items_stock_zero(session, item_idx: int): + with session.client.get(f"{STOCK_URL}/stock/find/{session.item_ids[item_idx]}", name="/stock/find/[item_id]", + catch_response=True) as response: + try: + stock_to_subtract = response.json()['stock'] + except json.JSONDecodeError: + response.failure("SERVER ERROR") + else: + session.client.post(f"{STOCK_URL}/stock/subtract/{session.item_ids[item_idx]}/{stock_to_subtract}", + name="/stock/subtract/[item_id]/[number]") -class LoadTest1(TaskSequence): +class LoadTest1(SequentialTaskSet): """ Scenario where a stock admin creates an item and adds stock to it """ @@ -99,14 +121,14 @@ def on_stop(self): """ on_stop is called when the TaskSet is stopping """ self.item_ids = list() - @seq_task(1) + @task def admin_creates_item(self): create_item(self) - @seq_task(2) + @task def admin_adds_stock_to_item(self): add_stock(self, 0) -class LoadTest2(TaskSequence): +class LoadTest2(SequentialTaskSet): """ Scenario where a user checks out an order with one item inside that an admin has added stock to before """ @@ -124,29 +146,29 @@ def on_stop(self): self.user_id = "" self.order_id = "" - @seq_task(1) + @task def admin_creates_item(self): create_item(self) - @seq_task(2) + @task def admin_adds_stock_to_item(self): add_stock(self, 0) - @seq_task(3) + @task def user_creates_account(self): create_user(self) - @seq_task(4) + @task def user_adds_balance(self): add_balance_to_user(self) - @seq_task(5) + @task def user_creates_order(self): create_order(self) - @seq_task(6) + @task def user_adds_item_to_order(self): add_item_to_order(self, 0) - @seq_task(7) + @task def user_checks_out_order(self): checkout_order(self) -class LoadTest3(TaskSequence): +class LoadTest3(SequentialTaskSet): """ Scenario where a user checks out an order with two items inside that an admin has added stock to before """ @@ -164,38 +186,38 @@ def on_stop(self): self.user_id = "" self.order_id = "" - @seq_task(1) + @task def admin_creates_item1(self): create_item(self) - @seq_task(2) + @task def admin_adds_stock_to_item1(self): add_stock(self, 0) - @seq_task(3) + @task def admin_creates_item2(self): create_item(self) - @seq_task(4) + @task def admin_adds_stock_to_item2(self): add_stock(self, 1) - @seq_task(5) + @task def user_creates_account(self): create_user(self) - @seq_task(6) + @task def user_adds_balance(self): add_balance_to_user(self) - @seq_task(7) + @task def user_creates_order(self): create_order(self) - @seq_task(8) + @task def user_adds_item1_to_order(self): add_item_to_order(self, 0) - @seq_task(9) + @task def user_adds_item2_to_order(self): add_item_to_order(self, 1) - @seq_task(10) + @task def user_checks_out_order(self): checkout_order(self) -class LoadTest4(TaskSequence): +class LoadTest4(SequentialTaskSet): """ Scenario where a user adds an item to an order, regrets it and removes it and then adds it back and checks out """ @@ -213,35 +235,35 @@ def on_stop(self): self.user_id = "" self.order_id = "" - @seq_task(1) + @task def admin_creates_item(self): create_item(self) - @seq_task(2) + @task def admin_adds_stock_to_item(self): add_stock(self, 0) - @seq_task(3) + @task def user_creates_account(self): create_user(self) - @seq_task(4) + @task def user_adds_balance(self): add_balance_to_user(self) - @seq_task(5) + @task def user_creates_order(self): create_order(self) - @seq_task(6) + @task def user_adds_item_to_order(self): add_item_to_order(self, 0) - @seq_task(7) + @task def user_removes_item_from_order(self): remove_item_from_order(self, 0) - @seq_task(8) + @task def user_adds_item_to_order_again(self): add_item_to_order(self, 0) - @seq_task(9) + @task def user_checks_out_order(self): checkout_order(self) -class LoadTest5(TaskSequence): +class LoadTest5(SequentialTaskSet): """ Scenario that is supposed to fail because the second item does not have enough stock """ @@ -259,41 +281,41 @@ def on_stop(self): self.user_id = "" self.order_id = "" - @seq_task(1) + @task def admin_creates_item1(self): create_item(self) - @seq_task(2) + @task def admin_adds_stock_to_item1(self): add_stock(self, 0) - @seq_task(3) + @task def admin_creates_item2(self): create_item(self) - @seq_task(4) + @task def admin_adds_stock_to_item2(self): add_stock(self, 1) - @seq_task(5) + @task def user_creates_account(self): create_user(self) - @seq_task(6) + @task def user_adds_balance(self): add_balance_to_user(self) - @seq_task(7) + @task def user_creates_order(self): create_order(self) - @seq_task(8) + @task def user_adds_item1_to_order(self): add_item_to_order(self, 0) - @seq_task(9) + @task def user_adds_item2_to_order(self): add_item_to_order(self, 1) - @seq_task(10) + @task def stock_admin_makes_item2s_stock_zero(self): make_items_stock_zero(self, 1) - @seq_task(11) + @task def user_checks_out_order(self): checkout_order_that_is_supposed_to_fail(self, 0) -class LoadTest6(TaskSequence): +class LoadTest6(SequentialTaskSet): """ Scenario that is supposed to fail because the user does not have enough credit """ @@ -311,27 +333,28 @@ def on_stop(self): self.user_id = "" self.order_id = "" - @seq_task(1) + @task def admin_creates_item(self): create_item(self) - @seq_task(2) + @task def admin_adds_stock_to_item(self): add_stock(self, 0) - @seq_task(3) + @task def user_creates_account(self): create_user(self) - @seq_task(4) + @task def user_creates_order(self): create_order(self) - @seq_task(5) + @task def user_adds_item_to_order(self): add_item_to_order(self, 0) - @seq_task(6) + @task def user_checks_out_order(self): checkout_order_that_is_supposed_to_fail(self, 1) -class LoadTests(TaskSet): - # [TaskSequence]: [weight of the TaskSequence] +class MicroservicesUser(HttpUser): + wait_time = between(1, 3) # how much time a user waits (seconds) to run another TaskSequence + # [SequentialTaskSet]: [weight of the SequentialTaskSet] tasks = { LoadTest1: 5, LoadTest2: 30, @@ -340,8 +363,3 @@ class LoadTests(TaskSet): LoadTest5: 10, LoadTest6: 10 } - - -class MicroservicesUser(HttpLocust): - task_set = LoadTests - wait_time = between(1, 15) # how much time a user waits (seconds) to run another TaskSequence From ab64743982d57a98504f5c448be3523e0c99a7fc Mon Sep 17 00:00:00 2001 From: Kyriakos Psarakis Date: Wed, 26 May 2021 01:13:07 +0300 Subject: [PATCH 3/7] Create requirements.txt --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..01dbae8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +locust==1.5.3 +requests==2.25.1 \ No newline at end of file From e16000c77363263c2d1f8a570b8a354fffa605fc Mon Sep 17 00:00:00 2001 From: Kyriakos Psarakis Date: Wed, 26 May 2021 01:13:38 +0300 Subject: [PATCH 4/7] update the consistency test to the new specs and make it public --- consistency-test/locustfile.py | 91 ++++++++++++++++++++++++ consistency-test/populate.py | 63 ++++++++++++++++ consistency-test/run_consistency_test.py | 32 +++++++++ consistency-test/verify.py | 91 ++++++++++++++++++++++++ 4 files changed, 277 insertions(+) create mode 100644 consistency-test/locustfile.py create mode 100644 consistency-test/populate.py create mode 100644 consistency-test/run_consistency_test.py create mode 100644 consistency-test/verify.py diff --git a/consistency-test/locustfile.py b/consistency-test/locustfile.py new file mode 100644 index 0000000..c287b38 --- /dev/null +++ b/consistency-test/locustfile.py @@ -0,0 +1,91 @@ +import pickle +import random +import logging +import json +import os +from typing import List, Union + +from locust import HttpUser, SequentialTaskSet, task, constant +from locust.exception import StopUser + +with open(os.path.join('..', 'urls.json')) as f: + urls = json.load(f) + ORDER_URL = urls['ORDER_URL'] + PAYMENT_URL = urls['PAYMENT_URL'] + STOCK_URL = urls['STOCK_URL'] + + +def create_order(session): + with session.client.post(f"{ORDER_URL}/orders/create/{session.user_id}", json={}, name="/orders/create/[user_id]", + catch_response=True) as response: + try: + session.order_id = response.json()['order_id'] + except json.JSONDecodeError: + response.failure("SERVER ERROR") + + +def add_item_to_order(session, item_idx: int): + with session.client.post(f"{ORDER_URL}/orders/addItem/{session.order_id}/{session.item_ids[item_idx]}", + name="/orders/addItem/[order_id]/[item_id]", json={}, catch_response=True) as response: + if 400 <= response.status_code < 500: + response.failure(response.text) + raise StopUser() + else: + response.success() + + +def checkout_order(session): + with session.client.post(f"{ORDER_URL}/orders/checkout/{session.order_id}", json={}, + name="/orders/checkout/[order_id]", catch_response=True) as response: + if 400 <= response.status_code < 500: + logging.info(f"CHECKOUT | ORDER: {session.order_id} USER: {session.user_id} FAIL __OUR_LOG__") + response.failure(response.text) + else: + logging.info(f"CHECKOUT | ORDER: {session.order_id} USER: {session.user_id} SUCCESS __OUR_LOG__") + response.success() + + +def load_pickle_file(file_name: str) -> Union[List[str], str]: + with open(file_name, 'rb') as pkl_file: + var = pickle.load(pkl_file) + return var + + +class ConsistencyTest(SequentialTaskSet): + """ + Scenario where a user checks out an order with one item inside that an admin has added stock to before + """ + + order_id: str + user_id: str + user_ids: List[str] + item_ids: List[str] + + def __init__(self, parent): + super().__init__(parent) + self.local_random = random.Random() + self.user_ids = load_pickle_file('tmp/user_ids.pkl') + tmp_item_ids = load_pickle_file('tmp/item_ids.pkl') + self.item_ids = tmp_item_ids if type(tmp_item_ids) is list else [str(tmp_item_ids)] + + def on_start(self): + self.user_id = str(self.local_random.choice(self.user_ids)) + self.order_id = "" + + def on_stop(self): + self.user_id = str(self.local_random.choice(self.user_ids)) + self.order_id = "" + + @task + def user_creates_order(self): create_order(self) + + @task + def user_adds_item_to_order(self): add_item_to_order(self, 0) + + @task + def user_checks_out_order(self): checkout_order(self) + + +class MicroservicesUser(HttpUser): + tasks = {ConsistencyTest: 100} + wait_time = constant(1) # how much time a user waits (seconds) to run another SequentialTaskSet diff --git a/consistency-test/populate.py b/consistency-test/populate.py new file mode 100644 index 0000000..e6e16b1 --- /dev/null +++ b/consistency-test/populate.py @@ -0,0 +1,63 @@ +import pickle +import json +import logging +import os +from typing import Union, List + +import requests +from multiprocessing.pool import ThreadPool +from itertools import repeat + +logging.basicConfig(level=logging.INFO, + format='%(levelname)s - %(asctime)s - %(name)s - %(message)s', + datefmt='%I:%M:%S') +logger = logging.getLogger(__name__) +NUMBER_0F_ITEMS = 100 +NUMBER_OF_USERS = 1000 + +with open(os.path.join('..', 'urls.json')) as f: + urls = json.load(f) + ORDER_URL = urls['ORDER_URL'] + PAYMENT_URL = urls['PAYMENT_URL'] + STOCK_URL = urls['STOCK_URL'] + + +def create_user_offline(balance: int) -> str: + user_id = requests.post(f"{PAYMENT_URL}/payment/create_user", json={}).json()['user_id'] + requests.post(f"{PAYMENT_URL}/payment/add_funds/{user_id}/{balance}", json={}) + return str(user_id) + + +def create_item_offline(stock_to_add: int, price: int = 1) -> str: + __item_id = requests.post(f"{STOCK_URL}/stock/item/create/{price}", json={}).json()['item_id'] + requests.post(f"{STOCK_URL}/stock/add/{__item_id}/{stock_to_add}", json={}) + return str(__item_id) + + +def create_items_offline(number_of_items: int, stock: int = 1) -> List[str]: + with ThreadPool(10) as pool: + __item_ids = list(pool.map(create_item_offline, repeat(stock, number_of_items))) + return __item_ids + + +def create_users_offline(number_of_users: int, credit: int = 1) -> List[str]: + with ThreadPool(10) as pool: + __user_ids = list(pool.map(create_user_offline, repeat(credit, number_of_users))) + return __user_ids + + +def write_pickle(file_name: str, var: Union[List[str], str]): + with open(file_name, 'wb') as output: + pickle.dump(var, output, pickle.HIGHEST_PROTOCOL) + + +def populate_databases(): + logger.info("Creating items ...") + item_id = create_item_offline(NUMBER_0F_ITEMS) # create item with 100 stock + write_pickle('tmp/item_ids.pkl', item_id) + logger.info("Items created") + + logger.info("Creating users ...") + user_ids = create_users_offline(NUMBER_OF_USERS) # create 1000 users + write_pickle('tmp/user_ids.pkl', user_ids) + logger.info("Users created") diff --git a/consistency-test/run_consistency_test.py b/consistency-test/run_consistency_test.py new file mode 100644 index 0000000..15ae6e1 --- /dev/null +++ b/consistency-test/run_consistency_test.py @@ -0,0 +1,32 @@ +import os +import shutil +import subprocess +import logging + +from verify import verify_systems_consistency +from populate import populate_databases + +logging.basicConfig(level=logging.INFO, + format='%(levelname)s - %(asctime)s - %(name)s - %(message)s', + datefmt='%I:%M:%S') +logger = logging.getLogger("Consistency test") + +logger.info("Creating tmp folder...") +current_file_directory: str = os.path.dirname(os.path.realpath(__file__)) +tmp_folder_path: str = os.path.join(current_file_directory, 'tmp') +tmp_folder_exists: bool = os.path.isdir(tmp_folder_path) + +if tmp_folder_exists: + shutil.rmtree(tmp_folder_path) +os.mkdir(tmp_folder_path) +logger.info("tmp folder created") +logger.info("Populating the databases...") +populate_databases() +logger.info("Databases populated") +logger.info("Starting the load test...") +subprocess.call(["locust", "-f", "locustfile.py", "--host=''", "--logfile=tmp/consistency-test.log", "--headless", + "-u", "1000", "-r", "1000", "--run-time=15s"], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) +logger.info("Load test completed") +logger.info("Starting the consistency evaluation...") +verify_systems_consistency() +logger.info("Consistency evaluation completed") diff --git a/consistency-test/verify.py b/consistency-test/verify.py new file mode 100644 index 0000000..e350a5a --- /dev/null +++ b/consistency-test/verify.py @@ -0,0 +1,91 @@ +import pickle +import re +import os +import json +import requests +import logging +from multiprocessing.pool import ThreadPool +from typing import Union, List, Dict, Tuple + +from populate import NUMBER_0F_ITEMS + +CORRECT_USER_STATE = 900 + +logging.basicConfig(level=logging.INFO, + format='%(levelname)s - %(asctime)s - %(name)s - %(message)s', + datefmt='%I:%M:%S') +logger = logging.getLogger(__name__) + +with open(os.path.join('..', 'urls.json')) as f: + urls = json.load(f) + ORDER_URL = urls['ORDER_URL'] + PAYMENT_URL = urls['PAYMENT_URL'] + STOCK_URL = urls['STOCK_URL'] + + +def load_pickle_file(file_name: str) -> Union[List[str], str]: + with open(file_name, 'rb') as pkl_file: + var = pickle.load(pkl_file) + return var + + +def get_user_credit(user_id: str) -> Tuple[str, int]: + credit = int(requests.get(f"{PAYMENT_URL}/payment/find_user/{user_id}", json={}).json()['credit']) + return user_id, credit + + +def get_user_credit_dict(user_id_list: List[str]) -> Dict[str, int]: + with ThreadPool(10) as pool: + user_id_credit = dict(pool.map(get_user_credit, user_id_list)) + return user_id_credit + + +def get_item_stock(item_id: str) -> Tuple[str, int]: + stock = int(requests.get(f"{STOCK_URL}/stock/find/{item_id}", json={}).json()['stock']) + return item_id, stock + + +def get_item_stock_dict(item_id_list: Union[List[str], str]) -> Dict[str, int]: + if type(item_id_list) is list: + with ThreadPool(10) as pool: + item_id_stock = dict(pool.map(get_item_stock, item_id_list)) + else: + item_id_stock = dict([get_item_stock(item_id_list)]) + return item_id_stock + + +def get_prior_user_state(): + user_state = dict() + for user_id in load_pickle_file('tmp/user_ids.pkl'): + user_state[str(user_id)] = 1 + return user_state + + +def parse_log(prior_user_state: Dict[str, int]): + i = 0 + with open('tmp/consistency-test.log', 'r') as log_file: + log_file = log_file.readlines() + for log in log_file: + if log.endswith('__OUR_LOG__\n'): + m = re.search('ORDER: (.*) USER: (.*) (.*) __OUR_LOG__', log) + user_id = str(m.group(2)) + status = m.group(3) + if status == 'SUCCESS': + i += 1 + if prior_user_state[user_id] == 0: + logger.info("NEGATIVE") + prior_user_state[user_id] = prior_user_state[user_id] - 1 + logger.info(f"Stock service inconsistencies in the logs: {i - NUMBER_0F_ITEMS}") + return prior_user_state + + +def verify_systems_consistency(): + pus: dict = parse_log(get_prior_user_state()) + uic: dict = get_user_credit_dict(load_pickle_file('tmp/user_ids.pkl')) + iis: dict = get_item_stock_dict(load_pickle_file('tmp/item_ids.pkl')) + server_side_items_bought: int = 100 - list(iis.values())[0] + logger.info(f"Stock service inconsistencies in the database: {server_side_items_bought - NUMBER_0F_ITEMS}") + logged_user_credit: int = sum(pus.values()) + logger.info(f"User service inconsistencies in the logs: {abs(CORRECT_USER_STATE - logged_user_credit)}") + server_side_user_credit: int = sum(list(uic.values())) + logger.info(f"User service inconsistencies in the database: {abs(CORRECT_USER_STATE - server_side_user_credit)}") From 94fe0b70680e9e9c648e6a0217e66ceebbb8d92b Mon Sep 17 00:00:00 2001 From: Kyriakos Psarakis Date: Wed, 26 May 2021 01:14:11 +0300 Subject: [PATCH 5/7] add a centralized service URLs file --- urls.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 urls.json diff --git a/urls.json b/urls.json new file mode 100644 index 0000000..7d645bc --- /dev/null +++ b/urls.json @@ -0,0 +1,5 @@ +{ + "ORDER_URL" : "http://127.0.0.1:5000", + "PAYMENT_URL" : "http://127.0.0.1:5001", + "STOCK_URL" : "http://127.0.0.1:5002" +} \ No newline at end of file From 7a6dbf619113aa1f1336e8a468dad1e3481da4fa Mon Sep 17 00:00:00 2001 From: Kyriakos Psarakis Date: Wed, 26 May 2021 18:11:29 +0300 Subject: [PATCH 6/7] Update README.md --- README.md | 72 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b70fd3e..e15ba5b 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,65 @@ # Instructions -## Running locust -* Install python 3.6 or greater -* Install locust using either: - * pip: `pip install locustio==0.14.6` - * conda: `conda install locust` -* Change the urls and ports in lines 8-11 in `locustfile.py` to correspond to your own + +## Setup +* Install python 3.6 or greater (tested with 3.9) +* Install the required packages using: `pip install -r requirements.txt` +* Change the URLs and ports in the `urls.json` file with your own + +```` +Note: For Windows users you might also need to install pywin32 +```` + +## Stress Test + +In the provided stress test we have created 6 scenarios: + +1) A stock admin creates an item and adds stock to it + +2) A user checks out an order with one item inside that an admin has added stock to before + +3) A user checks out an order with two items inside that an admin has added stock to before + +4) A user adds an item to an order, regrets it and removes it and then adds it back and checks out + +5) Scenario that is supposed to fail because the second item does not have enough stock + +6) Scenario that is supposed to fail because the user does not have enough credit + +To change the weight (task frequency) of the provided scenarios you can change the weights in the `tasks` definition (line 358) +With our locust file each user will make one request between 1 and 15 seconds (you can change that in line 356). + +``` +YOU CAN ALSO CREATE YOUR OWN SCENARIOS AS YOU LIKE +``` + +### Running * Open terminal and navigate to the `locustfile.py` folder -* Run script: `locust -f locustfile.py --host=""` +* Run script: `locust -f locustfile.py --host="localhost"` * Go to `http://localhost:8089/` -## Using the Locust UI + + +### Using the Locust UI Fill in an appropriate number of users that you want to test with. -With our locust file each user will make one request between 1 and 15 seconds (you can change that in line 347). -The hatch rate is how many users will spawn per second (locust suggests that you should use less than 100). -Leave the host field empty it will automatically find the right one (it corresponds to the locust master host). \ No newline at end of file +The hatch rate is how many users will spawn per second +(locust suggests that you should use less than 100 in local mode). + + +## Consistency Test + +In the provided consistency test we first populate the databases with 100 items with 1 stock that costs 1 credit +and 1000 users that have 1 credit. + +Then we concurrently send 1000 checkouts of 1 item with random user/item combinations. +The probabilities say that only 10% of the checkouts will succeed, and the expected state should be 0 stock across all +items and 100 credits subtracted from different users. + +Finally, the measurements are done in two phases: +1) Using logs to see whether the service sent the correct message to the clients +2) Querying the database to see if the actual state remained consistent + +### Running +* Run script `run_consistency_test.py` + +### Interpreting Results + +Wait for the script to finish and check how many inconsistencies you have in both the payment and stock services \ No newline at end of file From 9be45ddcba8ccb8ee23f2381042a95ee80235690 Mon Sep 17 00:00:00 2001 From: Kyriakos Psarakis Date: Wed, 26 May 2021 18:11:50 +0300 Subject: [PATCH 7/7] add minimal docs --- consistency-test/run_consistency_test.py | 13 ++++++++++++- consistency-test/verify.py | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/consistency-test/run_consistency_test.py b/consistency-test/run_consistency_test.py index 15ae6e1..a14559e 100644 --- a/consistency-test/run_consistency_test.py +++ b/consistency-test/run_consistency_test.py @@ -6,11 +6,15 @@ from verify import verify_systems_consistency from populate import populate_databases +STRESS_TEST_EXECUTION_TIME = 30 # Seconds + logging.basicConfig(level=logging.INFO, format='%(levelname)s - %(asctime)s - %(name)s - %(message)s', datefmt='%I:%M:%S') logger = logging.getLogger("Consistency test") + +# Create the tmp folder to store the logs, the users and the stock logger.info("Creating tmp folder...") current_file_directory: str = os.path.dirname(os.path.realpath(__file__)) tmp_folder_path: str = os.path.join(current_file_directory, 'tmp') @@ -20,13 +24,20 @@ shutil.rmtree(tmp_folder_path) os.mkdir(tmp_folder_path) logger.info("tmp folder created") + +# Populate the payment and stock databases logger.info("Populating the databases...") populate_databases() logger.info("Databases populated") + +# Run the load test logger.info("Starting the load test...") subprocess.call(["locust", "-f", "locustfile.py", "--host=''", "--logfile=tmp/consistency-test.log", "--headless", - "-u", "1000", "-r", "1000", "--run-time=15s"], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) + "-u", "1000", "-r", "1000", f"--run-time={STRESS_TEST_EXECUTION_TIME}s"], + stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) logger.info("Load test completed") + +# Verify the systems' consistency logger.info("Starting the consistency evaluation...") verify_systems_consistency() logger.info("Consistency evaluation completed") diff --git a/consistency-test/verify.py b/consistency-test/verify.py index e350a5a..f03f21c 100644 --- a/consistency-test/verify.py +++ b/consistency-test/verify.py @@ -86,6 +86,6 @@ def verify_systems_consistency(): server_side_items_bought: int = 100 - list(iis.values())[0] logger.info(f"Stock service inconsistencies in the database: {server_side_items_bought - NUMBER_0F_ITEMS}") logged_user_credit: int = sum(pus.values()) - logger.info(f"User service inconsistencies in the logs: {abs(CORRECT_USER_STATE - logged_user_credit)}") + logger.info(f"Payment service inconsistencies in the logs: {abs(CORRECT_USER_STATE - logged_user_credit)}") server_side_user_credit: int = sum(list(uic.values())) - logger.info(f"User service inconsistencies in the database: {abs(CORRECT_USER_STATE - server_side_user_credit)}") + logger.info(f"Payment service inconsistencies in the database: {abs(CORRECT_USER_STATE - server_side_user_credit)}")