diff --git a/frameworks/Java/spring-webflux/benchmark_config.json b/frameworks/Java/spring-webflux/benchmark_config.json index f7c9be500ac..e5c13f8e292 100644 --- a/frameworks/Java/spring-webflux/benchmark_config.json +++ b/frameworks/Java/spring-webflux/benchmark_config.json @@ -41,7 +41,8 @@ "database_os": "Linux", "display_name": "spring-webflux-mongo", "notes": "", - "versus": "spring" + "versus": "spring", + "tags": ["broken"] }, "pgclient": { "db_url": "/db", @@ -62,7 +63,8 @@ "database_os": "Linux", "display_name": "spring-webflux-pgclient", "notes": "", - "versus": "spring" + "versus": "spring", + "tags": ["broken"] }, "rxjdbc": { "db_url": "/db", @@ -83,7 +85,8 @@ "database_os": "Linux", "display_name": "spring-webflux-rxjdbc", "notes": "", - "versus": "spring" + "versus": "spring", + "tags": ["broken"] }, "jdbc": { "db_url": "/db", @@ -104,7 +107,8 @@ "database_os": "Linux", "display_name": "spring-webflux-jdbc", "notes": "", - "versus": "spring" + "versus": "spring", + "tags": ["broken"] } }] } diff --git a/frameworks/JavaScript/sailsjs/benchmark_config.json b/frameworks/JavaScript/sailsjs/benchmark_config.json index f2c048ac796..aebfb039b3f 100644 --- a/frameworks/JavaScript/sailsjs/benchmark_config.json +++ b/frameworks/JavaScript/sailsjs/benchmark_config.json @@ -22,7 +22,8 @@ "database_os": "Linux", "display_name": "Sails.js", "notes": "", - "versus": "nodejs" + "versus": "nodejs", + "tags": ["broken"] }, "postgres": { "db_url": "/postgres/db", diff --git a/frameworks/Kotlin/vertx-web-kotlin-coroutines/benchmark_config.json b/frameworks/Kotlin/vertx-web-kotlin-coroutines/benchmark_config.json index 2b8ebd26c83..d2bab14abb7 100644 --- a/frameworks/Kotlin/vertx-web-kotlin-coroutines/benchmark_config.json +++ b/frameworks/Kotlin/vertx-web-kotlin-coroutines/benchmark_config.json @@ -18,7 +18,8 @@ "database_os": "Linux", "display_name": "vertx-web-kotlin-coroutines", "notes": "", - "versus": "vertx-web" + "versus": "vertx-web", + "tags": ["broken"] }, "postgres": { "db_url": "/db", diff --git a/frameworks/PHP/php/benchmark_config.json b/frameworks/PHP/php/benchmark_config.json index e91e5e74fc8..33391560248 100644 --- a/frameworks/PHP/php/benchmark_config.json +++ b/frameworks/PHP/php/benchmark_config.json @@ -112,7 +112,8 @@ "database_os": "Linux", "display_name": "PHP Unit Nginx", "notes": "", - "versus": "php" + "versus": "php", + "tags": ["broken"] }, "pools": { "json_url": "/json.php", diff --git a/frameworks/Python/cherrypy/app.py b/frameworks/Python/cherrypy/app.py index 8fb12ef361d..bf090cfd69b 100755 --- a/frameworks/Python/cherrypy/app.py +++ b/frameworks/Python/cherrypy/app.py @@ -154,6 +154,8 @@ def fortunes(self): cherrypy.server.socket_host = '0.0.0.0' cherrypy.server.socket_port = 8080 cherrypy.server.thread_pool = workers + cherrypy.server.socket_queue_size = 25 + cherrypy.quickstart(CherryPyBenchmark(), '', { '/': { 'tools.db.on': True, diff --git a/frameworks/Python/cherrypy/benchmark_config.json b/frameworks/Python/cherrypy/benchmark_config.json index c0df9e76973..62e51834960 100644 --- a/frameworks/Python/cherrypy/benchmark_config.json +++ b/frameworks/Python/cherrypy/benchmark_config.json @@ -21,7 +21,8 @@ "os": "Linux", "database_os": "Linux", "display_name": "CherryPy [py2]", - "notes": "CPython 2.7" + "notes": "CPython 2.7", + "tags": ["broken"] }, "py3": { "json_url": "/json", diff --git a/frameworks/Python/fastapi/benchmark_config.json b/frameworks/Python/fastapi/benchmark_config.json index 56b961ba0e8..6fbceb1f0ae 100755 --- a/frameworks/Python/fastapi/benchmark_config.json +++ b/frameworks/Python/fastapi/benchmark_config.json @@ -169,7 +169,8 @@ "database_os": "Linux", "display_name": "FastAPI-nginx-unit", "notes": "", - "versus": "None" + "versus": "None", + "tags": ["broken"] }, "nginx-unit-orjson": { "json_url": "/json", @@ -190,7 +191,8 @@ "database_os": "Linux", "display_name": "FastAPI-nginx-unit-orjson", "notes": "", - "versus": "None" + "versus": "None", + "tags": ["broken"] }, "uvicorn": { "json_url": "/json", diff --git a/frameworks/Python/sanic/benchmark_config.json b/frameworks/Python/sanic/benchmark_config.json index b3c6799b42b..b1bdd529d60 100644 --- a/frameworks/Python/sanic/benchmark_config.json +++ b/frameworks/Python/sanic/benchmark_config.json @@ -23,7 +23,8 @@ "database_os": "Linux", "display_name": "Sanic", "notes": "", - "versus": "None" + "versus": "None", + "tags": ["broken"] } } ] diff --git a/frameworks/Rust/rocket/benchmark_config.json b/frameworks/Rust/rocket/benchmark_config.json index 2ffa525c65e..e4643fe6c08 100755 --- a/frameworks/Rust/rocket/benchmark_config.json +++ b/frameworks/Rust/rocket/benchmark_config.json @@ -22,7 +22,8 @@ "database_os": "Linux", "display_name": "Rocket", "notes": "", - "versus": "None" + "versus": "None", + "tags": ["broken"] }, "diesel": { "json_url": "/json", @@ -45,7 +46,8 @@ "database_os": "Linux", "display_name": "Rocket (Diesel)", "notes": "", - "versus": "None" + "versus": "None", + "tags": ["broken"] } }] } diff --git a/frameworks/Scala/snunit/benchmark_config.json b/frameworks/Scala/snunit/benchmark_config.json index f46cf29e0d8..7711ab00316 100755 --- a/frameworks/Scala/snunit/benchmark_config.json +++ b/frameworks/Scala/snunit/benchmark_config.json @@ -19,7 +19,8 @@ "database_os": "Linux", "display_name": "SNUnit", "notes": "", - "versus": "" + "versus": "", + "tags": ["broken"] } } ] diff --git a/scripts/tfb-fail-detector.py b/scripts/tfb-fail-detector.py new file mode 100644 index 00000000000..9eb217ba4fc --- /dev/null +++ b/scripts/tfb-fail-detector.py @@ -0,0 +1,78 @@ +""" +TFB Fail Detector Web Scraper +Scrapes data from the last 3 complete runs and outputs the frameworks that have failed all of them with the same errors. + +Developed using selenium==4.10.0 and the selenium/standalone-chrome:4.10.0 docker image. + +USAGE: python tfb-fail-detector.py +""" + +from collections import defaultdict +import re + +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By + +NUM_RUNS = 3 + +def get_driver(): + """Gets the selenium webdriver for interacting with the website""" + chrome_options = Options() + chrome_options.add_argument("--headless") + chrome_options.add_argument("--no-sandbox") + chrome_options.add_argument("--disable-dev-shm-usage") + chrome_prefs = {} + chrome_options.experimental_options["prefs"] = chrome_prefs + chrome_prefs["profile.default_content_settings"] = {"images": 2} + driver = webdriver.Chrome(options=chrome_options) + return driver + +def get_last_n_complete_runs(driver, n=3): + """Scrape the last n complete runs""" + rows = driver.find_elements(By.CSS_SELECTOR, "table.resultsTable > tbody > tr") + complete_runs = [] + for row in rows: + row_uuid = row.get_dom_attribute("data-uuid") + run_stats = row.find_element(By.CSS_SELECTOR, "td:nth-of-type(2)") + frameworks_tested_stats = re.search(r"([0-9]+)/([0-9]+) frameworks tested", run_stats.text) + if not frameworks_tested_stats: + # print("Unable to get info from run %s" % row_uuid) + continue + tested, total = frameworks_tested_stats.groups() + if tested != total: + # print("Found incomplete run %s. Tested: %s/%s" % (row_uuid, tested, total)) + continue + # print("Found complete run %s. Tested: %s/%s" % (row_uuid, tested, total)) + complete_runs.append(row_uuid) + if len(complete_runs) >= n: + return complete_runs + +def find_failing_frameworks(driver, run_uuids): + """Find frameworks that have failed all the given runs with the same error message""" + failing_frameworks = defaultdict(lambda: 0) + def process_failed_framework(framework_element): + framework = re.search(r"\[\S+] [a-zA-Z][a-zA-Z:\_ ]*$", framework_element.text) + framework_failure = framework.group(0) + failing_frameworks[framework_failure] += 1 + + for run_uuid in run_uuids: + driver.get("https://tfb-status.techempower.com/results/%s" % run_uuid) + assert "TFB Status" in driver.title + failure_list = driver.find_element(By.CLASS_NAME, "failures") + failures = failure_list.find_elements(By.TAG_NAME, "li") + for failure in failures: + process_failed_framework(failure) + return failing_frameworks + +if __name__ == '__main__': + driver = get_driver() + driver.get("https://tfb-status.techempower.com/") + assert "TFB Status" in driver.title + complete_runs = get_last_n_complete_runs(driver, NUM_RUNS) + failed_frameworks = find_failing_frameworks(driver, complete_runs) + + for failure_info, fail_count in failed_frameworks.items(): + if fail_count != NUM_RUNS: + continue + print(failure_info)