From ae98bce7487b4386bbe016d539c9613e75b7a2fa Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 28 May 2021 23:49:36 +0800 Subject: [PATCH 1/6] add scorper (#6608) --- frameworks/Nim/scorper/.gitignore | 7 +++++ frameworks/Nim/scorper/README.md | 20 +++++++++++++++ frameworks/Nim/scorper/benchmark_config.json | 26 +++++++++++++++++++ frameworks/Nim/scorper/config.nims | 0 frameworks/Nim/scorper/config.toml | 15 +++++++++++ frameworks/Nim/scorper/scorper.dockerfile | 13 ++++++++++ frameworks/Nim/scorper/scorper_bench.nim | 27 ++++++++++++++++++++ frameworks/Nim/scorper/scorper_bench.nimble | 15 +++++++++++ 8 files changed, 123 insertions(+) create mode 100644 frameworks/Nim/scorper/.gitignore create mode 100755 frameworks/Nim/scorper/README.md create mode 100755 frameworks/Nim/scorper/benchmark_config.json create mode 100644 frameworks/Nim/scorper/config.nims create mode 100644 frameworks/Nim/scorper/config.toml create mode 100644 frameworks/Nim/scorper/scorper.dockerfile create mode 100644 frameworks/Nim/scorper/scorper_bench.nim create mode 100644 frameworks/Nim/scorper/scorper_bench.nimble diff --git a/frameworks/Nim/scorper/.gitignore b/frameworks/Nim/scorper/.gitignore new file mode 100644 index 00000000000..452dc2c4a4a --- /dev/null +++ b/frameworks/Nim/scorper/.gitignore @@ -0,0 +1,7 @@ +# Binaries +* +!*.* +!*/ +*.out +nimbledeps +.DS_Store \ No newline at end of file diff --git a/frameworks/Nim/scorper/README.md b/frameworks/Nim/scorper/README.md new file mode 100755 index 00000000000..d9aeee85c22 --- /dev/null +++ b/frameworks/Nim/scorper/README.md @@ -0,0 +1,20 @@ +# scorper Benchmarking Test + +### Test Type Implementation Source Code + +* [JSON](./scorper_bench.nim) +* [PLAINTEXT](./scorper_bench.nim) + +## Important Libraries +The tests were run with: +* [Software](https://github.com/bung87/scorper) +* [Example](https://github.com/bung87/scorper/tree/devel/examples) + +## Test URLs +### JSON + +http://localhost:8080/json + +### PLAINTEXT + +http://localhost:8080/plaintext diff --git a/frameworks/Nim/scorper/benchmark_config.json b/frameworks/Nim/scorper/benchmark_config.json new file mode 100755 index 00000000000..77669f6cad2 --- /dev/null +++ b/frameworks/Nim/scorper/benchmark_config.json @@ -0,0 +1,26 @@ +{ + "framework": "scorper", + "tests": [ + { + "default": { + "json_url": "/json", + "plaintext_url": "/plaintext", + "port": 8080, + "approach": "Realistic", + "classification": "Micro", + "database": "None", + "framework": "scorper", + "language": "Nim", + "flavor": "None", + "orm": "None", + "platform": "None", + "webserver": "None", + "os": "Linux", + "database_os": "Linux", + "display_name": "scorper", + "notes": "", + "versus": "httpbeast" + } + } + ] +} diff --git a/frameworks/Nim/scorper/config.nims b/frameworks/Nim/scorper/config.nims new file mode 100644 index 00000000000..e69de29bb2d diff --git a/frameworks/Nim/scorper/config.toml b/frameworks/Nim/scorper/config.toml new file mode 100644 index 00000000000..1a7f5e417ef --- /dev/null +++ b/frameworks/Nim/scorper/config.toml @@ -0,0 +1,15 @@ +[framework] +name = "scorper" + +[main] +urls.plaintext = "/plaintext" +urls.json = "/json" +approach = "Realistic" +classification = "Micro" +database = "None" +database_os = "Linux" +os = "Linux" +orm = "None" +platform = "None" +webserver = "None" +versus = "httpbeast" diff --git a/frameworks/Nim/scorper/scorper.dockerfile b/frameworks/Nim/scorper/scorper.dockerfile new file mode 100644 index 00000000000..87c894971d0 --- /dev/null +++ b/frameworks/Nim/scorper/scorper.dockerfile @@ -0,0 +1,13 @@ +FROM nimlang/nim:alpine + +ENV PATH $PATH:/root/.nimble/bin + +ADD ./ /scorper +WORKDIR /scorper + +RUN nimble install -d -y +RUN nimble c -d:ChronosAsync -d:release --opt:speed -d:danger -o:scorper_bench_bin ./scorper_bench.nim + +EXPOSE 8080 + +CMD ./scorper_bench_bin diff --git a/frameworks/Nim/scorper/scorper_bench.nim b/frameworks/Nim/scorper/scorper_bench.nim new file mode 100644 index 00000000000..db8979c01a1 --- /dev/null +++ b/frameworks/Nim/scorper/scorper_bench.nim @@ -0,0 +1,27 @@ +import std / [macros, exitprocs] +import scorper + + +type AsyncCallback = proc (request: Request): Future[void] {.closure, gcsafe, + raises: [].} + +proc jsonHandler(req: Request) {.route("get", "/json"), async.} = + let headers = {"Content-type": "application/json"} + await req.resp("""{"message":"Hello, World!"}""", headers.newHttpHeaders()) + +proc plaintextHandler(req: Request) {.route("get", "/plaintext"), async.} = + let headers = {"Content-type": "text/plain"} + await req.resp("Hello, World!", headers.newHttpHeaders()) + +when isMainModule: + + let address = "0.0.0.0:8080" + let flags = {ReuseAddr} + let r = newRouter[AsyncCallback]() + r.addRoute(jsonHandler) + r.addRoute(plaintextHandler) + var server = newScorper(address, r, flags) + exitprocs.addExitProc proc() = waitFor server.closeWait() + server.start() + + waitFor server.join() diff --git a/frameworks/Nim/scorper/scorper_bench.nimble b/frameworks/Nim/scorper/scorper_bench.nimble new file mode 100644 index 00000000000..3c6595c78d8 --- /dev/null +++ b/frameworks/Nim/scorper/scorper_bench.nimble @@ -0,0 +1,15 @@ +# Package + +version = "0.1.0" +author = "bung87" +description = "A new awesome scorper package" +license = "MIT" +srcDir = "." +bin = @["scorper_bench"] + +backend = "c" + +# Dependencies + +requires "nim >= 1.2.0" +requires "scorper >= 1.0.6" From 5dbbd1f9457e62203573d7d30d4563af863d86c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=89=E5=88=80?= Date: Fri, 28 May 2021 23:50:00 +0800 Subject: [PATCH 2/6] Optimize code (#6609) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update smart-servlet to 0.1.3-SNAPSHOT * update aio-enhance to 1.0.3-SNAPSHOT * smart-servlet bugfix * bugfix * update smart-socket to 1.5.6-SNAPSHOT * remove file * update aio-enhance to 1.0.4-SNAPSHOT * 优化代码 * 优化代码 * update smart-socket to 1.5.6 * config threadNum * update smart-socket to 1.5.7-SNAPSHOT * 优化代码 --- .../smart-socket/src/main/java/org/smartboot/http/JsonUtil.java | 1 - 1 file changed, 1 deletion(-) diff --git a/frameworks/Java/smart-socket/src/main/java/org/smartboot/http/JsonUtil.java b/frameworks/Java/smart-socket/src/main/java/org/smartboot/http/JsonUtil.java index 0b507e9142d..d8491ef6118 100644 --- a/frameworks/Java/smart-socket/src/main/java/org/smartboot/http/JsonUtil.java +++ b/frameworks/Java/smart-socket/src/main/java/org/smartboot/http/JsonUtil.java @@ -19,7 +19,6 @@ public static void writeJsonBytes(HttpResponse httpResponse, Object obj) { try { stream.reset(null); stream.writeVal(obj.getClass(), obj); - stream.buffer(); Slice slice = stream.buffer(); httpResponse.setContentLength(slice.tail()); httpResponse.getOutputStream().write(slice.data(), 0, slice.tail()); From d059402440d90df9472cc3fbf2ab07996d58cb0e Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 28 May 2021 23:50:26 +0800 Subject: [PATCH 3/6] OfficeFloor adding performance improvements for SQL Client (#6607) * Removing Rapidoid as seems dead project (no commit last year) * Downloading from SourceForge * Bump maven-compiler-plugin in /frameworks/Java/officefloor/src Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.8.0 to 3.8.1. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.8.0...maven-compiler-plugin-3.8.1) Signed-off-by: dependabot-preview[bot] * Bump maven-shade-plugin in /frameworks/Java/officefloor/src Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.2.1 to 3.2.2. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.2.1...maven-shade-plugin-3.2.2) Signed-off-by: dependabot-preview[bot] * Fixing Raw OfficeFloor * Specifying Spring plugin version * Bump to OfficeFloor 3.21.0 * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.2.4.RELEASE to 2.2.5.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.2.4.RELEASE...v2.2.5.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.2.5.RELEASE to 2.2.6.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.2.5.RELEASE...v2.2.6.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.21.0 to 3.22.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump maven-shade-plugin in /frameworks/Java/officefloor/src Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.2.2 to 3.2.3. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.2.2...maven-shade-plugin-3.2.3) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.22.0 to 3.23.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.23.0 to 3.24.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.2.6.RELEASE to 2.2.7.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.2.6.RELEASE...v2.2.7.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.2.7.RELEASE to 2.3.0.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.2.7.RELEASE...v2.3.0.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.24.0 to 3.25.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump maven-shade-plugin in /frameworks/Java/officefloor/src Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.2.3 to 3.2.4. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.2.3...maven-shade-plugin-3.2.4) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.3.0.RELEASE to 2.3.1.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.3.0.RELEASE...v2.3.1.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.3.1.RELEASE to 2.3.2.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.3.1.RELEASE...v2.3.2.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.25.0 to 3.26.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/compare/release-3.25.0...release-3.26.0) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.3.2.RELEASE to 2.3.3.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.3.2.RELEASE...v2.3.3.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.26.0 to 3.27.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.3.3.RELEASE to 2.3.4.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.3.3.RELEASE...v2.3.4.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.27.0 to 3.28.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Fixing to run with v3.28.0 * Including Undertow * Using slim docker images * Fixing to add all supported tests * Increasing connection pool size * Write up the 503 errors and why * Fix grammer * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.3.4.RELEASE to 2.3.5.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.3.4.RELEASE...v2.3.5.RELEASE) Signed-off-by: dependabot-preview[bot] * Improving chances of full processing Increasing max thread counts to have thread per client. Also, shading jars correctly. Plus adding in DB test for raw * Fixing raw test to have database * Ensure random numbers to avoid cached entities * Ensuring spring random numbers to avoid entity caching * Adding queries for raw * officefloor-raw queries passing * office-raw supporting all requests * Fixed update test to ensure updates occur before responding * Adding officefloor-async This will demonstrate OfficeFloor running as a single thread asynchronous server * Upgrading to OfficeFloor 2.38.1 * Tidy up read me * Appropriate back pressure queue * Handle rate limiting * Providing appropriate throttling * Lowering threading to handle through put * officefloor-raw passing tests * Passing tests * OfficeFloor 3.28.2 * Fixing rate limiting * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.28.2 to 3.29.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Fixing for bump to 3.30.0 * Increasing max direct memory * Increasing threshold to enable verifications to pass for officefloor-raw * Start script to take available memory into consideration * Max direct memory aware of available memory Also, cleaning writer threads of buffers to avoid leaks. * Avoiding OOM by managing disable/enable reading * DB and Fortune passing * Ensuring free command is available * Removing TODOs * Increasing reactor buffer for multiple queries * Fixing for 3.30.1 * Passing benchmark tests * Avoiding OOM on reactor buffer sizes * Passing validate tests * Using defaults from 3.30.1 * Bump to 512 threads and connections * Reverting dockerfiles to provide appropriate parameters * openjdk:slim for apt-get available * Providing thread affinity to raw * Fixing rate limit throttling * Multiplexing queries over connections per socket * Fixing for large queries in validate * Fixing versions of maven and java * Providing thread affinity of DB connection thread * Fixing for Netty event loop thread * Tidying up compiler warnings * Single db pool to avoid additional threads * Use default LoopResources * Tidying up code for warnings * Further tidy up of code * Revert to latest pull request * Using parallel GC for better throughput * Fix up for thread local buffering improvements * OfficeFloor fortune raw Focus of officefloor-raw is raw performance of the OfficeFloor HTTP server. Using raw fortune to remove mustache overheads. * Tidy up loop * Server name (as per discussions) * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.3.5.RELEASE to 2.4.2. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.3.5.RELEASE...v2.4.2) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.30.2 to 3.31.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.31.0 to 3.32.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.4.2 to 2.4.3. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.4.2...v2.4.3) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.4.3 to 2.4.5. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.4.3...v2.4.5) Signed-off-by: dependabot-preview[bot] * Upgrade to GitHub-native Dependabot * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.32.0 to 3.35.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits/release-3.35.0) Signed-off-by: dependabot-preview[bot] * Bump to OfficeFloor 3.35.0 * Allow bump of all versions * Vertx server Also, renaming officefloor-raw to officefloor-r2dbc to make way for officefloor-sqlclient * Rename to R2dbc (from raw) * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.35.0 to 3.36.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot[bot] * Fixing meta-data * Can not propagate Dependabot to TechEmpower * Abstract WoOF from database driver This will allow introducing Vertx SQL Client * Fixing db port * Providing Vertx SQL Client implementation * Using OfficeFloorVertx for vertx Allows for tests to reset * Fixing link in read me * Reducing repetition in readme * Swapped OfficeFloor async to use Vertx SQL Client * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.4.5 to 2.5.0. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.4.5...v2.5.0) Signed-off-by: dependabot[bot] * Increasing SQL Client pool size to 512 Also, fixing update to sort to avoid deadlocks * Removing unnecessary logging * Removing dependabot configuration * Improving performance of Vertx This is by caching queries and using native communication. Also, adding further performance updates to update test by sorting updates * Fixing OfficeFloor-vertx name Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../Java/officefloor/benchmark_config.json | 4 +- .../java/net/officefloor/benchmark/Logic.java | 10 +- .../benchmark/SetupVertxSqlClient.java | 5 + .../benchmark/R2dbcOfficeFloorMain.java | 9 +- .../benchmark/SqlClientOfficeFloorMain.java | 338 +++++++++--------- 5 files changed, 198 insertions(+), 168 deletions(-) diff --git a/frameworks/Java/officefloor/benchmark_config.json b/frameworks/Java/officefloor/benchmark_config.json index df2a5b20d69..ffadb687d77 100755 --- a/frameworks/Java/officefloor/benchmark_config.json +++ b/frameworks/Java/officefloor/benchmark_config.json @@ -203,7 +203,7 @@ "webserver": "vertx", "os": "Linux", "database_os": "Linux", - "display_name": "OfficeFloor-undertow", + "display_name": "OfficeFloor-vertx", "notes": "", "versus": "vertx-postgres" }, @@ -232,4 +232,4 @@ } } ] -} \ No newline at end of file +} diff --git a/frameworks/Java/officefloor/src/woof_benchmark_async/src/main/java/net/officefloor/benchmark/Logic.java b/frameworks/Java/officefloor/src/woof_benchmark_async/src/main/java/net/officefloor/benchmark/Logic.java index f753a273022..b39093d3ddf 100644 --- a/frameworks/Java/officefloor/src/woof_benchmark_async/src/main/java/net/officefloor/benchmark/Logic.java +++ b/frameworks/Java/officefloor/src/woof_benchmark_async/src/main/java/net/officefloor/benchmark/Logic.java @@ -139,7 +139,15 @@ public void update(@HttpQueryParameter("queries") String queries, AsynchronousFl return null; } Row row = rows.next(); - return new World(row.getInteger(0), ThreadLocalRandom.current().nextInt(1, 10001)); + + // Ensure change to random number to trigger update + int previousRandomNumber = row.getInteger(1); + int newRandomNumber; + do { + newRandomNumber = ThreadLocalRandom.current().nextInt(1, 10001); + } while (previousRandomNumber == newRandomNumber); + + return new World(row.getInteger(0), newRandomNumber); })); } return CompositeFuture.all(futures).flatMap((compositeFuture) -> { diff --git a/frameworks/Java/officefloor/src/woof_benchmark_async/src/main/java/net/officefloor/benchmark/SetupVertxSqlClient.java b/frameworks/Java/officefloor/src/woof_benchmark_async/src/main/java/net/officefloor/benchmark/SetupVertxSqlClient.java index 0f64203a17d..1d7133bcdb5 100644 --- a/frameworks/Java/officefloor/src/woof_benchmark_async/src/main/java/net/officefloor/benchmark/SetupVertxSqlClient.java +++ b/frameworks/Java/officefloor/src/woof_benchmark_async/src/main/java/net/officefloor/benchmark/SetupVertxSqlClient.java @@ -28,9 +28,14 @@ public VertxSqlPoolConfigurer createService(ServiceContext context) throws Throw @Override public void configure(VertxSqlPoolConfigurerContext context) throws Exception { + + // Ensure adequate number of connections final int MAX_POOL_SIZE = 512; System.out.println("Setting max pool size to " + MAX_POOL_SIZE); context.getPoolOptions().setMaxSize(MAX_POOL_SIZE); + + // Configure options + context.getSqlConnectOptions().setCachePreparedStatements(true); } } \ No newline at end of file diff --git a/frameworks/Java/officefloor/src/woof_benchmark_r2dbc/src/main/java/net/officefloor/benchmark/R2dbcOfficeFloorMain.java b/frameworks/Java/officefloor/src/woof_benchmark_r2dbc/src/main/java/net/officefloor/benchmark/R2dbcOfficeFloorMain.java index 002a54fbbc0..b3524b75e78 100644 --- a/frameworks/Java/officefloor/src/woof_benchmark_r2dbc/src/main/java/net/officefloor/benchmark/R2dbcOfficeFloorMain.java +++ b/frameworks/Java/officefloor/src/woof_benchmark_r2dbc/src/main/java/net/officefloor/benchmark/R2dbcOfficeFloorMain.java @@ -223,7 +223,14 @@ public void update(int queryCount, HttpResponse response, ServerHttpConnection c Collections.sort(worlds, (a, b) -> a.id - b.id); Batch batch = conn.connection.createBatch(); for (World world : worlds) { - world.randomNumber = ThreadLocalRandom.current().nextInt(1, 10001); + + // Ensure change to random number to trigger update + int newRandomNumber; + do { + newRandomNumber = ThreadLocalRandom.current().nextInt(1, 10001); + } while (world.randomNumber == newRandomNumber); + world.randomNumber = newRandomNumber; + batch.add("UPDATE WORLD SET RANDOMNUMBER = " + world.randomNumber + " WHERE ID = " + world.id); } return Mono.from(batch.execute()).map((result) -> worlds); diff --git a/frameworks/Java/officefloor/src/woof_benchmark_sqlclient/src/main/java/net/officefloor/benchmark/SqlClientOfficeFloorMain.java b/frameworks/Java/officefloor/src/woof_benchmark_sqlclient/src/main/java/net/officefloor/benchmark/SqlClientOfficeFloorMain.java index 21305d6ac6d..a7ec0ea327d 100644 --- a/frameworks/Java/officefloor/src/woof_benchmark_sqlclient/src/main/java/net/officefloor/benchmark/SqlClientOfficeFloorMain.java +++ b/frameworks/Java/officefloor/src/woof_benchmark_sqlclient/src/main/java/net/officefloor/benchmark/SqlClientOfficeFloorMain.java @@ -1,6 +1,7 @@ package net.officefloor.benchmark; import io.vertx.core.Vertx; +import io.vertx.core.VertxOptions; import io.vertx.pgclient.PgConnectOptions; import io.vertx.pgclient.PgConnection; import io.vertx.sqlclient.Row; @@ -15,6 +16,7 @@ import java.net.Socket; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedDeque; @@ -27,169 +29,177 @@ */ public class SqlClientOfficeFloorMain implements DatabaseOperations { - /** - * Run application. - */ - public static void main(String[] args) throws Exception { - RawWoof.run(args, (socketCount, server, port, database, username, - password) -> new SqlClientOfficeFloorMain(socketCount, server, port, database, username, password)); - } - - /** - * {@link ThreadLocal} {@link PgConnection} instances. - */ - private final ThreadLocal threadLocalConnection; - - /** - * Instantiate. - * - * @param socketCount Number of server {@link Socket} instances. - * @param server Name of database server. - * @param port Port of database. - * @param database Name of database within server. - * @param username Username. - * @param password Password. - */ - public SqlClientOfficeFloorMain(int socketCount, String server, int port, String database, String username, - String password) { - - // Obtain the vertx - Vertx vertx = OfficeFloorVertx.getVertx(); - - // Create connection - PgConnectOptions connectOptions = new PgConnectOptions().setHost(server).setPort(port).setDatabase(database) - .setUser(username).setPassword(password); - - // Create thread local connection - this.threadLocalConnection = new ThreadLocal() { - @Override - protected PgConnection initialValue() { - try { - return OfficeFloorVertx.block(PgConnection.connect(vertx, connectOptions)); - } catch (Exception ex) { - throw new IllegalStateException("Failed to setup connection", ex); - } - } - }; - } - - /* - * ===================== DatabaseOperations ====================== - */ - - @Override - public void threadSetup(RequestHandler requestHandler) { - // Nothing thread specific to set up - } - - @Override - public void db(HttpResponse response, ServerHttpConnection connection, DatabaseOperationsContext context) { - this.threadLocalConnection.get().preparedQuery("SELECT ID, RANDOMNUMBER FROM WORLD WHERE ID=$1") - .execute(Tuple.of(ThreadLocalRandom.current().nextInt(1, 10001)), - result -> { - if (result.failed()) { - context.sendError(connection, result.cause()); - } else { - RowIterator rows = result.result().iterator(); - if (!rows.hasNext()) { - context.sendError(connection, 404); - } else { - Row row = rows.next(); - World world = new World(row.getInteger(0), row.getInteger(1)); - context.dbSend(response, connection, world); - } - } - }); - } - - @Override - public void queries(int queryCount, HttpResponse response, ServerHttpConnection connection, - DatabaseOperationsContext context) { - Queue worlds = new ConcurrentLinkedDeque<>(); - SqlConnection sqlConnection = this.threadLocalConnection.get(); - for (int i = 0; i < queryCount; i++) { - sqlConnection.preparedQuery("SELECT ID, RANDOMNUMBER FROM WORLD WHERE ID=$1") - .execute(Tuple.of(ThreadLocalRandom.current().nextInt(1, 10001)), - result -> { - if (result.failed()) { - context.sendError(connection, result.cause()); - } else { - RowIterator rows = result.result().iterator(); - if (!rows.hasNext()) { - context.sendError(connection, 404); - } else { - Row row = rows.next(); - World world = new World(row.getInteger(0), row.getInteger(1)); - worlds.add(world); - - if (worlds.size() == queryCount) { - context.queriesSend(response, connection, new ArrayList<>(worlds)); - } - } - } - }); - } - } - - @Override - public void fortunes(HttpResponse response, ServerHttpConnection connection, DatabaseOperationsContext context) { - this.threadLocalConnection.get().preparedQuery("SELECT ID, MESSAGE FROM FORTUNE") - .execute(result -> { - if (result.failed()) { - context.sendError(connection, result.cause()); - } else { - List fortunes = new ArrayList<>(16); - RowIterator rows = result.result().iterator(); - while (rows.hasNext()) { - Row row = rows.next(); - fortunes.add(new Fortune(row.getInteger(0), row.getString(1))); - } - context.fortunesSend(response, connection, fortunes); - } - }); - } - - @Override - public void update(int queryCount, HttpResponse response, ServerHttpConnection connection, - DatabaseOperationsContext context) { - Queue worlds = new ConcurrentLinkedDeque<>(); - SqlConnection sqlConnection = this.threadLocalConnection.get(); - for (int i = 0; i < queryCount; i++) { - sqlConnection.preparedQuery("SELECT ID, RANDOMNUMBER FROM WORLD WHERE ID=$1") - .execute(Tuple.of(ThreadLocalRandom.current().nextInt(1, 10001)), - result -> { - if (result.failed()) { - context.sendError(connection, result.cause()); - } else { - RowIterator rows = result.result().iterator(); - if (!rows.hasNext()) { - context.sendError(connection, 404); - } else { - Row row = rows.next(); - World world = new World(row.getInteger(0), ThreadLocalRandom.current().nextInt(1, 10001)); - worlds.add(world); - - if (worlds.size() == queryCount) { - - // All worlds obtained, so run update - List batch = new ArrayList<>(queryCount); - for (World update : worlds) { - batch.add(Tuple.of(update.randomNumber, update.id)); - } - sqlConnection.preparedQuery("UPDATE world SET randomnumber=$1 WHERE id=$2").executeBatch(batch, ar -> { - if (result.failed()) { - context.sendError(connection, result.cause()); - } else { - - // Updated, so send response - context.queriesSend(response, connection, new ArrayList<>(worlds)); - } - }); - } - } - } - }); - } - } + /** + * Run application. + */ + public static void main(String[] args) throws Exception { + RawWoof.run(args, (socketCount, server, port, database, username, + password) -> new SqlClientOfficeFloorMain(socketCount, server, port, database, username, password)); + } + + /** + * {@link ThreadLocal} {@link PgConnection} instances. + */ + private final ThreadLocal threadLocalConnection; + + /** + * Instantiate. + * + * @param socketCount Number of server {@link Socket} instances. + * @param server Name of database server. + * @param port Port of database. + * @param database Name of database within server. + * @param username Username. + * @param password Password. + */ + public SqlClientOfficeFloorMain(int socketCount, String server, int port, String database, String username, + String password) { + + // Obtain the vertx + Vertx vertx = Vertx.vertx(new VertxOptions().setPreferNativeTransport(true)); + + // Create connection + PgConnectOptions connectOptions = new PgConnectOptions().setHost(server).setPort(port).setDatabase(database) + .setUser(username).setPassword(password).setCachePreparedStatements(true); + + // Create thread local connection + this.threadLocalConnection = new ThreadLocal() { + @Override + protected PgConnection initialValue() { + try { + return OfficeFloorVertx.block(PgConnection.connect(vertx, connectOptions)); + } catch (Exception ex) { + throw new IllegalStateException("Failed to setup connection", ex); + } + } + }; + } + + /* + * ===================== DatabaseOperations ====================== + */ + + @Override + public void threadSetup(RequestHandler requestHandler) { + // Nothing thread specific to set up + } + + @Override + public void db(HttpResponse response, ServerHttpConnection connection, DatabaseOperationsContext context) { + this.threadLocalConnection.get().preparedQuery("SELECT ID, RANDOMNUMBER FROM WORLD WHERE ID=$1") + .execute(Tuple.of(ThreadLocalRandom.current().nextInt(1, 10001)), result -> { + if (result.failed()) { + context.sendError(connection, result.cause()); + } else { + RowIterator rows = result.result().iterator(); + if (!rows.hasNext()) { + context.sendError(connection, 404); + } else { + Row row = rows.next(); + World world = new World(row.getInteger(0), row.getInteger(1)); + context.dbSend(response, connection, world); + } + } + }); + } + + @Override + public void queries(int queryCount, HttpResponse response, ServerHttpConnection connection, + DatabaseOperationsContext context) { + Queue worlds = new ConcurrentLinkedDeque<>(); + SqlConnection sqlConnection = this.threadLocalConnection.get(); + for (int i = 0; i < queryCount; i++) { + sqlConnection.preparedQuery("SELECT ID, RANDOMNUMBER FROM WORLD WHERE ID=$1") + .execute(Tuple.of(ThreadLocalRandom.current().nextInt(1, 10001)), result -> { + if (result.failed()) { + context.sendError(connection, result.cause()); + } else { + RowIterator rows = result.result().iterator(); + if (!rows.hasNext()) { + context.sendError(connection, 404); + } else { + Row row = rows.next(); + World world = new World(row.getInteger(0), row.getInteger(1)); + worlds.add(world); + + if (worlds.size() == queryCount) { + context.queriesSend(response, connection, new ArrayList<>(worlds)); + } + } + } + }); + } + } + + @Override + public void fortunes(HttpResponse response, ServerHttpConnection connection, DatabaseOperationsContext context) { + this.threadLocalConnection.get().preparedQuery("SELECT ID, MESSAGE FROM FORTUNE").execute(result -> { + if (result.failed()) { + context.sendError(connection, result.cause()); + } else { + List fortunes = new ArrayList<>(16); + RowIterator rows = result.result().iterator(); + while (rows.hasNext()) { + Row row = rows.next(); + fortunes.add(new Fortune(row.getInteger(0), row.getString(1))); + } + context.fortunesSend(response, connection, fortunes); + } + }); + } + + @Override + public void update(int queryCount, HttpResponse response, ServerHttpConnection connection, + DatabaseOperationsContext context) { + Queue worlds = new ConcurrentLinkedDeque<>(); + SqlConnection sqlConnection = this.threadLocalConnection.get(); + for (int i = 0; i < queryCount; i++) { + sqlConnection.preparedQuery("SELECT ID, RANDOMNUMBER FROM WORLD WHERE ID=$1") + .execute(Tuple.of(ThreadLocalRandom.current().nextInt(1, 10001)), result -> { + if (result.failed()) { + context.sendError(connection, result.cause()); + } else { + RowIterator rows = result.result().iterator(); + if (!rows.hasNext()) { + context.sendError(connection, 404); + } else { + Row row = rows.next(); + + // Ensure change to random number to trigger update + int previousRandomNumber = row.getInteger(1); + int newRandomNumber; + do { + newRandomNumber = ThreadLocalRandom.current().nextInt(1, 10001); + } while (previousRandomNumber == newRandomNumber); + World world = new World(row.getInteger(0), newRandomNumber); + worlds.add(world); + + if (worlds.size() == queryCount) { + + // All worlds obtained, so run update + List batch = new ArrayList<>(queryCount); + for (World update : worlds) { + batch.add(Tuple.of(update.randomNumber, update.id)); + } + + // Sort to avoid deadlocks on updates + Collections.sort(batch, (a, b) -> a.getInteger(1) - b.getInteger(1)); + + sqlConnection.preparedQuery("UPDATE world SET randomnumber=$1 WHERE id=$2") + .executeBatch(batch, ar -> { + if (result.failed()) { + context.sendError(connection, result.cause()); + } else { + + // Updated, so send response + context.queriesSend(response, connection, new ArrayList<>(worlds)); + } + }); + } + } + } + }); + } + } } \ No newline at end of file From d71a836cfd072e4ece261017cc95d8688a339935 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 1 Jun 2021 08:08:26 -0700 Subject: [PATCH 4/6] Bump Finagle benchmarks (#6610) --- frameworks/Scala/finagle/build.sbt | 2 +- frameworks/Scala/finatra/build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frameworks/Scala/finagle/build.sbt b/frameworks/Scala/finagle/build.sbt index ca4d3cee2ed..084370990e8 100644 --- a/frameworks/Scala/finagle/build.sbt +++ b/frameworks/Scala/finagle/build.sbt @@ -1,4 +1,4 @@ -lazy val finagleVersion = "21.4.0" +lazy val finagleVersion = "21.5.0" name := "finagle-benchmark" scalaVersion := "2.12.12" diff --git a/frameworks/Scala/finatra/build.sbt b/frameworks/Scala/finatra/build.sbt index 52f5fb6b807..f9ca0163835 100644 --- a/frameworks/Scala/finatra/build.sbt +++ b/frameworks/Scala/finatra/build.sbt @@ -1,4 +1,4 @@ -lazy val finatraVersion = "21.4.0" +lazy val finatraVersion = "21.5.0" name := "techempower-benchmarks-finatra" organization := "com.twitter" From fbdfa4150f2b2c51788d10e5af8981bcb4089f57 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 1 Jun 2021 23:09:19 +0800 Subject: [PATCH 5/6] OfficeFloor - write response back on socket thread (#6612) * Removing Rapidoid as seems dead project (no commit last year) * Downloading from SourceForge * Bump maven-compiler-plugin in /frameworks/Java/officefloor/src Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.8.0 to 3.8.1. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.8.0...maven-compiler-plugin-3.8.1) Signed-off-by: dependabot-preview[bot] * Bump maven-shade-plugin in /frameworks/Java/officefloor/src Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.2.1 to 3.2.2. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.2.1...maven-shade-plugin-3.2.2) Signed-off-by: dependabot-preview[bot] * Fixing Raw OfficeFloor * Specifying Spring plugin version * Bump to OfficeFloor 3.21.0 * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.2.4.RELEASE to 2.2.5.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.2.4.RELEASE...v2.2.5.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.2.5.RELEASE to 2.2.6.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.2.5.RELEASE...v2.2.6.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.21.0 to 3.22.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump maven-shade-plugin in /frameworks/Java/officefloor/src Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.2.2 to 3.2.3. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.2.2...maven-shade-plugin-3.2.3) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.22.0 to 3.23.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.23.0 to 3.24.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.2.6.RELEASE to 2.2.7.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.2.6.RELEASE...v2.2.7.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.2.7.RELEASE to 2.3.0.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.2.7.RELEASE...v2.3.0.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.24.0 to 3.25.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump maven-shade-plugin in /frameworks/Java/officefloor/src Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.2.3 to 3.2.4. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.2.3...maven-shade-plugin-3.2.4) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.3.0.RELEASE to 2.3.1.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.3.0.RELEASE...v2.3.1.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.3.1.RELEASE to 2.3.2.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.3.1.RELEASE...v2.3.2.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.25.0 to 3.26.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/compare/release-3.25.0...release-3.26.0) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.3.2.RELEASE to 2.3.3.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.3.2.RELEASE...v2.3.3.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.26.0 to 3.27.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.3.3.RELEASE to 2.3.4.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.3.3.RELEASE...v2.3.4.RELEASE) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.27.0 to 3.28.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Fixing to run with v3.28.0 * Including Undertow * Using slim docker images * Fixing to add all supported tests * Increasing connection pool size * Write up the 503 errors and why * Fix grammer * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.3.4.RELEASE to 2.3.5.RELEASE. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.3.4.RELEASE...v2.3.5.RELEASE) Signed-off-by: dependabot-preview[bot] * Improving chances of full processing Increasing max thread counts to have thread per client. Also, shading jars correctly. Plus adding in DB test for raw * Fixing raw test to have database * Ensure random numbers to avoid cached entities * Ensuring spring random numbers to avoid entity caching * Adding queries for raw * officefloor-raw queries passing * office-raw supporting all requests * Fixed update test to ensure updates occur before responding * Adding officefloor-async This will demonstrate OfficeFloor running as a single thread asynchronous server * Upgrading to OfficeFloor 2.38.1 * Tidy up read me * Appropriate back pressure queue * Handle rate limiting * Providing appropriate throttling * Lowering threading to handle through put * officefloor-raw passing tests * Passing tests * OfficeFloor 3.28.2 * Fixing rate limiting * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.28.2 to 3.29.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Fixing for bump to 3.30.0 * Increasing max direct memory * Increasing threshold to enable verifications to pass for officefloor-raw * Start script to take available memory into consideration * Max direct memory aware of available memory Also, cleaning writer threads of buffers to avoid leaks. * Avoiding OOM by managing disable/enable reading * DB and Fortune passing * Ensuring free command is available * Removing TODOs * Increasing reactor buffer for multiple queries * Fixing for 3.30.1 * Passing benchmark tests * Avoiding OOM on reactor buffer sizes * Passing validate tests * Using defaults from 3.30.1 * Bump to 512 threads and connections * Reverting dockerfiles to provide appropriate parameters * openjdk:slim for apt-get available * Providing thread affinity to raw * Fixing rate limit throttling * Multiplexing queries over connections per socket * Fixing for large queries in validate * Fixing versions of maven and java * Providing thread affinity of DB connection thread * Fixing for Netty event loop thread * Tidying up compiler warnings * Single db pool to avoid additional threads * Use default LoopResources * Tidying up code for warnings * Further tidy up of code * Revert to latest pull request * Using parallel GC for better throughput * Fix up for thread local buffering improvements * OfficeFloor fortune raw Focus of officefloor-raw is raw performance of the OfficeFloor HTTP server. Using raw fortune to remove mustache overheads. * Tidy up loop * Server name (as per discussions) * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.3.5.RELEASE to 2.4.2. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.3.5.RELEASE...v2.4.2) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.30.2 to 3.31.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.31.0 to 3.32.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.4.2 to 2.4.3. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.4.2...v2.4.3) Signed-off-by: dependabot-preview[bot] * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.4.3 to 2.4.5. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.4.3...v2.4.5) Signed-off-by: dependabot-preview[bot] * Upgrade to GitHub-native Dependabot * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.32.0 to 3.35.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits/release-3.35.0) Signed-off-by: dependabot-preview[bot] * Bump to OfficeFloor 3.35.0 * Allow bump of all versions * Vertx server Also, renaming officefloor-raw to officefloor-r2dbc to make way for officefloor-sqlclient * Rename to R2dbc (from raw) * Bump net.officefloor:bom in /frameworks/Java/officefloor/src Bumps [net.officefloor:bom](https://github.com/officefloor/OfficeFloor) from 3.35.0 to 3.36.0. - [Release notes](https://github.com/officefloor/OfficeFloor/releases) - [Commits](https://github.com/officefloor/OfficeFloor/commits) Signed-off-by: dependabot[bot] * Fixing meta-data * Can not propagate Dependabot to TechEmpower * Abstract WoOF from database driver This will allow introducing Vertx SQL Client * Fixing db port * Providing Vertx SQL Client implementation * Using OfficeFloorVertx for vertx Allows for tests to reset * Fixing link in read me * Reducing repetition in readme * Swapped OfficeFloor async to use Vertx SQL Client * Bump spring-boot-maven-plugin in /frameworks/Java/officefloor/src Bumps [spring-boot-maven-plugin](https://github.com/spring-projects/spring-boot) from 2.4.5 to 2.5.0. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.4.5...v2.5.0) Signed-off-by: dependabot[bot] * Increasing SQL Client pool size to 512 Also, fixing update to sort to avoid deadlocks * Removing unnecessary logging * Removing dependabot configuration * Improving performance of Vertx This is by caching queries and using native communication. Also, adding further performance updates to update test by sorting updates * Fixing OfficeFloor-vertx name * Reducing load on DB callback threads * Database worker threads relative to number of socket threads * Further improvements to performance (by less contention) Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../benchmark/R2dbcOfficeFloorMain.java | 36 +- .../benchmark/SqlClientOfficeFloorMain.java | 73 +- .../benchmark/AbstractSendResponse.java | 101 +++ .../benchmark/DatabaseOperations.java | 30 +- .../benchmark/DatabaseOperationsContext.java | 72 -- .../officefloor/benchmark/DbSendResponse.java | 42 ++ .../benchmark/FortunesSendResponse.java | 81 +++ .../benchmark/QueriesSendResponse.java | 42 ++ .../net/officefloor/benchmark/RawWoof.java | 657 +++++++----------- .../benchmark/UpdateSendResponse.java | 42 ++ 10 files changed, 609 insertions(+), 567 deletions(-) create mode 100644 frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/AbstractSendResponse.java delete mode 100644 frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/DatabaseOperationsContext.java create mode 100644 frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/DbSendResponse.java create mode 100644 frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/FortunesSendResponse.java create mode 100644 frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/QueriesSendResponse.java create mode 100644 frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/UpdateSendResponse.java diff --git a/frameworks/Java/officefloor/src/woof_benchmark_r2dbc/src/main/java/net/officefloor/benchmark/R2dbcOfficeFloorMain.java b/frameworks/Java/officefloor/src/woof_benchmark_r2dbc/src/main/java/net/officefloor/benchmark/R2dbcOfficeFloorMain.java index b3524b75e78..00709063d96 100644 --- a/frameworks/Java/officefloor/src/woof_benchmark_r2dbc/src/main/java/net/officefloor/benchmark/R2dbcOfficeFloorMain.java +++ b/frameworks/Java/officefloor/src/woof_benchmark_r2dbc/src/main/java/net/officefloor/benchmark/R2dbcOfficeFloorMain.java @@ -12,8 +12,6 @@ import io.r2dbc.spi.ConnectionFactory; import io.r2dbc.spi.ConnectionFactoryOptions; import net.officefloor.server.RequestHandler; -import net.officefloor.server.http.HttpResponse; -import net.officefloor.server.http.ServerHttpConnection; import net.officefloor.server.http.parse.HttpRequestParser; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -120,12 +118,12 @@ public void threadSetup(RequestHandler requestHandler) { } @Override - public void db(HttpResponse response, ServerHttpConnection connection, DatabaseOperationsContext context) { + public void db(DbSendResponse sender) { // Determine if will overload queries RateLimitedConnection conn = this.threadLocalRateLimit.get().getAvailableConnection(1); if (conn == null) { - context.sendError(connection, context.getTransientResourceException()); + sender.sendOverloaded(); return; // rate limited } @@ -137,22 +135,21 @@ public void db(HttpResponse response, ServerHttpConnection connection, DatabaseO Integer number = row.get(1, Integer.class); return new World(id, number); }))).publishOn(conn.writeScheduler).subscribe(world -> { - context.dbSend(response, connection, world); + sender.sendDb(world); }, error -> { - context.sendError(connection, error); + sender.sendError(error); }, () -> { conn.processed(1); }); } @Override - public void queries(int queryCount, HttpResponse response, ServerHttpConnection connection, - DatabaseOperationsContext context) { + public void queries(int queryCount, QueriesSendResponse sender) { // Determine if will overload queries RateLimitedConnection conn = this.threadLocalRateLimit.get().getAvailableConnection(queryCount); if (conn == null) { - context.sendError(connection, context.getTransientResourceException()); + sender.sendOverloaded(); return; // rate limited } @@ -165,21 +162,21 @@ public void queries(int queryCount, HttpResponse response, ServerHttpConnection Integer number = row.get(1, Integer.class); return new World(id, number); }))).collectList().publishOn(conn.writeScheduler).subscribe(worlds -> { - context.queriesSend(response, connection, worlds); + sender.sendQueries(worlds.toArray(World[]::new)); }, error -> { - context.sendError(connection, error); + sender.sendError(error); }, () -> { conn.processed(queryCount); }); } @Override - public void fortunes(HttpResponse response, ServerHttpConnection connection, DatabaseOperationsContext context) { + public void fortunes(FortunesSendResponse sender) { // Determine if will overload queries RateLimitedConnection conn = this.threadLocalRateLimit.get().getAvailableConnection(1); if (conn == null) { - context.sendError(connection, context.getTransientResourceException()); + sender.sendOverloaded(); return; // rate limited } @@ -190,24 +187,23 @@ public void fortunes(HttpResponse response, ServerHttpConnection connection, Dat String message = row.get(1, String.class); return new Fortune(id, message); }))).collectList().publishOn(conn.writeScheduler).subscribe(fortunes -> { - context.fortunesSend(response, connection, fortunes); + sender.sendFortunes(fortunes); }, error -> { - context.sendError(connection, error); + sender.sendError(error); }, () -> { conn.processed(1); }); } @Override - public void update(int queryCount, HttpResponse response, ServerHttpConnection connection, - DatabaseOperationsContext context) { + public void update(int queryCount, UpdateSendResponse sender) { int executeQueryCount = queryCount + 1; // select all and update // Determine if will overload queries RateLimitedConnection conn = this.threadLocalRateLimit.get().getAvailableConnection(executeQueryCount); if (conn == null) { - context.sendError(connection, context.getTransientResourceException()); + sender.sendOverloaded(); return; // rate limited } @@ -235,9 +231,9 @@ public void update(int queryCount, HttpResponse response, ServerHttpConnection c } return Mono.from(batch.execute()).map((result) -> worlds); }).publishOn(conn.writeScheduler).subscribe(worlds -> { - context.updateSend(response, connection, worlds); + sender.sendUpdate(worlds.toArray(World[]::new)); }, error -> { - context.sendError(connection, error); + sender.sendError(error); }, () -> { conn.processed(executeQueryCount); }); diff --git a/frameworks/Java/officefloor/src/woof_benchmark_sqlclient/src/main/java/net/officefloor/benchmark/SqlClientOfficeFloorMain.java b/frameworks/Java/officefloor/src/woof_benchmark_sqlclient/src/main/java/net/officefloor/benchmark/SqlClientOfficeFloorMain.java index a7ec0ea327d..4b39c99c94a 100644 --- a/frameworks/Java/officefloor/src/woof_benchmark_sqlclient/src/main/java/net/officefloor/benchmark/SqlClientOfficeFloorMain.java +++ b/frameworks/Java/officefloor/src/woof_benchmark_sqlclient/src/main/java/net/officefloor/benchmark/SqlClientOfficeFloorMain.java @@ -1,5 +1,12 @@ package net.officefloor.benchmark; +import java.net.Socket; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; + import io.vertx.core.Vertx; import io.vertx.core.VertxOptions; import io.vertx.pgclient.PgConnectOptions; @@ -9,19 +16,9 @@ import io.vertx.sqlclient.SqlConnection; import io.vertx.sqlclient.Tuple; import net.officefloor.server.RequestHandler; -import net.officefloor.server.http.HttpResponse; -import net.officefloor.server.http.ServerHttpConnection; import net.officefloor.server.http.parse.HttpRequestParser; import net.officefloor.vertx.OfficeFloorVertx; -import java.net.Socket; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.ThreadLocalRandom; - /** * R2DBC server. * @@ -56,7 +53,9 @@ public SqlClientOfficeFloorMain(int socketCount, String server, int port, String String password) { // Obtain the vertx - Vertx vertx = Vertx.vertx(new VertxOptions().setPreferNativeTransport(true)); + int workerThreadCount = Math.max(1, socketCount / 4); + Vertx vertx = Vertx + .vertx(new VertxOptions().setPreferNativeTransport(true).setWorkerPoolSize(workerThreadCount)); // Create connection PgConnectOptions connectOptions = new PgConnectOptions().setHost(server).setPort(port).setDatabase(database) @@ -85,45 +84,45 @@ public void threadSetup(RequestHandler requestHandler) { } @Override - public void db(HttpResponse response, ServerHttpConnection connection, DatabaseOperationsContext context) { + public void db(DbSendResponse sender) { this.threadLocalConnection.get().preparedQuery("SELECT ID, RANDOMNUMBER FROM WORLD WHERE ID=$1") .execute(Tuple.of(ThreadLocalRandom.current().nextInt(1, 10001)), result -> { if (result.failed()) { - context.sendError(connection, result.cause()); + sender.sendError(result.cause()); } else { RowIterator rows = result.result().iterator(); if (!rows.hasNext()) { - context.sendError(connection, 404); + sender.sendError(404); } else { Row row = rows.next(); World world = new World(row.getInteger(0), row.getInteger(1)); - context.dbSend(response, connection, world); + sender.sendDb(world); } } }); } @Override - public void queries(int queryCount, HttpResponse response, ServerHttpConnection connection, - DatabaseOperationsContext context) { - Queue worlds = new ConcurrentLinkedDeque<>(); + public void queries(int queryCount, QueriesSendResponse sender) { + World[] worlds = new World[queryCount]; + AtomicInteger count = new AtomicInteger(0); SqlConnection sqlConnection = this.threadLocalConnection.get(); for (int i = 0; i < queryCount; i++) { sqlConnection.preparedQuery("SELECT ID, RANDOMNUMBER FROM WORLD WHERE ID=$1") .execute(Tuple.of(ThreadLocalRandom.current().nextInt(1, 10001)), result -> { if (result.failed()) { - context.sendError(connection, result.cause()); + sender.sendError(result.cause()); } else { RowIterator rows = result.result().iterator(); if (!rows.hasNext()) { - context.sendError(connection, 404); + sender.sendError(404); } else { Row row = rows.next(); - World world = new World(row.getInteger(0), row.getInteger(1)); - worlds.add(world); + int index = count.getAndIncrement(); + worlds[index] = new World(row.getInteger(0), row.getInteger(1)); - if (worlds.size() == queryCount) { - context.queriesSend(response, connection, new ArrayList<>(worlds)); + if ((index + 1) == queryCount) { + sender.sendQueries(worlds); } } } @@ -132,10 +131,10 @@ public void queries(int queryCount, HttpResponse response, ServerHttpConnection } @Override - public void fortunes(HttpResponse response, ServerHttpConnection connection, DatabaseOperationsContext context) { + public void fortunes(FortunesSendResponse sender) { this.threadLocalConnection.get().preparedQuery("SELECT ID, MESSAGE FROM FORTUNE").execute(result -> { if (result.failed()) { - context.sendError(connection, result.cause()); + sender.sendError(result.cause()); } else { List fortunes = new ArrayList<>(16); RowIterator rows = result.result().iterator(); @@ -143,25 +142,25 @@ public void fortunes(HttpResponse response, ServerHttpConnection connection, Dat Row row = rows.next(); fortunes.add(new Fortune(row.getInteger(0), row.getString(1))); } - context.fortunesSend(response, connection, fortunes); + sender.sendFortunes(fortunes); } }); } @Override - public void update(int queryCount, HttpResponse response, ServerHttpConnection connection, - DatabaseOperationsContext context) { - Queue worlds = new ConcurrentLinkedDeque<>(); + public void update(int queryCount, UpdateSendResponse sender) { + World[] worlds = new World[queryCount]; + AtomicInteger count = new AtomicInteger(0); SqlConnection sqlConnection = this.threadLocalConnection.get(); for (int i = 0; i < queryCount; i++) { sqlConnection.preparedQuery("SELECT ID, RANDOMNUMBER FROM WORLD WHERE ID=$1") .execute(Tuple.of(ThreadLocalRandom.current().nextInt(1, 10001)), result -> { if (result.failed()) { - context.sendError(connection, result.cause()); + sender.sendError(result.cause()); } else { RowIterator rows = result.result().iterator(); if (!rows.hasNext()) { - context.sendError(connection, 404); + sender.sendError(404); } else { Row row = rows.next(); @@ -171,10 +170,10 @@ public void update(int queryCount, HttpResponse response, ServerHttpConnection c do { newRandomNumber = ThreadLocalRandom.current().nextInt(1, 10001); } while (previousRandomNumber == newRandomNumber); - World world = new World(row.getInteger(0), newRandomNumber); - worlds.add(world); + int index = count.getAndIncrement(); + worlds[index] = new World(row.getInteger(0), newRandomNumber); - if (worlds.size() == queryCount) { + if ((index + 1) == queryCount) { // All worlds obtained, so run update List batch = new ArrayList<>(queryCount); @@ -188,11 +187,11 @@ public void update(int queryCount, HttpResponse response, ServerHttpConnection c sqlConnection.preparedQuery("UPDATE world SET randomnumber=$1 WHERE id=$2") .executeBatch(batch, ar -> { if (result.failed()) { - context.sendError(connection, result.cause()); + sender.sendError(result.cause()); } else { // Updated, so send response - context.queriesSend(response, connection, new ArrayList<>(worlds)); + sender.sendUpdate(worlds); } }); } diff --git a/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/AbstractSendResponse.java b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/AbstractSendResponse.java new file mode 100644 index 00000000000..d3ab9bb75b5 --- /dev/null +++ b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/AbstractSendResponse.java @@ -0,0 +1,101 @@ +package net.officefloor.benchmark; + +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.ByteBuffer; +import java.nio.channels.CancelledKeyException; +import java.nio.channels.ClosedChannelException; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import net.officefloor.server.RequestHandler; +import net.officefloor.server.http.HttpHeaderValue; +import net.officefloor.server.http.HttpResponse; +import net.officefloor.server.http.HttpStatus; +import net.officefloor.server.http.impl.ProcessAwareServerHttpConnectionManagedObject; +import net.officefloor.server.http.parse.HttpRequestParser; + +/** + * Handles sending {@link HttpResponse}. + * + * @author Daniel Sagenschneider + */ +public class AbstractSendResponse { + + public static final HttpHeaderValue TEXT_PLAIN = new HttpHeaderValue("text/plain"); + + public static final HttpHeaderValue APPLICATION_JSON = new HttpHeaderValue("application/json"); + + public static void send(ProcessAwareServerHttpConnectionManagedObject connection) throws IOException { + try { + connection.getServiceFlowCallback().run(null); + } catch (IOException ex) { + throw ex; + } catch (Throwable ex) { + throw new IOException(ex); + } + } + + protected final RequestHandler requestHandler; + + protected final ProcessAwareServerHttpConnectionManagedObject connection; + + protected final ObjectMapper objectMapper; + + public AbstractSendResponse(RequestHandler requestHandler, + ProcessAwareServerHttpConnectionManagedObject connection, ObjectMapper objectMapper) { + this.requestHandler = requestHandler; + this.connection = connection; + this.objectMapper = objectMapper; + } + + public void sendOverloaded() { + try { + HttpResponse response = this.connection.getResponse(); + response.reset(); + + // Send overloaded + response.setStatus(HttpStatus.SERVICE_UNAVAILABLE); + send(this.connection); + + } catch (CancelledKeyException | ClosedChannelException ex) { + // Ignore as disconnecting client + } catch (IOException ex) { + ex.printStackTrace(); + } + } + + public void sendError(Throwable failure) { + try { + HttpResponse response = this.connection.getResponse(); + response.reset(); + + // Send failure + response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR); + response.setContentType(TEXT_PLAIN, null); + failure.printStackTrace(new PrintWriter(response.getEntityWriter())); + send(this.connection); + + } catch (CancelledKeyException | ClosedChannelException ex) { + // Ignore as disconnecting client + } catch (IOException ex) { + ex.printStackTrace(); + } + } + + public void sendError(int status) { + try { + // Setup to send response + HttpResponse response = connection.getResponse(); + response.reset(); + + // Send error response + response.setStatus(HttpStatus.getHttpStatus(status)); + send(this.connection); + + } catch (IOException ex) { + ex.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/DatabaseOperations.java b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/DatabaseOperations.java index bc0176f2709..74a2a5c8aa9 100644 --- a/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/DatabaseOperations.java +++ b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/DatabaseOperations.java @@ -3,8 +3,6 @@ import java.net.Socket; import net.officefloor.server.RequestHandler; -import net.officefloor.server.http.HttpResponse; -import net.officefloor.server.http.ServerHttpConnection; import net.officefloor.server.http.parse.HttpRequestParser; /** @@ -22,42 +20,22 @@ public interface DatabaseOperations { /** * Undertakes the db. - * - * @param response {@link HttpResponse}. - * @param connection {@link ServerHttpConnection}. - * @param context {@link DatabaseOperationsContext}. */ - void db(HttpResponse response, ServerHttpConnection connection, DatabaseOperationsContext context); + void db(DbSendResponse sender); /** * Undertakes the queries. - * - * @param queryCount Query count. - * @param response {@link HttpResponse}. - * @param connection {@link ServerHttpConnection}. - * @param context {@link DatabaseOperationsContext}. */ - void queries(int queryCount, HttpResponse response, ServerHttpConnection connection, - DatabaseOperationsContext context); + void queries(int queriesCount, QueriesSendResponse sender); /** * Undertakes the fortunes. - * - * @param response {@link HttpResponse}. - * @param connection {@link ServerHttpConnection}. - * @param context {@link DatabaseOperationsContext}. */ - void fortunes(HttpResponse response, ServerHttpConnection connection, DatabaseOperationsContext context); + void fortunes(FortunesSendResponse sender); /** * Undertakes the update. - * - * @param queryCount Query count. - * @param response {@link HttpResponse}. - * @param connection {@link ServerHttpConnection}. - * @param context {@link DatabaseOperationsContext}. */ - void update(int queryCount, HttpResponse response, ServerHttpConnection connection, - DatabaseOperationsContext context); + void update(int updateCount, UpdateSendResponse sender); } diff --git a/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/DatabaseOperationsContext.java b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/DatabaseOperationsContext.java deleted file mode 100644 index ecb027148a5..00000000000 --- a/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/DatabaseOperationsContext.java +++ /dev/null @@ -1,72 +0,0 @@ -package net.officefloor.benchmark; - -import java.util.List; - -import net.officefloor.server.http.HttpResponse; -import net.officefloor.server.http.ServerHttpConnection; - -/** - * @author Daniel Sagenschneider - */ -public interface DatabaseOperationsContext { - - /** - * Obtains {@link Exception} for transient overload of resource. - * - * @return {@link Exception} for transient overload of resource. - */ - Exception getTransientResourceException(); - - /** - * Sends db response. - * - * @param response {@link HttpResponse}. - * @param connection {@link ServerHttpConnection}. - * @param world {@link World} to send. - */ - void dbSend(HttpResponse response, ServerHttpConnection connection, World world); - - /** - * Sends queries response. - * - * @param response {@link HttpResponse}. - * @param connection {@link ServerHttpConnection}. - * @param worlds {@link World} instances to send. - */ - void queriesSend(HttpResponse response, ServerHttpConnection connection, List worlds); - - /** - * Sends fortunes response. - * - * @param response {@link HttpResponse}. - * @param connection {@link ServerHttpConnection}. - * @param fortune {@link Fortune} instances to send. - */ - void fortunesSend(HttpResponse response, ServerHttpConnection connection, List fortunes); - - /** - * Sends update response. - * - * @param response {@link HttpResponse}. - * @param connection {@link ServerHttpConnection}. - * @param worlds {@link World} instances to send. - */ - void updateSend(HttpResponse response, ServerHttpConnection connection, List worlds); - - /** - * Sends error. - * - * @param connection {@link ServerHttpConnection}. - * @param failure Cause. - */ - void sendError(ServerHttpConnection connection, Throwable failure); - - /** - * Sends error. - * - * @param connection {@link ServerHttpConnection}. - * @param satus Status. - */ - void sendError(ServerHttpConnection connection, int status); - -} diff --git a/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/DbSendResponse.java b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/DbSendResponse.java new file mode 100644 index 00000000000..fa1656567bc --- /dev/null +++ b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/DbSendResponse.java @@ -0,0 +1,42 @@ +package net.officefloor.benchmark; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.CancelledKeyException; +import java.nio.channels.ClosedChannelException; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import net.officefloor.server.RequestHandler; +import net.officefloor.server.http.HttpResponse; +import net.officefloor.server.http.impl.ProcessAwareServerHttpConnectionManagedObject; +import net.officefloor.server.http.parse.HttpRequestParser; + +/** + * Sends the DB response. + * + * @author Daniel Sagenschneider + */ +public class DbSendResponse extends AbstractSendResponse { + + public DbSendResponse(RequestHandler requestHandler, + ProcessAwareServerHttpConnectionManagedObject connection, ObjectMapper objectMapper) { + super(requestHandler, connection, objectMapper); + } + + public void sendDb(World world) { + this.requestHandler.execute(() -> { + try { + HttpResponse response = this.connection.getResponse(); + response.setContentType(APPLICATION_JSON, null); + this.objectMapper.writeValue(response.getEntityWriter(), world); + send(this.connection); + } catch (CancelledKeyException | ClosedChannelException ex) { + // Ignore as disconnecting client + } catch (IOException ex) { + ex.printStackTrace(); + } + }); + } + +} diff --git a/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/FortunesSendResponse.java b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/FortunesSendResponse.java new file mode 100644 index 00000000000..4b4032443c7 --- /dev/null +++ b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/FortunesSendResponse.java @@ -0,0 +1,81 @@ +package net.officefloor.benchmark; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.CancelledKeyException; +import java.nio.channels.ClosedChannelException; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import org.apache.commons.text.StringEscapeUtils; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import net.officefloor.server.RequestHandler; +import net.officefloor.server.http.HttpHeaderValue; +import net.officefloor.server.http.HttpResponse; +import net.officefloor.server.http.ServerHttpConnection; +import net.officefloor.server.http.impl.ProcessAwareServerHttpConnectionManagedObject; +import net.officefloor.server.http.parse.HttpRequestParser; +import net.officefloor.server.stream.ServerWriter; + +/** + * Sends the Fortunes response. + * + * @author Daniel Sagenschneider + */ +public class FortunesSendResponse extends AbstractSendResponse { + + private static final HttpHeaderValue TEXT_HTML = new HttpHeaderValue("text/html;charset=utf-8"); + + private static final byte[] TEMPLATE_START = "Fortunes" + .getBytes(ServerHttpConnection.DEFAULT_HTTP_ENTITY_CHARSET); + + private static final byte[] FORTUNE_START = "".getBytes(ServerHttpConnection.DEFAULT_HTTP_ENTITY_CHARSET); + + private static final byte[] TEMPLATE_END = "
idmessage
".getBytes(ServerHttpConnection.DEFAULT_HTTP_ENTITY_CHARSET); + + private static final byte[] FORTUNE_MIDDLE = "".getBytes(ServerHttpConnection.DEFAULT_HTTP_ENTITY_CHARSET); + + private static final byte[] FORTUNE_END = "
" + .getBytes(ServerHttpConnection.DEFAULT_HTTP_ENTITY_CHARSET); + + private static Comparator SORT_FORTUNE = (a, b) -> a.message.compareTo(b.message); + + public FortunesSendResponse(RequestHandler requestHandler, + ProcessAwareServerHttpConnectionManagedObject connection, ObjectMapper objectMapper) { + super(requestHandler, connection, objectMapper); + } + + public void sendFortunes(List fortunes) { + this.requestHandler.execute(() -> { + try { + // Additional fortunes + fortunes.add(new Fortune(0, "Additional fortune added at request time.")); + Collections.sort(fortunes, SORT_FORTUNE); + + // Send response + HttpResponse response = this.connection.getResponse(); + response.setContentType(TEXT_HTML, null); + ServerWriter writer = response.getEntityWriter(); + writer.write(TEMPLATE_START); + for (Fortune fortune : fortunes) { + writer.write(FORTUNE_START); + int id = fortune.id; + writer.write(Integer.valueOf(id).toString()); + writer.write(FORTUNE_MIDDLE); + StringEscapeUtils.ESCAPE_HTML4.translate(fortune.message, writer); + writer.write(FORTUNE_END); + } + writer.write(TEMPLATE_END); + send(this.connection); + } catch (CancelledKeyException | ClosedChannelException ex) { + // Ignore as disconnecting client + } catch (IOException ex) { + ex.printStackTrace(); + } + }); + } + +} \ No newline at end of file diff --git a/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/QueriesSendResponse.java b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/QueriesSendResponse.java new file mode 100644 index 00000000000..12574ee0a6e --- /dev/null +++ b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/QueriesSendResponse.java @@ -0,0 +1,42 @@ +package net.officefloor.benchmark; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.CancelledKeyException; +import java.nio.channels.ClosedChannelException; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import net.officefloor.server.RequestHandler; +import net.officefloor.server.http.HttpResponse; +import net.officefloor.server.http.impl.ProcessAwareServerHttpConnectionManagedObject; +import net.officefloor.server.http.parse.HttpRequestParser; + +/** + * Sends the Queries response. + * + * @author Daniel Sagenschneider + */ +public class QueriesSendResponse extends AbstractSendResponse { + + public QueriesSendResponse(RequestHandler requestHandler, + ProcessAwareServerHttpConnectionManagedObject connection, ObjectMapper objectMapper) { + super(requestHandler, connection, objectMapper); + } + + public void sendQueries(World[] worlds) { + this.requestHandler.execute(() -> { + try { + HttpResponse response = this.connection.getResponse(); + response.setContentType(APPLICATION_JSON, null); + this.objectMapper.writeValue(response.getEntityWriter(), worlds); + send(this.connection); + } catch (CancelledKeyException | ClosedChannelException ex) { + // Ignore as disconnecting client + } catch (IOException ex) { + ex.printStackTrace(); + } + }); + } + +} \ No newline at end of file diff --git a/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/RawWoof.java b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/RawWoof.java index 9bcd66cdbbe..818c7623105 100644 --- a/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/RawWoof.java +++ b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/RawWoof.java @@ -18,23 +18,16 @@ package net.officefloor.benchmark; import java.io.IOException; -import java.io.PrintWriter; import java.nio.ByteBuffer; -import java.nio.channels.CancelledKeyException; -import java.nio.channels.ClosedChannelException; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.util.Collections; -import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; -import org.apache.commons.text.StringEscapeUtils; - import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.module.afterburner.AfterburnerModule; @@ -60,7 +53,6 @@ import net.officefloor.server.http.impl.ProcessAwareServerHttpConnectionManagedObject; import net.officefloor.server.http.parse.HttpRequestParser; import net.officefloor.server.http.parse.HttpRequestParser.HttpRequestParserMetaData; -import net.officefloor.server.stream.ServerWriter; import net.officefloor.server.stream.impl.ThreadLocalStreamBufferPool; /** @@ -71,411 +63,252 @@ */ public abstract class RawWoof { - /** - * {@link SocketManager}. - */ - public static SocketManager socketManager = null; - - /** - * {@link Logger}. - */ - private static Logger logger = Logger.getLogger(RawWoof.class.getName()); - - /** - * Run application. - * - * @param args Command line arguments. - * @param operationsFactory {@link DatabaseOperationsFactory}. - */ - public static void run(String[] args, DatabaseOperationsFactory operationsFactory) throws Exception { - - // Obtain the port from properties - int port = args.length > 0 ? Integer.parseInt(args[0]) : 8080; - - // Ensure previous socket manager shutdown (typically from tests) - if (socketManager != null) { - socketManager.shutdown(); - } - - // Indicate details - String server = System.getProperty("OFFICE.net_officefloor_jdbc_DataSourceManagedObjectSource.server", - "tfb-database"); - System.out.println("Starting server on port " + port + " talking to database " + server); - - // Create the server location - HttpServerLocation serverLocation = new HttpServerLocationImpl("localhost", port, -1); - - // Create a thread factory per logical CPU - ThreadCompletionListener[] threadCompletionListenerCapture = new ThreadCompletionListener[]{null}; - ThreadFactory[] executionStrategy = RawWoofThreadAffinity - .createThreadFactories(() -> (ThreadLocalStreamBufferPool) threadCompletionListenerCapture[0]); - System.out.println("Using " + executionStrategy.length + " executors"); - - // Create the socket manager - socketManager = HttpServerSocketManagedObjectSource.createSocketManager(executionStrategy, - (threadCompletionListener) -> threadCompletionListenerCapture[0] = threadCompletionListener); - - // Create the database operations - DatabaseOperations operations = operationsFactory.createDatabaseOperations(executionStrategy.length, server, - 5432, "hello_world", "benchmarkdbuser", "benchmarkdbpass"); - - // Create raw HTTP servicing - RawHttpServicerFactory serviceFactory = new RawHttpServicerFactory(serverLocation, operations); - socketManager.bindServerSocket(serverLocation.getClusterHttpPort(), null, null, serviceFactory, serviceFactory); - - // Setup Date - ScheduledExecutorService dateTimer = Executors.newScheduledThreadPool(1); - dateTimer.scheduleAtFixedRate(serviceFactory.updateDate, 0, 1, TimeUnit.SECONDS); + /** + * {@link SocketManager}. + */ + public static SocketManager socketManager = null; - // Start servicing - Runnable[] runnables = socketManager.getRunnables(); - for (int i = 0; i < runnables.length; i++) { - executionStrategy[i].newThread(runnables[i]).start(); - } - Thread.sleep(1000); // allow threads to start up + /** + * {@link Logger}. + */ + private static Logger logger = Logger.getLogger(RawWoof.class.getName()); - // Indicate running - System.out.println("OfficeFloor running on port " + serverLocation.getClusterHttpPort()); - } - - /** - * Raw {@link AbstractHttpServicerFactory}. - */ - private static class RawHttpServicerFactory extends AbstractHttpServicerFactory - implements DatabaseOperationsContext { - - private static HttpHeaderName NAME_SERVER = new HttpHeaderName("Server"); - - private static HttpHeaderValue VALUE_SERVER = new HttpHeaderValue("O"); - - private static HttpHeaderName NAME_DATE = new HttpHeaderName("Date"); - - private static byte[] HELLO_WORLD = "Hello, World!".getBytes(ServerHttpConnection.DEFAULT_HTTP_ENTITY_CHARSET); - - private static final HttpHeaderValue APPLICATION_JSON = new HttpHeaderValue("application/json"); - - private static final HttpHeaderValue TEXT_PLAIN = new HttpHeaderValue("text/plain"); - - private static final HttpHeaderValue TEXT_HTML = new HttpHeaderValue("text/html;charset=utf-8"); - - private static final String QUERIES_PATH_PREFIX = "/queries?queries="; - - private static final String UPDATE_PATH_PREFIX = "/update?queries="; - - private static final byte[] TEMPLATE_START = "Fortunes" - .getBytes(ServerHttpConnection.DEFAULT_HTTP_ENTITY_CHARSET); - - private static final byte[] FORTUNE_START = "" - .getBytes(ServerHttpConnection.DEFAULT_HTTP_ENTITY_CHARSET); - - private static final byte[] TEMPLATE_END = "
idmessage
" - .getBytes(ServerHttpConnection.DEFAULT_HTTP_ENTITY_CHARSET); - - private static final byte[] FORTUNE_MIDDLE = "" - .getBytes(ServerHttpConnection.DEFAULT_HTTP_ENTITY_CHARSET); - - private static final byte[] FORTUNE_END = "
" - .getBytes(ServerHttpConnection.DEFAULT_HTTP_ENTITY_CHARSET); - - private static final TransientResourceException THROTTLED = new TransientResourceException(); - - private static class TransientResourceException extends Exception { - private static final long serialVersionUID = 1L; - } - - /** - * Date {@link HttpHeaderValue}. - */ - private volatile HttpHeaderValue dateHttpHeader; - - private final Runnable updateDate = () -> { - String now = DateTimeFormatter.RFC_1123_DATE_TIME.format(ZonedDateTime.now(ZoneOffset.UTC)); - RawHttpServicerFactory.this.dateHttpHeader = new HttpHeaderValue(now); - }; - - /** - * {@link ObjectMapper}. - */ - private final ObjectMapper objectMapper = new ObjectMapper(); - - /** - * {@link ManagedObjectContext}. - */ - private static ManagedObjectContext managedObjectContext = new ManagedObjectContext() { - - @Override - public String getBoundName() { - return RawWoof.class.getSimpleName(); - } - - @Override - public Logger getLogger() { - return logger; - } - - @Override - public R run(ProcessSafeOperation operation) throws T { - return operation.run(); - } - }; - - private static int getQueryCount(String queries) { - try { - int count = Integer.parseInt(queries); - return (count < 1) ? 1 : (count > 500) ? 500 : count; - } catch (NumberFormatException ex) { - return 1; - } - } - - /** - * {@link DatabaseOperations}. - */ - private final DatabaseOperations databaseOperations; - - /** - * Instantiate. - * - * @param serverLocation {@link HttpServerLocation}. - * @param operations {@link DatabaseOperations}. - */ - public RawHttpServicerFactory(HttpServerLocation serverLocation, DatabaseOperations operations) { - super(serverLocation, false, new HttpRequestParserMetaData(100, 1000, 1000000), null, null, true); - this.objectMapper.registerModule(new AfterburnerModule()); - this.databaseOperations = operations; - } - - /** - * Sends the {@link HttpResponse}. - * - * @param connection {@link ServerHttpConnection}. - * @throws IOException If fails to send. - */ - protected void send(ServerHttpConnection connection) throws IOException { - try { - @SuppressWarnings("unchecked") - ProcessAwareServerHttpConnectionManagedObject rawConnection = (ProcessAwareServerHttpConnectionManagedObject) connection; - rawConnection.getServiceFlowCallback().run(null); - } catch (IOException ex) { - throw ex; - } catch (Throwable ex) { - throw new IOException(ex); - } - } - - /* - * =============== SocketServicerFactory ================= - */ - - @Override - public SocketServicer createSocketServicer( - RequestHandler requestHandler) { - - // Set up the thread - this.databaseOperations.threadSetup(requestHandler); - - // Continue on to create socket servicer - return super.createSocketServicer(requestHandler); - } - - /* - * ===================== HttpServicer ==================== - */ - - @Override - protected ProcessManager service(ProcessAwareServerHttpConnectionManagedObject connection) - throws IOException { - - // Configure context - connection.setManagedObjectContext(managedObjectContext); - - // Service the connection - HttpRequest request = connection.getRequest(); - HttpResponse response = connection.getResponse(); - - // Provider Server and Date - HttpResponseHeaders headers = response.getHeaders(); - headers.addHeader(NAME_SERVER, VALUE_SERVER); - headers.addHeader(NAME_DATE, this.dateHttpHeader); - - // Determine request - String requestUri = request.getUri(); - switch (requestUri) { - - case "/plaintext": - response.setContentType(TEXT_PLAIN, null); - response.getEntity().write(HELLO_WORLD); - this.send(connection); - break; - - case "/json": - response.setContentType(APPLICATION_JSON, null); - this.objectMapper.writeValue(response.getEntityWriter(), new Message("Hello, World!")); - this.send(connection); - break; - - case "/db": - this.databaseOperations.db(response, connection, this); - break; - - case "/fortunes": - this.databaseOperations.fortunes(response, connection, this); - break; - - default: - // Provide redirect - if (requestUri.startsWith(QUERIES_PATH_PREFIX)) { - // Obtain the number of queries - String queriesCountText = requestUri.substring(QUERIES_PATH_PREFIX.length()); - int queryCount = getQueryCount(queriesCountText); - - // Undertake queries - this.databaseOperations.queries(queryCount, response, connection, this); - - } else if (requestUri.startsWith(UPDATE_PATH_PREFIX)) { - // Obtain the number of queries - String queriesCountText = requestUri.substring(UPDATE_PATH_PREFIX.length()); - int queryCount = getQueryCount(queriesCountText); - - // Undertake update - this.databaseOperations.update(queryCount, response, connection, this); - - } else { - // Unknown request - response.setStatus(HttpStatus.NOT_FOUND); - this.send(connection); - } - break; - } - - // No process management - return null; - } - - /* - * ==================== DatabaseOperationsContext ===================== - */ - - @Override - public Exception getTransientResourceException() { - return THROTTLED; - } - - @Override - public void dbSend(HttpResponse response, ServerHttpConnection connection, World world) { - try { - response.setContentType(APPLICATION_JSON, null); - this.objectMapper.writeValue(response.getEntityWriter(), world); - this.send(connection); - } catch (CancelledKeyException | ClosedChannelException ex) { - // Ignore as disconnecting client - } catch (IOException ex) { - ex.printStackTrace(); - } - } - - @Override - public void queriesSend(HttpResponse response, ServerHttpConnection connection, List worlds) { - try { - response.setContentType(APPLICATION_JSON, null); - this.objectMapper.writeValue(response.getEntityWriter(), worlds); - this.send(connection); - } catch (CancelledKeyException | ClosedChannelException ex) { - // Ignore as disconnecting client - } catch (IOException ex) { - ex.printStackTrace(); - } - } - - @Override - public void fortunesSend(HttpResponse response, ServerHttpConnection connection, List fortunes) { - try { - // Additional fortunes - fortunes.add(new Fortune(0, "Additional fortune added at request time.")); - Collections.sort(fortunes, (a, b) -> a.message.compareTo(b.message)); - - // Send response - response.setContentType(TEXT_HTML, null); - ServerWriter writer = response.getEntityWriter(); - writer.write(TEMPLATE_START); - for (Fortune fortune : fortunes) { - writer.write(FORTUNE_START); - int id = fortune.id; - writer.write(Integer.valueOf(id).toString()); - writer.write(FORTUNE_MIDDLE); - StringEscapeUtils.ESCAPE_HTML4.translate(fortune.message, writer); - writer.write(FORTUNE_END); - } - writer.write(TEMPLATE_END); - this.send(connection); - } catch (CancelledKeyException | ClosedChannelException ex) { - // Ignore as disconnecting client - } catch (IOException ex) { - ex.printStackTrace(); - } - } - - @Override - public void updateSend(HttpResponse response, ServerHttpConnection connection, List worlds) { - try { - response.setContentType(APPLICATION_JSON, null); - this.objectMapper.writeValue(response.getEntityWriter(), worlds); - this.send(connection); - } catch (CancelledKeyException | ClosedChannelException ex) { - // Ignore as disconnecting client - } catch (IOException ex) { - ex.printStackTrace(); - } - } - - @Override - public void sendError(ServerHttpConnection connection, Throwable failure) { - try { - - // Setup to send response - HttpResponse response = connection.getResponse(); - response.reset(); - - // Determine type of error - if (failure instanceof TransientResourceException) { - - // Indicate overloaded - response.setStatus(HttpStatus.SERVICE_UNAVAILABLE); - - } else { - // Provide details of failure - response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR); - response.setContentType(TEXT_PLAIN, null); - failure.printStackTrace(new PrintWriter(response.getEntityWriter())); - } - - // Send error response - this.send(connection); - - } catch (CancelledKeyException | ClosedChannelException ex) { - // Ignore as disconnecting client - } catch (IOException ex) { - ex.printStackTrace(); - } - } - - @Override - public void sendError(ServerHttpConnection connection, int status) { - try { - // Setup to send response - HttpResponse response = connection.getResponse(); - response.reset(); - - // Flag error status - response.setStatus(HttpStatus.getHttpStatus(status)); - - // Send error response - this.send(connection); - - } catch (IOException ex) { - ex.printStackTrace(); - } - } - } + /** + * Run application. + * + * @param args Command line arguments. + * @param operationsFactory {@link DatabaseOperationsFactory}. + */ + public static void run(String[] args, DatabaseOperationsFactory operationsFactory) throws Exception { + + // Obtain the port from properties + int port = args.length > 0 ? Integer.parseInt(args[0]) : 8080; + + // Ensure previous socket manager shutdown (typically from tests) + if (socketManager != null) { + socketManager.shutdown(); + } + + // Indicate details + String server = System.getProperty("OFFICE.net_officefloor_jdbc_DataSourceManagedObjectSource.server", + "tfb-database"); + System.out.println("Starting server on port " + port + " talking to database " + server); + + // Create the server location + HttpServerLocation serverLocation = new HttpServerLocationImpl("localhost", port, -1); + + // Create a thread factory per logical CPU + ThreadCompletionListener[] threadCompletionListenerCapture = new ThreadCompletionListener[] { null }; + ThreadFactory[] executionStrategy = RawWoofThreadAffinity + .createThreadFactories(() -> (ThreadLocalStreamBufferPool) threadCompletionListenerCapture[0]); + System.out.println("Using " + executionStrategy.length + " executors"); + + // Create the socket manager + socketManager = HttpServerSocketManagedObjectSource.createSocketManager(executionStrategy, + (threadCompletionListener) -> threadCompletionListenerCapture[0] = threadCompletionListener); + + // Create the database operations + DatabaseOperations operations = operationsFactory.createDatabaseOperations(executionStrategy.length, server, + 5432, "hello_world", "benchmarkdbuser", "benchmarkdbpass"); + + // Create raw HTTP servicing + RawHttpServicerFactory serviceFactory = new RawHttpServicerFactory(serverLocation, operations); + socketManager.bindServerSocket(serverLocation.getClusterHttpPort(), null, null, serviceFactory, serviceFactory); + + // Setup Date + ScheduledExecutorService dateTimer = Executors.newScheduledThreadPool(1); + dateTimer.scheduleAtFixedRate(serviceFactory.updateDate, 0, 1, TimeUnit.SECONDS); + + // Start servicing + Runnable[] runnables = socketManager.getRunnables(); + for (int i = 0; i < runnables.length; i++) { + executionStrategy[i].newThread(runnables[i]).start(); + } + Thread.sleep(1000); // allow threads to start up + + // Indicate running + System.out.println("OfficeFloor running on port " + serverLocation.getClusterHttpPort()); + } + + /** + * Raw {@link AbstractHttpServicerFactory}. + */ + private static class RawHttpServicerFactory extends AbstractHttpServicerFactory { + + private static HttpHeaderName NAME_SERVER = new HttpHeaderName("Server"); + + private static HttpHeaderValue VALUE_SERVER = new HttpHeaderValue("O"); + + private static HttpHeaderName NAME_DATE = new HttpHeaderName("Date"); + + private static byte[] HELLO_WORLD = "Hello, World!".getBytes(ServerHttpConnection.DEFAULT_HTTP_ENTITY_CHARSET); + + private static final String QUERIES_PATH_PREFIX = "/queries?queries="; + + private static final String UPDATE_PATH_PREFIX = "/update?queries="; + + /** + * Date {@link HttpHeaderValue}. + */ + private volatile HttpHeaderValue dateHttpHeader; + + private final Runnable updateDate = () -> { + String now = DateTimeFormatter.RFC_1123_DATE_TIME.format(ZonedDateTime.now(ZoneOffset.UTC)); + RawHttpServicerFactory.this.dateHttpHeader = new HttpHeaderValue(now); + }; + + /** + * {@link ObjectMapper}. + */ + private final ObjectMapper objectMapper = new ObjectMapper(); + + /** + * {@link ManagedObjectContext}. + */ + private static ManagedObjectContext managedObjectContext = new ManagedObjectContext() { + + @Override + public String getBoundName() { + return RawWoof.class.getSimpleName(); + } + + @Override + public Logger getLogger() { + return logger; + } + + @Override + public R run(ProcessSafeOperation operation) throws T { + return operation.run(); + } + }; + + private static int getQueryCount(String queries) { + try { + int count = Integer.parseInt(queries); + return (count < 1) ? 1 : (count > 500) ? 500 : count; + } catch (NumberFormatException ex) { + return 1; + } + } + + /** + * {@link DatabaseOperations}. + */ + private final DatabaseOperations databaseOperations; + + /** + * {@link ThreadLocal} {@link RequestHandler}. + */ + private final ThreadLocal> threadLocalRequestHandler = new ThreadLocal<>(); + + /** + * Instantiate. + * + * @param serverLocation {@link HttpServerLocation}. + * @param operations {@link DatabaseOperations}. + */ + public RawHttpServicerFactory(HttpServerLocation serverLocation, DatabaseOperations operations) { + super(serverLocation, false, new HttpRequestParserMetaData(100, 1000, 1000000), null, null, true); + this.objectMapper.registerModule(new AfterburnerModule()); + this.databaseOperations = operations; + } + + /* + * =============== SocketServicerFactory ================= + */ + + @Override + public SocketServicer createSocketServicer( + RequestHandler requestHandler) { + + // Specify the thread local request handler + this.threadLocalRequestHandler.set(requestHandler); + + // Set up the thread + this.databaseOperations.threadSetup(requestHandler); + + // Continue on to create socket servicer + return super.createSocketServicer(requestHandler); + } + + /* + * ===================== HttpServicer ==================== + */ + + @Override + protected ProcessManager service(ProcessAwareServerHttpConnectionManagedObject connection) + throws IOException { + + // Configure context + connection.setManagedObjectContext(managedObjectContext); + + // Service the connection + HttpRequest request = connection.getRequest(); + HttpResponse response = connection.getResponse(); + + // Provider Server and Date + HttpResponseHeaders headers = response.getHeaders(); + headers.addHeader(NAME_SERVER, VALUE_SERVER); + headers.addHeader(NAME_DATE, this.dateHttpHeader); + + // Determine request + String requestUri = request.getUri(); + switch (requestUri) { + + case "/plaintext": + response.setContentType(AbstractSendResponse.TEXT_PLAIN, null); + response.getEntity().write(HELLO_WORLD); + AbstractSendResponse.send(connection); + break; + + case "/json": + response.setContentType(AbstractSendResponse.APPLICATION_JSON, null); + this.objectMapper.writeValue(response.getEntityWriter(), new Message("Hello, World!")); + AbstractSendResponse.send(connection); + break; + + case "/db": + this.databaseOperations + .db(new DbSendResponse(this.threadLocalRequestHandler.get(), connection, this.objectMapper)); + break; + + case "/fortunes": + this.databaseOperations.fortunes( + new FortunesSendResponse(this.threadLocalRequestHandler.get(), connection, this.objectMapper)); + break; + + default: + // Provide redirect + if (requestUri.startsWith(QUERIES_PATH_PREFIX)) { + // Obtain the number of queries + String queriesCountText = requestUri.substring(QUERIES_PATH_PREFIX.length()); + int queryCount = getQueryCount(queriesCountText); + + // Undertake queries + this.databaseOperations.queries(queryCount, new QueriesSendResponse( + this.threadLocalRequestHandler.get(), connection, this.objectMapper)); + + } else if (requestUri.startsWith(UPDATE_PATH_PREFIX)) { + // Obtain the number of queries + String queriesCountText = requestUri.substring(UPDATE_PATH_PREFIX.length()); + int queryCount = getQueryCount(queriesCountText); + + // Undertake update + this.databaseOperations.update(queryCount, new UpdateSendResponse( + this.threadLocalRequestHandler.get(), connection, this.objectMapper)); + + } else { + // Unknown request + response.setStatus(HttpStatus.NOT_FOUND); + AbstractSendResponse.send(connection); + } + break; + } + + // No process management + return null; + } + } } \ No newline at end of file diff --git a/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/UpdateSendResponse.java b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/UpdateSendResponse.java new file mode 100644 index 00000000000..cacefe42dd8 --- /dev/null +++ b/frameworks/Java/officefloor/src/woof_benchmark_woof/src/main/java/net/officefloor/benchmark/UpdateSendResponse.java @@ -0,0 +1,42 @@ +package net.officefloor.benchmark; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.CancelledKeyException; +import java.nio.channels.ClosedChannelException; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import net.officefloor.server.RequestHandler; +import net.officefloor.server.http.HttpResponse; +import net.officefloor.server.http.impl.ProcessAwareServerHttpConnectionManagedObject; +import net.officefloor.server.http.parse.HttpRequestParser; + +/** + * Sends the Update response. + * + * @author Daniel Sagenschneider + */ +public class UpdateSendResponse extends AbstractSendResponse { + + public UpdateSendResponse(RequestHandler requestHandler, + ProcessAwareServerHttpConnectionManagedObject connection, ObjectMapper objectMapper) { + super(requestHandler, connection, objectMapper); + } + + public void sendUpdate(World[] worlds) { + this.requestHandler.execute(() -> { + try { + HttpResponse response = this.connection.getResponse(); + response.setContentType(APPLICATION_JSON, null); + this.objectMapper.writeValue(response.getEntityWriter(), worlds); + send(this.connection); + } catch (CancelledKeyException | ClosedChannelException ex) { + // Ignore as disconnecting client + } catch (IOException ex) { + ex.printStackTrace(); + } + }); + } + +} \ No newline at end of file From 822f510f4eca055fbebd1007254230f6d4a50cc1 Mon Sep 17 00:00:00 2001 From: Laurents Meyer Date: Tue, 1 Jun 2021 17:12:33 +0200 Subject: [PATCH 6/6] Fix permission issue when using skip-name-resolve in conjunction with running a local MySQL database server and connecting via TCP. (#6611) --- toolset/databases/mysql/create.sql | 13 +++++++++++-- toolset/databases/mysql/mysql.dockerfile | 15 ++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/toolset/databases/mysql/create.sql b/toolset/databases/mysql/create.sql index ed15cb5ffdb..f1a5756d230 100644 --- a/toolset/databases/mysql/create.sql +++ b/toolset/databases/mysql/create.sql @@ -2,6 +2,15 @@ # http://stackoverflow.com/questions/37719818/the-server-time-zone-value-aest-is-unrecognized-or-represents-more-than-one-ti SET GLOBAL time_zone = '+00:00'; +CREATE USER 'benchmarkdbuser'@'%' IDENTIFIED WITH mysql_native_password BY 'benchmarkdbpass'; +CREATE USER 'benchmarkdbuser'@'localhost' IDENTIFIED WITH mysql_native_password BY 'benchmarkdbpass'; + +-- GitHub Actions/CI run the database server on the same system as the benchmarks. +-- Because we setup MySQL with the skip-name-resolve option, the IP address 127.0.0.1 might not be resolved to localhost +-- anymore. This does not seem to matter, as long as Unix sockets are being used (e.g. when setting up the docker image), +-- because the host is set to be localhost implicitly, but it matters for local TCP connections. +CREATE USER 'benchmarkdbuser'@'127.0.0.1' IDENTIFIED WITH mysql_native_password BY 'benchmarkdbpass'; + # modified from SO answer http://stackoverflow.com/questions/5125096/for-loop-in-mysql CREATE DATABASE hello_world; USE hello_world; @@ -12,10 +21,9 @@ CREATE TABLE world ( PRIMARY KEY (id) ) ENGINE=INNODB; -CREATE USER 'benchmarkdbuser'@'%' IDENTIFIED WITH mysql_native_password BY 'benchmarkdbpass'; -CREATE USER 'benchmarkdbuser'@'localhost' IDENTIFIED WITH mysql_native_password BY 'benchmarkdbpass'; GRANT ALL PRIVILEGES ON hello_world.world TO 'benchmarkdbuser'@'%'; GRANT ALL PRIVILEGES ON hello_world.world TO 'benchmarkdbuser'@'localhost'; +GRANT ALL PRIVILEGES ON hello_world.world TO 'benchmarkdbuser'@'127.0.0.1'; DELIMITER # CREATE PROCEDURE load_data() @@ -46,6 +54,7 @@ CREATE TABLE fortune ( ENGINE=INNODB; GRANT ALL PRIVILEGES ON hello_world.fortune TO 'benchmarkdbuser'@'%'; GRANT ALL PRIVILEGES ON hello_world.fortune TO 'benchmarkdbuser'@'localhost'; +GRANT ALL PRIVILEGES ON hello_world.fortune TO 'benchmarkdbuser'@'127.0.0.1'; INSERT INTO fortune (message) VALUES ('fortune: No such file or directory'); INSERT INTO fortune (message) VALUES ('A computer scientist is someone who fixes things that aren''t broken.'); diff --git a/toolset/databases/mysql/mysql.dockerfile b/toolset/databases/mysql/mysql.dockerfile index e579a9f7343..e4774dd15f5 100644 --- a/toolset/databases/mysql/mysql.dockerfile +++ b/toolset/databases/mysql/mysql.dockerfile @@ -32,15 +32,12 @@ RUN cp -R -p /var/lib/mysql /ssd/ RUN cp -R -p /var/log/mysql /ssd/log RUN mkdir -p /var/run/mysqld -# It may seem weird that we call `service mysql start` several times, but the RUN -# directive is a 1-time operation for building this image. Subsequent RUN calls -# do not see running processes from prior RUN calls; therefor, each command here -# that relies on the mysql server running will explicitly start the server and -# perform the work required. RUN chown -R mysql:mysql /var/lib/mysql /var/log/mysql /var/run/mysqld /ssd && \ - mysqld & \ - until mysql -uroot -psecret -e "exit"; do sleep 1; done && \ + (mysqld &) && \ + until mysqladmin -uroot -psecret ping; do sleep 1; done && \ mysqladmin -uroot -psecret flush-hosts && \ - mysql -uroot -psecret < create.sql + mysql -uroot -psecret < create.sql && \ + mysqladmin -uroot -psecret shutdown && \ + chown -R mysql:mysql /var/lib/mysql /var/log/mysql /var/run/mysqld /ssd -CMD chown -R mysql:mysql /var/lib/mysql /var/log/mysql /var/run/mysqld /ssd && mysqld +CMD ["mysqld"]