From 623a7db20613fbeb1faac078d761b6a7c59394c7 Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Fri, 15 Mar 2024 07:59:06 -0700 Subject: [PATCH 1/4] Fix/improve X-Forwarded-For/throttling testing --- tests/CMakeLists.txt | 41 +++++++++++++++++++++-------------------- tests/httpd.conf.in | 6 +++--- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cbbf99e1..f3e7486c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -754,26 +754,6 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH}) set_tests_properties(tile_expired_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES FIXTURES_REQUIRED services_started_${STORAGE_BACKEND} ) - set(TILE_URL_PATH "/tiles/${MAP_NAME}/3/0/3.${EXTENSION}") - set(HTTPD1_URL "http://${HTTPD1_HOST}:${HTTPD1_PORT}${TILE_URL_PATH}") - set(HTTPD2_URL "http://${HTTPD2_HOST}:${HTTPD2_PORT}${TILE_URL_PATH}") - add_test(NAME throttling_xforward_${MAP_NAME}_${STORAGE_BACKEND} - COMMAND ${BASH} -c " - TEST_CURL_CMD=\"${CURL_CMD} --header 'X-Forwarded-For: ${CTEST_HOST}, 127.0.0.1' --output /dev/null\" - for i in {0..10}; do - if ! \${TEST_CURL_CMD} ${HTTPD1_URL}; then - echo \"${HTTPD1_URL}\"; - fi - if ! \${TEST_CURL_CMD} ${HTTPD2_URL}; then - echo \"${HTTPD2_URL}\"; - fi - done - " - WORKING_DIRECTORY tests - ) - set_tests_properties(throttling_xforward_${MAP_NAME}_${STORAGE_BACKEND} PROPERTIES - FIXTURES_REQUIRED services_started_${STORAGE_BACKEND} - ) endforeach() # Generate file and URL paths for tiles @@ -977,6 +957,27 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH}) set_tests_properties(max_load_missing_${STORAGE_BACKEND} PROPERTIES FIXTURES_REQUIRED services_started_${STORAGE_BACKEND} ) + add_test(NAME throttling_xforward_${STORAGE_BACKEND} + COMMAND ${BASH} -c " + RESPONSE_CODE_CMD=\"${CURL_CMD} --write-out %{http_code}\" + for i in {0..19}; do + RESPONSE_CODE=$(\${RESPONSE_CODE_CMD} --header \"X-Forwarded-For: ${CTEST_HOST}, 127.0.0.1\" \ + http://${HTTPD2_HOST}:${HTTPD2_PORT}/tiles/webp/\$i/0/0.webp) + echo \"Response code: '\${RESPONSE_CODE}'\" + if [ \"\${RESPONSE_CODE}\" = \"503\" ]; then + echo 'Request being rejected'; + exit 0; + fi + done + echo 'Request was never rejected'; + exit 1; + " + WORKING_DIRECTORY tests + ) + set_tests_properties(throttling_xforward_${STORAGE_BACKEND} PROPERTIES + FIXTURES_REQUIRED services_started_${STORAGE_BACKEND} + RUN_SERIAL TRUE + ) endif() if(JQ_EXECUTABLE) diff --git a/tests/httpd.conf.in b/tests/httpd.conf.in index abc008a6..9fb0840a 100644 --- a/tests/httpd.conf.in +++ b/tests/httpd.conf.in @@ -58,7 +58,7 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d ModTileEnableDirtyURL Off ModTileEnableStats Off ModTileEnableStatusURL Off - ModTileEnableTileThrottling On + ModTileEnableTileThrottling Off ModTileEnableTileThrottlingXForward 1 ModTileMissingRequestTimeout 3 ModTileRenderdSocketAddr @RENDERD1_HOST@ @RENDERD1_PORT@ @@ -88,8 +88,8 @@ Redirect /renderd-example-map/leaflet/leaflet.min.js https://unpkg.com/leaflet/d ModTileMissingRequestTimeout 3 ModTileRenderdSocketName @RENDERD2_SOCKET@ ModTileRequestTimeout 3 - ModTileThrottlingRenders 1 0.2 - ModTileThrottlingTiles 1 1 + ModTileThrottlingRenders 10 0.2 + ModTileThrottlingTiles 10 0.2 ModTileTileDir @TILE_DIR@ ModTileVeryOldThreshold -10000000 From 3978feb77645239475633f45a3f00e1c2eb6b358 Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Fri, 15 Mar 2024 09:33:54 -0700 Subject: [PATCH 2/4] FreeBSD client IP is detected as ::1 --- .github/workflows/build-and-test.yml | 3 ++- docs/build/building_on_freebsd.md | 2 ++ tests/CMakeLists.txt | 31 ++++++++++++++++++---------- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index bb8cd08a..910dff34 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -208,7 +208,8 @@ jobs: continue-on-error: true env: CFLAGS: --coverage - CTEST_HOST: localhost + CTEST_CLIENT_HOST: ::1 + CTEST_SERVER_HOST: localhost CXXFLAGS: --coverage LIBRARY_PATH: /usr/local/lib TMPDIR: /tmp diff --git a/docs/build/building_on_freebsd.md b/docs/build/building_on_freebsd.md index 73a7cc04..1e61a410 100644 --- a/docs/build/building_on_freebsd.md +++ b/docs/build/building_on_freebsd.md @@ -34,6 +34,8 @@ sudo pkg install --yes \ # Download, Build, Test & Install `mod_tile` export CMAKE_BUILD_PARALLEL_LEVEL=$(sysctl -n hw.ncpu) +export CTEST_CLIENT_HOST="::1" +export CTEST_SERVER_HOST="localhost" export LIBRARY_PATH="/usr/local/lib" rm -rf /tmp/mod_tile_src /tmp/mod_tile_build mkdir /tmp/mod_tile_src /tmp/mod_tile_build diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f3e7486c..dbf97503 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,12 +36,21 @@ find_program(SLEEP_EXECUTABLE NAMES sleep REQUIRED) find_program(TOUCH_EXECUTABLE NAMES gtouch touch REQUIRED) # Sets the host to be used for CTest test services -if(DEFINED ENV{CTEST_HOST}) - # To the value of environment variable "CTEST_HOST" - set(CTEST_HOST "$ENV{CTEST_HOST}") +if(DEFINED ENV{CTEST_SERVER_HOST}) + # To the value of environment variable "CTEST_SERVER_HOST" + set(CTEST_SERVER_HOST "$ENV{CTEST_SERVER_HOST}") else() # Or to 0.0.0.0 by default - set(CTEST_HOST "0.0.0.0") + set(CTEST_SERVER_HOST "0.0.0.0") +endif() + +# Sets the host to be used for CTest client +if(DEFINED ENV{CTEST_CLIENT_HOST}) + # To the value of environment variable "CTEST_CLIENT_HOST" + set(CTEST_CLIENT_HOST "$ENV{CTEST_CLIENT_HOST}") +else() + # Or to 0.0.0.0 by default + set(CTEST_CLIENT_HOST "127.0.0.") endif() #----------------------------------------------------------------------------- @@ -51,15 +60,15 @@ endif() #----------------------------------------------------------------------------- set(DEFAULT_MAP_NAME "default") -set(HTTPD0_HOST "${CTEST_HOST}") +set(HTTPD0_HOST "${CTEST_SERVER_HOST}") set(HTTPD0_PORT_BASE "59000") -set(HTTPD1_HOST "${CTEST_HOST}") +set(HTTPD1_HOST "${CTEST_SERVER_HOST}") set(HTTPD1_PORT_BASE "59100") -set(HTTPD2_HOST "${CTEST_HOST}") +set(HTTPD2_HOST "${CTEST_SERVER_HOST}") set(HTTPD2_PORT_BASE "59200") -set(MEMCACHED_HOST "${CTEST_HOST}") +set(MEMCACHED_HOST "${CTEST_SERVER_HOST}") set(MEMCACHED_PORT_BASE "60000") -set(RENDERD1_HOST "${CTEST_HOST}") +set(RENDERD1_HOST "${CTEST_SERVER_HOST}") set(RENDERD1_PORT_BASE "59500") set(CURL_CMD "${CURL_EXECUTABLE} --fail --silent") @@ -960,8 +969,8 @@ foreach(STORAGE_BACKEND_INDEX RANGE ${STORAGE_BACKENDS_LENGTH}) add_test(NAME throttling_xforward_${STORAGE_BACKEND} COMMAND ${BASH} -c " RESPONSE_CODE_CMD=\"${CURL_CMD} --write-out %{http_code}\" - for i in {0..19}; do - RESPONSE_CODE=$(\${RESPONSE_CODE_CMD} --header \"X-Forwarded-For: ${CTEST_HOST}, 127.0.0.1\" \ + for i in {0..20}; do + RESPONSE_CODE=$(\${RESPONSE_CODE_CMD} --header \"X-Forwarded-For: ${CTEST_SERVER_HOST}, ${CTEST_CLIENT_HOST}\" \ http://${HTTPD2_HOST}:${HTTPD2_PORT}/tiles/webp/\$i/0/0.webp) echo \"Response code: '\${RESPONSE_CODE}'\" if [ \"\${RESPONSE_CODE}\" = \"503\" ]; then From 1a96bcc0ea2eb6d578b1f727d30e9c84903daf8d Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Fri, 15 Mar 2024 09:49:16 -0700 Subject: [PATCH 3/4] storage_mutex not used, was only used with Apache HTTP Server < v2.4 --- src/mod_tile.c | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/src/mod_tile.c b/src/mod_tile.c index 78fba343..d3fc76ed 100644 --- a/src/mod_tile.c +++ b/src/mod_tile.c @@ -90,7 +90,6 @@ char *shmfilename; char *shmfilename_delaypool; apr_global_mutex_t *stats_mutex; apr_global_mutex_t *delay_mutex; -apr_global_mutex_t *storage_mutex; char *mutexfilename; int layerCount = 0; @@ -1868,36 +1867,6 @@ static int mod_tile_post_config(apr_pool_t *pconf, apr_pool_t *plog, return HTTP_INTERNAL_SERVER_ERROR; } -#endif /* MOD_TILE_SET_MUTEX_PERMS */ - - /* - * Create another unique filename to lock upon. Note that - * depending on OS and locking mechanism of choice, the file - * may or may not be actually created. - */ - mutexfilename = apr_psprintf(pconf, "/tmp/httpd_mutex_storage.%ld", - (long int) getpid()); - - rs = apr_global_mutex_create(&storage_mutex, (const char *) mutexfilename, - APR_LOCK_DEFAULT, pconf); - - if (rs != APR_SUCCESS) { - ap_log_error(APLOG_MARK, APLOG_ERR, rs, s, - "Failed to create mutex on file %s", - mutexfilename); - return HTTP_INTERNAL_SERVER_ERROR; - } - -#ifdef MOD_TILE_SET_MUTEX_PERMS - rs = ap_unixd_set_global_mutex_perms(storage_mutex); - - if (rs != APR_SUCCESS) { - ap_log_error(APLOG_MARK, APLOG_CRIT, rs, s, - "Parent could not set permissions on mod_tile " - "mutex: check User and Group directives"); - return HTTP_INTERNAL_SERVER_ERROR; - } - #endif /* MOD_TILE_SET_MUTEX_PERMS */ return OK; From 0d8cc51885ffe7e3303057c873fe46feae154dc3 Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Fri, 15 Mar 2024 09:58:37 -0700 Subject: [PATCH 4/4] Fix typo --- tests/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index dbf97503..dedead2e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -49,8 +49,8 @@ if(DEFINED ENV{CTEST_CLIENT_HOST}) # To the value of environment variable "CTEST_CLIENT_HOST" set(CTEST_CLIENT_HOST "$ENV{CTEST_CLIENT_HOST}") else() - # Or to 0.0.0.0 by default - set(CTEST_CLIENT_HOST "127.0.0.") + # Or to 127.0.0.1 by default + set(CTEST_CLIENT_HOST "127.0.0.1") endif() #-----------------------------------------------------------------------------