Permalink
Browse files

CBD-294 Simplify/cleanup httpd layer for performance

Removed all the features not used in Couchbase Server
(vhosts, url rewriting, access log, etc).
These were increasing, very significantly, query response
times/latencies.

Tests repeated on Linux and Mac OS X, 2 and 4 nodes clusters,
with evperf and the erlang script view_query_perf.

Some sample results with view_query_perf below.

$ ./run --queries 25000 --workers 1 --query-url 'http://localhost:9500/default/_design/test2/_view/view1?limit=10' --output-times before
Spawning 1 workers, each will perform 25000 view queries
View query URL is:  http://localhost:9500/default/_design/test2/_view/view1?limit=10

Waiting for workers to finish...

All workers finished. Final statistics are:

    Average response time:   6.4329149200000275ms
    Highest response time:   168.963ms
    Lowest response time:    5.382ms
    Response time std dev:   1.4596464282289152ms
    # of errors:             0

Saving query response times to file before

$ ./run --queries 25000 --workers 1 --query-url 'http://localhost:9500/default/_design/test2/_view/view1?limit=10' --output-times after
Spawning 1 workers, each will perform 25000 view queries
View query URL is:  http://localhost:9500/default/_design/test2/_view/view1?limit=10

Waiting for workers to finish...

All workers finished. Final statistics are:

    Average response time:   5.704996960000008ms
    Highest response time:   161.67ms
    Lowest response time:    4.836ms
    Response time std dev:   1.463934100590158ms
    # of errors:             0

Saving query response times to file after

$ cat before | perl -MStatistics::Histogram -e '@data = <>; chomp @data; print get_histogram(\@data);'
Count: 25000
Range:  5.382 - 168.963; Mean:  6.433; Median:  6.310; Stddev:  1.460
Percentiles:  90th:  6.987; 95th:  7.342; 99th:  8.240
   5.382 -    7.861: 24472 #####################################################
   7.861 -   11.304:   486 #
  11.304 -   16.083:    28 |
  16.083 -   22.720:     3 |
  22.720 -   31.935:     7 |
  31.935 -   44.729:     1 |
  44.729 -   87.161:     1 |
  87.161 -  121.409:     1 |
 121.409 -  168.963:     1 |

$ cat after | perl -MStatistics::Histogram -e '@data = <>; chomp @data; print get_histogram(\@data);'
Count: 25000
Range:  4.836 - 161.670; Mean:  5.705; Median:  5.602; Stddev:  1.464
Percentiles:  90th:  6.191; 95th:  6.499; 99th:  7.333
   4.836 -    7.140: 24633 #####################################################
   7.140 -   10.354:   336 #
  10.354 -   14.837:    24 |
  14.837 -   29.811:     3 |
  29.811 -   41.976:     1 |
  41.976 -   58.945:     1 |
  58.945 -  161.670:     2 |

Change-Id: I7c029c334df5b2322837e1b213efabe4818b0361
Reviewed-on: http://review.couchbase.org/16460
Reviewed-by: Damien Katz <damien@couchbase.com>
Tested-by: Filipe David Borba Manana <fdmanana@gmail.com>
  • Loading branch information...
1 parent 0fab83a commit d6f27435cebe4aba87d276821f7238b4009e420c @fdmanana fdmanana committed May 27, 2012
@@ -6,7 +6,6 @@
database_dir = %localstatelibdir%
view_index_dir = %localstatelibdir%
util_driver_dir = %couchprivlibdir%
-max_document_size = 4294967296 ; 4 GB
os_process_timeout = 30000 ; 5 seconds. for view and external servers.
max_dbs_open = 100
delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned
@@ -19,10 +18,6 @@ uri_file = %localstaterundir%/couch.uri
; deflate_[N] - use zlib's deflate, N is the compression level which ranges from 1 (fastest,
; lowest compression ratio) to 9 (slowest, highest compression ratio)
file_compression = snappy
-; Higher values may give better read performance due to less read operations
-; and/or more OS page cache hits, but they can also increase overall response
-; time for writes when there are many attachment write requests in parallel.
-attachment_stream_buffer_size = 4096
; Maximum number of distinct view update/building processes at any point in time.
max_parallel_indexers = 4
max_parallel_replica_indexers = 2
@@ -37,10 +32,9 @@ checkpoint_after = 5242880 ; checkpoint after every N bytes were written
[httpd]
port = 5984
bind_address = 127.0.0.1
-authentication_handlers = {couch_httpd_oauth, oauth_authentication_handler}, {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler}
+authentication_handlers =
+; authentication_handlers = {couch_httpd_oauth, oauth_authentication_handler}, {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler}
default_handler = {couch_httpd_db, handle_request}
-secure_rewrites = true
-vhost_global_handlers = _utils, _uuids, _session, _oauth, _users
allow_jsonp = false
db_frontend = couch_db_frontend
; Options for the MochiWeb HTTP server.
@@ -56,21 +50,6 @@ file = %localstatelogdir%/couch.log
level = info
include_sasl = true
-; Enable apache httpd-style access logging.
-; The default format is the "extended" or "combined" format that
-; includes:
-; host ident authuser date request status bytes referer(sic) user-agent
-;
-; The alternative format is "common" that includes:
-; host ident authuser date request status bytes
-;
-; If a value is not defined, the log entry will be " - " for that value.
-
-[access_log]
-enable = false
-format = extended
-file = %localstatelogdir%/couch_access.log
-
[couch_httpd_auth]
authentication_db = _users
authentication_redirect = /_utils/session.html
@@ -94,17 +73,12 @@ os_process_limit = 50
view_manager={couch_view, start_link, []}
set_view_manager={couch_set_view, start_link, []}
index_merger_pool={lhttpc_manager, start_link, [[{connection_timeout, 90000}, {pool_size, 10000}, {name, couch_index_merger_connection_pool}]]}
-external_manager={couch_external_manager, start_link, []}
query_servers={couch_query_servers, start_link, []}
-vhosts={couch_httpd_vhost, start_link, []}
httpd={couch_httpd, start_link, []}
-stats_aggregator={couch_stats_aggregator, start, []}
-stats_collector={couch_stats_collector, start, []}
uuids={couch_uuids, start, []}
auth_cache={couch_auth_cache, start_link, []}
couch_set_view_ddoc_cache={couch_set_view_ddoc_cache, start_link, []}
replication_manager={couch_replication_manager, start_link, []}
-os_daemons={couch_os_daemons, start_link, []}
compaction_daemon={couch_compaction_daemon, start_link, []}
[httpd_global_handlers]
@@ -118,7 +92,6 @@ _config = {couch_httpd_misc_handlers, handle_config_req}
_replicate = {couch_httpd_replicator, handle_req}
_uuids = {couch_httpd_misc_handlers, handle_uuids_req}
_restart = {couch_httpd_misc_handlers, handle_restart_req}
-_stats = {couch_httpd_stats_handlers, handle_stats_req}
_log = {couch_httpd_misc_handlers, handle_log_req}
_session = {couch_httpd_auth, handle_session_req}
_oauth = {couch_httpd_oauth, handle_oauth_req}
@@ -133,32 +106,13 @@ _design = {couch_httpd_db, handle_design_req}
_temp_view = {couch_httpd_view, handle_temp_view_req}
_changes = {couch_httpd_db, handle_changes_req}
-; The external module takes an optional argument allowing you to narrow it to a
-; single script. Otherwise the script name is inferred from the first path section
-; after _external's own path.
-; _mypath = {couch_httpd_external, handle_external_req, <<"mykey">>}
-; _external = {couch_httpd_external, handle_external_req}
-
[httpd_design_handlers]
_view = {couch_httpd_view, handle_view_req}
_show = {couch_httpd_show, handle_doc_show_req}
_list = {couch_httpd_show, handle_view_list_req}
_info = {couch_httpd_db, handle_design_info_req}
-_rewrite = {couch_httpd_rewrite, handle_rewrite_req}
_update = {couch_httpd_show, handle_doc_update_req}
-; enable external as an httpd handler, then link it with commands here.
-; note, this api is still under consideration.
-; [external]
-; mykey = /path/to/mycommand
-
-; Here you can setup commands for CouchDB to manage
-; while it is alive. It will attempt to keep each command
-; alive if it exits.
-; [os_daemons]
-; some_daemon_name = /path/to/script -with args
-
-
[uuids]
; Known algorithms:
; random - 128 bits of random awesome
@@ -171,16 +125,6 @@ _update = {couch_httpd_show, handle_doc_update_req}
; First 14 characters are the time in hex. Last 18 are random.
algorithm = sequential
-[stats]
-; rate is in milliseconds
-rate = 1000
-; sample intervals are in seconds
-samples = [0, 60, 300, 900]
-
-[attachments]
-compression_level = 8 ; from 1 (lowest, fastest) to 9 (highest, slowest), 0 to disable compression
-compressible_types = text/*, application/javascript, application/json, application/xml
-
[replicator]
db = _replicator
; Maximum replicaton retry count can be a non-negative integer or "infinity".
@@ -168,7 +168,6 @@ design_doc_view(Req, SetName, DDocId, ViewName, FilteredPartitions, Keys) ->
throw({not_found, Reason})
end
end,
- couch_stats_collector:increment({httpd, view_reads}),
Result.
output_map_view(Req, View, Group, QueryArgs) ->
View
@@ -39,7 +39,6 @@ source_files = \
couch_changes.erl \
couch_compaction_daemon.erl \
couch_compress.erl \
- couch_compress_types.erl \
couch_config.erl \
couch_config_writer.erl \
couch_db.erl \
@@ -49,8 +48,6 @@ source_files = \
couch_drv.erl \
couch_ejson_compare.erl \
couch_event_sup.erl \
- couch_external_manager.erl \
- couch_external_server.erl \
couch_file.erl \
couch_db_frontend.erl \
couch_httpd.erl \
@@ -61,12 +58,8 @@ source_files = \
couch_httpd_view.erl \
couch_httpd_misc_handlers.erl \
couch_httpd_replicator.erl \
- couch_httpd_stats_handlers.erl \
- couch_httpd_vhost.erl \
couch_log.erl \
- couch_access_log.erl \
couch_native_process.erl \
- couch_os_daemons.erl \
couch_os_process.erl \
couch_primary_sup.erl \
couch_query_servers.erl \
@@ -80,8 +73,6 @@ source_files = \
couch_secondary_sup.erl \
couch_server.erl \
couch_server_sup.erl \
- couch_stats_aggregator.erl \
- couch_stats_collector.erl \
couch_task_status.erl \
couch_util.erl \
couch_uuids.erl \
@@ -116,7 +107,6 @@ compiled_files = \
couch_changes.beam \
couch_compaction_daemon.beam \
couch_compress.beam \
- couch_compress_types.beam \
couch_config.beam \
couch_config_writer.beam \
couch_db.beam \
@@ -126,8 +116,6 @@ compiled_files = \
couch_drv.beam \
couch_ejson_compare.beam \
couch_event_sup.beam \
- couch_external_manager.beam \
- couch_external_server.beam \
couch_file.beam \
couch_file_write_guard.beam \
couch_db_frontend.beam \
@@ -139,12 +127,8 @@ compiled_files = \
couch_httpd_view.beam \
couch_httpd_misc_handlers.beam \
couch_httpd_replicator.beam \
- couch_httpd_stats_handlers.beam \
- couch_httpd_vhost.beam \
couch_log.beam \
- couch_access_log.beam \
couch_native_process.beam \
- couch_os_daemons.beam \
couch_os_process.beam \
couch_primary_sup.beam \
couch_query_servers.beam \
@@ -158,8 +142,6 @@ compiled_files = \
couch_secondary_sup.beam \
couch_server.beam \
couch_server_sup.beam \
- couch_stats_aggregator.beam \
- couch_stats_collector.beam \
couch_task_status.beam \
couch_util.beam \
couch_uuids.beam \
@@ -6,18 +6,14 @@
couch_config,
couch_db_update,
couch_db_update_notifier_sup,
- couch_external_manager,
couch_httpd,
couch_log,
- couch_access_log,
couch_primary_services,
couch_query_servers,
couch_rep_sup,
couch_secondary_services,
couch_server,
couch_server_sup,
- couch_stats_aggregator,
- couch_stats_collector,
couch_task_status,
couch_view,
couch_file_write_guard
Oops, something went wrong.

0 comments on commit d6f2743

Please sign in to comment.