libcouchbase 2.0 compatibility and some re factoring #2

Merged
merged 28 commits into from Dec 18, 2012

Projects

None yet

2 participants

@aliyakamercan
Contributor

No description provided.

Ali Yakamercan and others added some commits Aug 21, 2012
@fauxsoup
Contributor

Why did you swap jiffy out for jsx? Jiffy is 10x faster according to my tests (tested on Ubuntu 12.04, erts-5.9.3, OTP R15B03).

@aliyakamercan
Contributor

How did you test it? It wasn't the case for me, i will swap it jiffy and run bench marks again. I made some more changes anyways, I will make new pull request with the latest changes. Feel free to close this one.

@fauxsoup
Contributor

I used timer:tc/1 with a large JSON line. Sample script to execute:

{ok, JSON} = file:read_file("sample.json").
TimingReport = fun(Module, Function, Timings) ->
    Sorted = lists:sort(Timings),
    Min = lists:min(Timings),
    Max = lists:max(Timings),
    Count = length(Timings),
    Sum = lists:sum(Timings),
    Avg = Sum / Count,
    U90 = lists:nth(trunc(Count * 0.9), Sorted),
    io:format("Report for ~p:~p~n", [Module, Function]),
    io:format("Min: ~p; Max: ~p; Avg: ~p; 90%: ~p~n", [Min, Max, Avg, U90])
end.
Test = fun(Module) ->
    Term = Module:decode(JSON),
    EncodeTimes = [ begin {Duration, _} = timer:tc(fun() -> Module:encode(Term) end), Duration end || _ <- lists:seq(1, 10000) ],
    DecodeTimes = [ begin {Duration, _} = timer:tc(fun() -> Module:decode(JSON) end), Duration end || _ <- lists:seq(1, 10000) ],
    TimingReport(Module, encode, EncodeTimes),
    TimingReport(Module, decode, DecodeTimes)
end.

Test(jiffy).
Test(jsx).

sample.json

{"web-app": {
  "servlet": [   
    {
      "servlet-name": "cofaxCDS",
      "servlet-class": "org.cofax.cds.CDSServlet",
      "init-param": {
        "configGlossary:installationAt": "Philadelphia, PA",
        "configGlossary:adminEmail": "ksm@pobox.com",
        "configGlossary:poweredBy": "Cofax",
        "configGlossary:poweredByIcon": "/images/cofax.gif",
        "configGlossary:staticPath": "/content/static",
        "templateProcessorClass": "org.cofax.WysiwygTemplate",
        "templateLoaderClass": "org.cofax.FilesTemplateLoader",
        "templatePath": "templates",
        "templateOverridePath": "",
        "defaultListTemplate": "listTemplate.htm",
        "defaultFileTemplate": "articleTemplate.htm",
        "useJSP": false,
        "jspListTemplate": "listTemplate.jsp",
        "jspFileTemplate": "articleTemplate.jsp",
        "cachePackageTagsTrack": 200,
        "cachePackageTagsStore": 200,
        "cachePackageTagsRefresh": 60,
        "cacheTemplatesTrack": 100,
        "cacheTemplatesStore": 50,
        "cacheTemplatesRefresh": 15,
        "cachePagesTrack": 200,
        "cachePagesStore": 100,
        "cachePagesRefresh": 10,
        "cachePagesDirtyRead": 10,
        "searchEngineListTemplate": "forSearchEnginesList.htm",
        "searchEngineFileTemplate": "forSearchEngines.htm",
        "searchEngineRobotsDb": "WEB-INF/robots.db",
        "useDataStore": true,
        "dataStoreClass": "org.cofax.SqlDataStore",
        "redirectionClass": "org.cofax.SqlRedirection",
        "dataStoreName": "cofax",
        "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver",
        "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon",
        "dataStoreUser": "sa",
        "dataStorePassword": "dataStoreTestQuery",
        "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';",
        "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log",
        "dataStoreInitConns": 10,
        "dataStoreMaxConns": 100,
        "dataStoreConnUsageLimit": 100,
        "dataStoreLogLevel": "debug",
        "maxUrlLength": 500}},
    {
      "servlet-name": "cofaxEmail",
      "servlet-class": "org.cofax.cds.EmailServlet",
      "init-param": {
      "mailHost": "mail1",
      "mailHostOverride": "mail2"}},
    {
      "servlet-name": "cofaxAdmin",
      "servlet-class": "org.cofax.cds.AdminServlet"},

    {
      "servlet-name": "fileServlet",
      "servlet-class": "org.cofax.cds.FileServlet"},
    {
      "servlet-name": "cofaxTools",
      "servlet-class": "org.cofax.cms.CofaxToolsServlet",
      "init-param": {
        "templatePath": "toolstemplates/",
        "log": 1,
        "logLocation": "/usr/local/tomcat/logs/CofaxTools.log",
        "logMaxSize": "",
        "dataLog": 1,
        "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log",
        "dataLogMaxSize": "",
        "removePageCache": "/content/admin/remove?cache=pages&id=",
        "removeTemplateCache": "/content/admin/remove?cache=templates&id=",
        "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder",
        "lookInContext": 1,
        "adminGroupID": 4,
        "betaServer": true}}],
  "servlet-mapping": {
    "cofaxCDS": "/",
    "cofaxEmail": "/cofaxutil/aemail/*",
    "cofaxAdmin": "/admin/*",
    "fileServlet": "/static/*",
    "cofaxTools": "/tools/*"},

  "taglib": {
    "taglib-uri": "cofax.tld",
    "taglib-location": "/WEB-INF/tlds/cofax.tld"}}}

Output on my desktop:
4> Test(jiffy).
Report for jiffy:encode
Min: 59; Max: 119; Avg: 68.5604; 90%: 69
Report for jiffy:decode
Min: 52; Max: 111; Avg: 61.672; 90%: 73
ok
5> Test(jsx).
Report for jsx:encode
Min: 180; Max: 1980; Avg: 219.542; 90%: 234
Report for jsx:decode
Min: 253; Max: 724; Avg: 310.9576; 90%: 335
ok

So 5x faster, not 10x, according to this test (note: I changed the JSON I was using; this example is a bit smaller and is not minified).

Of particular note is the worst-case scenario for JSX, which was 16.6x slower than the worst-case scenario for jiffy.

@aliyakamercan
Contributor
-module(test).

-export([start/2, f/3]).

start(Module, Num) ->
    process_flag(trap_exit, true),
    {ok, JSON} = file:read_file("sample.json"),
    Term = Module:decode(JSON),
    TS = now(),
    Decode = [begin spawn_link(test, f, [Module, decode, JSON]) end || _ <- lists:seq(1,Num)],
    Encode = [begin spawn_link(test, f, [Module, encode, Term]) end || _ <- lists:seq(1,Num)],
    is_it_done_yet(Decode, Encode, TS, Module).

is_it_done_yet([], [], TS, Module) ->
    io:format("~p: ~p milli seconds ~n", [Module, timer:now_diff(now(), TS)/1000]);
is_it_done_yet(D, E, TS, Module) ->
    receive
        {'EXIT',FromPid,_Reason} ->
            is_it_done_yet(lists:delete(FromPid, D), 
                                  lists:delete(FromPid, E), 
                                  TS, Module)
    end.

f(Module, OP, JT) ->
    io:format("Starting ~p:~p (~p)~n", [Module, OP, self()]),
    [begin Module:OP(JT) end || _ <- lists:seq(1, 1000) ],
    io:format("Exiting ~p:~p (~p)~n", [Module, OP, self()]).

I thought jsx would perform better when it is not sequential but apparently not. I couldn't even wait jsx to finish with 150 processes :). I will switch it back to jiffy.

Ali Yakamercan swap jsx with jiffy 9762bea
@fauxsoup fauxsoup merged commit 15da4fa into chitika:master Dec 18, 2012
@wcummings wcummings added a commit that referenced this pull request Sep 9, 2013
@wcummings wcummings + William Cummings closes #2
Check the rv from lcb_wait when connecting
d605654
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment