libcouchbase 2.0 compatibility and some re factoring #2

merged 28 commits into from Dec 18, 2012


None yet

2 participants


No description provided.

Ali Yakamercan and others added some commits Aug 21, 2012

Why did you swap jiffy out for jsx? Jiffy is 10x faster according to my tests (tested on Ubuntu 12.04, erts-5.9.3, OTP R15B03).


How did you test it? It wasn't the case for me, i will swap it jiffy and run bench marks again. I made some more changes anyways, I will make new pull request with the latest changes. Feel free to close this one.


I used timer:tc/1 with a large JSON line. Sample script to execute:

{ok, JSON} = file:read_file("sample.json").
TimingReport = fun(Module, Function, Timings) ->
    Sorted = lists:sort(Timings),
    Min = lists:min(Timings),
    Max = lists:max(Timings),
    Count = length(Timings),
    Sum = lists:sum(Timings),
    Avg = Sum / Count,
    U90 = lists:nth(trunc(Count * 0.9), Sorted),
    io:format("Report for ~p:~p~n", [Module, Function]),
    io:format("Min: ~p; Max: ~p; Avg: ~p; 90%: ~p~n", [Min, Max, Avg, U90])
Test = fun(Module) ->
    Term = Module:decode(JSON),
    EncodeTimes = [ begin {Duration, _} = timer:tc(fun() -> Module:encode(Term) end), Duration end || _ <- lists:seq(1, 10000) ],
    DecodeTimes = [ begin {Duration, _} = timer:tc(fun() -> Module:decode(JSON) end), Duration end || _ <- lists:seq(1, 10000) ],
    TimingReport(Module, encode, EncodeTimes),
    TimingReport(Module, decode, DecodeTimes)



{"web-app": {
  "servlet": [   
      "servlet-name": "cofaxCDS",
      "servlet-class": "org.cofax.cds.CDSServlet",
      "init-param": {
        "configGlossary:installationAt": "Philadelphia, PA",
        "configGlossary:adminEmail": "",
        "configGlossary:poweredBy": "Cofax",
        "configGlossary:poweredByIcon": "/images/cofax.gif",
        "configGlossary:staticPath": "/content/static",
        "templateProcessorClass": "org.cofax.WysiwygTemplate",
        "templateLoaderClass": "org.cofax.FilesTemplateLoader",
        "templatePath": "templates",
        "templateOverridePath": "",
        "defaultListTemplate": "listTemplate.htm",
        "defaultFileTemplate": "articleTemplate.htm",
        "useJSP": false,
        "jspListTemplate": "listTemplate.jsp",
        "jspFileTemplate": "articleTemplate.jsp",
        "cachePackageTagsTrack": 200,
        "cachePackageTagsStore": 200,
        "cachePackageTagsRefresh": 60,
        "cacheTemplatesTrack": 100,
        "cacheTemplatesStore": 50,
        "cacheTemplatesRefresh": 15,
        "cachePagesTrack": 200,
        "cachePagesStore": 100,
        "cachePagesRefresh": 10,
        "cachePagesDirtyRead": 10,
        "searchEngineListTemplate": "forSearchEnginesList.htm",
        "searchEngineFileTemplate": "forSearchEngines.htm",
        "searchEngineRobotsDb": "WEB-INF/robots.db",
        "useDataStore": true,
        "dataStoreClass": "org.cofax.SqlDataStore",
        "redirectionClass": "org.cofax.SqlRedirection",
        "dataStoreName": "cofax",
        "dataStoreDriver": "",
        "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon",
        "dataStoreUser": "sa",
        "dataStorePassword": "dataStoreTestQuery",
        "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';",
        "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log",
        "dataStoreInitConns": 10,
        "dataStoreMaxConns": 100,
        "dataStoreConnUsageLimit": 100,
        "dataStoreLogLevel": "debug",
        "maxUrlLength": 500}},
      "servlet-name": "cofaxEmail",
      "servlet-class": "org.cofax.cds.EmailServlet",
      "init-param": {
      "mailHost": "mail1",
      "mailHostOverride": "mail2"}},
      "servlet-name": "cofaxAdmin",
      "servlet-class": "org.cofax.cds.AdminServlet"},

      "servlet-name": "fileServlet",
      "servlet-class": "org.cofax.cds.FileServlet"},
      "servlet-name": "cofaxTools",
      "servlet-class": "org.cofax.cms.CofaxToolsServlet",
      "init-param": {
        "templatePath": "toolstemplates/",
        "log": 1,
        "logLocation": "/usr/local/tomcat/logs/CofaxTools.log",
        "logMaxSize": "",
        "dataLog": 1,
        "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log",
        "dataLogMaxSize": "",
        "removePageCache": "/content/admin/remove?cache=pages&id=",
        "removeTemplateCache": "/content/admin/remove?cache=templates&id=",
        "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder",
        "lookInContext": 1,
        "adminGroupID": 4,
        "betaServer": true}}],
  "servlet-mapping": {
    "cofaxCDS": "/",
    "cofaxEmail": "/cofaxutil/aemail/*",
    "cofaxAdmin": "/admin/*",
    "fileServlet": "/static/*",
    "cofaxTools": "/tools/*"},

  "taglib": {
    "taglib-uri": "cofax.tld",
    "taglib-location": "/WEB-INF/tlds/cofax.tld"}}}

Output on my desktop:
4> Test(jiffy).
Report for jiffy:encode
Min: 59; Max: 119; Avg: 68.5604; 90%: 69
Report for jiffy:decode
Min: 52; Max: 111; Avg: 61.672; 90%: 73
5> Test(jsx).
Report for jsx:encode
Min: 180; Max: 1980; Avg: 219.542; 90%: 234
Report for jsx:decode
Min: 253; Max: 724; Avg: 310.9576; 90%: 335

So 5x faster, not 10x, according to this test (note: I changed the JSON I was using; this example is a bit smaller and is not minified).

Of particular note is the worst-case scenario for JSX, which was 16.6x slower than the worst-case scenario for jiffy.


-export([start/2, f/3]).

start(Module, Num) ->
    process_flag(trap_exit, true),
    {ok, JSON} = file:read_file("sample.json"),
    Term = Module:decode(JSON),
    TS = now(),
    Decode = [begin spawn_link(test, f, [Module, decode, JSON]) end || _ <- lists:seq(1,Num)],
    Encode = [begin spawn_link(test, f, [Module, encode, Term]) end || _ <- lists:seq(1,Num)],
    is_it_done_yet(Decode, Encode, TS, Module).

is_it_done_yet([], [], TS, Module) ->
    io:format("~p: ~p milli seconds ~n", [Module, timer:now_diff(now(), TS)/1000]);
is_it_done_yet(D, E, TS, Module) ->
        {'EXIT',FromPid,_Reason} ->
            is_it_done_yet(lists:delete(FromPid, D), 
                                  lists:delete(FromPid, E), 
                                  TS, Module)

f(Module, OP, JT) ->
    io:format("Starting ~p:~p (~p)~n", [Module, OP, self()]),
    [begin Module:OP(JT) end || _ <- lists:seq(1, 1000) ],
    io:format("Exiting ~p:~p (~p)~n", [Module, OP, self()]).

I thought jsx would perform better when it is not sequential but apparently not. I couldn't even wait jsx to finish with 150 processes :). I will switch it back to jiffy.

Ali Yakamercan swap jsx with jiffy 9762bea
@fauxsoup fauxsoup merged commit 15da4fa into chitika:master Dec 18, 2012
@wcummings wcummings added a commit that referenced this pull request Sep 9, 2013
@wcummings wcummings + William Cummings closes #2
Check the rv from lcb_wait when connecting
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment