From dd653a153f29ff57838d11bd8e7e9e35665b1198 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Mon, 13 Apr 2015 00:53:15 +0800 Subject: [PATCH 01/31] Add timeline and histogram graphs for streaming statistics --- .rat-excludes | 1 + LICENSE | 30 +++ .../org/apache/spark/ui/static/d3.min.js | 5 + .../apache/spark/ui/static/streaming-page.css | 41 +++ .../apache/spark/ui/static/streaming-page.js | 120 +++++++++ .../org/apache/spark/util/Distribution.scala | 29 +++ .../spark/streaming/scheduler/BatchInfo.scala | 7 + .../ui/StreamingJobProgressListener.scala | 31 +++ .../spark/streaming/ui/StreamingPage.scala | 236 +++++++++++++++--- 9 files changed, 468 insertions(+), 32 deletions(-) create mode 100644 core/src/main/resources/org/apache/spark/ui/static/d3.min.js create mode 100644 core/src/main/resources/org/apache/spark/ui/static/streaming-page.css create mode 100644 core/src/main/resources/org/apache/spark/ui/static/streaming-page.js diff --git a/.rat-excludes b/.rat-excludes index 8aca5a7f7a967..950f762e18394 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -30,6 +30,7 @@ log4j-defaults.properties bootstrap-tooltip.js jquery-1.11.1.min.js sorttable.js +d3.min.js .*avsc .*txt .*json diff --git a/LICENSE b/LICENSE index 9b364a4d00079..81293efcafedd 100644 --- a/LICENSE +++ b/LICENSE @@ -643,6 +643,36 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +======================================================================== +For d3 (core/src/main/resources/org/apache/spark/ui/static/d3.min.js): +======================================================================== + +Copyright (c) 2010-2015, Michael Bostock +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* The name Michael Bostock may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ======================================================================== For Scala Interpreter classes (all .scala files in repl/src/main/scala diff --git a/core/src/main/resources/org/apache/spark/ui/static/d3.min.js b/core/src/main/resources/org/apache/spark/ui/static/d3.min.js new file mode 100644 index 0000000000000..34d5513ebfe11 --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/d3.min.js @@ -0,0 +1,5 @@ +!function(){function n(n){return n&&(n.ownerDocument||n.document||n).documentElement}function t(n){return n&&(n.ownerDocument&&n.ownerDocument.defaultView||n.document&&n||n.defaultView)}function e(n,t){return t>n?-1:n>t?1:n>=t?0:0/0}function r(n){return null===n?0/0:+n}function u(n){return!isNaN(n)}function i(n){return{left:function(t,e,r,u){for(arguments.length<3&&(r=0),arguments.length<4&&(u=t.length);u>r;){var i=r+u>>>1;n(t[i],e)<0?r=i+1:u=i}return r},right:function(t,e,r,u){for(arguments.length<3&&(r=0),arguments.length<4&&(u=t.length);u>r;){var i=r+u>>>1;n(t[i],e)>0?u=i:r=i+1}return r}}}function o(n){return n.length}function a(n){for(var t=1;n*t%1;)t*=10;return t}function c(n,t){for(var e in t)Object.defineProperty(n.prototype,e,{value:t[e],enumerable:!1})}function l(){this._=Object.create(null)}function s(n){return(n+="")===pa||n[0]===va?va+n:n}function f(n){return(n+="")[0]===va?n.slice(1):n}function h(n){return s(n)in this._}function g(n){return(n=s(n))in this._&&delete this._[n]}function p(){var n=[];for(var t in this._)n.push(f(t));return n}function v(){var n=0;for(var t in this._)++n;return n}function d(){for(var n in this._)return!1;return!0}function m(){this._=Object.create(null)}function y(n){return n}function M(n,t,e){return function(){var r=e.apply(t,arguments);return r===t?n:r}}function x(n,t){if(t in n)return t;t=t.charAt(0).toUpperCase()+t.slice(1);for(var e=0,r=da.length;r>e;++e){var u=da[e]+t;if(u in n)return u}}function b(){}function _(){}function w(n){function t(){for(var t,r=e,u=-1,i=r.length;++ue;e++)for(var u,i=n[e],o=0,a=i.length;a>o;o++)(u=i[o])&&t(u,o,e);return n}function Z(n){return ya(n,Sa),n}function V(n){var t,e;return function(r,u,i){var o,a=n[i].update,c=a.length;for(i!=e&&(e=i,t=0),u>=t&&(t=u+1);!(o=a[t])&&++t0&&(n=n.slice(0,a));var l=ka.get(n);return l&&(n=l,c=B),a?t?u:r:t?b:i}function $(n,t){return function(e){var r=ta.event;ta.event=e,t[0]=this.__data__;try{n.apply(this,t)}finally{ta.event=r}}}function B(n,t){var e=$(n,t);return function(n){var t=this,r=n.relatedTarget;r&&(r===t||8&r.compareDocumentPosition(t))||e.call(t,n)}}function W(e){var r=".dragsuppress-"+ ++Aa,u="click"+r,i=ta.select(t(e)).on("touchmove"+r,S).on("dragstart"+r,S).on("selectstart"+r,S);if(null==Ea&&(Ea="onselectstart"in e?!1:x(e.style,"userSelect")),Ea){var o=n(e).style,a=o[Ea];o[Ea]="none"}return function(n){if(i.on(r,null),Ea&&(o[Ea]=a),n){var t=function(){i.on(u,null)};i.on(u,function(){S(),t()},!0),setTimeout(t,0)}}}function J(n,e){e.changedTouches&&(e=e.changedTouches[0]);var r=n.ownerSVGElement||n;if(r.createSVGPoint){var u=r.createSVGPoint();if(0>Na){var i=t(n);if(i.scrollX||i.scrollY){r=ta.select("body").append("svg").style({position:"absolute",top:0,left:0,margin:0,padding:0,border:"none"},"important");var o=r[0][0].getScreenCTM();Na=!(o.f||o.e),r.remove()}}return Na?(u.x=e.pageX,u.y=e.pageY):(u.x=e.clientX,u.y=e.clientY),u=u.matrixTransform(n.getScreenCTM().inverse()),[u.x,u.y]}var a=n.getBoundingClientRect();return[e.clientX-a.left-n.clientLeft,e.clientY-a.top-n.clientTop]}function G(){return ta.event.changedTouches[0].identifier}function K(n){return n>0?1:0>n?-1:0}function Q(n,t,e){return(t[0]-n[0])*(e[1]-n[1])-(t[1]-n[1])*(e[0]-n[0])}function nt(n){return n>1?0:-1>n?qa:Math.acos(n)}function tt(n){return n>1?Ra:-1>n?-Ra:Math.asin(n)}function et(n){return((n=Math.exp(n))-1/n)/2}function rt(n){return((n=Math.exp(n))+1/n)/2}function ut(n){return((n=Math.exp(2*n))-1)/(n+1)}function it(n){return(n=Math.sin(n/2))*n}function ot(){}function at(n,t,e){return this instanceof at?(this.h=+n,this.s=+t,void(this.l=+e)):arguments.length<2?n instanceof at?new at(n.h,n.s,n.l):bt(""+n,_t,at):new at(n,t,e)}function ct(n,t,e){function r(n){return n>360?n-=360:0>n&&(n+=360),60>n?i+(o-i)*n/60:180>n?o:240>n?i+(o-i)*(240-n)/60:i}function u(n){return Math.round(255*r(n))}var i,o;return n=isNaN(n)?0:(n%=360)<0?n+360:n,t=isNaN(t)?0:0>t?0:t>1?1:t,e=0>e?0:e>1?1:e,o=.5>=e?e*(1+t):e+t-e*t,i=2*e-o,new mt(u(n+120),u(n),u(n-120))}function lt(n,t,e){return this instanceof lt?(this.h=+n,this.c=+t,void(this.l=+e)):arguments.length<2?n instanceof lt?new lt(n.h,n.c,n.l):n instanceof ft?gt(n.l,n.a,n.b):gt((n=wt((n=ta.rgb(n)).r,n.g,n.b)).l,n.a,n.b):new lt(n,t,e)}function st(n,t,e){return isNaN(n)&&(n=0),isNaN(t)&&(t=0),new ft(e,Math.cos(n*=Da)*t,Math.sin(n)*t)}function ft(n,t,e){return this instanceof ft?(this.l=+n,this.a=+t,void(this.b=+e)):arguments.length<2?n instanceof ft?new ft(n.l,n.a,n.b):n instanceof lt?st(n.h,n.c,n.l):wt((n=mt(n)).r,n.g,n.b):new ft(n,t,e)}function ht(n,t,e){var r=(n+16)/116,u=r+t/500,i=r-e/200;return u=pt(u)*Xa,r=pt(r)*$a,i=pt(i)*Ba,new mt(dt(3.2404542*u-1.5371385*r-.4985314*i),dt(-.969266*u+1.8760108*r+.041556*i),dt(.0556434*u-.2040259*r+1.0572252*i))}function gt(n,t,e){return n>0?new lt(Math.atan2(e,t)*Pa,Math.sqrt(t*t+e*e),n):new lt(0/0,0/0,n)}function pt(n){return n>.206893034?n*n*n:(n-4/29)/7.787037}function vt(n){return n>.008856?Math.pow(n,1/3):7.787037*n+4/29}function dt(n){return Math.round(255*(.00304>=n?12.92*n:1.055*Math.pow(n,1/2.4)-.055))}function mt(n,t,e){return this instanceof mt?(this.r=~~n,this.g=~~t,void(this.b=~~e)):arguments.length<2?n instanceof mt?new mt(n.r,n.g,n.b):bt(""+n,mt,ct):new mt(n,t,e)}function yt(n){return new mt(n>>16,n>>8&255,255&n)}function Mt(n){return yt(n)+""}function xt(n){return 16>n?"0"+Math.max(0,n).toString(16):Math.min(255,n).toString(16)}function bt(n,t,e){var r,u,i,o=0,a=0,c=0;if(r=/([a-z]+)\((.*)\)/i.exec(n))switch(u=r[2].split(","),r[1]){case"hsl":return e(parseFloat(u[0]),parseFloat(u[1])/100,parseFloat(u[2])/100);case"rgb":return t(kt(u[0]),kt(u[1]),kt(u[2]))}return(i=Ga.get(n.toLowerCase()))?t(i.r,i.g,i.b):(null==n||"#"!==n.charAt(0)||isNaN(i=parseInt(n.slice(1),16))||(4===n.length?(o=(3840&i)>>4,o=o>>4|o,a=240&i,a=a>>4|a,c=15&i,c=c<<4|c):7===n.length&&(o=(16711680&i)>>16,a=(65280&i)>>8,c=255&i)),t(o,a,c))}function _t(n,t,e){var r,u,i=Math.min(n/=255,t/=255,e/=255),o=Math.max(n,t,e),a=o-i,c=(o+i)/2;return a?(u=.5>c?a/(o+i):a/(2-o-i),r=n==o?(t-e)/a+(e>t?6:0):t==o?(e-n)/a+2:(n-t)/a+4,r*=60):(r=0/0,u=c>0&&1>c?0:r),new at(r,u,c)}function wt(n,t,e){n=St(n),t=St(t),e=St(e);var r=vt((.4124564*n+.3575761*t+.1804375*e)/Xa),u=vt((.2126729*n+.7151522*t+.072175*e)/$a),i=vt((.0193339*n+.119192*t+.9503041*e)/Ba);return ft(116*u-16,500*(r-u),200*(u-i))}function St(n){return(n/=255)<=.04045?n/12.92:Math.pow((n+.055)/1.055,2.4)}function kt(n){var t=parseFloat(n);return"%"===n.charAt(n.length-1)?Math.round(2.55*t):t}function Et(n){return"function"==typeof n?n:function(){return n}}function At(n){return function(t,e,r){return 2===arguments.length&&"function"==typeof e&&(r=e,e=null),Nt(t,e,n,r)}}function Nt(n,t,e,r){function u(){var n,t=c.status;if(!t&&zt(c)||t>=200&&300>t||304===t){try{n=e.call(i,c)}catch(r){return void o.error.call(i,r)}o.load.call(i,n)}else o.error.call(i,c)}var i={},o=ta.dispatch("beforesend","progress","load","error"),a={},c=new XMLHttpRequest,l=null;return!this.XDomainRequest||"withCredentials"in c||!/^(http(s)?:)?\/\//.test(n)||(c=new XDomainRequest),"onload"in c?c.onload=c.onerror=u:c.onreadystatechange=function(){c.readyState>3&&u()},c.onprogress=function(n){var t=ta.event;ta.event=n;try{o.progress.call(i,c)}finally{ta.event=t}},i.header=function(n,t){return n=(n+"").toLowerCase(),arguments.length<2?a[n]:(null==t?delete a[n]:a[n]=t+"",i)},i.mimeType=function(n){return arguments.length?(t=null==n?null:n+"",i):t},i.responseType=function(n){return arguments.length?(l=n,i):l},i.response=function(n){return e=n,i},["get","post"].forEach(function(n){i[n]=function(){return i.send.apply(i,[n].concat(ra(arguments)))}}),i.send=function(e,r,u){if(2===arguments.length&&"function"==typeof r&&(u=r,r=null),c.open(e,n,!0),null==t||"accept"in a||(a.accept=t+",*/*"),c.setRequestHeader)for(var s in a)c.setRequestHeader(s,a[s]);return null!=t&&c.overrideMimeType&&c.overrideMimeType(t),null!=l&&(c.responseType=l),null!=u&&i.on("error",u).on("load",function(n){u(null,n)}),o.beforesend.call(i,c),c.send(null==r?null:r),i},i.abort=function(){return c.abort(),i},ta.rebind(i,o,"on"),null==r?i:i.get(Ct(r))}function Ct(n){return 1===n.length?function(t,e){n(null==t?e:null)}:n}function zt(n){var t=n.responseType;return t&&"text"!==t?n.response:n.responseText}function qt(){var n=Lt(),t=Tt()-n;t>24?(isFinite(t)&&(clearTimeout(tc),tc=setTimeout(qt,t)),nc=0):(nc=1,rc(qt))}function Lt(){var n=Date.now();for(ec=Ka;ec;)n>=ec.t&&(ec.f=ec.c(n-ec.t)),ec=ec.n;return n}function Tt(){for(var n,t=Ka,e=1/0;t;)t.f?t=n?n.n=t.n:Ka=t.n:(t.t8?function(n){return n/e}:function(n){return n*e},symbol:n}}function Pt(n){var t=n.decimal,e=n.thousands,r=n.grouping,u=n.currency,i=r&&e?function(n,t){for(var u=n.length,i=[],o=0,a=r[0],c=0;u>0&&a>0&&(c+a+1>t&&(a=Math.max(1,t-c)),i.push(n.substring(u-=a,u+a)),!((c+=a+1)>t));)a=r[o=(o+1)%r.length];return i.reverse().join(e)}:y;return function(n){var e=ic.exec(n),r=e[1]||" ",o=e[2]||">",a=e[3]||"-",c=e[4]||"",l=e[5],s=+e[6],f=e[7],h=e[8],g=e[9],p=1,v="",d="",m=!1,y=!0;switch(h&&(h=+h.substring(1)),(l||"0"===r&&"="===o)&&(l=r="0",o="="),g){case"n":f=!0,g="g";break;case"%":p=100,d="%",g="f";break;case"p":p=100,d="%",g="r";break;case"b":case"o":case"x":case"X":"#"===c&&(v="0"+g.toLowerCase());case"c":y=!1;case"d":m=!0,h=0;break;case"s":p=-1,g="r"}"$"===c&&(v=u[0],d=u[1]),"r"!=g||h||(g="g"),null!=h&&("g"==g?h=Math.max(1,Math.min(21,h)):("e"==g||"f"==g)&&(h=Math.max(0,Math.min(20,h)))),g=oc.get(g)||Ut;var M=l&&f;return function(n){var e=d;if(m&&n%1)return"";var u=0>n||0===n&&0>1/n?(n=-n,"-"):"-"===a?"":a;if(0>p){var c=ta.formatPrefix(n,h);n=c.scale(n),e=c.symbol+d}else n*=p;n=g(n,h);var x,b,_=n.lastIndexOf(".");if(0>_){var w=y?n.lastIndexOf("e"):-1;0>w?(x=n,b=""):(x=n.substring(0,w),b=n.substring(w))}else x=n.substring(0,_),b=t+n.substring(_+1);!l&&f&&(x=i(x,1/0));var S=v.length+x.length+b.length+(M?0:u.length),k=s>S?new Array(S=s-S+1).join(r):"";return M&&(x=i(k+x,k.length?s-b.length:1/0)),u+=v,n=x+b,("<"===o?u+n+k:">"===o?k+u+n:"^"===o?k.substring(0,S>>=1)+u+n+k.substring(S):u+(M?n:k+n))+e}}}function Ut(n){return n+""}function jt(){this._=new Date(arguments.length>1?Date.UTC.apply(this,arguments):arguments[0])}function Ft(n,t,e){function r(t){var e=n(t),r=i(e,1);return r-t>t-e?e:r}function u(e){return t(e=n(new cc(e-1)),1),e}function i(n,e){return t(n=new cc(+n),e),n}function o(n,r,i){var o=u(n),a=[];if(i>1)for(;r>o;)e(o)%i||a.push(new Date(+o)),t(o,1);else for(;r>o;)a.push(new Date(+o)),t(o,1);return a}function a(n,t,e){try{cc=jt;var r=new jt;return r._=n,o(r,t,e)}finally{cc=Date}}n.floor=n,n.round=r,n.ceil=u,n.offset=i,n.range=o;var c=n.utc=Ht(n);return c.floor=c,c.round=Ht(r),c.ceil=Ht(u),c.offset=Ht(i),c.range=a,n}function Ht(n){return function(t,e){try{cc=jt;var r=new jt;return r._=t,n(r,e)._}finally{cc=Date}}}function Ot(n){function t(n){function t(t){for(var e,u,i,o=[],a=-1,c=0;++aa;){if(r>=l)return-1;if(u=t.charCodeAt(a++),37===u){if(o=t.charAt(a++),i=C[o in sc?t.charAt(a++):o],!i||(r=i(n,e,r))<0)return-1}else if(u!=e.charCodeAt(r++))return-1}return r}function r(n,t,e){_.lastIndex=0;var r=_.exec(t.slice(e));return r?(n.w=w.get(r[0].toLowerCase()),e+r[0].length):-1}function u(n,t,e){x.lastIndex=0;var r=x.exec(t.slice(e));return r?(n.w=b.get(r[0].toLowerCase()),e+r[0].length):-1}function i(n,t,e){E.lastIndex=0;var r=E.exec(t.slice(e));return r?(n.m=A.get(r[0].toLowerCase()),e+r[0].length):-1}function o(n,t,e){S.lastIndex=0;var r=S.exec(t.slice(e));return r?(n.m=k.get(r[0].toLowerCase()),e+r[0].length):-1}function a(n,t,r){return e(n,N.c.toString(),t,r)}function c(n,t,r){return e(n,N.x.toString(),t,r)}function l(n,t,r){return e(n,N.X.toString(),t,r)}function s(n,t,e){var r=M.get(t.slice(e,e+=2).toLowerCase());return null==r?-1:(n.p=r,e)}var f=n.dateTime,h=n.date,g=n.time,p=n.periods,v=n.days,d=n.shortDays,m=n.months,y=n.shortMonths;t.utc=function(n){function e(n){try{cc=jt;var t=new cc;return t._=n,r(t)}finally{cc=Date}}var r=t(n);return e.parse=function(n){try{cc=jt;var t=r.parse(n);return t&&t._}finally{cc=Date}},e.toString=r.toString,e},t.multi=t.utc.multi=ae;var M=ta.map(),x=Yt(v),b=Zt(v),_=Yt(d),w=Zt(d),S=Yt(m),k=Zt(m),E=Yt(y),A=Zt(y);p.forEach(function(n,t){M.set(n.toLowerCase(),t)});var N={a:function(n){return d[n.getDay()]},A:function(n){return v[n.getDay()]},b:function(n){return y[n.getMonth()]},B:function(n){return m[n.getMonth()]},c:t(f),d:function(n,t){return It(n.getDate(),t,2)},e:function(n,t){return It(n.getDate(),t,2)},H:function(n,t){return It(n.getHours(),t,2)},I:function(n,t){return It(n.getHours()%12||12,t,2)},j:function(n,t){return It(1+ac.dayOfYear(n),t,3)},L:function(n,t){return It(n.getMilliseconds(),t,3)},m:function(n,t){return It(n.getMonth()+1,t,2)},M:function(n,t){return It(n.getMinutes(),t,2)},p:function(n){return p[+(n.getHours()>=12)]},S:function(n,t){return It(n.getSeconds(),t,2)},U:function(n,t){return It(ac.sundayOfYear(n),t,2)},w:function(n){return n.getDay()},W:function(n,t){return It(ac.mondayOfYear(n),t,2)},x:t(h),X:t(g),y:function(n,t){return It(n.getFullYear()%100,t,2)},Y:function(n,t){return It(n.getFullYear()%1e4,t,4)},Z:ie,"%":function(){return"%"}},C={a:r,A:u,b:i,B:o,c:a,d:Qt,e:Qt,H:te,I:te,j:ne,L:ue,m:Kt,M:ee,p:s,S:re,U:Xt,w:Vt,W:$t,x:c,X:l,y:Wt,Y:Bt,Z:Jt,"%":oe};return t}function It(n,t,e){var r=0>n?"-":"",u=(r?-n:n)+"",i=u.length;return r+(e>i?new Array(e-i+1).join(t)+u:u)}function Yt(n){return new RegExp("^(?:"+n.map(ta.requote).join("|")+")","i")}function Zt(n){for(var t=new l,e=-1,r=n.length;++e68?1900:2e3)}function Kt(n,t,e){fc.lastIndex=0;var r=fc.exec(t.slice(e,e+2));return r?(n.m=r[0]-1,e+r[0].length):-1}function Qt(n,t,e){fc.lastIndex=0;var r=fc.exec(t.slice(e,e+2));return r?(n.d=+r[0],e+r[0].length):-1}function ne(n,t,e){fc.lastIndex=0;var r=fc.exec(t.slice(e,e+3));return r?(n.j=+r[0],e+r[0].length):-1}function te(n,t,e){fc.lastIndex=0;var r=fc.exec(t.slice(e,e+2));return r?(n.H=+r[0],e+r[0].length):-1}function ee(n,t,e){fc.lastIndex=0;var r=fc.exec(t.slice(e,e+2));return r?(n.M=+r[0],e+r[0].length):-1}function re(n,t,e){fc.lastIndex=0;var r=fc.exec(t.slice(e,e+2));return r?(n.S=+r[0],e+r[0].length):-1}function ue(n,t,e){fc.lastIndex=0;var r=fc.exec(t.slice(e,e+3));return r?(n.L=+r[0],e+r[0].length):-1}function ie(n){var t=n.getTimezoneOffset(),e=t>0?"-":"+",r=ga(t)/60|0,u=ga(t)%60;return e+It(r,"0",2)+It(u,"0",2)}function oe(n,t,e){hc.lastIndex=0;var r=hc.exec(t.slice(e,e+1));return r?e+r[0].length:-1}function ae(n){for(var t=n.length,e=-1;++e=0?1:-1,a=o*e,c=Math.cos(t),l=Math.sin(t),s=i*l,f=u*c+s*Math.cos(a),h=s*o*Math.sin(a);yc.add(Math.atan2(h,f)),r=n,u=c,i=l}var t,e,r,u,i;Mc.point=function(o,a){Mc.point=n,r=(t=o)*Da,u=Math.cos(a=(e=a)*Da/2+qa/4),i=Math.sin(a)},Mc.lineEnd=function(){n(t,e)}}function pe(n){var t=n[0],e=n[1],r=Math.cos(e);return[r*Math.cos(t),r*Math.sin(t),Math.sin(e)]}function ve(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]}function de(n,t){return[n[1]*t[2]-n[2]*t[1],n[2]*t[0]-n[0]*t[2],n[0]*t[1]-n[1]*t[0]]}function me(n,t){n[0]+=t[0],n[1]+=t[1],n[2]+=t[2]}function ye(n,t){return[n[0]*t,n[1]*t,n[2]*t]}function Me(n){var t=Math.sqrt(n[0]*n[0]+n[1]*n[1]+n[2]*n[2]);n[0]/=t,n[1]/=t,n[2]/=t}function xe(n){return[Math.atan2(n[1],n[0]),tt(n[2])]}function be(n,t){return ga(n[0]-t[0])a;++a)u.point((e=n[a])[0],e[1]);return void u.lineEnd()}var c=new qe(e,n,null,!0),l=new qe(e,null,c,!1);c.o=l,i.push(c),o.push(l),c=new qe(r,n,null,!1),l=new qe(r,null,c,!0),c.o=l,i.push(c),o.push(l)}}),o.sort(t),ze(i),ze(o),i.length){for(var a=0,c=e,l=o.length;l>a;++a)o[a].e=c=!c;for(var s,f,h=i[0];;){for(var g=h,p=!0;g.v;)if((g=g.n)===h)return;s=g.z,u.lineStart();do{if(g.v=g.o.v=!0,g.e){if(p)for(var a=0,l=s.length;l>a;++a)u.point((f=s[a])[0],f[1]);else r(g.x,g.n.x,1,u);g=g.n}else{if(p){s=g.p.z;for(var a=s.length-1;a>=0;--a)u.point((f=s[a])[0],f[1])}else r(g.x,g.p.x,-1,u);g=g.p}g=g.o,s=g.z,p=!p}while(!g.v);u.lineEnd()}}}function ze(n){if(t=n.length){for(var t,e,r=0,u=n[0];++r0){for(b||(i.polygonStart(),b=!0),i.lineStart();++o1&&2&t&&e.push(e.pop().concat(e.shift())),g.push(e.filter(Te))}var g,p,v,d=t(i),m=u.invert(r[0],r[1]),y={point:o,lineStart:c,lineEnd:l,polygonStart:function(){y.point=s,y.lineStart=f,y.lineEnd=h,g=[],p=[]},polygonEnd:function(){y.point=o,y.lineStart=c,y.lineEnd=l,g=ta.merge(g);var n=Fe(m,p);g.length?(b||(i.polygonStart(),b=!0),Ce(g,De,n,e,i)):n&&(b||(i.polygonStart(),b=!0),i.lineStart(),e(null,null,1,i),i.lineEnd()),b&&(i.polygonEnd(),b=!1),g=p=null},sphere:function(){i.polygonStart(),i.lineStart(),e(null,null,1,i),i.lineEnd(),i.polygonEnd()}},M=Re(),x=t(M),b=!1;return y}}function Te(n){return n.length>1}function Re(){var n,t=[];return{lineStart:function(){t.push(n=[])},point:function(t,e){n.push([t,e])},lineEnd:b,buffer:function(){var e=t;return t=[],n=null,e},rejoin:function(){t.length>1&&t.push(t.pop().concat(t.shift()))}}}function De(n,t){return((n=n.x)[0]<0?n[1]-Ra-Ca:Ra-n[1])-((t=t.x)[0]<0?t[1]-Ra-Ca:Ra-t[1])}function Pe(n){var t,e=0/0,r=0/0,u=0/0;return{lineStart:function(){n.lineStart(),t=1},point:function(i,o){var a=i>0?qa:-qa,c=ga(i-e);ga(c-qa)0?Ra:-Ra),n.point(u,r),n.lineEnd(),n.lineStart(),n.point(a,r),n.point(i,r),t=0):u!==a&&c>=qa&&(ga(e-u)Ca?Math.atan((Math.sin(t)*(i=Math.cos(r))*Math.sin(e)-Math.sin(r)*(u=Math.cos(t))*Math.sin(n))/(u*i*o)):(t+r)/2}function je(n,t,e,r){var u;if(null==n)u=e*Ra,r.point(-qa,u),r.point(0,u),r.point(qa,u),r.point(qa,0),r.point(qa,-u),r.point(0,-u),r.point(-qa,-u),r.point(-qa,0),r.point(-qa,u);else if(ga(n[0]-t[0])>Ca){var i=n[0]a;++a){var l=t[a],s=l.length;if(s)for(var f=l[0],h=f[0],g=f[1]/2+qa/4,p=Math.sin(g),v=Math.cos(g),d=1;;){d===s&&(d=0),n=l[d];var m=n[0],y=n[1]/2+qa/4,M=Math.sin(y),x=Math.cos(y),b=m-h,_=b>=0?1:-1,w=_*b,S=w>qa,k=p*M;if(yc.add(Math.atan2(k*_*Math.sin(w),v*x+k*Math.cos(w))),i+=S?b+_*La:b,S^h>=e^m>=e){var E=de(pe(f),pe(n));Me(E);var A=de(u,E);Me(A);var N=(S^b>=0?-1:1)*tt(A[2]);(r>N||r===N&&(E[0]||E[1]))&&(o+=S^b>=0?1:-1)}if(!d++)break;h=m,p=M,v=x,f=n}}return(-Ca>i||Ca>i&&0>yc)^1&o}function He(n){function t(n,t){return Math.cos(n)*Math.cos(t)>i}function e(n){var e,i,c,l,s;return{lineStart:function(){l=c=!1,s=1},point:function(f,h){var g,p=[f,h],v=t(f,h),d=o?v?0:u(f,h):v?u(f+(0>f?qa:-qa),h):0;if(!e&&(l=c=v)&&n.lineStart(),v!==c&&(g=r(e,p),(be(e,g)||be(p,g))&&(p[0]+=Ca,p[1]+=Ca,v=t(p[0],p[1]))),v!==c)s=0,v?(n.lineStart(),g=r(p,e),n.point(g[0],g[1])):(g=r(e,p),n.point(g[0],g[1]),n.lineEnd()),e=g;else if(a&&e&&o^v){var m;d&i||!(m=r(p,e,!0))||(s=0,o?(n.lineStart(),n.point(m[0][0],m[0][1]),n.point(m[1][0],m[1][1]),n.lineEnd()):(n.point(m[1][0],m[1][1]),n.lineEnd(),n.lineStart(),n.point(m[0][0],m[0][1])))}!v||e&&be(e,p)||n.point(p[0],p[1]),e=p,c=v,i=d},lineEnd:function(){c&&n.lineEnd(),e=null},clean:function(){return s|(l&&c)<<1}}}function r(n,t,e){var r=pe(n),u=pe(t),o=[1,0,0],a=de(r,u),c=ve(a,a),l=a[0],s=c-l*l;if(!s)return!e&&n;var f=i*c/s,h=-i*l/s,g=de(o,a),p=ye(o,f),v=ye(a,h);me(p,v);var d=g,m=ve(p,d),y=ve(d,d),M=m*m-y*(ve(p,p)-1);if(!(0>M)){var x=Math.sqrt(M),b=ye(d,(-m-x)/y);if(me(b,p),b=xe(b),!e)return b;var _,w=n[0],S=t[0],k=n[1],E=t[1];w>S&&(_=w,w=S,S=_);var A=S-w,N=ga(A-qa)A;if(!N&&k>E&&(_=k,k=E,E=_),C?N?k+E>0^b[1]<(ga(b[0]-w)qa^(w<=b[0]&&b[0]<=S)){var z=ye(d,(-m+x)/y);return me(z,p),[b,xe(z)]}}}function u(t,e){var r=o?n:qa-n,u=0;return-r>t?u|=1:t>r&&(u|=2),-r>e?u|=4:e>r&&(u|=8),u}var i=Math.cos(n),o=i>0,a=ga(i)>Ca,c=gr(n,6*Da);return Le(t,e,c,o?[0,-n]:[-qa,n-qa])}function Oe(n,t,e,r){return function(u){var i,o=u.a,a=u.b,c=o.x,l=o.y,s=a.x,f=a.y,h=0,g=1,p=s-c,v=f-l;if(i=n-c,p||!(i>0)){if(i/=p,0>p){if(h>i)return;g>i&&(g=i)}else if(p>0){if(i>g)return;i>h&&(h=i)}if(i=e-c,p||!(0>i)){if(i/=p,0>p){if(i>g)return;i>h&&(h=i)}else if(p>0){if(h>i)return;g>i&&(g=i)}if(i=t-l,v||!(i>0)){if(i/=v,0>v){if(h>i)return;g>i&&(g=i)}else if(v>0){if(i>g)return;i>h&&(h=i)}if(i=r-l,v||!(0>i)){if(i/=v,0>v){if(i>g)return;i>h&&(h=i)}else if(v>0){if(h>i)return;g>i&&(g=i)}return h>0&&(u.a={x:c+h*p,y:l+h*v}),1>g&&(u.b={x:c+g*p,y:l+g*v}),u}}}}}}function Ie(n,t,e,r){function u(r,u){return ga(r[0]-n)0?0:3:ga(r[0]-e)0?2:1:ga(r[1]-t)0?1:0:u>0?3:2}function i(n,t){return o(n.x,t.x)}function o(n,t){var e=u(n,1),r=u(t,1);return e!==r?e-r:0===e?t[1]-n[1]:1===e?n[0]-t[0]:2===e?n[1]-t[1]:t[0]-n[0]}return function(a){function c(n){for(var t=0,e=d.length,r=n[1],u=0;e>u;++u)for(var i,o=1,a=d[u],c=a.length,l=a[0];c>o;++o)i=a[o],l[1]<=r?i[1]>r&&Q(l,i,n)>0&&++t:i[1]<=r&&Q(l,i,n)<0&&--t,l=i;return 0!==t}function l(i,a,c,l){var s=0,f=0;if(null==i||(s=u(i,c))!==(f=u(a,c))||o(i,a)<0^c>0){do l.point(0===s||3===s?n:e,s>1?r:t);while((s=(s+c+4)%4)!==f)}else l.point(a[0],a[1])}function s(u,i){return u>=n&&e>=u&&i>=t&&r>=i}function f(n,t){s(n,t)&&a.point(n,t)}function h(){C.point=p,d&&d.push(m=[]),S=!0,w=!1,b=_=0/0}function g(){v&&(p(y,M),x&&w&&A.rejoin(),v.push(A.buffer())),C.point=f,w&&a.lineEnd()}function p(n,t){n=Math.max(-Tc,Math.min(Tc,n)),t=Math.max(-Tc,Math.min(Tc,t));var e=s(n,t);if(d&&m.push([n,t]),S)y=n,M=t,x=e,S=!1,e&&(a.lineStart(),a.point(n,t));else if(e&&w)a.point(n,t);else{var r={a:{x:b,y:_},b:{x:n,y:t}};N(r)?(w||(a.lineStart(),a.point(r.a.x,r.a.y)),a.point(r.b.x,r.b.y),e||a.lineEnd(),k=!1):e&&(a.lineStart(),a.point(n,t),k=!1)}b=n,_=t,w=e}var v,d,m,y,M,x,b,_,w,S,k,E=a,A=Re(),N=Oe(n,t,e,r),C={point:f,lineStart:h,lineEnd:g,polygonStart:function(){a=A,v=[],d=[],k=!0},polygonEnd:function(){a=E,v=ta.merge(v);var t=c([n,r]),e=k&&t,u=v.length;(e||u)&&(a.polygonStart(),e&&(a.lineStart(),l(null,null,1,a),a.lineEnd()),u&&Ce(v,i,t,l,a),a.polygonEnd()),v=d=m=null}};return C}}function Ye(n){var t=0,e=qa/3,r=ir(n),u=r(t,e);return u.parallels=function(n){return arguments.length?r(t=n[0]*qa/180,e=n[1]*qa/180):[t/qa*180,e/qa*180]},u}function Ze(n,t){function e(n,t){var e=Math.sqrt(i-2*u*Math.sin(t))/u;return[e*Math.sin(n*=u),o-e*Math.cos(n)]}var r=Math.sin(n),u=(r+Math.sin(t))/2,i=1+r*(2*u-r),o=Math.sqrt(i)/u;return e.invert=function(n,t){var e=o-t;return[Math.atan2(n,e)/u,tt((i-(n*n+e*e)*u*u)/(2*u))]},e}function Ve(){function n(n,t){Dc+=u*n-r*t,r=n,u=t}var t,e,r,u;Hc.point=function(i,o){Hc.point=n,t=r=i,e=u=o},Hc.lineEnd=function(){n(t,e)}}function Xe(n,t){Pc>n&&(Pc=n),n>jc&&(jc=n),Uc>t&&(Uc=t),t>Fc&&(Fc=t)}function $e(){function n(n,t){o.push("M",n,",",t,i)}function t(n,t){o.push("M",n,",",t),a.point=e}function e(n,t){o.push("L",n,",",t)}function r(){a.point=n}function u(){o.push("Z")}var i=Be(4.5),o=[],a={point:n,lineStart:function(){a.point=t},lineEnd:r,polygonStart:function(){a.lineEnd=u},polygonEnd:function(){a.lineEnd=r,a.point=n},pointRadius:function(n){return i=Be(n),a},result:function(){if(o.length){var n=o.join("");return o=[],n}}};return a}function Be(n){return"m0,"+n+"a"+n+","+n+" 0 1,1 0,"+-2*n+"a"+n+","+n+" 0 1,1 0,"+2*n+"z"}function We(n,t){_c+=n,wc+=t,++Sc}function Je(){function n(n,r){var u=n-t,i=r-e,o=Math.sqrt(u*u+i*i);kc+=o*(t+n)/2,Ec+=o*(e+r)/2,Ac+=o,We(t=n,e=r)}var t,e;Ic.point=function(r,u){Ic.point=n,We(t=r,e=u)}}function Ge(){Ic.point=We}function Ke(){function n(n,t){var e=n-r,i=t-u,o=Math.sqrt(e*e+i*i);kc+=o*(r+n)/2,Ec+=o*(u+t)/2,Ac+=o,o=u*n-r*t,Nc+=o*(r+n),Cc+=o*(u+t),zc+=3*o,We(r=n,u=t)}var t,e,r,u;Ic.point=function(i,o){Ic.point=n,We(t=r=i,e=u=o)},Ic.lineEnd=function(){n(t,e)}}function Qe(n){function t(t,e){n.moveTo(t+o,e),n.arc(t,e,o,0,La)}function e(t,e){n.moveTo(t,e),a.point=r}function r(t,e){n.lineTo(t,e)}function u(){a.point=t}function i(){n.closePath()}var o=4.5,a={point:t,lineStart:function(){a.point=e},lineEnd:u,polygonStart:function(){a.lineEnd=i},polygonEnd:function(){a.lineEnd=u,a.point=t},pointRadius:function(n){return o=n,a},result:b};return a}function nr(n){function t(n){return(a?r:e)(n)}function e(t){return rr(t,function(e,r){e=n(e,r),t.point(e[0],e[1])})}function r(t){function e(e,r){e=n(e,r),t.point(e[0],e[1])}function r(){M=0/0,S.point=i,t.lineStart()}function i(e,r){var i=pe([e,r]),o=n(e,r);u(M,x,y,b,_,w,M=o[0],x=o[1],y=e,b=i[0],_=i[1],w=i[2],a,t),t.point(M,x)}function o(){S.point=e,t.lineEnd()}function c(){r(),S.point=l,S.lineEnd=s}function l(n,t){i(f=n,h=t),g=M,p=x,v=b,d=_,m=w,S.point=i}function s(){u(M,x,y,b,_,w,g,p,f,v,d,m,a,t),S.lineEnd=o,o()}var f,h,g,p,v,d,m,y,M,x,b,_,w,S={point:e,lineStart:r,lineEnd:o,polygonStart:function(){t.polygonStart(),S.lineStart=c +},polygonEnd:function(){t.polygonEnd(),S.lineStart=r}};return S}function u(t,e,r,a,c,l,s,f,h,g,p,v,d,m){var y=s-t,M=f-e,x=y*y+M*M;if(x>4*i&&d--){var b=a+g,_=c+p,w=l+v,S=Math.sqrt(b*b+_*_+w*w),k=Math.asin(w/=S),E=ga(ga(w)-1)i||ga((y*z+M*q)/x-.5)>.3||o>a*g+c*p+l*v)&&(u(t,e,r,a,c,l,N,C,E,b/=S,_/=S,w,d,m),m.point(N,C),u(N,C,E,b,_,w,s,f,h,g,p,v,d,m))}}var i=.5,o=Math.cos(30*Da),a=16;return t.precision=function(n){return arguments.length?(a=(i=n*n)>0&&16,t):Math.sqrt(i)},t}function tr(n){var t=nr(function(t,e){return n([t*Pa,e*Pa])});return function(n){return or(t(n))}}function er(n){this.stream=n}function rr(n,t){return{point:t,sphere:function(){n.sphere()},lineStart:function(){n.lineStart()},lineEnd:function(){n.lineEnd()},polygonStart:function(){n.polygonStart()},polygonEnd:function(){n.polygonEnd()}}}function ur(n){return ir(function(){return n})()}function ir(n){function t(n){return n=a(n[0]*Da,n[1]*Da),[n[0]*h+c,l-n[1]*h]}function e(n){return n=a.invert((n[0]-c)/h,(l-n[1])/h),n&&[n[0]*Pa,n[1]*Pa]}function r(){a=Ae(o=lr(m,M,x),i);var n=i(v,d);return c=g-n[0]*h,l=p+n[1]*h,u()}function u(){return s&&(s.valid=!1,s=null),t}var i,o,a,c,l,s,f=nr(function(n,t){return n=i(n,t),[n[0]*h+c,l-n[1]*h]}),h=150,g=480,p=250,v=0,d=0,m=0,M=0,x=0,b=Lc,_=y,w=null,S=null;return t.stream=function(n){return s&&(s.valid=!1),s=or(b(o,f(_(n)))),s.valid=!0,s},t.clipAngle=function(n){return arguments.length?(b=null==n?(w=n,Lc):He((w=+n)*Da),u()):w},t.clipExtent=function(n){return arguments.length?(S=n,_=n?Ie(n[0][0],n[0][1],n[1][0],n[1][1]):y,u()):S},t.scale=function(n){return arguments.length?(h=+n,r()):h},t.translate=function(n){return arguments.length?(g=+n[0],p=+n[1],r()):[g,p]},t.center=function(n){return arguments.length?(v=n[0]%360*Da,d=n[1]%360*Da,r()):[v*Pa,d*Pa]},t.rotate=function(n){return arguments.length?(m=n[0]%360*Da,M=n[1]%360*Da,x=n.length>2?n[2]%360*Da:0,r()):[m*Pa,M*Pa,x*Pa]},ta.rebind(t,f,"precision"),function(){return i=n.apply(this,arguments),t.invert=i.invert&&e,r()}}function or(n){return rr(n,function(t,e){n.point(t*Da,e*Da)})}function ar(n,t){return[n,t]}function cr(n,t){return[n>qa?n-La:-qa>n?n+La:n,t]}function lr(n,t,e){return n?t||e?Ae(fr(n),hr(t,e)):fr(n):t||e?hr(t,e):cr}function sr(n){return function(t,e){return t+=n,[t>qa?t-La:-qa>t?t+La:t,e]}}function fr(n){var t=sr(n);return t.invert=sr(-n),t}function hr(n,t){function e(n,t){var e=Math.cos(t),a=Math.cos(n)*e,c=Math.sin(n)*e,l=Math.sin(t),s=l*r+a*u;return[Math.atan2(c*i-s*o,a*r-l*u),tt(s*i+c*o)]}var r=Math.cos(n),u=Math.sin(n),i=Math.cos(t),o=Math.sin(t);return e.invert=function(n,t){var e=Math.cos(t),a=Math.cos(n)*e,c=Math.sin(n)*e,l=Math.sin(t),s=l*i-c*o;return[Math.atan2(c*i+l*o,a*r+s*u),tt(s*r-a*u)]},e}function gr(n,t){var e=Math.cos(n),r=Math.sin(n);return function(u,i,o,a){var c=o*t;null!=u?(u=pr(e,u),i=pr(e,i),(o>0?i>u:u>i)&&(u+=o*La)):(u=n+o*La,i=n-.5*c);for(var l,s=u;o>0?s>i:i>s;s-=c)a.point((l=xe([e,-r*Math.cos(s),-r*Math.sin(s)]))[0],l[1])}}function pr(n,t){var e=pe(t);e[0]-=n,Me(e);var r=nt(-e[1]);return((-e[2]<0?-r:r)+2*Math.PI-Ca)%(2*Math.PI)}function vr(n,t,e){var r=ta.range(n,t-Ca,e).concat(t);return function(n){return r.map(function(t){return[n,t]})}}function dr(n,t,e){var r=ta.range(n,t-Ca,e).concat(t);return function(n){return r.map(function(t){return[t,n]})}}function mr(n){return n.source}function yr(n){return n.target}function Mr(n,t,e,r){var u=Math.cos(t),i=Math.sin(t),o=Math.cos(r),a=Math.sin(r),c=u*Math.cos(n),l=u*Math.sin(n),s=o*Math.cos(e),f=o*Math.sin(e),h=2*Math.asin(Math.sqrt(it(r-t)+u*o*it(e-n))),g=1/Math.sin(h),p=h?function(n){var t=Math.sin(n*=h)*g,e=Math.sin(h-n)*g,r=e*c+t*s,u=e*l+t*f,o=e*i+t*a;return[Math.atan2(u,r)*Pa,Math.atan2(o,Math.sqrt(r*r+u*u))*Pa]}:function(){return[n*Pa,t*Pa]};return p.distance=h,p}function xr(){function n(n,u){var i=Math.sin(u*=Da),o=Math.cos(u),a=ga((n*=Da)-t),c=Math.cos(a);Yc+=Math.atan2(Math.sqrt((a=o*Math.sin(a))*a+(a=r*i-e*o*c)*a),e*i+r*o*c),t=n,e=i,r=o}var t,e,r;Zc.point=function(u,i){t=u*Da,e=Math.sin(i*=Da),r=Math.cos(i),Zc.point=n},Zc.lineEnd=function(){Zc.point=Zc.lineEnd=b}}function br(n,t){function e(t,e){var r=Math.cos(t),u=Math.cos(e),i=n(r*u);return[i*u*Math.sin(t),i*Math.sin(e)]}return e.invert=function(n,e){var r=Math.sqrt(n*n+e*e),u=t(r),i=Math.sin(u),o=Math.cos(u);return[Math.atan2(n*i,r*o),Math.asin(r&&e*i/r)]},e}function _r(n,t){function e(n,t){o>0?-Ra+Ca>t&&(t=-Ra+Ca):t>Ra-Ca&&(t=Ra-Ca);var e=o/Math.pow(u(t),i);return[e*Math.sin(i*n),o-e*Math.cos(i*n)]}var r=Math.cos(n),u=function(n){return Math.tan(qa/4+n/2)},i=n===t?Math.sin(n):Math.log(r/Math.cos(t))/Math.log(u(t)/u(n)),o=r*Math.pow(u(n),i)/i;return i?(e.invert=function(n,t){var e=o-t,r=K(i)*Math.sqrt(n*n+e*e);return[Math.atan2(n,e)/i,2*Math.atan(Math.pow(o/r,1/i))-Ra]},e):Sr}function wr(n,t){function e(n,t){var e=i-t;return[e*Math.sin(u*n),i-e*Math.cos(u*n)]}var r=Math.cos(n),u=n===t?Math.sin(n):(r-Math.cos(t))/(t-n),i=r/u+n;return ga(u)u;u++){for(;r>1&&Q(n[e[r-2]],n[e[r-1]],n[u])<=0;)--r;e[r++]=u}return e.slice(0,r)}function zr(n,t){return n[0]-t[0]||n[1]-t[1]}function qr(n,t,e){return(e[0]-t[0])*(n[1]-t[1])<(e[1]-t[1])*(n[0]-t[0])}function Lr(n,t,e,r){var u=n[0],i=e[0],o=t[0]-u,a=r[0]-i,c=n[1],l=e[1],s=t[1]-c,f=r[1]-l,h=(a*(c-l)-f*(u-i))/(f*o-a*s);return[u+h*o,c+h*s]}function Tr(n){var t=n[0],e=n[n.length-1];return!(t[0]-e[0]||t[1]-e[1])}function Rr(){tu(this),this.edge=this.site=this.circle=null}function Dr(n){var t=el.pop()||new Rr;return t.site=n,t}function Pr(n){Xr(n),Qc.remove(n),el.push(n),tu(n)}function Ur(n){var t=n.circle,e=t.x,r=t.cy,u={x:e,y:r},i=n.P,o=n.N,a=[n];Pr(n);for(var c=i;c.circle&&ga(e-c.circle.x)s;++s)l=a[s],c=a[s-1],Kr(l.edge,c.site,l.site,u);c=a[0],l=a[f-1],l.edge=Jr(c.site,l.site,null,u),Vr(c),Vr(l)}function jr(n){for(var t,e,r,u,i=n.x,o=n.y,a=Qc._;a;)if(r=Fr(a,o)-i,r>Ca)a=a.L;else{if(u=i-Hr(a,o),!(u>Ca)){r>-Ca?(t=a.P,e=a):u>-Ca?(t=a,e=a.N):t=e=a;break}if(!a.R){t=a;break}a=a.R}var c=Dr(n);if(Qc.insert(t,c),t||e){if(t===e)return Xr(t),e=Dr(t.site),Qc.insert(c,e),c.edge=e.edge=Jr(t.site,c.site),Vr(t),void Vr(e);if(!e)return void(c.edge=Jr(t.site,c.site));Xr(t),Xr(e);var l=t.site,s=l.x,f=l.y,h=n.x-s,g=n.y-f,p=e.site,v=p.x-s,d=p.y-f,m=2*(h*d-g*v),y=h*h+g*g,M=v*v+d*d,x={x:(d*y-g*M)/m+s,y:(h*M-v*y)/m+f};Kr(e.edge,l,p,x),c.edge=Jr(l,n,null,x),e.edge=Jr(n,p,null,x),Vr(t),Vr(e)}}function Fr(n,t){var e=n.site,r=e.x,u=e.y,i=u-t;if(!i)return r;var o=n.P;if(!o)return-1/0;e=o.site;var a=e.x,c=e.y,l=c-t;if(!l)return a;var s=a-r,f=1/i-1/l,h=s/l;return f?(-h+Math.sqrt(h*h-2*f*(s*s/(-2*l)-c+l/2+u-i/2)))/f+r:(r+a)/2}function Hr(n,t){var e=n.N;if(e)return Fr(e,t);var r=n.site;return r.y===t?r.x:1/0}function Or(n){this.site=n,this.edges=[]}function Ir(n){for(var t,e,r,u,i,o,a,c,l,s,f=n[0][0],h=n[1][0],g=n[0][1],p=n[1][1],v=Kc,d=v.length;d--;)if(i=v[d],i&&i.prepare())for(a=i.edges,c=a.length,o=0;c>o;)s=a[o].end(),r=s.x,u=s.y,l=a[++o%c].start(),t=l.x,e=l.y,(ga(r-t)>Ca||ga(u-e)>Ca)&&(a.splice(o,0,new Qr(Gr(i.site,s,ga(r-f)Ca?{x:f,y:ga(t-f)Ca?{x:ga(e-p)Ca?{x:h,y:ga(t-h)Ca?{x:ga(e-g)=-za)){var g=c*c+l*l,p=s*s+f*f,v=(f*g-l*p)/h,d=(c*p-s*g)/h,f=d+a,m=rl.pop()||new Zr;m.arc=n,m.site=u,m.x=v+o,m.y=f+Math.sqrt(v*v+d*d),m.cy=f,n.circle=m;for(var y=null,M=tl._;M;)if(m.yd||d>=a)return;if(h>p){if(i){if(i.y>=l)return}else i={x:d,y:c};e={x:d,y:l}}else{if(i){if(i.yr||r>1)if(h>p){if(i){if(i.y>=l)return}else i={x:(c-u)/r,y:c};e={x:(l-u)/r,y:l}}else{if(i){if(i.yg){if(i){if(i.x>=a)return}else i={x:o,y:r*o+u};e={x:a,y:r*a+u}}else{if(i){if(i.xi||f>o||r>h||u>g)){if(p=n.point){var p,v=t-n.x,d=e-n.y,m=v*v+d*d;if(c>m){var y=Math.sqrt(c=m);r=t-y,u=e-y,i=t+y,o=e+y,a=p}}for(var M=n.nodes,x=.5*(s+h),b=.5*(f+g),_=t>=x,w=e>=b,S=w<<1|_,k=S+4;k>S;++S)if(n=M[3&S])switch(3&S){case 0:l(n,s,f,x,b);break;case 1:l(n,x,f,h,b);break;case 2:l(n,s,b,x,g);break;case 3:l(n,x,b,h,g)}}}(n,r,u,i,o),a}function gu(n,t){n=ta.rgb(n),t=ta.rgb(t);var e=n.r,r=n.g,u=n.b,i=t.r-e,o=t.g-r,a=t.b-u;return function(n){return"#"+xt(Math.round(e+i*n))+xt(Math.round(r+o*n))+xt(Math.round(u+a*n))}}function pu(n,t){var e,r={},u={};for(e in n)e in t?r[e]=mu(n[e],t[e]):u[e]=n[e];for(e in t)e in n||(u[e]=t[e]);return function(n){for(e in r)u[e]=r[e](n);return u}}function vu(n,t){return n=+n,t=+t,function(e){return n*(1-e)+t*e}}function du(n,t){var e,r,u,i=il.lastIndex=ol.lastIndex=0,o=-1,a=[],c=[];for(n+="",t+="";(e=il.exec(n))&&(r=ol.exec(t));)(u=r.index)>i&&(u=t.slice(i,u),a[o]?a[o]+=u:a[++o]=u),(e=e[0])===(r=r[0])?a[o]?a[o]+=r:a[++o]=r:(a[++o]=null,c.push({i:o,x:vu(e,r)})),i=ol.lastIndex;return ir;++r)a[(e=c[r]).i]=e.x(n);return a.join("")})}function mu(n,t){for(var e,r=ta.interpolators.length;--r>=0&&!(e=ta.interpolators[r](n,t)););return e}function yu(n,t){var e,r=[],u=[],i=n.length,o=t.length,a=Math.min(n.length,t.length);for(e=0;a>e;++e)r.push(mu(n[e],t[e]));for(;i>e;++e)u[e]=n[e];for(;o>e;++e)u[e]=t[e];return function(n){for(e=0;a>e;++e)u[e]=r[e](n);return u}}function Mu(n){return function(t){return 0>=t?0:t>=1?1:n(t)}}function xu(n){return function(t){return 1-n(1-t)}}function bu(n){return function(t){return.5*(.5>t?n(2*t):2-n(2-2*t))}}function _u(n){return n*n}function wu(n){return n*n*n}function Su(n){if(0>=n)return 0;if(n>=1)return 1;var t=n*n,e=t*n;return 4*(.5>n?e:3*(n-t)+e-.75)}function ku(n){return function(t){return Math.pow(t,n)}}function Eu(n){return 1-Math.cos(n*Ra)}function Au(n){return Math.pow(2,10*(n-1))}function Nu(n){return 1-Math.sqrt(1-n*n)}function Cu(n,t){var e;return arguments.length<2&&(t=.45),arguments.length?e=t/La*Math.asin(1/n):(n=1,e=t/4),function(r){return 1+n*Math.pow(2,-10*r)*Math.sin((r-e)*La/t)}}function zu(n){return n||(n=1.70158),function(t){return t*t*((n+1)*t-n)}}function qu(n){return 1/2.75>n?7.5625*n*n:2/2.75>n?7.5625*(n-=1.5/2.75)*n+.75:2.5/2.75>n?7.5625*(n-=2.25/2.75)*n+.9375:7.5625*(n-=2.625/2.75)*n+.984375}function Lu(n,t){n=ta.hcl(n),t=ta.hcl(t);var e=n.h,r=n.c,u=n.l,i=t.h-e,o=t.c-r,a=t.l-u;return isNaN(o)&&(o=0,r=isNaN(r)?t.c:r),isNaN(i)?(i=0,e=isNaN(e)?t.h:e):i>180?i-=360:-180>i&&(i+=360),function(n){return st(e+i*n,r+o*n,u+a*n)+""}}function Tu(n,t){n=ta.hsl(n),t=ta.hsl(t);var e=n.h,r=n.s,u=n.l,i=t.h-e,o=t.s-r,a=t.l-u;return isNaN(o)&&(o=0,r=isNaN(r)?t.s:r),isNaN(i)?(i=0,e=isNaN(e)?t.h:e):i>180?i-=360:-180>i&&(i+=360),function(n){return ct(e+i*n,r+o*n,u+a*n)+""}}function Ru(n,t){n=ta.lab(n),t=ta.lab(t);var e=n.l,r=n.a,u=n.b,i=t.l-e,o=t.a-r,a=t.b-u;return function(n){return ht(e+i*n,r+o*n,u+a*n)+""}}function Du(n,t){return t-=n,function(e){return Math.round(n+t*e)}}function Pu(n){var t=[n.a,n.b],e=[n.c,n.d],r=ju(t),u=Uu(t,e),i=ju(Fu(e,t,-u))||0;t[0]*e[1]180?s+=360:s-l>180&&(l+=360),u.push({i:r.push(r.pop()+"rotate(",null,")")-2,x:vu(l,s)})):s&&r.push(r.pop()+"rotate("+s+")"),f!=h?u.push({i:r.push(r.pop()+"skewX(",null,")")-2,x:vu(f,h)}):h&&r.push(r.pop()+"skewX("+h+")"),g[0]!=p[0]||g[1]!=p[1]?(e=r.push(r.pop()+"scale(",null,",",null,")"),u.push({i:e-4,x:vu(g[0],p[0])},{i:e-2,x:vu(g[1],p[1])})):(1!=p[0]||1!=p[1])&&r.push(r.pop()+"scale("+p+")"),e=u.length,function(n){for(var t,i=-1;++i=0;)e.push(u[r])}function Qu(n,t){for(var e=[n],r=[];null!=(n=e.pop());)if(r.push(n),(i=n.children)&&(u=i.length))for(var u,i,o=-1;++oe;++e)(t=n[e][1])>u&&(r=e,u=t);return r}function si(n){return n.reduce(fi,0)}function fi(n,t){return n+t[1]}function hi(n,t){return gi(n,Math.ceil(Math.log(t.length)/Math.LN2+1))}function gi(n,t){for(var e=-1,r=+n[0],u=(n[1]-r)/t,i=[];++e<=t;)i[e]=u*e+r;return i}function pi(n){return[ta.min(n),ta.max(n)]}function vi(n,t){return n.value-t.value}function di(n,t){var e=n._pack_next;n._pack_next=t,t._pack_prev=n,t._pack_next=e,e._pack_prev=t}function mi(n,t){n._pack_next=t,t._pack_prev=n}function yi(n,t){var e=t.x-n.x,r=t.y-n.y,u=n.r+t.r;return.999*u*u>e*e+r*r}function Mi(n){function t(n){s=Math.min(n.x-n.r,s),f=Math.max(n.x+n.r,f),h=Math.min(n.y-n.r,h),g=Math.max(n.y+n.r,g)}if((e=n.children)&&(l=e.length)){var e,r,u,i,o,a,c,l,s=1/0,f=-1/0,h=1/0,g=-1/0;if(e.forEach(xi),r=e[0],r.x=-r.r,r.y=0,t(r),l>1&&(u=e[1],u.x=u.r,u.y=0,t(u),l>2))for(i=e[2],wi(r,u,i),t(i),di(r,i),r._pack_prev=i,di(i,u),u=r._pack_next,o=3;l>o;o++){wi(r,u,i=e[o]);var p=0,v=1,d=1;for(a=u._pack_next;a!==u;a=a._pack_next,v++)if(yi(a,i)){p=1;break}if(1==p)for(c=r._pack_prev;c!==a._pack_prev&&!yi(c,i);c=c._pack_prev,d++);p?(d>v||v==d&&u.ro;o++)i=e[o],i.x-=m,i.y-=y,M=Math.max(M,i.r+Math.sqrt(i.x*i.x+i.y*i.y));n.r=M,e.forEach(bi)}}function xi(n){n._pack_next=n._pack_prev=n}function bi(n){delete n._pack_next,delete n._pack_prev}function _i(n,t,e,r){var u=n.children;if(n.x=t+=r*n.x,n.y=e+=r*n.y,n.r*=r,u)for(var i=-1,o=u.length;++i=0;)t=u[i],t.z+=e,t.m+=e,e+=t.s+(r+=t.c)}function Ci(n,t,e){return n.a.parent===t.parent?n.a:e}function zi(n){return 1+ta.max(n,function(n){return n.y})}function qi(n){return n.reduce(function(n,t){return n+t.x},0)/n.length}function Li(n){var t=n.children;return t&&t.length?Li(t[0]):n}function Ti(n){var t,e=n.children;return e&&(t=e.length)?Ti(e[t-1]):n}function Ri(n){return{x:n.x,y:n.y,dx:n.dx,dy:n.dy}}function Di(n,t){var e=n.x+t[3],r=n.y+t[0],u=n.dx-t[1]-t[3],i=n.dy-t[0]-t[2];return 0>u&&(e+=u/2,u=0),0>i&&(r+=i/2,i=0),{x:e,y:r,dx:u,dy:i}}function Pi(n){var t=n[0],e=n[n.length-1];return e>t?[t,e]:[e,t]}function Ui(n){return n.rangeExtent?n.rangeExtent():Pi(n.range())}function ji(n,t,e,r){var u=e(n[0],n[1]),i=r(t[0],t[1]);return function(n){return i(u(n))}}function Fi(n,t){var e,r=0,u=n.length-1,i=n[r],o=n[u];return i>o&&(e=r,r=u,u=e,e=i,i=o,o=e),n[r]=t.floor(i),n[u]=t.ceil(o),n}function Hi(n){return n?{floor:function(t){return Math.floor(t/n)*n},ceil:function(t){return Math.ceil(t/n)*n}}:ml}function Oi(n,t,e,r){var u=[],i=[],o=0,a=Math.min(n.length,t.length)-1;for(n[a]2?Oi:ji,c=r?Iu:Ou;return o=u(n,t,c,e),a=u(t,n,c,mu),i}function i(n){return o(n)}var o,a;return i.invert=function(n){return a(n)},i.domain=function(t){return arguments.length?(n=t.map(Number),u()):n},i.range=function(n){return arguments.length?(t=n,u()):t},i.rangeRound=function(n){return i.range(n).interpolate(Du)},i.clamp=function(n){return arguments.length?(r=n,u()):r},i.interpolate=function(n){return arguments.length?(e=n,u()):e},i.ticks=function(t){return Xi(n,t)},i.tickFormat=function(t,e){return $i(n,t,e)},i.nice=function(t){return Zi(n,t),u()},i.copy=function(){return Ii(n,t,e,r)},u()}function Yi(n,t){return ta.rebind(n,t,"range","rangeRound","interpolate","clamp")}function Zi(n,t){return Fi(n,Hi(Vi(n,t)[2]))}function Vi(n,t){null==t&&(t=10);var e=Pi(n),r=e[1]-e[0],u=Math.pow(10,Math.floor(Math.log(r/t)/Math.LN10)),i=t/r*u;return.15>=i?u*=10:.35>=i?u*=5:.75>=i&&(u*=2),e[0]=Math.ceil(e[0]/u)*u,e[1]=Math.floor(e[1]/u)*u+.5*u,e[2]=u,e}function Xi(n,t){return ta.range.apply(ta,Vi(n,t))}function $i(n,t,e){var r=Vi(n,t);if(e){var u=ic.exec(e);if(u.shift(),"s"===u[8]){var i=ta.formatPrefix(Math.max(ga(r[0]),ga(r[1])));return u[7]||(u[7]="."+Bi(i.scale(r[2]))),u[8]="f",e=ta.format(u.join("")),function(n){return e(i.scale(n))+i.symbol}}u[7]||(u[7]="."+Wi(u[8],r)),e=u.join("")}else e=",."+Bi(r[2])+"f";return ta.format(e)}function Bi(n){return-Math.floor(Math.log(n)/Math.LN10+.01)}function Wi(n,t){var e=Bi(t[2]);return n in yl?Math.abs(e-Bi(Math.max(ga(t[0]),ga(t[1]))))+ +("e"!==n):e-2*("%"===n)}function Ji(n,t,e,r){function u(n){return(e?Math.log(0>n?0:n):-Math.log(n>0?0:-n))/Math.log(t)}function i(n){return e?Math.pow(t,n):-Math.pow(t,-n)}function o(t){return n(u(t))}return o.invert=function(t){return i(n.invert(t))},o.domain=function(t){return arguments.length?(e=t[0]>=0,n.domain((r=t.map(Number)).map(u)),o):r},o.base=function(e){return arguments.length?(t=+e,n.domain(r.map(u)),o):t},o.nice=function(){var t=Fi(r.map(u),e?Math:xl);return n.domain(t),r=t.map(i),o},o.ticks=function(){var n=Pi(r),o=[],a=n[0],c=n[1],l=Math.floor(u(a)),s=Math.ceil(u(c)),f=t%1?2:t;if(isFinite(s-l)){if(e){for(;s>l;l++)for(var h=1;f>h;h++)o.push(i(l)*h);o.push(i(l))}else for(o.push(i(l));l++0;h--)o.push(i(l)*h);for(l=0;o[l]c;s--);o=o.slice(l,s)}return o},o.tickFormat=function(n,t){if(!arguments.length)return Ml;arguments.length<2?t=Ml:"function"!=typeof t&&(t=ta.format(t));var r,a=Math.max(.1,n/o.ticks().length),c=e?(r=1e-12,Math.ceil):(r=-1e-12,Math.floor);return function(n){return n/i(c(u(n)+r))<=a?t(n):""}},o.copy=function(){return Ji(n.copy(),t,e,r)},Yi(o,n)}function Gi(n,t,e){function r(t){return n(u(t))}var u=Ki(t),i=Ki(1/t);return r.invert=function(t){return i(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain((e=t.map(Number)).map(u)),r):e},r.ticks=function(n){return Xi(e,n)},r.tickFormat=function(n,t){return $i(e,n,t)},r.nice=function(n){return r.domain(Zi(e,n))},r.exponent=function(o){return arguments.length?(u=Ki(t=o),i=Ki(1/t),n.domain(e.map(u)),r):t},r.copy=function(){return Gi(n.copy(),t,e)},Yi(r,n)}function Ki(n){return function(t){return 0>t?-Math.pow(-t,n):Math.pow(t,n)}}function Qi(n,t){function e(e){return i[((u.get(e)||("range"===t.t?u.set(e,n.push(e)):0/0))-1)%i.length]}function r(t,e){return ta.range(n.length).map(function(n){return t+e*n})}var u,i,o;return e.domain=function(r){if(!arguments.length)return n;n=[],u=new l;for(var i,o=-1,a=r.length;++oe?[0/0,0/0]:[e>0?a[e-1]:n[0],et?0/0:t/i+n,[t,t+1/i]},r.copy=function(){return to(n,t,e)},u()}function eo(n,t){function e(e){return e>=e?t[ta.bisect(n,e)]:void 0}return e.domain=function(t){return arguments.length?(n=t,e):n},e.range=function(n){return arguments.length?(t=n,e):t},e.invertExtent=function(e){return e=t.indexOf(e),[n[e-1],n[e]]},e.copy=function(){return eo(n,t)},e}function ro(n){function t(n){return+n}return t.invert=t,t.domain=t.range=function(e){return arguments.length?(n=e.map(t),t):n},t.ticks=function(t){return Xi(n,t)},t.tickFormat=function(t,e){return $i(n,t,e)},t.copy=function(){return ro(n)},t}function uo(){return 0}function io(n){return n.innerRadius}function oo(n){return n.outerRadius}function ao(n){return n.startAngle}function co(n){return n.endAngle}function lo(n){return n&&n.padAngle}function so(n,t,e,r){return(n-e)*t-(t-r)*n>0?0:1}function fo(n,t,e,r,u){var i=n[0]-t[0],o=n[1]-t[1],a=(u?r:-r)/Math.sqrt(i*i+o*o),c=a*o,l=-a*i,s=n[0]+c,f=n[1]+l,h=t[0]+c,g=t[1]+l,p=(s+h)/2,v=(f+g)/2,d=h-s,m=g-f,y=d*d+m*m,M=e-r,x=s*g-h*f,b=(0>m?-1:1)*Math.sqrt(M*M*y-x*x),_=(x*m-d*b)/y,w=(-x*d-m*b)/y,S=(x*m+d*b)/y,k=(-x*d+m*b)/y,E=_-p,A=w-v,N=S-p,C=k-v;return E*E+A*A>N*N+C*C&&(_=S,w=k),[[_-c,w-l],[_*e/M,w*e/M]]}function ho(n){function t(t){function o(){l.push("M",i(n(s),a))}for(var c,l=[],s=[],f=-1,h=t.length,g=Et(e),p=Et(r);++f1&&u.push("H",r[0]),u.join("")}function mo(n){for(var t=0,e=n.length,r=n[0],u=[r[0],",",r[1]];++t1){a=t[1],i=n[c],c++,r+="C"+(u[0]+o[0])+","+(u[1]+o[1])+","+(i[0]-a[0])+","+(i[1]-a[1])+","+i[0]+","+i[1];for(var l=2;l9&&(u=3*t/Math.sqrt(u),o[a]=u*e,o[a+1]=u*r));for(a=-1;++a<=c;)u=(n[Math.min(c,a+1)][0]-n[Math.max(0,a-1)][0])/(6*(1+o[a]*o[a])),i.push([u||0,o[a]*u||0]);return i}function To(n){return n.length<3?go(n):n[0]+_o(n,Lo(n))}function Ro(n){for(var t,e,r,u=-1,i=n.length;++ur)return s();var u=i[i.active];u&&(--i.count,delete i[i.active],u.event&&u.event.interrupt.call(n,n.__data__,u.index)),i.active=r,o.event&&o.event.start.call(n,n.__data__,t),o.tween.forEach(function(e,r){(r=r.call(n,n.__data__,t))&&v.push(r)}),h=o.ease,f=o.duration,ta.timer(function(){return p.c=l(e||1)?Ne:l,1},0,a)}function l(e){if(i.active!==r)return 1;for(var u=e/f,a=h(u),c=v.length;c>0;)v[--c].call(n,a);return u>=1?(o.event&&o.event.end.call(n,n.__data__,t),s()):void 0}function s(){return--i.count?delete i[r]:delete n[e],1}var f,h,g=o.delay,p=ec,v=[];return p.t=g+a,u>=g?c(u-g):void(p.c=c)},0,a)}}function Bo(n,t,e){n.attr("transform",function(n){var r=t(n);return"translate("+(isFinite(r)?r:e(n))+",0)"})}function Wo(n,t,e){n.attr("transform",function(n){var r=t(n);return"translate(0,"+(isFinite(r)?r:e(n))+")"})}function Jo(n){return n.toISOString()}function Go(n,t,e){function r(t){return n(t)}function u(n,e){var r=n[1]-n[0],u=r/e,i=ta.bisect(Vl,u);return i==Vl.length?[t.year,Vi(n.map(function(n){return n/31536e6}),e)[2]]:i?t[u/Vl[i-1]1?{floor:function(t){for(;e(t=n.floor(t));)t=Ko(t-1);return t},ceil:function(t){for(;e(t=n.ceil(t));)t=Ko(+t+1);return t}}:n))},r.ticks=function(n,t){var e=Pi(r.domain()),i=null==n?u(e,10):"number"==typeof n?u(e,n):!n.range&&[{range:n},t];return i&&(n=i[0],t=i[1]),n.range(e[0],Ko(+e[1]+1),1>t?1:t)},r.tickFormat=function(){return e},r.copy=function(){return Go(n.copy(),t,e)},Yi(r,n)}function Ko(n){return new Date(n)}function Qo(n){return JSON.parse(n.responseText)}function na(n){var t=ua.createRange();return t.selectNode(ua.body),t.createContextualFragment(n.responseText)}var ta={version:"3.5.5"},ea=[].slice,ra=function(n){return ea.call(n)},ua=this.document;if(ua)try{ra(ua.documentElement.childNodes)[0].nodeType}catch(ia){ra=function(n){for(var t=n.length,e=new Array(t);t--;)e[t]=n[t];return e}}if(Date.now||(Date.now=function(){return+new Date}),ua)try{ua.createElement("DIV").style.setProperty("opacity",0,"")}catch(oa){var aa=this.Element.prototype,ca=aa.setAttribute,la=aa.setAttributeNS,sa=this.CSSStyleDeclaration.prototype,fa=sa.setProperty;aa.setAttribute=function(n,t){ca.call(this,n,t+"")},aa.setAttributeNS=function(n,t,e){la.call(this,n,t,e+"")},sa.setProperty=function(n,t,e){fa.call(this,n,t+"",e)}}ta.ascending=e,ta.descending=function(n,t){return n>t?-1:t>n?1:t>=n?0:0/0},ta.min=function(n,t){var e,r,u=-1,i=n.length;if(1===arguments.length){for(;++u=r){e=r;break}for(;++ur&&(e=r)}else{for(;++u=r){e=r;break}for(;++ur&&(e=r)}return e},ta.max=function(n,t){var e,r,u=-1,i=n.length;if(1===arguments.length){for(;++u=r){e=r;break}for(;++ue&&(e=r)}else{for(;++u=r){e=r;break}for(;++ue&&(e=r)}return e},ta.extent=function(n,t){var e,r,u,i=-1,o=n.length;if(1===arguments.length){for(;++i=r){e=u=r;break}for(;++ir&&(e=r),r>u&&(u=r))}else{for(;++i=r){e=u=r;break}for(;++ir&&(e=r),r>u&&(u=r))}return[e,u]},ta.sum=function(n,t){var e,r=0,i=n.length,o=-1;if(1===arguments.length)for(;++o1?c/(s-1):void 0},ta.deviation=function(){var n=ta.variance.apply(this,arguments);return n?Math.sqrt(n):n};var ha=i(e);ta.bisectLeft=ha.left,ta.bisect=ta.bisectRight=ha.right,ta.bisector=function(n){return i(1===n.length?function(t,r){return e(n(t),r)}:n)},ta.shuffle=function(n,t,e){(i=arguments.length)<3&&(e=n.length,2>i&&(t=0));for(var r,u,i=e-t;i;)u=Math.random()*i--|0,r=n[i+t],n[i+t]=n[u+t],n[u+t]=r;return n},ta.permute=function(n,t){for(var e=t.length,r=new Array(e);e--;)r[e]=n[t[e]];return r},ta.pairs=function(n){for(var t,e=0,r=n.length-1,u=n[0],i=new Array(0>r?0:r);r>e;)i[e]=[t=u,u=n[++e]];return i},ta.zip=function(){if(!(r=arguments.length))return[];for(var n=-1,t=ta.min(arguments,o),e=new Array(t);++n=0;)for(r=n[u],t=r.length;--t>=0;)e[--o]=r[t];return e};var ga=Math.abs;ta.range=function(n,t,e){if(arguments.length<3&&(e=1,arguments.length<2&&(t=n,n=0)),(t-n)/e===1/0)throw new Error("infinite range");var r,u=[],i=a(ga(e)),o=-1;if(n*=i,t*=i,e*=i,0>e)for(;(r=n+e*++o)>t;)u.push(r/i);else for(;(r=n+e*++o)=i.length)return r?r.call(u,o):e?o.sort(e):o;for(var c,s,f,h,g=-1,p=o.length,v=i[a++],d=new l;++g=i.length)return n;var r=[],u=o[e++];return n.forEach(function(n,u){r.push({key:n,values:t(u,e)})}),u?r.sort(function(n,t){return u(n.key,t.key)}):r}var e,r,u={},i=[],o=[];return u.map=function(t,e){return n(e,t,0)},u.entries=function(e){return t(n(ta.map,e,0),0)},u.key=function(n){return i.push(n),u},u.sortKeys=function(n){return o[i.length-1]=n,u},u.sortValues=function(n){return e=n,u},u.rollup=function(n){return r=n,u},u},ta.set=function(n){var t=new m;if(n)for(var e=0,r=n.length;r>e;++e)t.add(n[e]);return t},c(m,{has:h,add:function(n){return this._[s(n+="")]=!0,n},remove:g,values:p,size:v,empty:d,forEach:function(n){for(var t in this._)n.call(this,f(t))}}),ta.behavior={},ta.rebind=function(n,t){for(var e,r=1,u=arguments.length;++r=0&&(r=n.slice(e+1),n=n.slice(0,e)),n)return arguments.length<2?this[n].on(r):this[n].on(r,t);if(2===arguments.length){if(null==t)for(n in this)this.hasOwnProperty(n)&&this[n].on(r,null);return this}},ta.event=null,ta.requote=function(n){return n.replace(ma,"\\$&")};var ma=/[\\\^\$\*\+\?\|\[\]\(\)\.\{\}]/g,ya={}.__proto__?function(n,t){n.__proto__=t}:function(n,t){for(var e in t)n[e]=t[e]},Ma=function(n,t){return t.querySelector(n)},xa=function(n,t){return t.querySelectorAll(n)},ba=function(n,t){var e=n.matches||n[x(n,"matchesSelector")];return(ba=function(n,t){return e.call(n,t)})(n,t)};"function"==typeof Sizzle&&(Ma=function(n,t){return Sizzle(n,t)[0]||null},xa=Sizzle,ba=Sizzle.matchesSelector),ta.selection=function(){return ta.select(ua.documentElement)};var _a=ta.selection.prototype=[];_a.select=function(n){var t,e,r,u,i=[];n=N(n);for(var o=-1,a=this.length;++o=0&&(e=n.slice(0,t),n=n.slice(t+1)),wa.hasOwnProperty(e)?{space:wa[e],local:n}:n}},_a.attr=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node();return n=ta.ns.qualify(n),n.local?e.getAttributeNS(n.space,n.local):e.getAttribute(n)}for(t in n)this.each(z(t,n[t]));return this}return this.each(z(n,t))},_a.classed=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node(),r=(n=T(n)).length,u=-1;if(t=e.classList){for(;++uu){if("string"!=typeof n){2>u&&(e="");for(r in n)this.each(P(r,n[r],e));return this}if(2>u){var i=this.node();return t(i).getComputedStyle(i,null).getPropertyValue(n)}r=""}return this.each(P(n,e,r))},_a.property=function(n,t){if(arguments.length<2){if("string"==typeof n)return this.node()[n];for(t in n)this.each(U(t,n[t]));return this}return this.each(U(n,t))},_a.text=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.textContent=null==t?"":t}:null==n?function(){this.textContent=""}:function(){this.textContent=n}):this.node().textContent},_a.html=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.innerHTML=null==t?"":t}:null==n?function(){this.innerHTML=""}:function(){this.innerHTML=n}):this.node().innerHTML},_a.append=function(n){return n=j(n),this.select(function(){return this.appendChild(n.apply(this,arguments))})},_a.insert=function(n,t){return n=j(n),t=N(t),this.select(function(){return this.insertBefore(n.apply(this,arguments),t.apply(this,arguments)||null)})},_a.remove=function(){return this.each(F)},_a.data=function(n,t){function e(n,e){var r,u,i,o=n.length,f=e.length,h=Math.min(o,f),g=new Array(f),p=new Array(f),v=new Array(o);if(t){var d,m=new l,y=new Array(o);for(r=-1;++rr;++r)p[r]=H(e[r]);for(;o>r;++r)v[r]=n[r]}p.update=g,p.parentNode=g.parentNode=v.parentNode=n.parentNode,a.push(p),c.push(g),s.push(v)}var r,u,i=-1,o=this.length;if(!arguments.length){for(n=new Array(o=(r=this[0]).length);++ii;i++){u.push(t=[]),t.parentNode=(e=this[i]).parentNode;for(var a=0,c=e.length;c>a;a++)(r=e[a])&&n.call(r,r.__data__,a,i)&&t.push(r)}return A(u)},_a.order=function(){for(var n=-1,t=this.length;++n=0;)(e=r[u])&&(i&&i!==e.nextSibling&&i.parentNode.insertBefore(e,i),i=e);return this},_a.sort=function(n){n=I.apply(this,arguments);for(var t=-1,e=this.length;++tn;n++)for(var e=this[n],r=0,u=e.length;u>r;r++){var i=e[r];if(i)return i}return null},_a.size=function(){var n=0;return Y(this,function(){++n}),n};var Sa=[];ta.selection.enter=Z,ta.selection.enter.prototype=Sa,Sa.append=_a.append,Sa.empty=_a.empty,Sa.node=_a.node,Sa.call=_a.call,Sa.size=_a.size,Sa.select=function(n){for(var t,e,r,u,i,o=[],a=-1,c=this.length;++ar){if("string"!=typeof n){2>r&&(t=!1);for(e in n)this.each(X(e,n[e],t));return this}if(2>r)return(r=this.node()["__on"+n])&&r._;e=!1}return this.each(X(n,t,e))};var ka=ta.map({mouseenter:"mouseover",mouseleave:"mouseout"});ua&&ka.forEach(function(n){"on"+n in ua&&ka.remove(n)});var Ea,Aa=0;ta.mouse=function(n){return J(n,k())};var Na=this.navigator&&/WebKit/.test(this.navigator.userAgent)?-1:0;ta.touch=function(n,t,e){if(arguments.length<3&&(e=t,t=k().changedTouches),t)for(var r,u=0,i=t.length;i>u;++u)if((r=t[u]).identifier===e)return J(n,r)},ta.behavior.drag=function(){function n(){this.on("mousedown.drag",i).on("touchstart.drag",o)}function e(n,t,e,i,o){return function(){function a(){var n,e,r=t(h,v);r&&(n=r[0]-M[0],e=r[1]-M[1],p|=n|e,M=r,g({type:"drag",x:r[0]+l[0],y:r[1]+l[1],dx:n,dy:e}))}function c(){t(h,v)&&(m.on(i+d,null).on(o+d,null),y(p&&ta.event.target===f),g({type:"dragend"}))}var l,s=this,f=ta.event.target,h=s.parentNode,g=r.of(s,arguments),p=0,v=n(),d=".drag"+(null==v?"":"-"+v),m=ta.select(e(f)).on(i+d,a).on(o+d,c),y=W(f),M=t(h,v);u?(l=u.apply(s,arguments),l=[l.x-M[0],l.y-M[1]]):l=[0,0],g({type:"dragstart"})}}var r=E(n,"drag","dragstart","dragend"),u=null,i=e(b,ta.mouse,t,"mousemove","mouseup"),o=e(G,ta.touch,y,"touchmove","touchend");return n.origin=function(t){return arguments.length?(u=t,n):u},ta.rebind(n,r,"on")},ta.touches=function(n,t){return arguments.length<2&&(t=k().touches),t?ra(t).map(function(t){var e=J(n,t);return e.identifier=t.identifier,e}):[]};var Ca=1e-6,za=Ca*Ca,qa=Math.PI,La=2*qa,Ta=La-Ca,Ra=qa/2,Da=qa/180,Pa=180/qa,Ua=Math.SQRT2,ja=2,Fa=4;ta.interpolateZoom=function(n,t){function e(n){var t=n*y;if(m){var e=rt(v),o=i/(ja*h)*(e*ut(Ua*t+v)-et(v));return[r+o*l,u+o*s,i*e/rt(Ua*t+v)]}return[r+n*l,u+n*s,i*Math.exp(Ua*t)]}var r=n[0],u=n[1],i=n[2],o=t[0],a=t[1],c=t[2],l=o-r,s=a-u,f=l*l+s*s,h=Math.sqrt(f),g=(c*c-i*i+Fa*f)/(2*i*ja*h),p=(c*c-i*i-Fa*f)/(2*c*ja*h),v=Math.log(Math.sqrt(g*g+1)-g),d=Math.log(Math.sqrt(p*p+1)-p),m=d-v,y=(m||Math.log(c/i))/Ua;return e.duration=1e3*y,e},ta.behavior.zoom=function(){function n(n){n.on(q,f).on(Oa+".zoom",g).on("dblclick.zoom",p).on(R,h)}function e(n){return[(n[0]-k.x)/k.k,(n[1]-k.y)/k.k]}function r(n){return[n[0]*k.k+k.x,n[1]*k.k+k.y]}function u(n){k.k=Math.max(N[0],Math.min(N[1],n))}function i(n,t){t=r(t),k.x+=n[0]-t[0],k.y+=n[1]-t[1]}function o(t,e,r,o){t.__chart__={x:k.x,y:k.y,k:k.k},u(Math.pow(2,o)),i(d=e,r),t=ta.select(t),C>0&&(t=t.transition().duration(C)),t.call(n.event)}function a(){b&&b.domain(x.range().map(function(n){return(n-k.x)/k.k}).map(x.invert)),w&&w.domain(_.range().map(function(n){return(n-k.y)/k.k}).map(_.invert))}function c(n){z++||n({type:"zoomstart"})}function l(n){a(),n({type:"zoom",scale:k.k,translate:[k.x,k.y]})}function s(n){--z||n({type:"zoomend"}),d=null}function f(){function n(){f=1,i(ta.mouse(u),g),l(a)}function r(){h.on(L,null).on(T,null),p(f&&ta.event.target===o),s(a)}var u=this,o=ta.event.target,a=D.of(u,arguments),f=0,h=ta.select(t(u)).on(L,n).on(T,r),g=e(ta.mouse(u)),p=W(u);Dl.call(u),c(a)}function h(){function n(){var n=ta.touches(p);return g=k.k,n.forEach(function(n){n.identifier in d&&(d[n.identifier]=e(n))}),n}function t(){var t=ta.event.target;ta.select(t).on(x,r).on(b,a),_.push(t);for(var e=ta.event.changedTouches,u=0,i=e.length;i>u;++u)d[e[u].identifier]=null;var c=n(),l=Date.now();if(1===c.length){if(500>l-M){var s=c[0];o(p,s,d[s.identifier],Math.floor(Math.log(k.k)/Math.LN2)+1),S()}M=l}else if(c.length>1){var s=c[0],f=c[1],h=s[0]-f[0],g=s[1]-f[1];m=h*h+g*g}}function r(){var n,t,e,r,o=ta.touches(p);Dl.call(p);for(var a=0,c=o.length;c>a;++a,r=null)if(e=o[a],r=d[e.identifier]){if(t)break;n=e,t=r}if(r){var s=(s=e[0]-n[0])*s+(s=e[1]-n[1])*s,f=m&&Math.sqrt(s/m);n=[(n[0]+e[0])/2,(n[1]+e[1])/2],t=[(t[0]+r[0])/2,(t[1]+r[1])/2],u(f*g)}M=null,i(n,t),l(v)}function a(){if(ta.event.touches.length){for(var t=ta.event.changedTouches,e=0,r=t.length;r>e;++e)delete d[t[e].identifier];for(var u in d)return void n()}ta.selectAll(_).on(y,null),w.on(q,f).on(R,h),E(),s(v)}var g,p=this,v=D.of(p,arguments),d={},m=0,y=".zoom-"+ta.event.changedTouches[0].identifier,x="touchmove"+y,b="touchend"+y,_=[],w=ta.select(p),E=W(p);t(),c(v),w.on(q,null).on(R,t)}function g(){var n=D.of(this,arguments);y?clearTimeout(y):(v=e(d=m||ta.mouse(this)),Dl.call(this),c(n)),y=setTimeout(function(){y=null,s(n)},50),S(),u(Math.pow(2,.002*Ha())*k.k),i(d,v),l(n)}function p(){var n=ta.mouse(this),t=Math.log(k.k)/Math.LN2;o(this,n,e(n),ta.event.shiftKey?Math.ceil(t)-1:Math.floor(t)+1)}var v,d,m,y,M,x,b,_,w,k={x:0,y:0,k:1},A=[960,500],N=Ia,C=250,z=0,q="mousedown.zoom",L="mousemove.zoom",T="mouseup.zoom",R="touchstart.zoom",D=E(n,"zoomstart","zoom","zoomend");return Oa||(Oa="onwheel"in ua?(Ha=function(){return-ta.event.deltaY*(ta.event.deltaMode?120:1)},"wheel"):"onmousewheel"in ua?(Ha=function(){return ta.event.wheelDelta},"mousewheel"):(Ha=function(){return-ta.event.detail},"MozMousePixelScroll")),n.event=function(n){n.each(function(){var n=D.of(this,arguments),t=k;Tl?ta.select(this).transition().each("start.zoom",function(){k=this.__chart__||{x:0,y:0,k:1},c(n)}).tween("zoom:zoom",function(){var e=A[0],r=A[1],u=d?d[0]:e/2,i=d?d[1]:r/2,o=ta.interpolateZoom([(u-k.x)/k.k,(i-k.y)/k.k,e/k.k],[(u-t.x)/t.k,(i-t.y)/t.k,e/t.k]);return function(t){var r=o(t),a=e/r[2];this.__chart__=k={x:u-r[0]*a,y:i-r[1]*a,k:a},l(n)}}).each("interrupt.zoom",function(){s(n)}).each("end.zoom",function(){s(n)}):(this.__chart__=k,c(n),l(n),s(n))})},n.translate=function(t){return arguments.length?(k={x:+t[0],y:+t[1],k:k.k},a(),n):[k.x,k.y]},n.scale=function(t){return arguments.length?(k={x:k.x,y:k.y,k:+t},a(),n):k.k},n.scaleExtent=function(t){return arguments.length?(N=null==t?Ia:[+t[0],+t[1]],n):N},n.center=function(t){return arguments.length?(m=t&&[+t[0],+t[1]],n):m},n.size=function(t){return arguments.length?(A=t&&[+t[0],+t[1]],n):A},n.duration=function(t){return arguments.length?(C=+t,n):C},n.x=function(t){return arguments.length?(b=t,x=t.copy(),k={x:0,y:0,k:1},n):b},n.y=function(t){return arguments.length?(w=t,_=t.copy(),k={x:0,y:0,k:1},n):w},ta.rebind(n,D,"on")};var Ha,Oa,Ia=[0,1/0];ta.color=ot,ot.prototype.toString=function(){return this.rgb()+""},ta.hsl=at;var Ya=at.prototype=new ot;Ya.brighter=function(n){return n=Math.pow(.7,arguments.length?n:1),new at(this.h,this.s,this.l/n)},Ya.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new at(this.h,this.s,n*this.l)},Ya.rgb=function(){return ct(this.h,this.s,this.l)},ta.hcl=lt;var Za=lt.prototype=new ot;Za.brighter=function(n){return new lt(this.h,this.c,Math.min(100,this.l+Va*(arguments.length?n:1)))},Za.darker=function(n){return new lt(this.h,this.c,Math.max(0,this.l-Va*(arguments.length?n:1)))},Za.rgb=function(){return st(this.h,this.c,this.l).rgb()},ta.lab=ft;var Va=18,Xa=.95047,$a=1,Ba=1.08883,Wa=ft.prototype=new ot;Wa.brighter=function(n){return new ft(Math.min(100,this.l+Va*(arguments.length?n:1)),this.a,this.b)},Wa.darker=function(n){return new ft(Math.max(0,this.l-Va*(arguments.length?n:1)),this.a,this.b)},Wa.rgb=function(){return ht(this.l,this.a,this.b)},ta.rgb=mt;var Ja=mt.prototype=new ot;Ja.brighter=function(n){n=Math.pow(.7,arguments.length?n:1);var t=this.r,e=this.g,r=this.b,u=30;return t||e||r?(t&&u>t&&(t=u),e&&u>e&&(e=u),r&&u>r&&(r=u),new mt(Math.min(255,t/n),Math.min(255,e/n),Math.min(255,r/n))):new mt(u,u,u)},Ja.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new mt(n*this.r,n*this.g,n*this.b)},Ja.hsl=function(){return _t(this.r,this.g,this.b)},Ja.toString=function(){return"#"+xt(this.r)+xt(this.g)+xt(this.b)};var Ga=ta.map({aliceblue:15792383,antiquewhite:16444375,aqua:65535,aquamarine:8388564,azure:15794175,beige:16119260,bisque:16770244,black:0,blanchedalmond:16772045,blue:255,blueviolet:9055202,brown:10824234,burlywood:14596231,cadetblue:6266528,chartreuse:8388352,chocolate:13789470,coral:16744272,cornflowerblue:6591981,cornsilk:16775388,crimson:14423100,cyan:65535,darkblue:139,darkcyan:35723,darkgoldenrod:12092939,darkgray:11119017,darkgreen:25600,darkgrey:11119017,darkkhaki:12433259,darkmagenta:9109643,darkolivegreen:5597999,darkorange:16747520,darkorchid:10040012,darkred:9109504,darksalmon:15308410,darkseagreen:9419919,darkslateblue:4734347,darkslategray:3100495,darkslategrey:3100495,darkturquoise:52945,darkviolet:9699539,deeppink:16716947,deepskyblue:49151,dimgray:6908265,dimgrey:6908265,dodgerblue:2003199,firebrick:11674146,floralwhite:16775920,forestgreen:2263842,fuchsia:16711935,gainsboro:14474460,ghostwhite:16316671,gold:16766720,goldenrod:14329120,gray:8421504,green:32768,greenyellow:11403055,grey:8421504,honeydew:15794160,hotpink:16738740,indianred:13458524,indigo:4915330,ivory:16777200,khaki:15787660,lavender:15132410,lavenderblush:16773365,lawngreen:8190976,lemonchiffon:16775885,lightblue:11393254,lightcoral:15761536,lightcyan:14745599,lightgoldenrodyellow:16448210,lightgray:13882323,lightgreen:9498256,lightgrey:13882323,lightpink:16758465,lightsalmon:16752762,lightseagreen:2142890,lightskyblue:8900346,lightslategray:7833753,lightslategrey:7833753,lightsteelblue:11584734,lightyellow:16777184,lime:65280,limegreen:3329330,linen:16445670,magenta:16711935,maroon:8388608,mediumaquamarine:6737322,mediumblue:205,mediumorchid:12211667,mediumpurple:9662683,mediumseagreen:3978097,mediumslateblue:8087790,mediumspringgreen:64154,mediumturquoise:4772300,mediumvioletred:13047173,midnightblue:1644912,mintcream:16121850,mistyrose:16770273,moccasin:16770229,navajowhite:16768685,navy:128,oldlace:16643558,olive:8421376,olivedrab:7048739,orange:16753920,orangered:16729344,orchid:14315734,palegoldenrod:15657130,palegreen:10025880,paleturquoise:11529966,palevioletred:14381203,papayawhip:16773077,peachpuff:16767673,peru:13468991,pink:16761035,plum:14524637,powderblue:11591910,purple:8388736,rebeccapurple:6697881,red:16711680,rosybrown:12357519,royalblue:4286945,saddlebrown:9127187,salmon:16416882,sandybrown:16032864,seagreen:3050327,seashell:16774638,sienna:10506797,silver:12632256,skyblue:8900331,slateblue:6970061,slategray:7372944,slategrey:7372944,snow:16775930,springgreen:65407,steelblue:4620980,tan:13808780,teal:32896,thistle:14204888,tomato:16737095,turquoise:4251856,violet:15631086,wheat:16113331,white:16777215,whitesmoke:16119285,yellow:16776960,yellowgreen:10145074});Ga.forEach(function(n,t){Ga.set(n,yt(t))}),ta.functor=Et,ta.xhr=At(y),ta.dsv=function(n,t){function e(n,e,i){arguments.length<3&&(i=e,e=null);var o=Nt(n,t,null==e?r:u(e),i);return o.row=function(n){return arguments.length?o.response(null==(e=n)?r:u(n)):e},o}function r(n){return e.parse(n.responseText)}function u(n){return function(t){return e.parse(t.responseText,n)}}function i(t){return t.map(o).join(n)}function o(n){return a.test(n)?'"'+n.replace(/\"/g,'""')+'"':n}var a=new RegExp('["'+n+"\n]"),c=n.charCodeAt(0);return e.parse=function(n,t){var r;return e.parseRows(n,function(n,e){if(r)return r(n,e-1);var u=new Function("d","return {"+n.map(function(n,t){return JSON.stringify(n)+": d["+t+"]"}).join(",")+"}");r=t?function(n,e){return t(u(n),e)}:u})},e.parseRows=function(n,t){function e(){if(s>=l)return o;if(u)return u=!1,i;var t=s;if(34===n.charCodeAt(t)){for(var e=t;e++s;){var r=n.charCodeAt(s++),a=1;if(10===r)u=!0;else if(13===r)u=!0,10===n.charCodeAt(s)&&(++s,++a);else if(r!==c)continue;return n.slice(t,s-a)}return n.slice(t)}for(var r,u,i={},o={},a=[],l=n.length,s=0,f=0;(r=e())!==o;){for(var h=[];r!==i&&r!==o;)h.push(r),r=e();t&&null==(h=t(h,f++))||a.push(h)}return a},e.format=function(t){if(Array.isArray(t[0]))return e.formatRows(t);var r=new m,u=[];return t.forEach(function(n){for(var t in n)r.has(t)||u.push(r.add(t))}),[u.map(o).join(n)].concat(t.map(function(t){return u.map(function(n){return o(t[n])}).join(n)})).join("\n")},e.formatRows=function(n){return n.map(i).join("\n")},e},ta.csv=ta.dsv(",","text/csv"),ta.tsv=ta.dsv(" ","text/tab-separated-values");var Ka,Qa,nc,tc,ec,rc=this[x(this,"requestAnimationFrame")]||function(n){setTimeout(n,17)};ta.timer=function(n,t,e){var r=arguments.length;2>r&&(t=0),3>r&&(e=Date.now());var u=e+t,i={c:n,t:u,f:!1,n:null};Qa?Qa.n=i:Ka=i,Qa=i,nc||(tc=clearTimeout(tc),nc=1,rc(qt))},ta.timer.flush=function(){Lt(),Tt()},ta.round=function(n,t){return t?Math.round(n*(t=Math.pow(10,t)))/t:Math.round(n)};var uc=["y","z","a","f","p","n","\xb5","m","","k","M","G","T","P","E","Z","Y"].map(Dt);ta.formatPrefix=function(n,t){var e=0;return n&&(0>n&&(n*=-1),t&&(n=ta.round(n,Rt(n,t))),e=1+Math.floor(1e-12+Math.log(n)/Math.LN10),e=Math.max(-24,Math.min(24,3*Math.floor((e-1)/3)))),uc[8+e/3]};var ic=/(?:([^{])?([<>=^]))?([+\- ])?([$#])?(0)?(\d+)?(,)?(\.-?\d+)?([a-z%])?/i,oc=ta.map({b:function(n){return n.toString(2)},c:function(n){return String.fromCharCode(n)},o:function(n){return n.toString(8)},x:function(n){return n.toString(16)},X:function(n){return n.toString(16).toUpperCase()},g:function(n,t){return n.toPrecision(t)},e:function(n,t){return n.toExponential(t)},f:function(n,t){return n.toFixed(t)},r:function(n,t){return(n=ta.round(n,Rt(n,t))).toFixed(Math.max(0,Math.min(20,Rt(n*(1+1e-15),t))))}}),ac=ta.time={},cc=Date;jt.prototype={getDate:function(){return this._.getUTCDate()},getDay:function(){return this._.getUTCDay()},getFullYear:function(){return this._.getUTCFullYear()},getHours:function(){return this._.getUTCHours()},getMilliseconds:function(){return this._.getUTCMilliseconds()},getMinutes:function(){return this._.getUTCMinutes()},getMonth:function(){return this._.getUTCMonth()},getSeconds:function(){return this._.getUTCSeconds()},getTime:function(){return this._.getTime()},getTimezoneOffset:function(){return 0},valueOf:function(){return this._.valueOf()},setDate:function(){lc.setUTCDate.apply(this._,arguments)},setDay:function(){lc.setUTCDay.apply(this._,arguments)},setFullYear:function(){lc.setUTCFullYear.apply(this._,arguments)},setHours:function(){lc.setUTCHours.apply(this._,arguments)},setMilliseconds:function(){lc.setUTCMilliseconds.apply(this._,arguments)},setMinutes:function(){lc.setUTCMinutes.apply(this._,arguments)},setMonth:function(){lc.setUTCMonth.apply(this._,arguments)},setSeconds:function(){lc.setUTCSeconds.apply(this._,arguments)},setTime:function(){lc.setTime.apply(this._,arguments)}};var lc=Date.prototype;ac.year=Ft(function(n){return n=ac.day(n),n.setMonth(0,1),n},function(n,t){n.setFullYear(n.getFullYear()+t)},function(n){return n.getFullYear()}),ac.years=ac.year.range,ac.years.utc=ac.year.utc.range,ac.day=Ft(function(n){var t=new cc(2e3,0);return t.setFullYear(n.getFullYear(),n.getMonth(),n.getDate()),t},function(n,t){n.setDate(n.getDate()+t)},function(n){return n.getDate()-1}),ac.days=ac.day.range,ac.days.utc=ac.day.utc.range,ac.dayOfYear=function(n){var t=ac.year(n);return Math.floor((n-t-6e4*(n.getTimezoneOffset()-t.getTimezoneOffset()))/864e5)},["sunday","monday","tuesday","wednesday","thursday","friday","saturday"].forEach(function(n,t){t=7-t;var e=ac[n]=Ft(function(n){return(n=ac.day(n)).setDate(n.getDate()-(n.getDay()+t)%7),n},function(n,t){n.setDate(n.getDate()+7*Math.floor(t))},function(n){var e=ac.year(n).getDay();return Math.floor((ac.dayOfYear(n)+(e+t)%7)/7)-(e!==t)});ac[n+"s"]=e.range,ac[n+"s"].utc=e.utc.range,ac[n+"OfYear"]=function(n){var e=ac.year(n).getDay();return Math.floor((ac.dayOfYear(n)+(e+t)%7)/7)}}),ac.week=ac.sunday,ac.weeks=ac.sunday.range,ac.weeks.utc=ac.sunday.utc.range,ac.weekOfYear=ac.sundayOfYear;var sc={"-":"",_:" ",0:"0"},fc=/^\s*\d+/,hc=/^%/;ta.locale=function(n){return{numberFormat:Pt(n),timeFormat:Ot(n)}};var gc=ta.locale({decimal:".",thousands:",",grouping:[3],currency:["$",""],dateTime:"%a %b %e %X %Y",date:"%m/%d/%Y",time:"%H:%M:%S",periods:["AM","PM"],days:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],shortDays:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],months:["January","February","March","April","May","June","July","August","September","October","November","December"],shortMonths:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]});ta.format=gc.numberFormat,ta.geo={},ce.prototype={s:0,t:0,add:function(n){le(n,this.t,pc),le(pc.s,this.s,this),this.s?this.t+=pc.t:this.s=pc.t +},reset:function(){this.s=this.t=0},valueOf:function(){return this.s}};var pc=new ce;ta.geo.stream=function(n,t){n&&vc.hasOwnProperty(n.type)?vc[n.type](n,t):se(n,t)};var vc={Feature:function(n,t){se(n.geometry,t)},FeatureCollection:function(n,t){for(var e=n.features,r=-1,u=e.length;++rn?4*qa+n:n,Mc.lineStart=Mc.lineEnd=Mc.point=b}};ta.geo.bounds=function(){function n(n,t){M.push(x=[s=n,h=n]),f>t&&(f=t),t>g&&(g=t)}function t(t,e){var r=pe([t*Da,e*Da]);if(m){var u=de(m,r),i=[u[1],-u[0],0],o=de(i,u);Me(o),o=xe(o);var c=t-p,l=c>0?1:-1,v=o[0]*Pa*l,d=ga(c)>180;if(d^(v>l*p&&l*t>v)){var y=o[1]*Pa;y>g&&(g=y)}else if(v=(v+360)%360-180,d^(v>l*p&&l*t>v)){var y=-o[1]*Pa;f>y&&(f=y)}else f>e&&(f=e),e>g&&(g=e);d?p>t?a(s,t)>a(s,h)&&(h=t):a(t,h)>a(s,h)&&(s=t):h>=s?(s>t&&(s=t),t>h&&(h=t)):t>p?a(s,t)>a(s,h)&&(h=t):a(t,h)>a(s,h)&&(s=t)}else n(t,e);m=r,p=t}function e(){b.point=t}function r(){x[0]=s,x[1]=h,b.point=n,m=null}function u(n,e){if(m){var r=n-p;y+=ga(r)>180?r+(r>0?360:-360):r}else v=n,d=e;Mc.point(n,e),t(n,e)}function i(){Mc.lineStart()}function o(){u(v,d),Mc.lineEnd(),ga(y)>Ca&&(s=-(h=180)),x[0]=s,x[1]=h,m=null}function a(n,t){return(t-=n)<0?t+360:t}function c(n,t){return n[0]-t[0]}function l(n,t){return t[0]<=t[1]?t[0]<=n&&n<=t[1]:nyc?(s=-(h=180),f=-(g=90)):y>Ca?g=90:-Ca>y&&(f=-90),x[0]=s,x[1]=h}};return function(n){g=h=-(s=f=1/0),M=[],ta.geo.stream(n,b);var t=M.length;if(t){M.sort(c);for(var e,r=1,u=M[0],i=[u];t>r;++r)e=M[r],l(e[0],u)||l(e[1],u)?(a(u[0],e[1])>a(u[0],u[1])&&(u[1]=e[1]),a(e[0],u[1])>a(u[0],u[1])&&(u[0]=e[0])):i.push(u=e);for(var o,e,p=-1/0,t=i.length-1,r=0,u=i[t];t>=r;u=e,++r)e=i[r],(o=a(u[1],e[0]))>p&&(p=o,s=e[0],h=u[1])}return M=x=null,1/0===s||1/0===f?[[0/0,0/0],[0/0,0/0]]:[[s,f],[h,g]]}}(),ta.geo.centroid=function(n){xc=bc=_c=wc=Sc=kc=Ec=Ac=Nc=Cc=zc=0,ta.geo.stream(n,qc);var t=Nc,e=Cc,r=zc,u=t*t+e*e+r*r;return za>u&&(t=kc,e=Ec,r=Ac,Ca>bc&&(t=_c,e=wc,r=Sc),u=t*t+e*e+r*r,za>u)?[0/0,0/0]:[Math.atan2(e,t)*Pa,tt(r/Math.sqrt(u))*Pa]};var xc,bc,_c,wc,Sc,kc,Ec,Ac,Nc,Cc,zc,qc={sphere:b,point:_e,lineStart:Se,lineEnd:ke,polygonStart:function(){qc.lineStart=Ee},polygonEnd:function(){qc.lineStart=Se}},Lc=Le(Ne,Pe,je,[-qa,-qa/2]),Tc=1e9;ta.geo.clipExtent=function(){var n,t,e,r,u,i,o={stream:function(n){return u&&(u.valid=!1),u=i(n),u.valid=!0,u},extent:function(a){return arguments.length?(i=Ie(n=+a[0][0],t=+a[0][1],e=+a[1][0],r=+a[1][1]),u&&(u.valid=!1,u=null),o):[[n,t],[e,r]]}};return o.extent([[0,0],[960,500]])},(ta.geo.conicEqualArea=function(){return Ye(Ze)}).raw=Ze,ta.geo.albers=function(){return ta.geo.conicEqualArea().rotate([96,0]).center([-.6,38.7]).parallels([29.5,45.5]).scale(1070)},ta.geo.albersUsa=function(){function n(n){var i=n[0],o=n[1];return t=null,e(i,o),t||(r(i,o),t)||u(i,o),t}var t,e,r,u,i=ta.geo.albers(),o=ta.geo.conicEqualArea().rotate([154,0]).center([-2,58.5]).parallels([55,65]),a=ta.geo.conicEqualArea().rotate([157,0]).center([-3,19.9]).parallels([8,18]),c={point:function(n,e){t=[n,e]}};return n.invert=function(n){var t=i.scale(),e=i.translate(),r=(n[0]-e[0])/t,u=(n[1]-e[1])/t;return(u>=.12&&.234>u&&r>=-.425&&-.214>r?o:u>=.166&&.234>u&&r>=-.214&&-.115>r?a:i).invert(n)},n.stream=function(n){var t=i.stream(n),e=o.stream(n),r=a.stream(n);return{point:function(n,u){t.point(n,u),e.point(n,u),r.point(n,u)},sphere:function(){t.sphere(),e.sphere(),r.sphere()},lineStart:function(){t.lineStart(),e.lineStart(),r.lineStart()},lineEnd:function(){t.lineEnd(),e.lineEnd(),r.lineEnd()},polygonStart:function(){t.polygonStart(),e.polygonStart(),r.polygonStart()},polygonEnd:function(){t.polygonEnd(),e.polygonEnd(),r.polygonEnd()}}},n.precision=function(t){return arguments.length?(i.precision(t),o.precision(t),a.precision(t),n):i.precision()},n.scale=function(t){return arguments.length?(i.scale(t),o.scale(.35*t),a.scale(t),n.translate(i.translate())):i.scale()},n.translate=function(t){if(!arguments.length)return i.translate();var l=i.scale(),s=+t[0],f=+t[1];return e=i.translate(t).clipExtent([[s-.455*l,f-.238*l],[s+.455*l,f+.238*l]]).stream(c).point,r=o.translate([s-.307*l,f+.201*l]).clipExtent([[s-.425*l+Ca,f+.12*l+Ca],[s-.214*l-Ca,f+.234*l-Ca]]).stream(c).point,u=a.translate([s-.205*l,f+.212*l]).clipExtent([[s-.214*l+Ca,f+.166*l+Ca],[s-.115*l-Ca,f+.234*l-Ca]]).stream(c).point,n},n.scale(1070)};var Rc,Dc,Pc,Uc,jc,Fc,Hc={point:b,lineStart:b,lineEnd:b,polygonStart:function(){Dc=0,Hc.lineStart=Ve},polygonEnd:function(){Hc.lineStart=Hc.lineEnd=Hc.point=b,Rc+=ga(Dc/2)}},Oc={point:Xe,lineStart:b,lineEnd:b,polygonStart:b,polygonEnd:b},Ic={point:We,lineStart:Je,lineEnd:Ge,polygonStart:function(){Ic.lineStart=Ke},polygonEnd:function(){Ic.point=We,Ic.lineStart=Je,Ic.lineEnd=Ge}};ta.geo.path=function(){function n(n){return n&&("function"==typeof a&&i.pointRadius(+a.apply(this,arguments)),o&&o.valid||(o=u(i)),ta.geo.stream(n,o)),i.result()}function t(){return o=null,n}var e,r,u,i,o,a=4.5;return n.area=function(n){return Rc=0,ta.geo.stream(n,u(Hc)),Rc},n.centroid=function(n){return _c=wc=Sc=kc=Ec=Ac=Nc=Cc=zc=0,ta.geo.stream(n,u(Ic)),zc?[Nc/zc,Cc/zc]:Ac?[kc/Ac,Ec/Ac]:Sc?[_c/Sc,wc/Sc]:[0/0,0/0]},n.bounds=function(n){return jc=Fc=-(Pc=Uc=1/0),ta.geo.stream(n,u(Oc)),[[Pc,Uc],[jc,Fc]]},n.projection=function(n){return arguments.length?(u=(e=n)?n.stream||tr(n):y,t()):e},n.context=function(n){return arguments.length?(i=null==(r=n)?new $e:new Qe(n),"function"!=typeof a&&i.pointRadius(a),t()):r},n.pointRadius=function(t){return arguments.length?(a="function"==typeof t?t:(i.pointRadius(+t),+t),n):a},n.projection(ta.geo.albersUsa()).context(null)},ta.geo.transform=function(n){return{stream:function(t){var e=new er(t);for(var r in n)e[r]=n[r];return e}}},er.prototype={point:function(n,t){this.stream.point(n,t)},sphere:function(){this.stream.sphere()},lineStart:function(){this.stream.lineStart()},lineEnd:function(){this.stream.lineEnd()},polygonStart:function(){this.stream.polygonStart()},polygonEnd:function(){this.stream.polygonEnd()}},ta.geo.projection=ur,ta.geo.projectionMutator=ir,(ta.geo.equirectangular=function(){return ur(ar)}).raw=ar.invert=ar,ta.geo.rotation=function(n){function t(t){return t=n(t[0]*Da,t[1]*Da),t[0]*=Pa,t[1]*=Pa,t}return n=lr(n[0]%360*Da,n[1]*Da,n.length>2?n[2]*Da:0),t.invert=function(t){return t=n.invert(t[0]*Da,t[1]*Da),t[0]*=Pa,t[1]*=Pa,t},t},cr.invert=ar,ta.geo.circle=function(){function n(){var n="function"==typeof r?r.apply(this,arguments):r,t=lr(-n[0]*Da,-n[1]*Da,0).invert,u=[];return e(null,null,1,{point:function(n,e){u.push(n=t(n,e)),n[0]*=Pa,n[1]*=Pa}}),{type:"Polygon",coordinates:[u]}}var t,e,r=[0,0],u=6;return n.origin=function(t){return arguments.length?(r=t,n):r},n.angle=function(r){return arguments.length?(e=gr((t=+r)*Da,u*Da),n):t},n.precision=function(r){return arguments.length?(e=gr(t*Da,(u=+r)*Da),n):u},n.angle(90)},ta.geo.distance=function(n,t){var e,r=(t[0]-n[0])*Da,u=n[1]*Da,i=t[1]*Da,o=Math.sin(r),a=Math.cos(r),c=Math.sin(u),l=Math.cos(u),s=Math.sin(i),f=Math.cos(i);return Math.atan2(Math.sqrt((e=f*o)*e+(e=l*s-c*f*a)*e),c*s+l*f*a)},ta.geo.graticule=function(){function n(){return{type:"MultiLineString",coordinates:t()}}function t(){return ta.range(Math.ceil(i/d)*d,u,d).map(h).concat(ta.range(Math.ceil(l/m)*m,c,m).map(g)).concat(ta.range(Math.ceil(r/p)*p,e,p).filter(function(n){return ga(n%d)>Ca}).map(s)).concat(ta.range(Math.ceil(a/v)*v,o,v).filter(function(n){return ga(n%m)>Ca}).map(f))}var e,r,u,i,o,a,c,l,s,f,h,g,p=10,v=p,d=90,m=360,y=2.5;return n.lines=function(){return t().map(function(n){return{type:"LineString",coordinates:n}})},n.outline=function(){return{type:"Polygon",coordinates:[h(i).concat(g(c).slice(1),h(u).reverse().slice(1),g(l).reverse().slice(1))]}},n.extent=function(t){return arguments.length?n.majorExtent(t).minorExtent(t):n.minorExtent()},n.majorExtent=function(t){return arguments.length?(i=+t[0][0],u=+t[1][0],l=+t[0][1],c=+t[1][1],i>u&&(t=i,i=u,u=t),l>c&&(t=l,l=c,c=t),n.precision(y)):[[i,l],[u,c]]},n.minorExtent=function(t){return arguments.length?(r=+t[0][0],e=+t[1][0],a=+t[0][1],o=+t[1][1],r>e&&(t=r,r=e,e=t),a>o&&(t=a,a=o,o=t),n.precision(y)):[[r,a],[e,o]]},n.step=function(t){return arguments.length?n.majorStep(t).minorStep(t):n.minorStep()},n.majorStep=function(t){return arguments.length?(d=+t[0],m=+t[1],n):[d,m]},n.minorStep=function(t){return arguments.length?(p=+t[0],v=+t[1],n):[p,v]},n.precision=function(t){return arguments.length?(y=+t,s=vr(a,o,90),f=dr(r,e,y),h=vr(l,c,90),g=dr(i,u,y),n):y},n.majorExtent([[-180,-90+Ca],[180,90-Ca]]).minorExtent([[-180,-80-Ca],[180,80+Ca]])},ta.geo.greatArc=function(){function n(){return{type:"LineString",coordinates:[t||r.apply(this,arguments),e||u.apply(this,arguments)]}}var t,e,r=mr,u=yr;return n.distance=function(){return ta.geo.distance(t||r.apply(this,arguments),e||u.apply(this,arguments))},n.source=function(e){return arguments.length?(r=e,t="function"==typeof e?null:e,n):r},n.target=function(t){return arguments.length?(u=t,e="function"==typeof t?null:t,n):u},n.precision=function(){return arguments.length?n:0},n},ta.geo.interpolate=function(n,t){return Mr(n[0]*Da,n[1]*Da,t[0]*Da,t[1]*Da)},ta.geo.length=function(n){return Yc=0,ta.geo.stream(n,Zc),Yc};var Yc,Zc={sphere:b,point:b,lineStart:xr,lineEnd:b,polygonStart:b,polygonEnd:b},Vc=br(function(n){return Math.sqrt(2/(1+n))},function(n){return 2*Math.asin(n/2)});(ta.geo.azimuthalEqualArea=function(){return ur(Vc)}).raw=Vc;var Xc=br(function(n){var t=Math.acos(n);return t&&t/Math.sin(t)},y);(ta.geo.azimuthalEquidistant=function(){return ur(Xc)}).raw=Xc,(ta.geo.conicConformal=function(){return Ye(_r)}).raw=_r,(ta.geo.conicEquidistant=function(){return Ye(wr)}).raw=wr;var $c=br(function(n){return 1/n},Math.atan);(ta.geo.gnomonic=function(){return ur($c)}).raw=$c,Sr.invert=function(n,t){return[n,2*Math.atan(Math.exp(t))-Ra]},(ta.geo.mercator=function(){return kr(Sr)}).raw=Sr;var Bc=br(function(){return 1},Math.asin);(ta.geo.orthographic=function(){return ur(Bc)}).raw=Bc;var Wc=br(function(n){return 1/(1+n)},function(n){return 2*Math.atan(n)});(ta.geo.stereographic=function(){return ur(Wc)}).raw=Wc,Er.invert=function(n,t){return[-t,2*Math.atan(Math.exp(n))-Ra]},(ta.geo.transverseMercator=function(){var n=kr(Er),t=n.center,e=n.rotate;return n.center=function(n){return n?t([-n[1],n[0]]):(n=t(),[n[1],-n[0]])},n.rotate=function(n){return n?e([n[0],n[1],n.length>2?n[2]+90:90]):(n=e(),[n[0],n[1],n[2]-90])},e([0,0,90])}).raw=Er,ta.geom={},ta.geom.hull=function(n){function t(n){if(n.length<3)return[];var t,u=Et(e),i=Et(r),o=n.length,a=[],c=[];for(t=0;o>t;t++)a.push([+u.call(this,n[t],t),+i.call(this,n[t],t),t]);for(a.sort(zr),t=0;o>t;t++)c.push([a[t][0],-a[t][1]]);var l=Cr(a),s=Cr(c),f=s[0]===l[0],h=s[s.length-1]===l[l.length-1],g=[];for(t=l.length-1;t>=0;--t)g.push(n[a[l[t]][2]]);for(t=+f;t=r&&l.x<=i&&l.y>=u&&l.y<=o?[[r,o],[i,o],[i,u],[r,u]]:[];s.point=n[a]}),t}function e(n){return n.map(function(n,t){return{x:Math.round(i(n,t)/Ca)*Ca,y:Math.round(o(n,t)/Ca)*Ca,i:t}})}var r=Ar,u=Nr,i=r,o=u,a=ul;return n?t(n):(t.links=function(n){return iu(e(n)).edges.filter(function(n){return n.l&&n.r}).map(function(t){return{source:n[t.l.i],target:n[t.r.i]}})},t.triangles=function(n){var t=[];return iu(e(n)).cells.forEach(function(e,r){for(var u,i,o=e.site,a=e.edges.sort(Yr),c=-1,l=a.length,s=a[l-1].edge,f=s.l===o?s.r:s.l;++c=l,h=r>=s,g=h<<1|f;n.leaf=!1,n=n.nodes[g]||(n.nodes[g]=su()),f?u=l:a=l,h?o=s:c=s,i(n,t,e,r,u,o,a,c)}var s,f,h,g,p,v,d,m,y,M=Et(a),x=Et(c);if(null!=t)v=t,d=e,m=r,y=u;else if(m=y=-(v=d=1/0),f=[],h=[],p=n.length,o)for(g=0;p>g;++g)s=n[g],s.xm&&(m=s.x),s.y>y&&(y=s.y),f.push(s.x),h.push(s.y);else for(g=0;p>g;++g){var b=+M(s=n[g],g),_=+x(s,g);v>b&&(v=b),d>_&&(d=_),b>m&&(m=b),_>y&&(y=_),f.push(b),h.push(_)}var w=m-v,S=y-d;w>S?y=d+w:m=v+S;var k=su();if(k.add=function(n){i(k,n,+M(n,++g),+x(n,g),v,d,m,y)},k.visit=function(n){fu(n,k,v,d,m,y)},k.find=function(n){return hu(k,n[0],n[1],v,d,m,y)},g=-1,null==t){for(;++g=0?n.slice(0,t):n,r=t>=0?n.slice(t+1):"in";return e=cl.get(e)||al,r=ll.get(r)||y,Mu(r(e.apply(null,ea.call(arguments,1))))},ta.interpolateHcl=Lu,ta.interpolateHsl=Tu,ta.interpolateLab=Ru,ta.interpolateRound=Du,ta.transform=function(n){var t=ua.createElementNS(ta.ns.prefix.svg,"g");return(ta.transform=function(n){if(null!=n){t.setAttribute("transform",n);var e=t.transform.baseVal.consolidate()}return new Pu(e?e.matrix:sl)})(n)},Pu.prototype.toString=function(){return"translate("+this.translate+")rotate("+this.rotate+")skewX("+this.skew+")scale("+this.scale+")"};var sl={a:1,b:0,c:0,d:1,e:0,f:0};ta.interpolateTransform=Hu,ta.layout={},ta.layout.bundle=function(){return function(n){for(var t=[],e=-1,r=n.length;++ea*a/d){if(p>c){var l=t.charge/c;n.px-=i*l,n.py-=o*l}return!0}if(t.point&&c&&p>c){var l=t.pointCharge/c;n.px-=i*l,n.py-=o*l}}return!t.charge}}function t(n){n.px=ta.event.x,n.py=ta.event.y,a.resume()}var e,r,u,i,o,a={},c=ta.dispatch("start","tick","end"),l=[1,1],s=.9,f=fl,h=hl,g=-30,p=gl,v=.1,d=.64,m=[],M=[];return a.tick=function(){if((r*=.99)<.005)return c.end({type:"end",alpha:r=0}),!0;var t,e,a,f,h,p,d,y,x,b=m.length,_=M.length;for(e=0;_>e;++e)a=M[e],f=a.source,h=a.target,y=h.x-f.x,x=h.y-f.y,(p=y*y+x*x)&&(p=r*i[e]*((p=Math.sqrt(p))-u[e])/p,y*=p,x*=p,h.x-=y*(d=f.weight/(h.weight+f.weight)),h.y-=x*d,f.x+=y*(d=1-d),f.y+=x*d);if((d=r*v)&&(y=l[0]/2,x=l[1]/2,e=-1,d))for(;++e0?n:0:n>0&&(c.start({type:"start",alpha:r=n}),ta.timer(a.tick)),a):r},a.start=function(){function n(n,r){if(!e){for(e=new Array(c),a=0;c>a;++a)e[a]=[];for(a=0;s>a;++a){var u=M[a];e[u.source.index].push(u.target),e[u.target.index].push(u.source)}}for(var i,o=e[t],a=-1,l=o.length;++at;++t)(r=m[t]).index=t,r.weight=0;for(t=0;s>t;++t)r=M[t],"number"==typeof r.source&&(r.source=m[r.source]),"number"==typeof r.target&&(r.target=m[r.target]),++r.source.weight,++r.target.weight;for(t=0;c>t;++t)r=m[t],isNaN(r.x)&&(r.x=n("x",p)),isNaN(r.y)&&(r.y=n("y",v)),isNaN(r.px)&&(r.px=r.x),isNaN(r.py)&&(r.py=r.y);if(u=[],"function"==typeof f)for(t=0;s>t;++t)u[t]=+f.call(this,M[t],t);else for(t=0;s>t;++t)u[t]=f;if(i=[],"function"==typeof h)for(t=0;s>t;++t)i[t]=+h.call(this,M[t],t);else for(t=0;s>t;++t)i[t]=h;if(o=[],"function"==typeof g)for(t=0;c>t;++t)o[t]=+g.call(this,m[t],t);else for(t=0;c>t;++t)o[t]=g;return a.resume()},a.resume=function(){return a.alpha(.1)},a.stop=function(){return a.alpha(0)},a.drag=function(){return e||(e=ta.behavior.drag().origin(y).on("dragstart.force",Xu).on("drag.force",t).on("dragend.force",$u)),arguments.length?void this.on("mouseover.force",Bu).on("mouseout.force",Wu).call(e):e},ta.rebind(a,c,"on")};var fl=20,hl=1,gl=1/0;ta.layout.hierarchy=function(){function n(u){var i,o=[u],a=[];for(u.depth=0;null!=(i=o.pop());)if(a.push(i),(l=e.call(n,i,i.depth))&&(c=l.length)){for(var c,l,s;--c>=0;)o.push(s=l[c]),s.parent=i,s.depth=i.depth+1;r&&(i.value=0),i.children=l}else r&&(i.value=+r.call(n,i,i.depth)||0),delete i.children;return Qu(u,function(n){var e,u;t&&(e=n.children)&&e.sort(t),r&&(u=n.parent)&&(u.value+=n.value)}),a}var t=ei,e=ni,r=ti;return n.sort=function(e){return arguments.length?(t=e,n):t},n.children=function(t){return arguments.length?(e=t,n):e},n.value=function(t){return arguments.length?(r=t,n):r},n.revalue=function(t){return r&&(Ku(t,function(n){n.children&&(n.value=0)}),Qu(t,function(t){var e;t.children||(t.value=+r.call(n,t,t.depth)||0),(e=t.parent)&&(e.value+=t.value)})),t},n},ta.layout.partition=function(){function n(t,e,r,u){var i=t.children;if(t.x=e,t.y=t.depth*u,t.dx=r,t.dy=u,i&&(o=i.length)){var o,a,c,l=-1;for(r=t.value?r/t.value:0;++lf?-1:1),p=(f-c*g)/ta.sum(l),v=ta.range(c),d=[];return null!=e&&v.sort(e===pl?function(n,t){return l[t]-l[n]}:function(n,t){return e(o[n],o[t])}),v.forEach(function(n){d[n]={data:o[n],value:a=l[n],startAngle:s,endAngle:s+=a*p+g,padAngle:h}}),d}var t=Number,e=pl,r=0,u=La,i=0;return n.value=function(e){return arguments.length?(t=e,n):t},n.sort=function(t){return arguments.length?(e=t,n):e},n.startAngle=function(t){return arguments.length?(r=t,n):r},n.endAngle=function(t){return arguments.length?(u=t,n):u},n.padAngle=function(t){return arguments.length?(i=t,n):i},n};var pl={};ta.layout.stack=function(){function n(a,c){if(!(h=a.length))return a;var l=a.map(function(e,r){return t.call(n,e,r)}),s=l.map(function(t){return t.map(function(t,e){return[i.call(n,t,e),o.call(n,t,e)]})}),f=e.call(n,s,c);l=ta.permute(l,f),s=ta.permute(s,f);var h,g,p,v,d=r.call(n,s,c),m=l[0].length;for(p=0;m>p;++p)for(u.call(n,l[0][p],v=d[p],s[0][p][1]),g=1;h>g;++g)u.call(n,l[g][p],v+=s[g-1][p][1],s[g][p][1]);return a}var t=y,e=ai,r=ci,u=oi,i=ui,o=ii;return n.values=function(e){return arguments.length?(t=e,n):t},n.order=function(t){return arguments.length?(e="function"==typeof t?t:vl.get(t)||ai,n):e},n.offset=function(t){return arguments.length?(r="function"==typeof t?t:dl.get(t)||ci,n):r},n.x=function(t){return arguments.length?(i=t,n):i},n.y=function(t){return arguments.length?(o=t,n):o},n.out=function(t){return arguments.length?(u=t,n):u},n};var vl=ta.map({"inside-out":function(n){var t,e,r=n.length,u=n.map(li),i=n.map(si),o=ta.range(r).sort(function(n,t){return u[n]-u[t]}),a=0,c=0,l=[],s=[];for(t=0;r>t;++t)e=o[t],c>a?(a+=i[e],l.push(e)):(c+=i[e],s.push(e));return s.reverse().concat(l)},reverse:function(n){return ta.range(n.length).reverse()},"default":ai}),dl=ta.map({silhouette:function(n){var t,e,r,u=n.length,i=n[0].length,o=[],a=0,c=[];for(e=0;i>e;++e){for(t=0,r=0;u>t;t++)r+=n[t][e][1];r>a&&(a=r),o.push(r)}for(e=0;i>e;++e)c[e]=(a-o[e])/2;return c},wiggle:function(n){var t,e,r,u,i,o,a,c,l,s=n.length,f=n[0],h=f.length,g=[];for(g[0]=c=l=0,e=1;h>e;++e){for(t=0,u=0;s>t;++t)u+=n[t][e][1];for(t=0,i=0,a=f[e][0]-f[e-1][0];s>t;++t){for(r=0,o=(n[t][e][1]-n[t][e-1][1])/(2*a);t>r;++r)o+=(n[r][e][1]-n[r][e-1][1])/a;i+=o*n[t][e][1]}g[e]=c-=u?i/u*a:0,l>c&&(l=c)}for(e=0;h>e;++e)g[e]-=l;return g},expand:function(n){var t,e,r,u=n.length,i=n[0].length,o=1/u,a=[];for(e=0;i>e;++e){for(t=0,r=0;u>t;t++)r+=n[t][e][1];if(r)for(t=0;u>t;t++)n[t][e][1]/=r;else for(t=0;u>t;t++)n[t][e][1]=o}for(e=0;i>e;++e)a[e]=0;return a},zero:ci});ta.layout.histogram=function(){function n(n,i){for(var o,a,c=[],l=n.map(e,this),s=r.call(this,l,i),f=u.call(this,s,l,i),i=-1,h=l.length,g=f.length-1,p=t?1:1/h;++i0)for(i=-1;++i=s[0]&&a<=s[1]&&(o=c[ta.bisect(f,a,1,g)-1],o.y+=p,o.push(n[i]));return c}var t=!0,e=Number,r=pi,u=hi;return n.value=function(t){return arguments.length?(e=t,n):e},n.range=function(t){return arguments.length?(r=Et(t),n):r},n.bins=function(t){return arguments.length?(u="number"==typeof t?function(n){return gi(n,t)}:Et(t),n):u},n.frequency=function(e){return arguments.length?(t=!!e,n):t},n},ta.layout.pack=function(){function n(n,i){var o=e.call(this,n,i),a=o[0],c=u[0],l=u[1],s=null==t?Math.sqrt:"function"==typeof t?t:function(){return t};if(a.x=a.y=0,Qu(a,function(n){n.r=+s(n.value)}),Qu(a,Mi),r){var f=r*(t?1:Math.max(2*a.r/c,2*a.r/l))/2;Qu(a,function(n){n.r+=f}),Qu(a,Mi),Qu(a,function(n){n.r-=f})}return _i(a,c/2,l/2,t?1:1/Math.max(2*a.r/c,2*a.r/l)),o}var t,e=ta.layout.hierarchy().sort(vi),r=0,u=[1,1];return n.size=function(t){return arguments.length?(u=t,n):u},n.radius=function(e){return arguments.length?(t=null==e||"function"==typeof e?e:+e,n):t},n.padding=function(t){return arguments.length?(r=+t,n):r},Gu(n,e)},ta.layout.tree=function(){function n(n,u){var s=o.call(this,n,u),f=s[0],h=t(f);if(Qu(h,e),h.parent.m=-h.z,Ku(h,r),l)Ku(f,i);else{var g=f,p=f,v=f;Ku(f,function(n){n.xp.x&&(p=n),n.depth>v.depth&&(v=n)});var d=a(g,p)/2-g.x,m=c[0]/(p.x+a(p,g)/2+d),y=c[1]/(v.depth||1);Ku(f,function(n){n.x=(n.x+d)*m,n.y=n.depth*y})}return s}function t(n){for(var t,e={A:null,children:[n]},r=[e];null!=(t=r.pop());)for(var u,i=t.children,o=0,a=i.length;a>o;++o)r.push((i[o]=u={_:i[o],parent:t,children:(u=i[o].children)&&u.slice()||[],A:null,a:null,z:0,m:0,c:0,s:0,t:null,i:o}).a=u);return e.children[0]}function e(n){var t=n.children,e=n.parent.children,r=n.i?e[n.i-1]:null;if(t.length){Ni(n);var i=(t[0].z+t[t.length-1].z)/2;r?(n.z=r.z+a(n._,r._),n.m=n.z-i):n.z=i}else r&&(n.z=r.z+a(n._,r._));n.parent.A=u(n,r,n.parent.A||e[0])}function r(n){n._.x=n.z+n.parent.m,n.m+=n.parent.m}function u(n,t,e){if(t){for(var r,u=n,i=n,o=t,c=u.parent.children[0],l=u.m,s=i.m,f=o.m,h=c.m;o=Ei(o),u=ki(u),o&&u;)c=ki(c),i=Ei(i),i.a=n,r=o.z+f-u.z-l+a(o._,u._),r>0&&(Ai(Ci(o,n,e),n,r),l+=r,s+=r),f+=o.m,l+=u.m,h+=c.m,s+=i.m;o&&!Ei(i)&&(i.t=o,i.m+=f-s),u&&!ki(c)&&(c.t=u,c.m+=l-h,e=n)}return e}function i(n){n.x*=c[0],n.y=n.depth*c[1]}var o=ta.layout.hierarchy().sort(null).value(null),a=Si,c=[1,1],l=null;return n.separation=function(t){return arguments.length?(a=t,n):a},n.size=function(t){return arguments.length?(l=null==(c=t)?i:null,n):l?null:c},n.nodeSize=function(t){return arguments.length?(l=null==(c=t)?null:i,n):l?c:null},Gu(n,o)},ta.layout.cluster=function(){function n(n,i){var o,a=t.call(this,n,i),c=a[0],l=0;Qu(c,function(n){var t=n.children;t&&t.length?(n.x=qi(t),n.y=zi(t)):(n.x=o?l+=e(n,o):0,n.y=0,o=n)});var s=Li(c),f=Ti(c),h=s.x-e(s,f)/2,g=f.x+e(f,s)/2;return Qu(c,u?function(n){n.x=(n.x-c.x)*r[0],n.y=(c.y-n.y)*r[1]}:function(n){n.x=(n.x-h)/(g-h)*r[0],n.y=(1-(c.y?n.y/c.y:1))*r[1]}),a}var t=ta.layout.hierarchy().sort(null).value(null),e=Si,r=[1,1],u=!1;return n.separation=function(t){return arguments.length?(e=t,n):e},n.size=function(t){return arguments.length?(u=null==(r=t),n):u?null:r},n.nodeSize=function(t){return arguments.length?(u=null!=(r=t),n):u?r:null},Gu(n,t)},ta.layout.treemap=function(){function n(n,t){for(var e,r,u=-1,i=n.length;++ut?0:t),e.area=isNaN(r)||0>=r?0:r}function t(e){var i=e.children;if(i&&i.length){var o,a,c,l=f(e),s=[],h=i.slice(),p=1/0,v="slice"===g?l.dx:"dice"===g?l.dy:"slice-dice"===g?1&e.depth?l.dy:l.dx:Math.min(l.dx,l.dy);for(n(h,l.dx*l.dy/e.value),s.area=0;(c=h.length)>0;)s.push(o=h[c-1]),s.area+=o.area,"squarify"!==g||(a=r(s,v))<=p?(h.pop(),p=a):(s.area-=s.pop().area,u(s,v,l,!1),v=Math.min(l.dx,l.dy),s.length=s.area=0,p=1/0);s.length&&(u(s,v,l,!0),s.length=s.area=0),i.forEach(t)}}function e(t){var r=t.children;if(r&&r.length){var i,o=f(t),a=r.slice(),c=[];for(n(a,o.dx*o.dy/t.value),c.area=0;i=a.pop();)c.push(i),c.area+=i.area,null!=i.z&&(u(c,i.z?o.dx:o.dy,o,!a.length),c.length=c.area=0);r.forEach(e)}}function r(n,t){for(var e,r=n.area,u=0,i=1/0,o=-1,a=n.length;++oe&&(i=e),e>u&&(u=e));return r*=r,t*=t,r?Math.max(t*u*p/r,r/(t*i*p)):1/0}function u(n,t,e,r){var u,i=-1,o=n.length,a=e.x,l=e.y,s=t?c(n.area/t):0;if(t==e.dx){for((r||s>e.dy)&&(s=e.dy);++ie.dx)&&(s=e.dx);++ie&&(t=1),1>e&&(n=0),function(){var e,r,u;do e=2*Math.random()-1,r=2*Math.random()-1,u=e*e+r*r;while(!u||u>1);return n+t*e*Math.sqrt(-2*Math.log(u)/u)}},logNormal:function(){var n=ta.random.normal.apply(ta,arguments);return function(){return Math.exp(n())}},bates:function(n){var t=ta.random.irwinHall(n);return function(){return t()/n}},irwinHall:function(n){return function(){for(var t=0,e=0;n>e;e++)t+=Math.random();return t}}},ta.scale={};var ml={floor:y,ceil:y};ta.scale.linear=function(){return Ii([0,1],[0,1],mu,!1)};var yl={s:1,g:1,p:1,r:1,e:1};ta.scale.log=function(){return Ji(ta.scale.linear().domain([0,1]),10,!0,[1,10])};var Ml=ta.format(".0e"),xl={floor:function(n){return-Math.ceil(-n)},ceil:function(n){return-Math.floor(-n)}};ta.scale.pow=function(){return Gi(ta.scale.linear(),1,[0,1])},ta.scale.sqrt=function(){return ta.scale.pow().exponent(.5)},ta.scale.ordinal=function(){return Qi([],{t:"range",a:[[]]})},ta.scale.category10=function(){return ta.scale.ordinal().range(bl)},ta.scale.category20=function(){return ta.scale.ordinal().range(_l)},ta.scale.category20b=function(){return ta.scale.ordinal().range(wl)},ta.scale.category20c=function(){return ta.scale.ordinal().range(Sl)};var bl=[2062260,16744206,2924588,14034728,9725885,9197131,14907330,8355711,12369186,1556175].map(Mt),_l=[2062260,11454440,16744206,16759672,2924588,10018698,14034728,16750742,9725885,12955861,9197131,12885140,14907330,16234194,8355711,13092807,12369186,14408589,1556175,10410725].map(Mt),wl=[3750777,5395619,7040719,10264286,6519097,9216594,11915115,13556636,9202993,12426809,15186514,15190932,8666169,11356490,14049643,15177372,8077683,10834324,13528509,14589654].map(Mt),Sl=[3244733,7057110,10406625,13032431,15095053,16616764,16625259,16634018,3253076,7652470,10607003,13101504,7695281,10394312,12369372,14342891,6513507,9868950,12434877,14277081].map(Mt);ta.scale.quantile=function(){return no([],[])},ta.scale.quantize=function(){return to(0,1,[0,1])},ta.scale.threshold=function(){return eo([.5],[0,1])},ta.scale.identity=function(){return ro([0,1])},ta.svg={},ta.svg.arc=function(){function n(){var n=Math.max(0,+e.apply(this,arguments)),l=Math.max(0,+r.apply(this,arguments)),s=o.apply(this,arguments)-Ra,f=a.apply(this,arguments)-Ra,h=Math.abs(f-s),g=s>f?0:1;if(n>l&&(p=l,l=n,n=p),h>=Ta)return t(l,g)+(n?t(n,1-g):"")+"Z";var p,v,d,m,y,M,x,b,_,w,S,k,E=0,A=0,N=[];if((m=(+c.apply(this,arguments)||0)/2)&&(d=i===kl?Math.sqrt(n*n+l*l):+i.apply(this,arguments),g||(A*=-1),l&&(A=tt(d/l*Math.sin(m))),n&&(E=tt(d/n*Math.sin(m)))),l){y=l*Math.cos(s+A),M=l*Math.sin(s+A),x=l*Math.cos(f-A),b=l*Math.sin(f-A);var C=Math.abs(f-s-2*A)<=qa?0:1;if(A&&so(y,M,x,b)===g^C){var z=(s+f)/2;y=l*Math.cos(z),M=l*Math.sin(z),x=b=null}}else y=M=0;if(n){_=n*Math.cos(f-E),w=n*Math.sin(f-E),S=n*Math.cos(s+E),k=n*Math.sin(s+E);var q=Math.abs(s-f+2*E)<=qa?0:1;if(E&&so(_,w,S,k)===1-g^q){var L=(s+f)/2;_=n*Math.cos(L),w=n*Math.sin(L),S=k=null}}else _=w=0;if((p=Math.min(Math.abs(l-n)/2,+u.apply(this,arguments)))>.001){v=l>n^g?0:1;var T=null==S?[_,w]:null==x?[y,M]:Lr([y,M],[S,k],[x,b],[_,w]),R=y-T[0],D=M-T[1],P=x-T[0],U=b-T[1],j=1/Math.sin(Math.acos((R*P+D*U)/(Math.sqrt(R*R+D*D)*Math.sqrt(P*P+U*U)))/2),F=Math.sqrt(T[0]*T[0]+T[1]*T[1]);if(null!=x){var H=Math.min(p,(l-F)/(j+1)),O=fo(null==S?[_,w]:[S,k],[y,M],l,H,g),I=fo([x,b],[_,w],l,H,g);p===H?N.push("M",O[0],"A",H,",",H," 0 0,",v," ",O[1],"A",l,",",l," 0 ",1-g^so(O[1][0],O[1][1],I[1][0],I[1][1]),",",g," ",I[1],"A",H,",",H," 0 0,",v," ",I[0]):N.push("M",O[0],"A",H,",",H," 0 1,",v," ",I[0])}else N.push("M",y,",",M);if(null!=S){var Y=Math.min(p,(n-F)/(j-1)),Z=fo([y,M],[S,k],n,-Y,g),V=fo([_,w],null==x?[y,M]:[x,b],n,-Y,g);p===Y?N.push("L",V[0],"A",Y,",",Y," 0 0,",v," ",V[1],"A",n,",",n," 0 ",g^so(V[1][0],V[1][1],Z[1][0],Z[1][1]),",",1-g," ",Z[1],"A",Y,",",Y," 0 0,",v," ",Z[0]):N.push("L",V[0],"A",Y,",",Y," 0 0,",v," ",Z[0])}else N.push("L",_,",",w)}else N.push("M",y,",",M),null!=x&&N.push("A",l,",",l," 0 ",C,",",g," ",x,",",b),N.push("L",_,",",w),null!=S&&N.push("A",n,",",n," 0 ",q,",",1-g," ",S,",",k);return N.push("Z"),N.join("")}function t(n,t){return"M0,"+n+"A"+n+","+n+" 0 1,"+t+" 0,"+-n+"A"+n+","+n+" 0 1,"+t+" 0,"+n}var e=io,r=oo,u=uo,i=kl,o=ao,a=co,c=lo;return n.innerRadius=function(t){return arguments.length?(e=Et(t),n):e},n.outerRadius=function(t){return arguments.length?(r=Et(t),n):r},n.cornerRadius=function(t){return arguments.length?(u=Et(t),n):u},n.padRadius=function(t){return arguments.length?(i=t==kl?kl:Et(t),n):i},n.startAngle=function(t){return arguments.length?(o=Et(t),n):o},n.endAngle=function(t){return arguments.length?(a=Et(t),n):a},n.padAngle=function(t){return arguments.length?(c=Et(t),n):c},n.centroid=function(){var n=(+e.apply(this,arguments)+ +r.apply(this,arguments))/2,t=(+o.apply(this,arguments)+ +a.apply(this,arguments))/2-Ra;return[Math.cos(t)*n,Math.sin(t)*n]},n};var kl="auto";ta.svg.line=function(){return ho(y)};var El=ta.map({linear:go,"linear-closed":po,step:vo,"step-before":mo,"step-after":yo,basis:So,"basis-open":ko,"basis-closed":Eo,bundle:Ao,cardinal:bo,"cardinal-open":Mo,"cardinal-closed":xo,monotone:To});El.forEach(function(n,t){t.key=n,t.closed=/-closed$/.test(n)});var Al=[0,2/3,1/3,0],Nl=[0,1/3,2/3,0],Cl=[0,1/6,2/3,1/6];ta.svg.line.radial=function(){var n=ho(Ro);return n.radius=n.x,delete n.x,n.angle=n.y,delete n.y,n},mo.reverse=yo,yo.reverse=mo,ta.svg.area=function(){return Do(y)},ta.svg.area.radial=function(){var n=Do(Ro);return n.radius=n.x,delete n.x,n.innerRadius=n.x0,delete n.x0,n.outerRadius=n.x1,delete n.x1,n.angle=n.y,delete n.y,n.startAngle=n.y0,delete n.y0,n.endAngle=n.y1,delete n.y1,n},ta.svg.chord=function(){function n(n,a){var c=t(this,i,n,a),l=t(this,o,n,a);return"M"+c.p0+r(c.r,c.p1,c.a1-c.a0)+(e(c,l)?u(c.r,c.p1,c.r,c.p0):u(c.r,c.p1,l.r,l.p0)+r(l.r,l.p1,l.a1-l.a0)+u(l.r,l.p1,c.r,c.p0))+"Z"}function t(n,t,e,r){var u=t.call(n,e,r),i=a.call(n,u,r),o=c.call(n,u,r)-Ra,s=l.call(n,u,r)-Ra;return{r:i,a0:o,a1:s,p0:[i*Math.cos(o),i*Math.sin(o)],p1:[i*Math.cos(s),i*Math.sin(s)]}}function e(n,t){return n.a0==t.a0&&n.a1==t.a1}function r(n,t,e){return"A"+n+","+n+" 0 "+ +(e>qa)+",1 "+t}function u(n,t,e,r){return"Q 0,0 "+r}var i=mr,o=yr,a=Po,c=ao,l=co;return n.radius=function(t){return arguments.length?(a=Et(t),n):a},n.source=function(t){return arguments.length?(i=Et(t),n):i},n.target=function(t){return arguments.length?(o=Et(t),n):o},n.startAngle=function(t){return arguments.length?(c=Et(t),n):c},n.endAngle=function(t){return arguments.length?(l=Et(t),n):l},n},ta.svg.diagonal=function(){function n(n,u){var i=t.call(this,n,u),o=e.call(this,n,u),a=(i.y+o.y)/2,c=[i,{x:i.x,y:a},{x:o.x,y:a},o];return c=c.map(r),"M"+c[0]+"C"+c[1]+" "+c[2]+" "+c[3]}var t=mr,e=yr,r=Uo;return n.source=function(e){return arguments.length?(t=Et(e),n):t},n.target=function(t){return arguments.length?(e=Et(t),n):e},n.projection=function(t){return arguments.length?(r=t,n):r},n},ta.svg.diagonal.radial=function(){var n=ta.svg.diagonal(),t=Uo,e=n.projection;return n.projection=function(n){return arguments.length?e(jo(t=n)):t},n},ta.svg.symbol=function(){function n(n,r){return(zl.get(t.call(this,n,r))||Oo)(e.call(this,n,r))}var t=Ho,e=Fo;return n.type=function(e){return arguments.length?(t=Et(e),n):t},n.size=function(t){return arguments.length?(e=Et(t),n):e},n};var zl=ta.map({circle:Oo,cross:function(n){var t=Math.sqrt(n/5)/2;return"M"+-3*t+","+-t+"H"+-t+"V"+-3*t+"H"+t+"V"+-t+"H"+3*t+"V"+t+"H"+t+"V"+3*t+"H"+-t+"V"+t+"H"+-3*t+"Z"},diamond:function(n){var t=Math.sqrt(n/(2*Ll)),e=t*Ll;return"M0,"+-t+"L"+e+",0 0,"+t+" "+-e+",0Z"},square:function(n){var t=Math.sqrt(n)/2;return"M"+-t+","+-t+"L"+t+","+-t+" "+t+","+t+" "+-t+","+t+"Z"},"triangle-down":function(n){var t=Math.sqrt(n/ql),e=t*ql/2;return"M0,"+e+"L"+t+","+-e+" "+-t+","+-e+"Z"},"triangle-up":function(n){var t=Math.sqrt(n/ql),e=t*ql/2;return"M0,"+-e+"L"+t+","+e+" "+-t+","+e+"Z"}});ta.svg.symbolTypes=zl.keys();var ql=Math.sqrt(3),Ll=Math.tan(30*Da);_a.transition=function(n){for(var t,e,r=Tl||++Ul,u=Xo(n),i=[],o=Rl||{time:Date.now(),ease:Su,delay:0,duration:250},a=-1,c=this.length;++ai;i++){u.push(t=[]);for(var e=this[i],a=0,c=e.length;c>a;a++)(r=e[a])&&n.call(r,r.__data__,a,i)&&t.push(r)}return Yo(u,this.namespace,this.id)},Pl.tween=function(n,t){var e=this.id,r=this.namespace;return arguments.length<2?this.node()[r][e].tween.get(n):Y(this,null==t?function(t){t[r][e].tween.remove(n)}:function(u){u[r][e].tween.set(n,t)})},Pl.attr=function(n,t){function e(){this.removeAttribute(a)}function r(){this.removeAttributeNS(a.space,a.local)}function u(n){return null==n?e:(n+="",function(){var t,e=this.getAttribute(a);return e!==n&&(t=o(e,n),function(n){this.setAttribute(a,t(n))})})}function i(n){return null==n?r:(n+="",function(){var t,e=this.getAttributeNS(a.space,a.local);return e!==n&&(t=o(e,n),function(n){this.setAttributeNS(a.space,a.local,t(n))})})}if(arguments.length<2){for(t in n)this.attr(t,n[t]);return this}var o="transform"==n?Hu:mu,a=ta.ns.qualify(n);return Zo(this,"attr."+n,t,a.local?i:u)},Pl.attrTween=function(n,t){function e(n,e){var r=t.call(this,n,e,this.getAttribute(u));return r&&function(n){this.setAttribute(u,r(n))}}function r(n,e){var r=t.call(this,n,e,this.getAttributeNS(u.space,u.local));return r&&function(n){this.setAttributeNS(u.space,u.local,r(n))}}var u=ta.ns.qualify(n);return this.tween("attr."+n,u.local?r:e)},Pl.style=function(n,e,r){function u(){this.style.removeProperty(n)}function i(e){return null==e?u:(e+="",function(){var u,i=t(this).getComputedStyle(this,null).getPropertyValue(n);return i!==e&&(u=mu(i,e),function(t){this.style.setProperty(n,u(t),r)})})}var o=arguments.length;if(3>o){if("string"!=typeof n){2>o&&(e="");for(r in n)this.style(r,n[r],e);return this}r=""}return Zo(this,"style."+n,e,i)},Pl.styleTween=function(n,e,r){function u(u,i){var o=e.call(this,u,i,t(this).getComputedStyle(this,null).getPropertyValue(n));return o&&function(t){this.style.setProperty(n,o(t),r)}}return arguments.length<3&&(r=""),this.tween("style."+n,u)},Pl.text=function(n){return Zo(this,"text",n,Vo)},Pl.remove=function(){var n=this.namespace;return this.each("end.transition",function(){var t;this[n].count<2&&(t=this.parentNode)&&t.removeChild(this)})},Pl.ease=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].ease:("function"!=typeof n&&(n=ta.ease.apply(ta,arguments)),Y(this,function(r){r[e][t].ease=n}))},Pl.delay=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].delay:Y(this,"function"==typeof n?function(r,u,i){r[e][t].delay=+n.call(r,r.__data__,u,i)}:(n=+n,function(r){r[e][t].delay=n}))},Pl.duration=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].duration:Y(this,"function"==typeof n?function(r,u,i){r[e][t].duration=Math.max(1,n.call(r,r.__data__,u,i))}:(n=Math.max(1,n),function(r){r[e][t].duration=n}))},Pl.each=function(n,t){var e=this.id,r=this.namespace;if(arguments.length<2){var u=Rl,i=Tl;try{Tl=e,Y(this,function(t,u,i){Rl=t[r][e],n.call(t,t.__data__,u,i)})}finally{Rl=u,Tl=i}}else Y(this,function(u){var i=u[r][e];(i.event||(i.event=ta.dispatch("start","end","interrupt"))).on(n,t)});return this},Pl.transition=function(){for(var n,t,e,r,u=this.id,i=++Ul,o=this.namespace,a=[],c=0,l=this.length;l>c;c++){a.push(n=[]);for(var t=this[c],s=0,f=t.length;f>s;s++)(e=t[s])&&(r=e[o][u],$o(e,s,o,i,{time:r.time,ease:r.ease,delay:r.delay+r.duration,duration:r.duration})),n.push(e)}return Yo(a,o,i)},ta.svg.axis=function(){function n(n){n.each(function(){var n,l=ta.select(this),s=this.__chart__||e,f=this.__chart__=e.copy(),h=null==c?f.ticks?f.ticks.apply(f,a):f.domain():c,g=null==t?f.tickFormat?f.tickFormat.apply(f,a):y:t,p=l.selectAll(".tick").data(h,f),v=p.enter().insert("g",".domain").attr("class","tick").style("opacity",Ca),d=ta.transition(p.exit()).style("opacity",Ca).remove(),m=ta.transition(p.order()).style("opacity",1),M=Math.max(u,0)+o,x=Ui(f),b=l.selectAll(".domain").data([0]),_=(b.enter().append("path").attr("class","domain"),ta.transition(b));v.append("line"),v.append("text");var w,S,k,E,A=v.select("line"),N=m.select("line"),C=p.select("text").text(g),z=v.select("text"),q=m.select("text"),L="top"===r||"left"===r?-1:1;if("bottom"===r||"top"===r?(n=Bo,w="x",k="y",S="x2",E="y2",C.attr("dy",0>L?"0em":".71em").style("text-anchor","middle"),_.attr("d","M"+x[0]+","+L*i+"V0H"+x[1]+"V"+L*i)):(n=Wo,w="y",k="x",S="y2",E="x2",C.attr("dy",".32em").style("text-anchor",0>L?"end":"start"),_.attr("d","M"+L*i+","+x[0]+"H0V"+x[1]+"H"+L*i)),A.attr(E,L*u),z.attr(k,L*M),N.attr(S,0).attr(E,L*u),q.attr(w,0).attr(k,L*M),f.rangeBand){var T=f,R=T.rangeBand()/2;s=f=function(n){return T(n)+R}}else s.rangeBand?s=f:d.call(n,f,s);v.call(n,s,f),m.call(n,f,f)})}var t,e=ta.scale.linear(),r=jl,u=6,i=6,o=3,a=[10],c=null;return n.scale=function(t){return arguments.length?(e=t,n):e},n.orient=function(t){return arguments.length?(r=t in Fl?t+"":jl,n):r},n.ticks=function(){return arguments.length?(a=arguments,n):a},n.tickValues=function(t){return arguments.length?(c=t,n):c},n.tickFormat=function(e){return arguments.length?(t=e,n):t},n.tickSize=function(t){var e=arguments.length;return e?(u=+t,i=+arguments[e-1],n):u},n.innerTickSize=function(t){return arguments.length?(u=+t,n):u},n.outerTickSize=function(t){return arguments.length?(i=+t,n):i},n.tickPadding=function(t){return arguments.length?(o=+t,n):o},n.tickSubdivide=function(){return arguments.length&&n},n};var jl="bottom",Fl={top:1,right:1,bottom:1,left:1};ta.svg.brush=function(){function n(t){t.each(function(){var t=ta.select(this).style("pointer-events","all").style("-webkit-tap-highlight-color","rgba(0,0,0,0)").on("mousedown.brush",i).on("touchstart.brush",i),o=t.selectAll(".background").data([0]);o.enter().append("rect").attr("class","background").style("visibility","hidden").style("cursor","crosshair"),t.selectAll(".extent").data([0]).enter().append("rect").attr("class","extent").style("cursor","move");var a=t.selectAll(".resize").data(v,y);a.exit().remove(),a.enter().append("g").attr("class",function(n){return"resize "+n}).style("cursor",function(n){return Hl[n]}).append("rect").attr("x",function(n){return/[ew]$/.test(n)?-3:null}).attr("y",function(n){return/^[ns]/.test(n)?-3:null}).attr("width",6).attr("height",6).style("visibility","hidden"),a.style("display",n.empty()?"none":null);var c,f=ta.transition(t),h=ta.transition(o);l&&(c=Ui(l),h.attr("x",c[0]).attr("width",c[1]-c[0]),r(f)),s&&(c=Ui(s),h.attr("y",c[0]).attr("height",c[1]-c[0]),u(f)),e(f)})}function e(n){n.selectAll(".resize").attr("transform",function(n){return"translate("+f[+/e$/.test(n)]+","+h[+/^s/.test(n)]+")"})}function r(n){n.select(".extent").attr("x",f[0]),n.selectAll(".extent,.n>rect,.s>rect").attr("width",f[1]-f[0])}function u(n){n.select(".extent").attr("y",h[0]),n.selectAll(".extent,.e>rect,.w>rect").attr("height",h[1]-h[0])}function i(){function i(){32==ta.event.keyCode&&(C||(M=null,q[0]-=f[1],q[1]-=h[1],C=2),S())}function v(){32==ta.event.keyCode&&2==C&&(q[0]+=f[1],q[1]+=h[1],C=0,S())}function d(){var n=ta.mouse(b),t=!1;x&&(n[0]+=x[0],n[1]+=x[1]),C||(ta.event.altKey?(M||(M=[(f[0]+f[1])/2,(h[0]+h[1])/2]),q[0]=f[+(n[0]s?(u=r,r=s):u=s),v[0]!=r||v[1]!=u?(e?a=null:o=null,v[0]=r,v[1]=u,!0):void 0}function y(){d(),k.style("pointer-events","all").selectAll(".resize").style("display",n.empty()?"none":null),ta.select("body").style("cursor",null),L.on("mousemove.brush",null).on("mouseup.brush",null).on("touchmove.brush",null).on("touchend.brush",null).on("keydown.brush",null).on("keyup.brush",null),z(),w({type:"brushend"})}var M,x,b=this,_=ta.select(ta.event.target),w=c.of(b,arguments),k=ta.select(b),E=_.datum(),A=!/^(n|s)$/.test(E)&&l,N=!/^(e|w)$/.test(E)&&s,C=_.classed("extent"),z=W(b),q=ta.mouse(b),L=ta.select(t(b)).on("keydown.brush",i).on("keyup.brush",v);if(ta.event.changedTouches?L.on("touchmove.brush",d).on("touchend.brush",y):L.on("mousemove.brush",d).on("mouseup.brush",y),k.interrupt().selectAll("*").interrupt(),C)q[0]=f[0]-q[0],q[1]=h[0]-q[1];else if(E){var T=+/w$/.test(E),R=+/^n/.test(E);x=[f[1-T]-q[0],h[1-R]-q[1]],q[0]=f[T],q[1]=h[R]}else ta.event.altKey&&(M=q.slice());k.style("pointer-events","none").selectAll(".resize").style("display",null),ta.select("body").style("cursor",_.style("cursor")),w({type:"brushstart"}),d()}var o,a,c=E(n,"brushstart","brush","brushend"),l=null,s=null,f=[0,0],h=[0,0],g=!0,p=!0,v=Ol[0];return n.event=function(n){n.each(function(){var n=c.of(this,arguments),t={x:f,y:h,i:o,j:a},e=this.__chart__||t;this.__chart__=t,Tl?ta.select(this).transition().each("start.brush",function(){o=e.i,a=e.j,f=e.x,h=e.y,n({type:"brushstart"})}).tween("brush:brush",function(){var e=yu(f,t.x),r=yu(h,t.y);return o=a=null,function(u){f=t.x=e(u),h=t.y=r(u),n({type:"brush",mode:"resize"})}}).each("end.brush",function(){o=t.i,a=t.j,n({type:"brush",mode:"resize"}),n({type:"brushend"})}):(n({type:"brushstart"}),n({type:"brush",mode:"resize"}),n({type:"brushend"}))})},n.x=function(t){return arguments.length?(l=t,v=Ol[!l<<1|!s],n):l},n.y=function(t){return arguments.length?(s=t,v=Ol[!l<<1|!s],n):s},n.clamp=function(t){return arguments.length?(l&&s?(g=!!t[0],p=!!t[1]):l?g=!!t:s&&(p=!!t),n):l&&s?[g,p]:l?g:s?p:null},n.extent=function(t){var e,r,u,i,c;return arguments.length?(l&&(e=t[0],r=t[1],s&&(e=e[0],r=r[0]),o=[e,r],l.invert&&(e=l(e),r=l(r)),e>r&&(c=e,e=r,r=c),(e!=f[0]||r!=f[1])&&(f=[e,r])),s&&(u=t[0],i=t[1],l&&(u=u[1],i=i[1]),a=[u,i],s.invert&&(u=s(u),i=s(i)),u>i&&(c=u,u=i,i=c),(u!=h[0]||i!=h[1])&&(h=[u,i])),n):(l&&(o?(e=o[0],r=o[1]):(e=f[0],r=f[1],l.invert&&(e=l.invert(e),r=l.invert(r)),e>r&&(c=e,e=r,r=c))),s&&(a?(u=a[0],i=a[1]):(u=h[0],i=h[1],s.invert&&(u=s.invert(u),i=s.invert(i)),u>i&&(c=u,u=i,i=c))),l&&s?[[e,u],[r,i]]:l?[e,r]:s&&[u,i])},n.clear=function(){return n.empty()||(f=[0,0],h=[0,0],o=a=null),n},n.empty=function(){return!!l&&f[0]==f[1]||!!s&&h[0]==h[1]},ta.rebind(n,c,"on")};var Hl={n:"ns-resize",e:"ew-resize",s:"ns-resize",w:"ew-resize",nw:"nwse-resize",ne:"nesw-resize",se:"nwse-resize",sw:"nesw-resize"},Ol=[["n","e","s","w","nw","ne","se","sw"],["e","w"],["n","s"],[]],Il=ac.format=gc.timeFormat,Yl=Il.utc,Zl=Yl("%Y-%m-%dT%H:%M:%S.%LZ");Il.iso=Date.prototype.toISOString&&+new Date("2000-01-01T00:00:00.000Z")?Jo:Zl,Jo.parse=function(n){var t=new Date(n);return isNaN(t)?null:t},Jo.toString=Zl.toString,ac.second=Ft(function(n){return new cc(1e3*Math.floor(n/1e3))},function(n,t){n.setTime(n.getTime()+1e3*Math.floor(t))},function(n){return n.getSeconds()}),ac.seconds=ac.second.range,ac.seconds.utc=ac.second.utc.range,ac.minute=Ft(function(n){return new cc(6e4*Math.floor(n/6e4))},function(n,t){n.setTime(n.getTime()+6e4*Math.floor(t))},function(n){return n.getMinutes()}),ac.minutes=ac.minute.range,ac.minutes.utc=ac.minute.utc.range,ac.hour=Ft(function(n){var t=n.getTimezoneOffset()/60;return new cc(36e5*(Math.floor(n/36e5-t)+t))},function(n,t){n.setTime(n.getTime()+36e5*Math.floor(t))},function(n){return n.getHours()}),ac.hours=ac.hour.range,ac.hours.utc=ac.hour.utc.range,ac.month=Ft(function(n){return n=ac.day(n),n.setDate(1),n},function(n,t){n.setMonth(n.getMonth()+t)},function(n){return n.getMonth()}),ac.months=ac.month.range,ac.months.utc=ac.month.utc.range;var Vl=[1e3,5e3,15e3,3e4,6e4,3e5,9e5,18e5,36e5,108e5,216e5,432e5,864e5,1728e5,6048e5,2592e6,7776e6,31536e6],Xl=[[ac.second,1],[ac.second,5],[ac.second,15],[ac.second,30],[ac.minute,1],[ac.minute,5],[ac.minute,15],[ac.minute,30],[ac.hour,1],[ac.hour,3],[ac.hour,6],[ac.hour,12],[ac.day,1],[ac.day,2],[ac.week,1],[ac.month,1],[ac.month,3],[ac.year,1]],$l=Il.multi([[".%L",function(n){return n.getMilliseconds()}],[":%S",function(n){return n.getSeconds()}],["%I:%M",function(n){return n.getMinutes()}],["%I %p",function(n){return n.getHours()}],["%a %d",function(n){return n.getDay()&&1!=n.getDate()}],["%b %d",function(n){return 1!=n.getDate()}],["%B",function(n){return n.getMonth()}],["%Y",Ne]]),Bl={range:function(n,t,e){return ta.range(Math.ceil(n/e)*e,+t,e).map(Ko)},floor:y,ceil:y};Xl.year=ac.year,ac.scale=function(){return Go(ta.scale.linear(),Xl,$l)};var Wl=Xl.map(function(n){return[n[0].utc,n[1]]}),Jl=Yl.multi([[".%L",function(n){return n.getUTCMilliseconds()}],[":%S",function(n){return n.getUTCSeconds()}],["%I:%M",function(n){return n.getUTCMinutes()}],["%I %p",function(n){return n.getUTCHours()}],["%a %d",function(n){return n.getUTCDay()&&1!=n.getUTCDate()}],["%b %d",function(n){return 1!=n.getUTCDate()}],["%B",function(n){return n.getUTCMonth()}],["%Y",Ne]]);Wl.year=ac.year.utc,ac.scale.utc=function(){return Go(ta.scale.linear(),Wl,Jl)},ta.text=At(function(n){return n.responseText}),ta.json=function(n,t){return Nt(n,"application/json",Qo,t)},ta.html=function(n,t){return Nt(n,"text/html",na,t)},ta.xml=At(function(n){return n.responseXML}),"function"==typeof define&&define.amd?define(ta):"object"==typeof module&&module.exports&&(module.exports=ta),this.d3=ta}(); \ No newline at end of file diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css new file mode 100644 index 0000000000000..019d706fa0a8f --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.graph { + font: 10px sans-serif; +} + +.axis path, .axis line { + fill: none; + stroke: #000; + shape-rendering: crispEdges; +} + +.line { + fill: none; + stroke: steelblue; + stroke-width: 1.5px; +} + +.bar rect { + fill: steelblue; + shape-rendering: crispEdges; +} + +.bar text { + fill: #000; +} diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js new file mode 100644 index 0000000000000..04bd39b8733b3 --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +function drawTimeline(id, data, unit) { + var margin = {top: 20, right: 20, bottom: 70, left: 50}; + var width = 500 - margin.left - margin.right; + var height = 250 - margin.top - margin.bottom; + + var x = d3.time.scale().range([0, width]); + var y = d3.scale.linear().range([height, 0]); + + var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(d3.time.format("%H:%M:%S")); + var yAxis = d3.svg.axis().scale(y).orient("left"); + + var line = d3.svg.line() + .x(function(d) { return x(new Date(d.x)); }) + .y(function(d) { return y(d.y); }); + + var svg = d3.select(id).append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + x.domain(d3.extent(data, function(d) { return d.x; })); + y.domain(d3.extent(data, function(d) { return d.y; })); + + svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + height + ")") + .call(xAxis) + .selectAll("text") + .style("text-anchor", "end") + .attr("dx", "-.8em") + .attr("dy", ".15em") + .attr("transform", "rotate(-65)"); + + svg.append("g") + .attr("class", "y axis") + .call(yAxis) + .append("text") + .attr("x", 25) + .attr("y", -5) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text(unit); + + svg.append("path") + .datum(data) + .attr("class", "line") + .attr("d", line); +} + +function drawDistribution(id, values, unit) { + var margin = {top: 10, right: 30, bottom: 30, left: 50}; + var width = 500 - margin.left - margin.right; + var height = 250 - margin.top - margin.bottom; + + var y = d3.scale.linear() + .domain(d3.extent(values, function(d) { return d; })) + .range([height, 0]); + var data = d3.layout.histogram().bins(y.ticks(10))(values); + console.log(values) + console.log(data) + var barHeight = height / data.length + + var x = d3.scale.linear() + .domain([0, d3.max(data, function(d) { return d.y; })]) + .range([0, width]); + + var xAxis = d3.svg.axis().scale(x).orient("bottom"); + var yAxis = d3.svg.axis().scale(y).orient("left"); + + var svg = d3.select(id).append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + var bar = svg.selectAll(".bar") + .data(data) + .enter().append("g") + .attr("class", "bar") + + bar.append("rect") + .attr("x", 1) + .attr("y", function(d) { return y(d.x) - margin.bottom + margin.top; }) + .attr("width", function(d) { return x(d.y); }) + .attr("height", function(d) { return (height - margin.bottom) / data.length; }); + + bar.append("text") + .attr("dy", ".75em") + .attr("x", function(d) { return x(d.y) + 10; }) + .attr("y", function(d) { return y(d.x) + 16 - margin.bottom; }) + .attr("text-anchor", "middle") + .text(function(d) { return d.y; }); + + svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + height + ")") + .call(xAxis); + + svg.append("g") + .attr("class", "y axis") + .call(yAxis); +} \ No newline at end of file diff --git a/core/src/main/scala/org/apache/spark/util/Distribution.scala b/core/src/main/scala/org/apache/spark/util/Distribution.scala index 9aea8efa38c7a..5931e7bcad9e9 100644 --- a/core/src/main/scala/org/apache/spark/util/Distribution.scala +++ b/core/src/main/scala/org/apache/spark/util/Distribution.scala @@ -20,6 +20,7 @@ package org.apache.spark.util import java.io.PrintStream import scala.collection.immutable.IndexedSeq +import scala.collection.mutable.ArrayBuffer /** * Util for getting some stats from a small sample of numeric values, with some handy @@ -67,6 +68,34 @@ private[spark] class Distribution(val data: Array[Double], val startIdx: Int, va out.println(statCounter) showQuantiles(out) } + + def histogram(bins: Int): Seq[(Double, Int)] = { + require(bins > 0) + val stat = statCounter + val binSize = (stat.max - stat.min) / (bins - 1) + val points = new Array[Double](bins) + for (i <- 0 until (bins - 1)) { + points(i) = stat.min + binSize * i + } + points(bins - 1) = stat.max + val counts = new Array[Int](bins) + for (i <- startIdx until endIdx) { + val v = data(i) + var j = 0 + var find = false + while (!find && j < bins - 1) { + if (v < points(j) + binSize / 2) { + counts(j) += 1 + find = true + } + j += 1 + } + if (!find) { + counts(bins - 1) += 1 + } + } + points.zip(counts) + } } private[spark] object Distribution { diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/BatchInfo.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/BatchInfo.scala index 92dc113f397ca..9d21be50a566f 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/BatchInfo.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/BatchInfo.scala @@ -58,4 +58,11 @@ case class BatchInfo( */ def totalDelay: Option[Long] = schedulingDelay.zip(processingDelay) .map(x => x._1 + x._2).headOption + + /** + * The number of recorders received by the receivers in this batch. + */ + def numRecords: Long = receivedBlockInfo.map { case (_, infos) => + infos.map(_.numRecords).sum + }.sum } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala index be1e8686cf9fa..428daf129b745 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala @@ -39,6 +39,7 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) private var totalReceivedRecords = 0L private var totalProcessedRecords = 0L private val receiverInfos = new HashMap[Int, ReceiverInfo] + private val receiverLastErrorTime = new HashMap[Int, Long] val batchDuration = ssc.graph.batchDuration.milliseconds @@ -51,6 +52,7 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) override def onReceiverError(receiverError: StreamingListenerReceiverError) { synchronized { receiverInfos(receiverError.receiverInfo.streamId) = receiverError.receiverInfo + receiverLastErrorTime(receiverError.receiverInfo.streamId) = System.currentTimeMillis() } } @@ -149,6 +151,31 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) }.toMap } + def receivedRecordsWithBatchTime: Map[Int, Seq[(Long, Double)]] = synchronized { + val latestBlockInfos = retainedBatches.take(batchInfoLimit).map { batchInfo => + (batchInfo.batchTime.milliseconds, batchInfo.receivedBlockInfo) + } + (0 until numReceivers).map { receiverId => + val blockInfoOfParticularReceiver = latestBlockInfos.map { + case (batchTime, receivedBlockInfo) => + (batchTime, receivedBlockInfo.get(receiverId).getOrElse(Array.empty)) + } + val recordsOfParticularReceiver = blockInfoOfParticularReceiver.map { + case (batchTime, blockInfo) => + // calculate records per second for each batch + (batchTime, blockInfo.map(_.numRecords).sum.toDouble * 1000 / batchDuration) + } + (receiverId, recordsOfParticularReceiver) + }.toMap + } + + + def allReceivedRecordsWithBatchTime: Seq[(Long, Double)] = synchronized { + retainedBatches.take(batchInfoLimit).map { batchInfo => + (batchInfo.batchTime.milliseconds, batchInfo.numRecords.toDouble * 1000 / batchDuration) + } + } + def lastReceivedBatchRecords: Map[Int, Long] = synchronized { val lastReceivedBlockInfoOption = lastReceivedBatch.map(_.receivedBlockInfo) lastReceivedBlockInfoOption.map { lastReceivedBlockInfo => @@ -164,6 +191,10 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) receiverInfos.get(receiverId) } + def receiverLastErrorTimeo(receiverId: Int): Option[Long] = synchronized { + receiverLastErrorTime.get(receiverId) + } + def lastCompletedBatch: Option[BatchInfo] = synchronized { completedBatchInfos.sortBy(_.batchTime)(Time.ordering).lastOption } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index 07fa285642eec..e200ee8f812e3 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -20,9 +20,11 @@ package org.apache.spark.streaming.ui import java.util.Calendar import javax.servlet.http.HttpServletRequest -import scala.xml.Node +import scala.collection.mutable.ArrayBuffer +import scala.xml.{Node, Unparsed} import org.apache.spark.Logging +import org.apache.spark.streaming.Time import org.apache.spark.ui._ import org.apache.spark.ui.UIUtils._ import org.apache.spark.util.Distribution @@ -31,6 +33,8 @@ import org.apache.spark.util.Distribution private[ui] class StreamingPage(parent: StreamingTab) extends WebUIPage("") with Logging { + import StreamingPage._ + private val listener = parent.listener private val startTime = System.currentTimeMillis() private val emptyCell = "-" @@ -38,48 +42,207 @@ private[ui] class StreamingPage(parent: StreamingTab) /** Render the page */ def render(request: HttpServletRequest): Seq[Node] = { val content = listener.synchronized { - generateBasicStats() ++

++ -

Statistics over last {listener.retainedCompletedBatches.size} processed batches

++ - generateReceiverStats() ++ - generateBatchStatsTable() ++ + generateLoadResources() ++ + generateBasicStats() ++ + generateStatTable() ++ generateBatchListTables() } - UIUtils.headerSparkPage("Streaming", content, parent, Some(5000)) + UIUtils.headerSparkPage("Streaming Statistics", content, parent, Some(5000)) + } + + private def generateLoadResources(): Seq[Node] = { + // scalastyle:off + + + + // scalastyle:on } /** Generate basic stats of the streaming program */ private def generateBasicStats(): Seq[Node] = { val timeSinceStart = System.currentTimeMillis() - startTime +
Running batches of + + {formatDurationVerbose(listener.batchDuration)} + + for + + {formatDurationVerbose(timeSinceStart)} + + since + + {UIUtils.formatDate(startTime)} + . +
+ } + + private def generateStatTable(): Seq[Node] = { + val avgSchedulingDelay = listener.schedulingDelayDistribution.map(_.statCounter.mean.toLong) + val avgProcessingTime = listener.processingDelayDistribution.map(_.statCounter.mean.toLong) + val avgTotalDelay = listener.totalDelayDistribution.map(_.statCounter.mean.toLong) + val avgReceiverEvents = + listener.receivedRecordsDistributions.mapValues(_.map(_.statCounter.mean.toLong)) + val avgAllReceiverEvents = avgReceiverEvents.values.flatMap(x => x).sum + val receivedRecords = listener.allReceivedRecordsWithBatchTime + + val completedBatches = listener.retainedCompletedBatches + val processingTime = completedBatches. + flatMap(batchInfo => batchInfo.processingDelay.map(batchInfo.batchTime.milliseconds -> _)). + sortBy(_._1) + val schedulingDelay = completedBatches. + flatMap(batchInfo => batchInfo.schedulingDelay.map(batchInfo.batchTime.milliseconds -> _)). + sortBy(_._1) + val totalDelay = completedBatches. + flatMap(batchInfo => batchInfo.totalDelay.map(batchInfo.batchTime.milliseconds -> _)). + sortBy(_._1) + val jsCollector = ArrayBuffer[String]() + + val triangleJs = + s"""$$('#inputs-table').toggle('collapsed'); + |if ($$(this).html() == '$BLACK_RIGHT_TRIANGLE_HTML') + |$$(this).html('$BLACK_DOWN_TRIANGLE_HTML'); + |else $$(this).html('$BLACK_RIGHT_TRIANGLE_HTML');""".stripMargin.replaceAll("\\n", "") + + val table = + // scalastyle:off + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TimelinesDistributions
+
+ {Unparsed(BLACK_RIGHT_TRIANGLE_HTML)} + Input Rate +
+
Avg: {avgAllReceiverEvents} events/sec
+ 		{generateTimeline(jsCollector, "all-receiver-events-timeline", receivedRecords, "#")}{generateDistribution(jsCollector, "all-receiver-events-distribution", receivedRecords.map(_._2), "#")}
+
Processing Time
+
Avg: {formatDurationOption(avgProcessingTime)}
+ 		{generateTimeline(jsCollector, "processing-time-timeline", processingTime, "ms")}{generateDistribution(jsCollector, "processing-time-distribution", processingTime.map(_._2), "ms")}
+
Scheduling Delay
+
Avg: {formatDurationOption(avgSchedulingDelay)}
+ 		{generateTimeline(jsCollector, "scheduling-delay-timeline", schedulingDelay, "ms")}{generateDistribution(jsCollector, "scheduling-delay-distribution", schedulingDelay.map(_._2), "ms")}
+
Total Delay
+
Avg: {formatDurationOption(avgTotalDelay)}
+ 		{generateTimeline(jsCollector, "total-delay-timeline", totalDelay, "ms")}{generateDistribution(jsCollector, "total-delay-distribution", totalDelay.map(_._2), "ms")}
+ // scalastyle:on + + val js = + s""" + |$$(document).ready(function(){ + | ${jsCollector.mkString("\n")} + |});""".stripMargin + table ++ + } + + private def generateInputReceiversTable(jsCollector: ArrayBuffer[String]): Seq[Node] = { + val content = listener.receivedRecordsDistributions.map { case (receiverId, distribution) => + generateInputReceiverRow(jsCollector, receiverId, distribution) + }.reduce(_ ++ _) + + + + + + + + + + + + + {content} + +
StatusLocationLast Error TimeLast Error Message
+ } + + private def generateInputReceiverRow( + jsCollector: ArrayBuffer[String], receiverId: Int, distribution: Option[Distribution]): + Seq[Node] = { + val avgReceiverEvents = distribution.map(_.statCounter.mean.toLong) + val receiverInfo = listener.receiverInfo(receiverId) + val receiverName = receiverInfo.map(_.name).getOrElse(s"Receiver-$receiverId") + val receiverActive = receiverInfo.map { info => + if (info.active) "ACTIVE" else "INACTIVE" + }.getOrElse(emptyCell) + val receiverLocation = receiverInfo.map(_.location).getOrElse(emptyCell) + val receiverLastError = listener.receiverInfo(receiverId).map { info => + val msg = s"${info.lastErrorMessage} - ${info.lastError}" + if (msg.size > 100) msg.take(97) + "..." else msg + }.getOrElse(emptyCell) + val receiverLastErrorTime = + listener.receiverLastErrorTimeo(receiverId).map(UIUtils.formatDate).getOrElse(emptyCell) + val receivedRecords = listener.receivedRecordsWithBatchTime.get(receiverId).getOrElse(Seq()) + // scalastyle:off -
    -
  • - Started at: {UIUtils.formatDate(startTime)} -
    • -
  • - Time since start: {formatDurationVerbose(timeSinceStart)} -
    • -
  • - Network receivers: {listener.numReceivers} -
    • -
  • - Batch interval: {formatDurationVerbose(listener.batchDuration)} -
    • -
  • - Completed batches: {listener.numTotalCompletedBatches} -
    • -
  • - Active batches: {listener.numUnprocessedBatches} -
    • -
  • - Received events: {listener.numTotalReceivedRecords} -
    • -
  • - Processed events: {listener.numTotalProcessedRecords} -
    • -
+ + +
+ {receiverName} +
+
Avg: {avgReceiverEvents.map(_.toString).getOrElse(emptyCell)} events/sec
+ + {receiverActive} + {receiverLocation} + {receiverLastErrorTime} + {receiverLastError} + + + + {generateTimeline(jsCollector, s"receiver-$receiverId-events-timeline", receivedRecords, "#")} + + {generateDistribution(jsCollector, s"receiver-$receiverId-events-distribution", receivedRecords.map(_._2), "#")} + // scalastyle:on } + private def generateTimeline( + jsCollector: ArrayBuffer[String], divId: String, data: Seq[(Long, _)], unit: String): + Seq[Node] = { + val jsForData = data.map { case (x, y) => + s"""{"x": $x, "y": $y}""" + }.mkString("[", ",", "]") + jsCollector += s"drawTimeline('#$divId', $jsForData, '$unit');" + +
+ } + + private def generateDistribution( + jsCollector: ArrayBuffer[String], divId: String, data: Seq[_], unit: String): Seq[Node] = { + val jsForData = data.mkString("[", ",", "]") + jsCollector += s"drawDistribution('#$divId', $jsForData, '$unit');" + +
+ } + + /** Generate stats of data received by the receivers in the streaming program */ private def generateReceiverStats(): Seq[Node] = { val receivedRecordDistributions = listener.receivedRecordsDistributions @@ -173,6 +336,10 @@ private[ui] class StreamingPage(parent: StreamingTab) content } + def generateTimeRow(title: String, times: Seq[(Time, Long)]): Unit = { + + } + /** * Returns a human-readable string representing a duration such as "5 second 35 ms" @@ -216,3 +383,8 @@ private[ui] class StreamingPage(parent: StreamingTab) } } +private object StreamingPage { + val BLACK_RIGHT_TRIANGLE_HTML = "▶" + val BLACK_DOWN_TRIANGLE_HTML = "▼" +} + From b82ed1e54564ef51e5ec678414b63c4a35bc7b15 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Fri, 17 Apr 2015 00:09:19 +0800 Subject: [PATCH 02/31] Update the graphs as per comments --- .../apache/spark/ui/static/streaming-page.css | 14 +- .../apache/spark/ui/static/streaming-page.js | 210 ++++++--- .../ui/StreamingJobProgressListener.scala | 10 +- .../spark/streaming/ui/StreamingPage.scala | 417 ++++++++++-------- 4 files changed, 386 insertions(+), 265 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css index 019d706fa0a8f..3c082e76a9ad1 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css @@ -15,6 +15,7 @@ * limitations under the License. */ + .graph { font: 10px sans-serif; } @@ -25,17 +26,12 @@ shape-rendering: crispEdges; } +.axis text { + fill: gray; +} + .line { fill: none; stroke: steelblue; stroke-width: 1.5px; } - -.bar rect { - fill: steelblue; - shape-rendering: crispEdges; -} - -.bar text { - fill: #000; -} diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index 04bd39b8733b3..054e068d896a6 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -15,106 +15,180 @@ * limitations under the License. */ -function drawTimeline(id, data, unit) { - var margin = {top: 20, right: 20, bottom: 70, left: 50}; + +// An invisible div to show details of a point in the graph +var graphTooltip = d3.select("body").append("div") + .style("position", "absolute") + .style("z-index", "10") + .style("visibility", "hidden") + .text("a simple tooltip"); + +// Show "text" at location (x, y) +function showGraphTooltip(text, x, y) { + graphTooltip.style("visibility", "visible") + .style("top", x + "px") + .style("left", y + "px") + .text(text); +} + +// Hide "graphTooltip" +function hideGraphTooltip() { + graphTooltip.style("visibility", "hidden"); +} + +/** + * @param id the `id` used in the html `div` tag + * @param data the data for the timeline graph + * @param minY the min value of Y axis + * @param maxY the max value of Y axis + * @param unitY the unit of Y axis + */ +function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { + var margin = {top: 20, right: 30, bottom: 30, left: 50}; var width = 500 - margin.left - margin.right; - var height = 250 - margin.top - margin.bottom; + var height = 150 - margin.top - margin.bottom; - var x = d3.time.scale().range([0, width]); - var y = d3.scale.linear().range([height, 0]); + var x = d3.time.scale().domain([minX, maxX]).range([0, width]); + var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); - var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(d3.time.format("%H:%M:%S")); - var yAxis = d3.svg.axis().scale(y).orient("left"); + var timeFormat = d3.time.format("%H:%M:%S") + var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(timeFormat); + var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5); var line = d3.svg.line() .x(function(d) { return x(new Date(d.x)); }) .y(function(d) { return y(d.y); }); var svg = d3.select(id).append("svg") - .attr("width", width + margin.left + margin.right) - .attr("height", height + margin.top + margin.bottom) - .append("g") - .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); - x.domain(d3.extent(data, function(d) { return d.x; })); - y.domain(d3.extent(data, function(d) { return d.y; })); + // Only show the first and last time in the graph + xAxis.tickValues(x.domain()); svg.append("g") - .attr("class", "x axis") - .attr("transform", "translate(0," + height + ")") - .call(xAxis) - .selectAll("text") - .style("text-anchor", "end") - .attr("dx", "-.8em") - .attr("dy", ".15em") - .attr("transform", "rotate(-65)"); + .attr("class", "x axis") + .attr("transform", "translate(0," + height + ")") + .call(xAxis) svg.append("g") - .attr("class", "y axis") - .call(yAxis) - .append("text") - .attr("x", 25) - .attr("y", -5) - .attr("dy", ".71em") - .style("text-anchor", "end") - .text(unit); + .attr("class", "y axis") + .call(yAxis) + .append("text") + .text(unitY); svg.append("path") - .datum(data) - .attr("class", "line") - .attr("d", line); + .datum(data) + .attr("class", "line") + .attr("d", line); + + // Add points to the line. However, we make it invisible at first. But when the user moves mouse + // over a point, it will be displayed with its detail. + svg.selectAll(".point") + .data(data) + .enter().append("circle") + .attr("stroke", "white") // white and opacity = 0 make it invisible + .attr("fill", "white") + .attr("opacity", "0") + .attr("cx", function(d) { return x(d.x); }) + .attr("cy", function(d) { return y(d.y); }) + .attr("r", function(d) { return 3; }) + .on('mouseover', function(d) { + var tip = d.y + " " + unitY + " at " + timeFormat(new Date(d.x)); + showGraphTooltip(tip, event.pageY-25, event.pageX); + // show the point + d3.select(this) + .attr("stroke", "steelblue") + .attr("fill", "steelblue") + .attr("opacity", "1"); + }) + .on('mouseout', function() { + hideGraphTooltip(); + // hide the point + d3.select(this) + .attr("stroke", "white") + .attr("fill", "white") + .attr("opacity", "0"); + }); } -function drawDistribution(id, values, unit) { - var margin = {top: 10, right: 30, bottom: 30, left: 50}; - var width = 500 - margin.left - margin.right; - var height = 250 - margin.top - margin.bottom; +/** + * @param id the `id` used in the html `div` tag + * @param data the data for the distribution graph + * @param minY the min value of Y axis + * @param maxY the max value of Y axis + * @param unitY the unit of Y axis + */ +function drawDistribution(id, values, minY, maxY, unitY) { + var margin = {top: 20, right: 30, bottom: 30, left: 50}; + var width = 300 - margin.left - margin.right; + var height = 150 - margin.top - margin.bottom; + + var binCount = values.length > 100 ? 100 : values.length; + var formatBinValue = d3.format(",.2f"); - var y = d3.scale.linear() - .domain(d3.extent(values, function(d) { return d; })) - .range([height, 0]); - var data = d3.layout.histogram().bins(y.ticks(10))(values); - console.log(values) - console.log(data) - var barHeight = height / data.length + var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); + var data = d3.layout.histogram().range([minY, maxY]).bins(binCount)(values); var x = d3.scale.linear() .domain([0, d3.max(data, function(d) { return d.y; })]) .range([0, width]); - var xAxis = d3.svg.axis().scale(x).orient("bottom"); - var yAxis = d3.svg.axis().scale(y).orient("left"); + var xAxis = d3.svg.axis().scale(x).orient("bottom").ticks(5); + var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5); + + var line = d3.svg.line() + .x(function(d) { return x(d.y); }) + .y(function(d) { return y(d.x); }); var svg = d3.select(id).append("svg") .attr("width", width + margin.left + margin.right) .attr("height", height + margin.top + margin.bottom) - .append("g") - .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); - - var bar = svg.selectAll(".bar") - .data(data) - .enter().append("g") - .attr("class", "bar") - - bar.append("rect") - .attr("x", 1) - .attr("y", function(d) { return y(d.x) - margin.bottom + margin.top; }) - .attr("width", function(d) { return x(d.y); }) - .attr("height", function(d) { return (height - margin.bottom) / data.length; }); - - bar.append("text") - .attr("dy", ".75em") - .attr("x", function(d) { return x(d.y) + 10; }) - .attr("y", function(d) { return y(d.x) + 16 - margin.bottom; }) - .attr("text-anchor", "middle") - .text(function(d) { return d.y; }); + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); svg.append("g") .attr("class", "x axis") .attr("transform", "translate(0," + height + ")") - .call(xAxis); + .call(xAxis) svg.append("g") .attr("class", "y axis") - .call(yAxis); -} \ No newline at end of file + .call(yAxis) + .append("text") + .text(unitY); + + svg.append("path") + .datum(data) + .attr("class", "line") + .attr("d", line); + + // Add points to the line. However, we make it invisible at first. But when the user moves mouse + // over a point, it will be displayed with its detail. + svg.selectAll(".point") + .data(data) + .enter().append("circle") + .attr("stroke", "white") // white and opacity = 0 make it invisible + .attr("fill", "white") + .attr("opacity", "0") + .attr("cx", function(d) { return x(d.y); }) + .attr("cy", function(d) { return y(d.x); }) + .attr("r", function(d) { return 3; }) + .on('mouseover', function(d) { + var tip = "[" + formatBinValue(d.x) + ", " + formatBinValue(d.x + d.dx) + "): " + d.y; + showGraphTooltip(tip, event.pageY, event.pageX + 10); + d3.select(this) + .attr("stroke", "steelblue") + .attr("fill", "steelblue") + .attr("opacity", "1"); + }) + .on('mouseout', function() { + hideGraphTooltip(); + d3.select(this) + .attr("stroke", "white") + .attr("fill", "white") + .attr("opacity", "0"); + }); +} diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala index 428daf129b745..23db722db941a 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala @@ -34,7 +34,7 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) private val waitingBatchInfos = new HashMap[Time, BatchInfo] private val runningBatchInfos = new HashMap[Time, BatchInfo] private val completedBatchInfos = new Queue[BatchInfo] - private val batchInfoLimit = ssc.conf.getInt("spark.streaming.ui.retainedBatches", 100) + private val batchInfoLimit = ssc.conf.getInt("spark.streaming.ui.retainedBatches", 1000) private var totalCompletedBatches = 0L private var totalReceivedRecords = 0L private var totalProcessedRecords = 0L @@ -170,7 +170,11 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) } - def allReceivedRecordsWithBatchTime: Seq[(Long, Double)] = synchronized { + /** + * Return event rate for all receivers together in each batch time. These are pairs of + * an event rate with a batch time. + */ + def eventRateForAllReceivers: Seq[(Long, Double)] = synchronized { retainedBatches.take(batchInfoLimit).map { batchInfo => (batchInfo.batchTime.milliseconds, batchInfo.numRecords.toDouble * 1000 / batchDuration) } @@ -203,7 +207,7 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) retainedBatches.lastOption } - private def retainedBatches: Seq[BatchInfo] = { + def retainedBatches: Seq[BatchInfo] = { (waitingBatchInfos.values.toSeq ++ runningBatchInfos.values.toSeq ++ completedBatchInfos).sortBy(_.batchTime)(Time.ordering) } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index e200ee8f812e3..3ff9e9a530c58 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -17,18 +17,68 @@ package org.apache.spark.streaming.ui -import java.util.Calendar import javax.servlet.http.HttpServletRequest import scala.collection.mutable.ArrayBuffer import scala.xml.{Node, Unparsed} import org.apache.spark.Logging -import org.apache.spark.streaming.Time import org.apache.spark.ui._ import org.apache.spark.ui.UIUtils._ import org.apache.spark.util.Distribution +/** + * @param divId the `id` used in the html `div` tag + * @param data the data for the timeline graph + * @param minY the min value of Y axis + * @param maxY the max value of Y axis + * @param unitY the unit of Y axis + */ +private[ui] case class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: Long, maxX: Long, + minY: Long, maxY: Long, unitY: String) { + + def toHtmlAndJs: (Seq[Node], String) = { + val jsForData = data.map { case (x, y) => + s"""{"x": $x, "y": $y}""" + }.mkString("[", ",", "]") + + (
, + s"drawTimeline('#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY');") + } +} + +/** + * @param divId the `id` used in the html `div` tag + * @param data the data for the distribution graph + * @param minY the min value of Y axis + * @param maxY the max value of Y axis + * @param unitY the unit of Y axis + */ +private[ui] case class DistributionUIData( + divId: String, data: Seq[_], minY: Long, maxY: Long, unitY: String) { + + def toHtmlAndJs: (Seq[Node], String) = { + val jsForData = data.mkString("[", ",", "]") + + (
, + s"drawDistribution('#$divId', $jsForData, $minY, $maxY, '$unitY');") + } +} + +private[ui] case class LongStreamingUIData(data: Seq[(Long, Long)]) { + + val avg: Option[Long] = if (data.isEmpty) None else Some(data.map(_._2).sum / data.size) + + val max: Option[Long] = if (data.isEmpty) None else Some(data.map(_._2).max) +} + +private[ui] case class DoubleStreamingUIData(data: Seq[(Long, Double)]) { + + val avg: Option[Double] = if (data.isEmpty) None else Some(data.map(_._2).sum / data.size) + + val max: Option[Double] = if (data.isEmpty) None else Some(data.map(_._2).max) +} + /** Page for Spark Web UI that shows statistics of a streaming job */ private[ui] class StreamingPage(parent: StreamingTab) extends WebUIPage("") with Logging { @@ -41,12 +91,14 @@ private[ui] class StreamingPage(parent: StreamingTab) /** Render the page */ def render(request: HttpServletRequest): Seq[Node] = { - val content = listener.synchronized { - generateLoadResources() ++ - generateBasicStats() ++ - generateStatTable() ++ - generateBatchListTables() - } + val resources = generateLoadResources() + val basicInfo = generateBasicInfo() + val content = resources ++ + basicInfo ++ + listener.synchronized { + generateStatTable() ++ + generateBatchListTables() + } UIUtils.headerSparkPage("Streaming Statistics", content, parent, Some(5000)) } @@ -58,8 +110,8 @@ private[ui] class StreamingPage(parent: StreamingTab) // scalastyle:on } - /** Generate basic stats of the streaming program */ - private def generateBasicStats(): Seq[Node] = { + /** Generate basic information of the streaming program */ + private def generateBasicInfo(): Seq[Node] = { val timeSinceStart = System.currentTimeMillis() - startTime
Running batches of @@ -77,32 +129,128 @@ private[ui] class StreamingPage(parent: StreamingTab) } private def generateStatTable(): Seq[Node] = { - val avgSchedulingDelay = listener.schedulingDelayDistribution.map(_.statCounter.mean.toLong) - val avgProcessingTime = listener.processingDelayDistribution.map(_.statCounter.mean.toLong) - val avgTotalDelay = listener.totalDelayDistribution.map(_.statCounter.mean.toLong) - val avgReceiverEvents = - listener.receivedRecordsDistributions.mapValues(_.map(_.statCounter.mean.toLong)) - val avgAllReceiverEvents = avgReceiverEvents.values.flatMap(x => x).sum - val receivedRecords = listener.allReceivedRecordsWithBatchTime - - val completedBatches = listener.retainedCompletedBatches - val processingTime = completedBatches. - flatMap(batchInfo => batchInfo.processingDelay.map(batchInfo.batchTime.milliseconds -> _)). - sortBy(_._1) - val schedulingDelay = completedBatches. - flatMap(batchInfo => batchInfo.schedulingDelay.map(batchInfo.batchTime.milliseconds -> _)). - sortBy(_._1) - val totalDelay = completedBatches. - flatMap(batchInfo => batchInfo.totalDelay.map(batchInfo.batchTime.milliseconds -> _)). - sortBy(_._1) + val batchInfos = listener.retainedBatches + + val batchTimes = batchInfos.map(_.batchTime.milliseconds) + val minBatchTime = if (batchTimes.isEmpty) startTime else batchTimes.min + val maxBatchTime = if (batchTimes.isEmpty) startTime else batchTimes.max + + val eventRateForAllReceivers = DoubleStreamingUIData(batchInfos.map { batchInfo => + (batchInfo.batchTime.milliseconds, batchInfo.numRecords * 1000.0 / listener.batchDuration) + }) + + val schedulingDelay = LongStreamingUIData(batchInfos.flatMap { batchInfo => + batchInfo.schedulingDelay.map(batchInfo.batchTime.milliseconds -> _) + }) + val processingTime = LongStreamingUIData(batchInfos.flatMap { batchInfo => + batchInfo.processingDelay.map(batchInfo.batchTime.milliseconds -> _) + }) + val totalDelay = LongStreamingUIData(batchInfos.flatMap { batchInfo => + batchInfo.totalDelay.map(batchInfo.batchTime.milliseconds -> _) + }) + val jsCollector = ArrayBuffer[String]() + // Use the max value of "schedulingDelay", "processingTime", and "totalDelay" to make the + // Y axis ranges same. + val maxTime = + (for (m1 <- schedulingDelay.max; m2 <- processingTime.max; m3 <- totalDelay.max) yield + m1 max m2 max m3).getOrElse(0L) + List(1, 2, 3).sum + // Should start at 0 + val minTime = 0L + + // Use the max input rate for all receivers' graphs to make the Y axis ranges same. + // If it's not an integral number, just use its ceil integral number. + val maxEventRate = eventRateForAllReceivers.max.map(_.ceil.toLong).getOrElse(0L) + val minEventRate = 0L + val triangleJs = s"""$$('#inputs-table').toggle('collapsed'); |if ($$(this).html() == '$BLACK_RIGHT_TRIANGLE_HTML') |$$(this).html('$BLACK_DOWN_TRIANGLE_HTML'); |else $$(this).html('$BLACK_RIGHT_TRIANGLE_HTML');""".stripMargin.replaceAll("\\n", "") + val timelineDataForEventRateOfAllReceivers = + TimelineUIData( + "all-receiver-events-timeline", + eventRateForAllReceivers.data, + minBatchTime, + maxBatchTime, + minEventRate, + maxEventRate, + "events/sec").toHtmlAndJs + jsCollector += timelineDataForEventRateOfAllReceivers._2 + + val distributionDataForEventRateOfAllReceivers = + DistributionUIData( + "all-receiver-events-distribution", + eventRateForAllReceivers.data.map(_._2), + minEventRate, + maxEventRate, + "events/sec").toHtmlAndJs + jsCollector += distributionDataForEventRateOfAllReceivers._2 + + val timelineDataForSchedulingDelay = + TimelineUIData( + "scheduling-delay-timeline", + schedulingDelay.data, + minBatchTime, + maxBatchTime, + minTime, + maxTime, + "ms").toHtmlAndJs + jsCollector += timelineDataForSchedulingDelay._2 + + val distributionDataForSchedulingDelay = + DistributionUIData( + "scheduling-delay-distribution", + schedulingDelay.data.map(_._2), + minTime, + maxTime, + "ms").toHtmlAndJs + jsCollector += distributionDataForSchedulingDelay._2 + + val timelineDataForProcessingTime = + TimelineUIData( + "processing-time-timeline", + processingTime.data, + minBatchTime, + maxBatchTime, + minTime, + maxTime, + "ms").toHtmlAndJs + jsCollector += timelineDataForProcessingTime._2 + + val distributionDataForProcessingTime = + DistributionUIData( + "processing-time-distribution", + processingTime.data.map(_._2), + minTime, + maxTime, + "ms").toHtmlAndJs + jsCollector += distributionDataForProcessingTime._2 + + val timelineDataForTotalDelay = + TimelineUIData( + "total-delay-timeline", + totalDelay.data, + minBatchTime, + maxBatchTime, + minTime, + maxTime, + "ms").toHtmlAndJs + jsCollector += timelineDataForTotalDelay._2 + + val distributionDataForTotalDelay = + DistributionUIData( + "total-delay-distribution", + totalDelay.data.map(_._2), + minTime, + maxTime, + "ms").toHtmlAndJs + jsCollector += distributionDataForTotalDelay._2 + val table = // scalastyle:off @@ -111,44 +259,44 @@ private[ui] class StreamingPage(parent: StreamingTab) - - - + + - - - + + - - - + + - - - + +
+
{Unparsed(BLACK_RIGHT_TRIANGLE_HTML)} Input Rate
-
Avg: {avgAllReceiverEvents} events/sec
+
Avg: {eventRateForAllReceivers.avg.map(_.formatted("%.2f")).getOrElse(emptyCell)} events/sec
{generateTimeline(jsCollector, "all-receiver-events-timeline", receivedRecords, "#")}{generateDistribution(jsCollector, "all-receiver-events-distribution", receivedRecords.map(_._2), "#")}{timelineDataForEventRateOfAllReceivers._1}{distributionDataForEventRateOfAllReceivers._1}
-
Processing Time
-
Avg: {formatDurationOption(avgProcessingTime)}
+ 		+
Scheduling Delay
+
Avg: {formatDurationOption(schedulingDelay.avg)}
{generateTimeline(jsCollector, "processing-time-timeline", processingTime, "ms")}{generateDistribution(jsCollector, "processing-time-distribution", processingTime.map(_._2), "ms")}{timelineDataForSchedulingDelay._1}{distributionDataForSchedulingDelay._1}
-
Scheduling Delay
-
Avg: {formatDurationOption(avgSchedulingDelay)}
+ 		+
Processing Time
+
Avg: {formatDurationOption(processingTime.avg)}
{generateTimeline(jsCollector, "scheduling-delay-timeline", schedulingDelay, "ms")}{generateDistribution(jsCollector, "scheduling-delay-distribution", schedulingDelay.map(_._2), "ms")}{timelineDataForProcessingTime._1}{distributionDataForProcessingTime._1}
+
Total Delay
-
Avg: {formatDurationOption(avgTotalDelay)}
+
Avg: {formatDurationOption(totalDelay.avg)}
{generateTimeline(jsCollector, "total-delay-timeline", totalDelay, "ms")}{generateDistribution(jsCollector, "total-delay-distribution", totalDelay.map(_._2), "ms")}{timelineDataForTotalDelay._1}{distributionDataForTotalDelay._1}
@@ -162,9 +310,14 @@ private[ui] class StreamingPage(parent: StreamingTab) table ++ } - private def generateInputReceiversTable(jsCollector: ArrayBuffer[String]): Seq[Node] = { + private def generateInputReceiversTable( + jsCollector: ArrayBuffer[String], + minX: Long, + maxX: Long, + minY: Long, + maxY: Long): Seq[Node] = { val content = listener.receivedRecordsDistributions.map { case (receiverId, distribution) => - generateInputReceiverRow(jsCollector, receiverId, distribution) + generateInputReceiverRow(jsCollector, receiverId, distribution, minX, maxX, minY, maxY) }.reduce(_ ++ _) @@ -184,8 +337,13 @@ private[ui] class StreamingPage(parent: StreamingTab) } private def generateInputReceiverRow( - jsCollector: ArrayBuffer[String], receiverId: Int, distribution: Option[Distribution]): - Seq[Node] = { + jsCollector: ArrayBuffer[String], + receiverId: Int, + distribution: Option[Distribution], + minX: Long, + maxX: Long, + minY: Long, + maxY: Long): Seq[Node] = { val avgReceiverEvents = distribution.map(_.statCounter.mean.toLong) val receiverInfo = listener.receiverInfo(receiverId) val receiverName = receiverInfo.map(_.name).getOrElse(s"Receiver-$receiverId") @@ -201,9 +359,29 @@ private[ui] class StreamingPage(parent: StreamingTab) listener.receiverLastErrorTimeo(receiverId).map(UIUtils.formatDate).getOrElse(emptyCell) val receivedRecords = listener.receivedRecordsWithBatchTime.get(receiverId).getOrElse(Seq()) + val timelineForEventRate = + TimelineUIData( + s"receiver-$receiverId-events-timeline", + receivedRecords, + minX, + maxX, + minY, + maxY, + "events/sec").toHtmlAndJs + jsCollector += timelineForEventRate._2 + + val distributionForEventsRate = + DistributionUIData( + s"receiver-$receiverId-events-distribution", + receivedRecords.map(_._2), + minY, + maxY, + "events/sec").toHtmlAndJs + jsCollector += distributionForEventsRate._2 + // scalastyle:off - - + // scalastyle:on } - private def generateTimeline( - jsCollector: ArrayBuffer[String], divId: String, data: Seq[(Long, _)], unit: String): - Seq[Node] = { - val jsForData = data.map { case (x, y) => - s"""{"x": $x, "y": $y}""" - }.mkString("[", ",", "]") - jsCollector += s"drawTimeline('#$divId', $jsForData, '$unit');" - -
- } - - private def generateDistribution( - jsCollector: ArrayBuffer[String], divId: String, data: Seq[_], unit: String): Seq[Node] = { - val jsForData = data.mkString("[", ",", "]") - jsCollector += s"drawDistribution('#$divId', $jsForData, '$unit');" - -
- } - - - /** Generate stats of data received by the receivers in the streaming program */ - private def generateReceiverStats(): Seq[Node] = { - val receivedRecordDistributions = listener.receivedRecordsDistributions - val lastBatchReceivedRecord = listener.lastReceivedBatchRecords - val table = if (receivedRecordDistributions.size > 0) { - val headerRow = Seq( - "Receiver", - "Status", - "Location", - "Events in last batch\n[" + formatDate(Calendar.getInstance().getTime()) + "]", - "Minimum rate\n[events/sec]", - "Median rate\n[events/sec]", - "Maximum rate\n[events/sec]", - "Last Error" - ) - val dataRows = (0 until listener.numReceivers).map { receiverId => - val receiverInfo = listener.receiverInfo(receiverId) - val receiverName = receiverInfo.map(_.name).getOrElse(s"Receiver-$receiverId") - val receiverActive = receiverInfo.map { info => - if (info.active) "ACTIVE" else "INACTIVE" - }.getOrElse(emptyCell) - val receiverLocation = receiverInfo.map(_.location).getOrElse(emptyCell) - val receiverLastBatchRecords = formatNumber(lastBatchReceivedRecord(receiverId)) - val receivedRecordStats = receivedRecordDistributions(receiverId).map { d => - d.getQuantiles(Seq(0.0, 0.5, 1.0)).map(r => formatNumber(r.toLong)) - }.getOrElse { - Seq(emptyCell, emptyCell, emptyCell, emptyCell, emptyCell) - } - val receiverLastError = listener.receiverInfo(receiverId).map { info => - val msg = s"${info.lastErrorMessage} - ${info.lastError}" - if (msg.size > 100) msg.take(97) + "..." else msg - }.getOrElse(emptyCell) - Seq(receiverName, receiverActive, receiverLocation, receiverLastBatchRecords) ++ - receivedRecordStats ++ Seq(receiverLastError) - } - Some(listingTable(headerRow, dataRows)) - } else { - None - } - - val content = -
Receiver Statistics
++ -
{table.getOrElse("No receivers")}
- - content - } - - /** Generate stats of batch jobs of the streaming program */ - private def generateBatchStatsTable(): Seq[Node] = { - val numBatches = listener.retainedCompletedBatches.size - val lastCompletedBatch = listener.lastCompletedBatch - val table = if (numBatches > 0) { - val processingDelayQuantilesRow = { - Seq( - "Processing Time", - formatDurationOption(lastCompletedBatch.flatMap(_.processingDelay)) - ) ++ getQuantiles(listener.processingDelayDistribution) - } - val schedulingDelayQuantilesRow = { - Seq( - "Scheduling Delay", - formatDurationOption(lastCompletedBatch.flatMap(_.schedulingDelay)) - ) ++ getQuantiles(listener.schedulingDelayDistribution) - } - val totalDelayQuantilesRow = { - Seq( - "Total Delay", - formatDurationOption(lastCompletedBatch.flatMap(_.totalDelay)) - ) ++ getQuantiles(listener.totalDelayDistribution) - } - val headerRow = Seq("Metric", "Last batch", "Minimum", "25th percentile", - "Median", "75th percentile", "Maximum") - val dataRows: Seq[Seq[String]] = Seq( - processingDelayQuantilesRow, - schedulingDelayQuantilesRow, - totalDelayQuantilesRow - ) - Some(listingTable(headerRow, dataRows)) - } else { - None - } - - val content = -
Batch Processing Statistics
++ -
-
    - {table.getOrElse("No statistics have been generated yet.")} -
-
- - content - } - - def generateTimeRow(title: String, times: Seq[(Time, Long)]): Unit = { - - } - - /** * Returns a human-readable string representing a duration such as "5 second 35 ms" */ @@ -348,19 +408,6 @@ private[ui] class StreamingPage(parent: StreamingTab) msOption.map(formatDurationVerbose).getOrElse(emptyCell) } - /** Get quantiles for any time distribution */ - private def getQuantiles(timeDistributionOption: Option[Distribution]) = { - timeDistributionOption.get.getQuantiles().map { ms => formatDurationVerbose(ms.toLong) } - } - - /** Generate HTML table from string data */ - private def listingTable(headers: Seq[String], data: Seq[Seq[String]]) = { - def generateDataRow(data: Seq[String]): Seq[Node] = { - {data.map(d => )} - } - UIUtils.listingTable(headers, generateDataRow, data, fixedWidth = true) - } - private def generateBatchListTables(): Seq[Node] = { val runningBatches = listener.runningBatches.sortBy(_.batchTime.milliseconds).reverse val waitingBatches = listener.waitingBatches.sortBy(_.batchTime.milliseconds).reverse From a0458f9d7223c88788520abcd59126ce9c152d9f Mon Sep 17 00:00:00 2001 From: zsxwing Date: Wed, 22 Apr 2015 01:18:38 +0800 Subject: [PATCH 03/31] Clean the codes --- .../streaming/ui/StreamingJobProgressListener.scala | 13 +------------ .../apache/spark/streaming/ui/StreamingPage.scala | 2 +- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala index 23db722db941a..de7e5fbe0b67d 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala @@ -169,17 +169,6 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) }.toMap } - - /** - * Return event rate for all receivers together in each batch time. These are pairs of - * an event rate with a batch time. - */ - def eventRateForAllReceivers: Seq[(Long, Double)] = synchronized { - retainedBatches.take(batchInfoLimit).map { batchInfo => - (batchInfo.batchTime.milliseconds, batchInfo.numRecords.toDouble * 1000 / batchDuration) - } - } - def lastReceivedBatchRecords: Map[Int, Long] = synchronized { val lastReceivedBlockInfoOption = lastReceivedBatch.map(_.receivedBlockInfo) lastReceivedBlockInfoOption.map { lastReceivedBlockInfo => @@ -195,7 +184,7 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) receiverInfos.get(receiverId) } - def receiverLastErrorTimeo(receiverId: Int): Option[Long] = synchronized { + def receiverLastErrorTime(receiverId: Int): Option[Long] = synchronized { receiverLastErrorTime.get(receiverId) } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index 3ff9e9a530c58..843d2102bbbe5 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -356,7 +356,7 @@ private[ui] class StreamingPage(parent: StreamingTab) if (msg.size > 100) msg.take(97) + "..." else msg }.getOrElse(emptyCell) val receiverLastErrorTime = - listener.receiverLastErrorTimeo(receiverId).map(UIUtils.formatDate).getOrElse(emptyCell) + listener.receiverLastErrorTime(receiverId).map(UIUtils.formatDate).getOrElse(emptyCell) val receivedRecords = listener.receivedRecordsWithBatchTime.get(receiverId).getOrElse(Seq()) val timelineForEventRate = From ad93295d1b43881776643fb30698d5172bf6db16 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Wed, 22 Apr 2015 08:12:14 +0800 Subject: [PATCH 04/31] Remove unnecessary codes --- .../org/apache/spark/util/Distribution.scala | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/Distribution.scala b/core/src/main/scala/org/apache/spark/util/Distribution.scala index 5931e7bcad9e9..9aea8efa38c7a 100644 --- a/core/src/main/scala/org/apache/spark/util/Distribution.scala +++ b/core/src/main/scala/org/apache/spark/util/Distribution.scala @@ -20,7 +20,6 @@ package org.apache.spark.util import java.io.PrintStream import scala.collection.immutable.IndexedSeq -import scala.collection.mutable.ArrayBuffer /** * Util for getting some stats from a small sample of numeric values, with some handy @@ -68,34 +67,6 @@ private[spark] class Distribution(val data: Array[Double], val startIdx: Int, va out.println(statCounter) showQuantiles(out) } - - def histogram(bins: Int): Seq[(Double, Int)] = { - require(bins > 0) - val stat = statCounter - val binSize = (stat.max - stat.min) / (bins - 1) - val points = new Array[Double](bins) - for (i <- 0 until (bins - 1)) { - points(i) = stat.min + binSize * i - } - points(bins - 1) = stat.max - val counts = new Array[Int](bins) - for (i <- startIdx until endIdx) { - val v = data(i) - var j = 0 - var find = false - while (!find && j < bins - 1) { - if (v < points(j) + binSize / 2) { - counts(j) += 1 - find = true - } - j += 1 - } - if (!find) { - counts(bins - 1) += 1 - } - } - points.zip(counts) - } } private[spark] object Distribution { From ddf741a35feaa7bccf7c99289171e4120278e1aa Mon Sep 17 00:00:00 2001 From: zsxwing Date: Wed, 22 Apr 2015 13:01:43 +0800 Subject: [PATCH 05/31] Fix the unit test --- .../org/apache/spark/streaming/ui/StreamingPage.scala | 2 +- .../org/apache/spark/streaming/UISeleniumSuite.scala | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index 843d2102bbbe5..2baccafa00993 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -318,7 +318,7 @@ private[ui] class StreamingPage(parent: StreamingTab) maxY: Long): Seq[Node] = { val content = listener.receivedRecordsDistributions.map { case (receiverId, distribution) => generateInputReceiverRow(jsCollector, receiverId, distribution, minX, maxX, minY, maxY) - }.reduce(_ ++ _) + }.foldLeft[Seq[Node]](Nil)(_ ++ _)
+
{receiverName}
@@ -216,131 +394,13 @@ private[ui] class StreamingPage(parent: StreamingTab)
- {generateTimeline(jsCollector, s"receiver-$receiverId-events-timeline", receivedRecords, "#")} + {timelineForEventRate._1} {generateDistribution(jsCollector, s"receiver-$receiverId-events-distribution", receivedRecords.map(_._2), "#")}{distributionForEventsRate._1}
{d}
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala index 205ddf6dbe9b0..a2f4f3fe19897 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala @@ -72,9 +72,8 @@ class UISeleniumSuite eventually(timeout(10 seconds), interval(50 milliseconds)) { // check whether streaming page exists go to (sparkUI.appUIAddress.stripSuffix("/") + "/streaming") - val statisticText = findAll(cssSelector("li strong")).map(_.text).toSeq - statisticText should contain("Network receivers:") - statisticText should contain("Batch interval:") + val h3Text = findAll(cssSelector("h3")).map(_.text).toSeq + h3Text should contain("Streaming Statistics") val h4Text = findAll(cssSelector("h4")).map(_.text).toSeq h4Text should contain("Active Batches (0)") @@ -97,9 +96,8 @@ class UISeleniumSuite eventually(timeout(10 seconds), interval(50 milliseconds)) { go to (sparkUI.appUIAddress.stripSuffix("/") + "/streaming") - val statisticText = findAll(cssSelector("li strong")).map(_.text).toSeq - statisticText should not contain ("Network receivers:") - statisticText should not contain ("Batch interval:") + val h3Text = findAll(cssSelector("h3")).map(_.text).toSeq + h3Text should not contain("Streaming Statistics") } } } From 5688702960e7de0f9f87684d49dd0e0445326182 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Wed, 22 Apr 2015 17:00:54 +0800 Subject: [PATCH 06/31] Fix the unit test --- .../spark/streaming/ui/StreamingJobProgressListenerSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala index 94b1985116feb..d26e9cbad2efd 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala @@ -100,7 +100,7 @@ class StreamingJobProgressListenerSuite extends TestSuiteBase with Matchers { test("Remove the old completed batches when exceeding the limit") { val ssc = setupStreams(input, operation) - val limit = ssc.conf.getInt("spark.streaming.ui.retainedBatches", 100) + val limit = ssc.conf.getInt("spark.streaming.ui.retainedBatches", 1000) val listener = new StreamingJobProgressListener(ssc) val receivedBlockInfo = Map( From 881c9072296bbd93d056ad6f78db8cd26b849493 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Wed, 22 Apr 2015 20:34:09 +0800 Subject: [PATCH 07/31] Use histogram for distribution --- .../apache/spark/ui/static/streaming-page.css | 11 ++- .../apache/spark/ui/static/streaming-page.js | 85 ++++++++++--------- 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css index 3c082e76a9ad1..0dbe9d8369703 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css @@ -22,7 +22,7 @@ .axis path, .axis line { fill: none; - stroke: #000; + stroke: gray; shape-rendering: crispEdges; } @@ -35,3 +35,12 @@ stroke: steelblue; stroke-width: 1.5px; } + +.bar rect { + fill: steelblue; + shape-rendering: crispEdges; +} + +.bar rect:hover { + fill: orange; +} diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index 054e068d896a6..e689583ab404c 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -21,13 +21,15 @@ var graphTooltip = d3.select("body").append("div") .style("position", "absolute") .style("z-index", "10") .style("visibility", "hidden") - .text("a simple tooltip"); + .text(""); // Show "text" at location (x, y) function showGraphTooltip(text, x, y) { + var left = x; + var top = y; graphTooltip.style("visibility", "visible") - .style("top", x + "px") - .style("left", y + "px") + .style("top", top + "px") + .style("left", left + "px") .text(text); } @@ -52,7 +54,14 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); var timeFormat = d3.time.format("%H:%M:%S") - var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(timeFormat); + var xAxis = d3.svg.axis().scale(x).orient("bottom").ticks(10) + .tickFormat(function(d) { + if (d.getTime() == minX || d.getTime() == maxX) { + return timeFormat(d); + } else { + return "x"; + } + }); var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5); var line = d3.svg.line() @@ -77,6 +86,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { .attr("class", "y axis") .call(yAxis) .append("text") + .attr("transform", "translate(0," + (-3) + ")") .text(unitY); svg.append("path") @@ -97,7 +107,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { .attr("r", function(d) { return 3; }) .on('mouseover', function(d) { var tip = d.y + " " + unitY + " at " + timeFormat(new Date(d.x)); - showGraphTooltip(tip, event.pageY-25, event.pageX); + showGraphTooltip(tip, d3.event.pageX + 5, d3.event.pageY - 25); // show the point d3.select(this) .attr("stroke", "steelblue") @@ -126,7 +136,8 @@ function drawDistribution(id, values, minY, maxY, unitY) { var width = 300 - margin.left - margin.right; var height = 150 - margin.top - margin.bottom; - var binCount = values.length > 100 ? 100 : values.length; + //var binCount = values.length > 100 ? 100 : values.length; + var binCount = 10; var formatBinValue = d3.format(",.2f"); var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); @@ -139,10 +150,6 @@ function drawDistribution(id, values, minY, maxY, unitY) { var xAxis = d3.svg.axis().scale(x).orient("bottom").ticks(5); var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5); - var line = d3.svg.line() - .x(function(d) { return x(d.y); }) - .y(function(d) { return y(d.x); }); - var svg = d3.select(id).append("svg") .attr("width", width + margin.left + margin.right) .attr("height", height + margin.top + margin.bottom) @@ -158,37 +165,35 @@ function drawDistribution(id, values, minY, maxY, unitY) { .attr("class", "y axis") .call(yAxis) .append("text") + .attr("transform", "translate(0," + (-3) + ")") .text(unitY); - svg.append("path") - .datum(data) - .attr("class", "line") - .attr("d", line); - - // Add points to the line. However, we make it invisible at first. But when the user moves mouse - // over a point, it will be displayed with its detail. - svg.selectAll(".point") + var bar = svg.selectAll(".bar") .data(data) - .enter().append("circle") - .attr("stroke", "white") // white and opacity = 0 make it invisible - .attr("fill", "white") - .attr("opacity", "0") - .attr("cx", function(d) { return x(d.y); }) - .attr("cy", function(d) { return y(d.x); }) - .attr("r", function(d) { return 3; }) - .on('mouseover', function(d) { - var tip = "[" + formatBinValue(d.x) + ", " + formatBinValue(d.x + d.dx) + "): " + d.y; - showGraphTooltip(tip, event.pageY, event.pageX + 10); - d3.select(this) - .attr("stroke", "steelblue") - .attr("fill", "steelblue") - .attr("opacity", "1"); - }) - .on('mouseout', function() { - hideGraphTooltip(); - d3.select(this) - .attr("stroke", "white") - .attr("fill", "white") - .attr("opacity", "0"); - }); + .enter().append("g") + .attr("transform", function(d) { return "translate(0," + (y(d.x) - height + y(d.dx)) + ")";}) + .attr("class", "bar").append("rect") + .attr("width", function(d) { return x(d.y); }) + .attr("height", function(d) { return height - y(d.dx); }) + .on('mouseover', function(d) { + var tip = "[" + formatBinValue(d.x) + ", " + formatBinValue(d.x + d.dx) + "): " + d.y; + + // Calculate the location for tip + var scrollTop = document.documentElement.scrollTop || document.body.scrollTop; + var scrollLeft = document.documentElement.scrollLeft || document.body.scrollLeft; + var target = d3.event.target; + var matrix = target.getScreenCTM(); + var targetBBox = target.getBBox(); + var point = svg.node().ownerSVGElement.createSVGPoint(); + point.x = targetBBox.x; + point.y = targetBBox.y; + var bbox = point.matrixTransform(matrix); + var tipX = bbox.x + scrollLeft + x(d.y) + 2; + var tipY = bbox.y + scrollTop - 6; + + showGraphTooltip(tip, tipX, tipY); + }) + .on('mouseout', function() { + hideGraphTooltip(); + }); } From d78672aaf070ae3e4d4571b7016a2bdebba932c0 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Thu, 23 Apr 2015 18:09:24 +0800 Subject: [PATCH 08/31] Make the X axis use the same range --- .../apache/spark/ui/static/streaming-page.js | 14 ++- .../spark/streaming/ui/StreamingPage.scala | 101 ++++++++++-------- 2 files changed, 68 insertions(+), 47 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index e689583ab404c..5557a6288534c 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -16,6 +16,10 @@ */ +var globalMinX = 0; +var globalMaxX = 0; +var binCount = 10; + // An invisible div to show details of a point in the graph var graphTooltip = d3.select("body").append("div") .style("position", "absolute") @@ -137,14 +141,14 @@ function drawDistribution(id, values, minY, maxY, unitY) { var height = 150 - margin.top - margin.bottom; //var binCount = values.length > 100 ? 100 : values.length; - var binCount = 10; var formatBinValue = d3.format(",.2f"); var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); var data = d3.layout.histogram().range([minY, maxY]).bins(binCount)(values); var x = d3.scale.linear() - .domain([0, d3.max(data, function(d) { return d.y; })]) + .domain([globalMinX, globalMaxX]) + //.domain([0, d3.max(data, function(d) { return d.y; })]) .range([0, width]); var xAxis = d3.svg.axis().scale(x).orient("bottom").ticks(5); @@ -197,3 +201,9 @@ function drawDistribution(id, values, minY, maxY, unitY) { hideGraphTooltip(); }); } + +function prepareDistribution(values, minY, maxY) { + var data = d3.layout.histogram().range([minY, maxY]).bins(binCount)(values); + var maxBarSize = d3.max(data, function(d) { return d.y; }); + globalMaxX = maxBarSize > globalMaxX? maxBarSize : globalMaxX; +} diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index 2baccafa00993..e4ad4b25bb53f 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -37,13 +37,14 @@ import org.apache.spark.util.Distribution private[ui] case class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: Long, maxX: Long, minY: Long, maxY: Long, unitY: String) { - def toHtmlAndJs: (Seq[Node], String) = { + def toHtml(jsCollector: JsCollector): Seq[Node] = { val jsForData = data.map { case (x, y) => s"""{"x": $x, "y": $y}""" }.mkString("[", ",", "]") - - (
, + jsCollector.addStatement( s"drawTimeline('#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY');") + +
} } @@ -57,11 +58,12 @@ private[ui] case class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: private[ui] case class DistributionUIData( divId: String, data: Seq[_], minY: Long, maxY: Long, unitY: String) { - def toHtmlAndJs: (Seq[Node], String) = { + def toHtml(jsCollector: JsCollector): Seq[Node] = { val jsForData = data.mkString("[", ",", "]") + jsCollector.addPreparedStatement(s"prepareDistribution($jsForData, $minY, $maxY);") + jsCollector.addStatement(s"drawDistribution('#$divId', $jsForData, $minY, $maxY, '$unitY');") - (
, - s"drawDistribution('#$divId', $jsForData, $minY, $maxY, '$unitY');") +
} } @@ -149,7 +151,7 @@ private[ui] class StreamingPage(parent: StreamingTab) batchInfo.totalDelay.map(batchInfo.batchTime.milliseconds -> _) }) - val jsCollector = ArrayBuffer[String]() + val jsCollector = new JsCollector // Use the max value of "schedulingDelay", "processingTime", and "totalDelay" to make the // Y axis ranges same. @@ -179,8 +181,7 @@ private[ui] class StreamingPage(parent: StreamingTab) maxBatchTime, minEventRate, maxEventRate, - "events/sec").toHtmlAndJs - jsCollector += timelineDataForEventRateOfAllReceivers._2 + "events/sec").toHtml(jsCollector) val distributionDataForEventRateOfAllReceivers = DistributionUIData( @@ -188,8 +189,7 @@ private[ui] class StreamingPage(parent: StreamingTab) eventRateForAllReceivers.data.map(_._2), minEventRate, maxEventRate, - "events/sec").toHtmlAndJs - jsCollector += distributionDataForEventRateOfAllReceivers._2 + "events/sec").toHtml(jsCollector) val timelineDataForSchedulingDelay = TimelineUIData( @@ -199,8 +199,7 @@ private[ui] class StreamingPage(parent: StreamingTab) maxBatchTime, minTime, maxTime, - "ms").toHtmlAndJs - jsCollector += timelineDataForSchedulingDelay._2 + "ms").toHtml(jsCollector) val distributionDataForSchedulingDelay = DistributionUIData( @@ -208,8 +207,7 @@ private[ui] class StreamingPage(parent: StreamingTab) schedulingDelay.data.map(_._2), minTime, maxTime, - "ms").toHtmlAndJs - jsCollector += distributionDataForSchedulingDelay._2 + "ms").toHtml(jsCollector) val timelineDataForProcessingTime = TimelineUIData( @@ -219,8 +217,7 @@ private[ui] class StreamingPage(parent: StreamingTab) maxBatchTime, minTime, maxTime, - "ms").toHtmlAndJs - jsCollector += timelineDataForProcessingTime._2 + "ms").toHtml(jsCollector) val distributionDataForProcessingTime = DistributionUIData( @@ -228,8 +225,7 @@ private[ui] class StreamingPage(parent: StreamingTab) processingTime.data.map(_._2), minTime, maxTime, - "ms").toHtmlAndJs - jsCollector += distributionDataForProcessingTime._2 + "ms").toHtml(jsCollector) val timelineDataForTotalDelay = TimelineUIData( @@ -239,8 +235,7 @@ private[ui] class StreamingPage(parent: StreamingTab) maxBatchTime, minTime, maxTime, - "ms").toHtmlAndJs - jsCollector += timelineDataForTotalDelay._2 + "ms").toHtml(jsCollector) val distributionDataForTotalDelay = DistributionUIData( @@ -248,8 +243,7 @@ private[ui] class StreamingPage(parent: StreamingTab) totalDelay.data.map(_._2), minTime, maxTime, - "ms").toHtmlAndJs - jsCollector += distributionDataForTotalDelay._2 + "ms").toHtml(jsCollector) val table = // scalastyle:off @@ -266,8 +260,8 @@ private[ui] class StreamingPage(parent: StreamingTab)
Avg: {eventRateForAllReceivers.avg.map(_.formatted("%.2f")).getOrElse(emptyCell)} events/sec
- - + + - - + + - - + + - - + +
{timelineDataForEventRateOfAllReceivers._1}{distributionDataForEventRateOfAllReceivers._1}{timelineDataForEventRateOfAllReceivers}{distributionDataForEventRateOfAllReceivers}
Processing Time
Avg: {formatDurationOption(processingTime.avg)}
{timelineDataForProcessingTime._1}{distributionDataForProcessingTime._1}{timelineDataForProcessingTime}{distributionDataForProcessingTime}
Total Delay
Avg: {formatDurationOption(totalDelay.avg)}
{timelineDataForTotalDelay._1}{distributionDataForTotalDelay._1}{timelineDataForTotalDelay}{distributionDataForTotalDelay}
// scalastyle:on - val js = - s""" - |$$(document).ready(function(){ - | ${jsCollector.mkString("\n")} - |});""".stripMargin - table ++ + table ++ jsCollector.toHtml } private def generateInputReceiversTable( - jsCollector: ArrayBuffer[String], + jsCollector: JsCollector, minX: Long, maxX: Long, minY: Long, @@ -337,7 +326,7 @@ private[ui] class StreamingPage(parent: StreamingTab) } private def generateInputReceiverRow( - jsCollector: ArrayBuffer[String], + jsCollector: JsCollector, receiverId: Int, distribution: Option[Distribution], minX: Long, @@ -367,8 +356,7 @@ private[ui] class StreamingPage(parent: StreamingTab) maxX, minY, maxY, - "events/sec").toHtmlAndJs - jsCollector += timelineForEventRate._2 + "events/sec").toHtml(jsCollector) val distributionForEventsRate = DistributionUIData( @@ -376,8 +364,7 @@ private[ui] class StreamingPage(parent: StreamingTab) receivedRecords.map(_._2), minY, maxY, - "events/sec").toHtmlAndJs - jsCollector += distributionForEventsRate._2 + "events/sec").toHtml(jsCollector) // scalastyle:off @@ -394,9 +381,9 @@ private[ui] class StreamingPage(parent: StreamingTab) - {timelineForEventRate._1} + {timelineForEventRate} - {distributionForEventsRate._1} + {distributionForEventsRate} // scalastyle:on } @@ -435,3 +422,27 @@ private object StreamingPage { val BLACK_DOWN_TRIANGLE_HTML = "▼" } +private[ui] class JsCollector { + private val preparedStatements = ArrayBuffer[String]() + private val statements = ArrayBuffer[String]() + + def addPreparedStatement(js: String): Unit = { + preparedStatements += js + } + + def addStatement(js: String): Unit = { + statements += js + } + + def toHtml: Seq[Node] = { + val js = + s""" + |$$(document).ready(function(){ + | ${preparedStatements.mkString("\n")} + | ${statements.mkString("\n")} + |});""".stripMargin + + + } +} + From c23ce10f0467de95a6edc684b35f66be545091cf Mon Sep 17 00:00:00 2001 From: zsxwing Date: Thu, 23 Apr 2015 21:23:23 +0800 Subject: [PATCH 09/31] Make two graphs close --- .../apache/spark/ui/static/streaming-page.css | 8 ++++++ .../apache/spark/ui/static/streaming-page.js | 17 +++++------- .../spark/streaming/ui/StreamingPage.scala | 26 +++++++++---------- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css index 0dbe9d8369703..fe07cec058d32 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css @@ -44,3 +44,11 @@ .bar rect:hover { fill: orange; } + +.timeline { + width: 500px; +} + +.distribution { + width: 300px; +} diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index 5557a6288534c..772719cec967f 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -50,7 +50,8 @@ function hideGraphTooltip() { * @param unitY the unit of Y axis */ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { - var margin = {top: 20, right: 30, bottom: 30, left: 50}; + d3.select(d3.select(id).node().parentNode).style("padding", "8px 0 8px 8px").style("border-right", "0px solid white"); + var margin = {top: 20, right: 27, bottom: 30, left: 50}; var width = 500 - margin.left - margin.right; var height = 150 - margin.top - margin.bottom; @@ -58,14 +59,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); var timeFormat = d3.time.format("%H:%M:%S") - var xAxis = d3.svg.axis().scale(x).orient("bottom").ticks(10) - .tickFormat(function(d) { - if (d.getTime() == minX || d.getTime() == maxX) { - return timeFormat(d); - } else { - return "x"; - } - }); + var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(timeFormat); var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5); var line = d3.svg.line() @@ -136,7 +130,8 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { * @param unitY the unit of Y axis */ function drawDistribution(id, values, minY, maxY, unitY) { - var margin = {top: 20, right: 30, bottom: 30, left: 50}; + d3.select(d3.select(id).node().parentNode).style("padding", "8px 8px 8px 0").style("border-left", "0px solid white"); + var margin = {top: 20, right: 30, bottom: 30, left: 10}; var width = 300 - margin.left - margin.right; var height = 150 - margin.top - margin.bottom; @@ -152,7 +147,7 @@ function drawDistribution(id, values, minY, maxY, unitY) { .range([0, width]); var xAxis = d3.svg.axis().scale(x).orient("bottom").ticks(5); - var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5); + var yAxis = d3.svg.axis().scale(y).orient("left").ticks(0).tickFormat(function(d) { return ""; }); var svg = d3.select(id).append("svg") .attr("width", width + margin.left + margin.right) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index e4ad4b25bb53f..aa4a246a62470 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -260,8 +260,8 @@ private[ui] class StreamingPage(parent: StreamingTab)
Avg: {eventRateForAllReceivers.avg.map(_.formatted("%.2f")).getOrElse(emptyCell)} events/sec
- {timelineDataForEventRateOfAllReceivers} - {distributionDataForEventRateOfAllReceivers} + {timelineDataForEventRateOfAllReceivers} + {distributionDataForEventRateOfAllReceivers} @@ -273,24 +273,24 @@ private[ui] class StreamingPage(parent: StreamingTab)
Scheduling Delay
Avg: {formatDurationOption(schedulingDelay.avg)}
- {timelineDataForSchedulingDelay} - {distributionDataForSchedulingDelay} + {timelineDataForSchedulingDelay} + {distributionDataForSchedulingDelay}
Processing Time
Avg: {formatDurationOption(processingTime.avg)}
- {timelineDataForProcessingTime} - {distributionDataForProcessingTime} + {timelineDataForProcessingTime} + {distributionDataForProcessingTime}
Total Delay
Avg: {formatDurationOption(totalDelay.avg)}
- {timelineDataForTotalDelay} - {distributionDataForTotalDelay} + {timelineDataForTotalDelay} + {distributionDataForTotalDelay} @@ -313,9 +313,9 @@ private[ui] class StreamingPage(parent: StreamingTab) - Status - Location - Last Error Time +
Status
+
Location
+
Last Error Time
Last Error Message @@ -380,10 +380,10 @@ private[ui] class StreamingPage(parent: StreamingTab) {receiverLastError} - + {timelineForEventRate} - {distributionForEventsRate} + {distributionForEventsRate} // scalastyle:on } From fed8219dc04270a03fdbc653274e8441a74c7576 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Mon, 27 Apr 2015 16:10:59 -0700 Subject: [PATCH 10/31] Move the x axis at the top and show a better tooltip --- .../org/apache/spark/ui/static/streaming-page.css | 3 ++- .../org/apache/spark/ui/static/streaming-page.js | 14 ++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css index fe07cec058d32..ca73d1592e1a5 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css @@ -42,7 +42,8 @@ } .bar rect:hover { - fill: orange; + //fill: rgb(49, 91, 125); + fill: rgb(100, 185, 255); } .timeline { diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index 772719cec967f..2e03b89c12660 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -22,6 +22,8 @@ var binCount = 10; // An invisible div to show details of a point in the graph var graphTooltip = d3.select("body").append("div") + .attr("class", "label") + .style("display", "inline-block") .style("position", "absolute") .style("z-index", "10") .style("visibility", "hidden") @@ -146,7 +148,7 @@ function drawDistribution(id, values, minY, maxY, unitY) { //.domain([0, d3.max(data, function(d) { return d.y; })]) .range([0, width]); - var xAxis = d3.svg.axis().scale(x).orient("bottom").ticks(5); + var xAxis = d3.svg.axis().scale(x).orient("top").ticks(5); var yAxis = d3.svg.axis().scale(y).orient("left").ticks(0).tickFormat(function(d) { return ""; }); var svg = d3.select(id).append("svg") @@ -157,15 +159,15 @@ function drawDistribution(id, values, minY, maxY, unitY) { svg.append("g") .attr("class", "x axis") - .attr("transform", "translate(0," + height + ")") + .attr("transform", "translate(0," + 0 + ")") .call(xAxis) svg.append("g") .attr("class", "y axis") .call(yAxis) - .append("text") - .attr("transform", "translate(0," + (-3) + ")") - .text(unitY); + // .append("text") + // .attr("transform", "translate(0," + (-3) + ")") + // .text(unitY); var bar = svg.selectAll(".bar") .data(data) @@ -175,7 +177,7 @@ function drawDistribution(id, values, minY, maxY, unitY) { .attr("width", function(d) { return x(d.y); }) .attr("height", function(d) { return height - y(d.dx); }) .on('mouseover', function(d) { - var tip = "[" + formatBinValue(d.x) + ", " + formatBinValue(d.x + d.dx) + "): " + d.y; + var tip = d.y + " between " + formatBinValue(d.x) + " " + unitY + " and " + formatBinValue(d.x + d.dx) + " " + unitY; // Calculate the location for tip var scrollTop = document.documentElement.scrollTop || document.body.scrollTop; From 04c750016e711448e6281789e39ef482030050db Mon Sep 17 00:00:00 2001 From: zsxwing Date: Mon, 27 Apr 2015 18:50:33 -0700 Subject: [PATCH 11/31] Make the server and client use the same timezone --- .../apache/spark/ui/static/streaming-page.js | 34 +++++---- .../ui/StreamingJobProgressListener.scala | 2 +- .../spark/streaming/ui/StreamingPage.scala | 73 +++++++++++++++---- 3 files changed, 78 insertions(+), 31 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index 2e03b89c12660..5b39ec6d198f0 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -16,8 +16,9 @@ */ -var globalMinX = 0; -var globalMaxX = 0; +var timelineMarginLeft = 50; +var distributionMinX = 0; +var distributionMaxX = 0; var binCount = 10; // An invisible div to show details of a point in the graph @@ -53,19 +54,18 @@ function hideGraphTooltip() { */ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { d3.select(d3.select(id).node().parentNode).style("padding", "8px 0 8px 8px").style("border-right", "0px solid white"); - var margin = {top: 20, right: 27, bottom: 30, left: 50}; + var margin = {top: 20, right: 27, bottom: 30, left: timelineMarginLeft}; var width = 500 - margin.left - margin.right; var height = 150 - margin.top - margin.bottom; - var x = d3.time.scale().domain([minX, maxX]).range([0, width]); + var x = d3.scale.linear().domain([minX, maxX]).range([0, width]); var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); - var timeFormat = d3.time.format("%H:%M:%S") - var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(timeFormat); + var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(function(d) { return timeFormat[d]; }); var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5); var line = d3.svg.line() - .x(function(d) { return x(new Date(d.x)); }) + .x(function(d) { return x(d.x); }) .y(function(d) { return y(d.y); }); var svg = d3.select(id).append("svg") @@ -106,7 +106,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { .attr("cy", function(d) { return y(d.y); }) .attr("r", function(d) { return 3; }) .on('mouseover', function(d) { - var tip = d.y + " " + unitY + " at " + timeFormat(new Date(d.x)); + var tip = d.y + " " + unitY + " at " + timeFormat[d.x]; showGraphTooltip(tip, d3.event.pageX + 5, d3.event.pageY - 25); // show the point d3.select(this) @@ -137,15 +137,13 @@ function drawDistribution(id, values, minY, maxY, unitY) { var width = 300 - margin.left - margin.right; var height = 150 - margin.top - margin.bottom; - //var binCount = values.length > 100 ? 100 : values.length; var formatBinValue = d3.format(",.2f"); var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); var data = d3.layout.histogram().range([minY, maxY]).bins(binCount)(values); var x = d3.scale.linear() - .domain([globalMinX, globalMaxX]) - //.domain([0, d3.max(data, function(d) { return d.y; })]) + .domain([distributionMinX, distributionMaxX]) .range([0, width]); var xAxis = d3.svg.axis().scale(x).orient("top").ticks(5); @@ -165,9 +163,6 @@ function drawDistribution(id, values, minY, maxY, unitY) { svg.append("g") .attr("class", "y axis") .call(yAxis) - // .append("text") - // .attr("transform", "translate(0," + (-3) + ")") - // .text(unitY); var bar = svg.selectAll(".bar") .data(data) @@ -199,8 +194,17 @@ function drawDistribution(id, values, minY, maxY, unitY) { }); } +function prepareTimeline(minY, maxY) { + var y = d3.scale.linear().domain([0, maxY]).tickFormat(5); + console.log(y(maxY)); + var numOfChars = y(maxY).length; + var maxPx = numOfChars * 8 + 10; + // Make sure we have enough space to show the ticks in the y axis of timeline + timelineMarginLeft = maxPx > timelineMarginLeft? maxPx : timelineMarginLeft; +} + function prepareDistribution(values, minY, maxY) { var data = d3.layout.histogram().range([minY, maxY]).bins(binCount)(values); var maxBarSize = d3.max(data, function(d) { return d.y; }); - globalMaxX = maxBarSize > globalMaxX? maxBarSize : globalMaxX; + distributionMaxX = maxBarSize > distributionMaxX? maxBarSize : distributionMaxX; } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala index de7e5fbe0b67d..0b5402a6d9fce 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala @@ -196,7 +196,7 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) retainedBatches.lastOption } - def retainedBatches: Seq[BatchInfo] = { + def retainedBatches: Seq[BatchInfo] = synchronized { (waitingBatchInfos.values.toSeq ++ runningBatchInfos.values.toSeq ++ completedBatchInfos).sortBy(_.batchTime)(Time.ordering) } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index aa4a246a62470..6abdb98abb5dd 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -17,6 +17,8 @@ package org.apache.spark.streaming.ui +import java.text.SimpleDateFormat +import java.util.Date import javax.servlet.http.HttpServletRequest import scala.collection.mutable.ArrayBuffer @@ -30,6 +32,8 @@ import org.apache.spark.util.Distribution /** * @param divId the `id` used in the html `div` tag * @param data the data for the timeline graph + * @param minX the min value of X axis + * @param maxX the max value of X axis * @param minY the min value of Y axis * @param maxY the max value of Y axis * @param unitY the unit of Y axis @@ -41,6 +45,7 @@ private[ui] case class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: val jsForData = data.map { case (x, y) => s"""{"x": $x, "y": $y}""" }.mkString("[", ",", "]") + jsCollector.addPreparedStatement(s"prepareTimeline($minY, $maxY);") jsCollector.addStatement( s"drawTimeline('#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY');") @@ -67,17 +72,21 @@ private[ui] case class DistributionUIData( } } -private[ui] case class LongStreamingUIData(data: Seq[(Long, Long)]) { +private[ui] case class MillisecondsStatUIData(data: Seq[(Long, Long)]) { val avg: Option[Long] = if (data.isEmpty) None else Some(data.map(_._2).sum / data.size) + val formattedAvg: String = StreamingPage.formatDurationOption(avg) + val max: Option[Long] = if (data.isEmpty) None else Some(data.map(_._2).max) } -private[ui] case class DoubleStreamingUIData(data: Seq[(Long, Double)]) { +private[ui] case class DoubleStatUIData(data: Seq[(Long, Double)]) { val avg: Option[Double] = if (data.isEmpty) None else Some(data.map(_._2).sum / data.size) + val formattedAvg: String = avg.map(_.formatted("%.2f")).getOrElse("-") + val max: Option[Double] = if (data.isEmpty) None else Some(data.map(_._2).max) } @@ -89,7 +98,6 @@ private[ui] class StreamingPage(parent: StreamingTab) private val listener = parent.listener private val startTime = System.currentTimeMillis() - private val emptyCell = "-" /** Render the page */ def render(request: HttpServletRequest): Seq[Node] = { @@ -104,6 +112,9 @@ private[ui] class StreamingPage(parent: StreamingTab) UIUtils.headerSparkPage("Streaming Statistics", content, parent, Some(5000)) } + /** + * Generate html that will load css/js files for StreamingPage + */ private def generateLoadResources(): Seq[Node] = { // scalastyle:off @@ -130,6 +141,16 @@ private[ui] class StreamingPage(parent: StreamingTab) } + private def generateTimeMap(times: Seq[Long]): Seq[Node] = { + val dateFormat = new SimpleDateFormat("HH:mm:ss") + val js = "var timeFormat = {};\n" + times.map { time => + val formattedTime = dateFormat.format(new Date(time)) + s"timeFormat[$time] = '$formattedTime';" + }.mkString("\n") + + + } + private def generateStatTable(): Seq[Node] = { val batchInfos = listener.retainedBatches @@ -137,17 +158,17 @@ private[ui] class StreamingPage(parent: StreamingTab) val minBatchTime = if (batchTimes.isEmpty) startTime else batchTimes.min val maxBatchTime = if (batchTimes.isEmpty) startTime else batchTimes.max - val eventRateForAllReceivers = DoubleStreamingUIData(batchInfos.map { batchInfo => + val eventRateForAllReceivers = DoubleStatUIData(batchInfos.map { batchInfo => (batchInfo.batchTime.milliseconds, batchInfo.numRecords * 1000.0 / listener.batchDuration) }) - val schedulingDelay = LongStreamingUIData(batchInfos.flatMap { batchInfo => + val schedulingDelay = MillisecondsStatUIData(batchInfos.flatMap { batchInfo => batchInfo.schedulingDelay.map(batchInfo.batchTime.milliseconds -> _) }) - val processingTime = LongStreamingUIData(batchInfos.flatMap { batchInfo => + val processingTime = MillisecondsStatUIData(batchInfos.flatMap { batchInfo => batchInfo.processingDelay.map(batchInfo.batchTime.milliseconds -> _) }) - val totalDelay = LongStreamingUIData(batchInfos.flatMap { batchInfo => + val totalDelay = MillisecondsStatUIData(batchInfos.flatMap { batchInfo => batchInfo.totalDelay.map(batchInfo.batchTime.milliseconds -> _) }) @@ -167,6 +188,7 @@ private[ui] class StreamingPage(parent: StreamingTab) val maxEventRate = eventRateForAllReceivers.max.map(_.ceil.toLong).getOrElse(0L) val minEventRate = 0L + // JavaScript to show/hide the receiver sub table. val triangleJs = s"""$$('#inputs-table').toggle('collapsed'); |if ($$(this).html() == '$BLACK_RIGHT_TRIANGLE_HTML') @@ -258,7 +280,7 @@ private[ui] class StreamingPage(parent: StreamingTab) {Unparsed(BLACK_RIGHT_TRIANGLE_HTML)} Input Rate -
Avg: {eventRateForAllReceivers.avg.map(_.formatted("%.2f")).getOrElse(emptyCell)} events/sec
+
Avg: {eventRateForAllReceivers.formattedAvg} events/sec
{timelineDataForEventRateOfAllReceivers} {distributionDataForEventRateOfAllReceivers} @@ -271,7 +293,7 @@ private[ui] class StreamingPage(parent: StreamingTab)
Scheduling Delay
-
Avg: {formatDurationOption(schedulingDelay.avg)}
+
Avg: {schedulingDelay.formattedAvg}
{timelineDataForSchedulingDelay} {distributionDataForSchedulingDelay} @@ -279,7 +301,7 @@ private[ui] class StreamingPage(parent: StreamingTab)
Processing Time
-
Avg: {formatDurationOption(processingTime.avg)}
+
Avg: {processingTime.formattedAvg}
{timelineDataForProcessingTime} {distributionDataForProcessingTime} @@ -287,7 +309,7 @@ private[ui] class StreamingPage(parent: StreamingTab)
Total Delay
-
Avg: {formatDurationOption(totalDelay.avg)}
+
Avg: {totalDelay.formattedAvg}
{timelineDataForTotalDelay} {distributionDataForTotalDelay} @@ -296,7 +318,7 @@ private[ui] class StreamingPage(parent: StreamingTab) // scalastyle:on - table ++ jsCollector.toHtml + generateTimeMap(batchTimes) ++ table ++ jsCollector.toHtml } private def generateInputReceiversTable( @@ -366,7 +388,6 @@ private[ui] class StreamingPage(parent: StreamingTab) maxY, "events/sec").toHtml(jsCollector) - // scalastyle:off
@@ -385,7 +406,6 @@ private[ui] class StreamingPage(parent: StreamingTab) {distributionForEventsRate} - // scalastyle:on } /** @@ -417,13 +437,33 @@ private[ui] class StreamingPage(parent: StreamingTab) } } -private object StreamingPage { +private[ui] object StreamingPage { val BLACK_RIGHT_TRIANGLE_HTML = "▶" val BLACK_DOWN_TRIANGLE_HTML = "▼" + + val emptyCell = "-" + + /** + * Returns a human-readable string representing a duration such as "5 second 35 ms" + */ + def formatDurationOption(msOption: Option[Long]): String = { + msOption.map(formatDurationVerbose).getOrElse(emptyCell) + } } +/** + * A helper class that allows the user to add JavaScript statements which will be executed when the + * DOM has finished loading. + */ private[ui] class JsCollector { + /** + * JavaScript statements that will execute before `statements` + */ private val preparedStatements = ArrayBuffer[String]() + + /** + * JavaScript statements that will execute after `preparedStatements` + */ private val statements = ArrayBuffer[String]() def addPreparedStatement(js: String): Unit = { @@ -434,6 +474,9 @@ private[ui] class JsCollector { statements += js } + /** + * Generate a html snippet that will execute all scripts when the DOM has finished loading. + */ def toHtml: Seq[Node] = { val js = s""" From 4c0b43f3618b600f9e77ace3280db34b3fa0aa18 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Tue, 28 Apr 2015 10:54:55 -0700 Subject: [PATCH 12/31] Update Streaming UI --- .../spark/ui/static/bootstrap-tooltip.js | 135 ++++++++++++++---- .../apache/spark/ui/static/streaming-page.css | 2 +- .../apache/spark/ui/static/streaming-page.js | 29 +++- .../spark/streaming/ui/StreamingPage.scala | 10 +- 4 files changed, 132 insertions(+), 44 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/bootstrap-tooltip.js b/core/src/main/resources/org/apache/spark/ui/static/bootstrap-tooltip.js index 2934181c1006a..acd6096e6743e 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/bootstrap-tooltip.js +++ b/core/src/main/resources/org/apache/spark/ui/static/bootstrap-tooltip.js @@ -1,9 +1,9 @@ /* =========================================================== - * bootstrap-tooltip.js v2.2.2 - * http://twitter.github.com/bootstrap/javascript.html#tooltips + * bootstrap-tooltip.js v2.3.2 + * http://getbootstrap.com/2.3.2/javascript.html#tooltips * Inspired by the original jQuery.tipsy by Jason Frame * =========================================================== - * Copyright 2012 Twitter, Inc. + * Copyright 2013 Twitter, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,19 +38,27 @@ , init: function (type, element, options) { var eventIn , eventOut + , triggers + , trigger + , i this.type = type this.$element = $(element) this.options = this.getOptions(options) this.enabled = true - if (this.options.trigger == 'click') { - this.$element.on('click.' + this.type, this.options.selector, $.proxy(this.toggle, this)) - } else if (this.options.trigger != 'manual') { - eventIn = this.options.trigger == 'hover' ? 'mouseenter' : 'focus' - eventOut = this.options.trigger == 'hover' ? 'mouseleave' : 'blur' - this.$element.on(eventIn + '.' + this.type, this.options.selector, $.proxy(this.enter, this)) - this.$element.on(eventOut + '.' + this.type, this.options.selector, $.proxy(this.leave, this)) + triggers = this.options.trigger.split(' ') + + for (i = triggers.length; i--;) { + trigger = triggers[i] + if (trigger == 'click') { + this.$element.on('click.' + this.type, this.options.selector, $.proxy(this.toggle, this)) + } else if (trigger != 'manual') { + eventIn = trigger == 'hover' ? 'mouseenter' : 'focus' + eventOut = trigger == 'hover' ? 'mouseleave' : 'blur' + this.$element.on(eventIn + '.' + this.type, this.options.selector, $.proxy(this.enter, this)) + this.$element.on(eventOut + '.' + this.type, this.options.selector, $.proxy(this.leave, this)) + } } this.options.selector ? @@ -59,7 +67,7 @@ } , getOptions: function (options) { - options = $.extend({}, $.fn[this.type].defaults, options, this.$element.data()) + options = $.extend({}, $.fn[this.type].defaults, this.$element.data(), options) if (options.delay && typeof options.delay == 'number') { options.delay = { @@ -72,7 +80,15 @@ } , enter: function (e) { - var self = $(e.currentTarget)[this.type](this._options).data(this.type) + var defaults = $.fn[this.type].defaults + , options = {} + , self + + this._options && $.each(this._options, function (key, value) { + if (defaults[key] != value) options[key] = value + }, this) + + self = $(e.currentTarget)[this.type](options).data(this.type) if (!self.options.delay || !self.options.delay.show) return self.show() @@ -97,14 +113,16 @@ , show: function () { var $tip - , inside , pos , actualWidth , actualHeight , placement , tp + , e = $.Event('show') if (this.hasContent() && this.enabled) { + this.$element.trigger(e) + if (e.isDefaultPrevented()) return $tip = this.tip() this.setContent() @@ -116,19 +134,18 @@ this.options.placement.call(this, $tip[0], this.$element[0]) : this.options.placement - inside = /in/.test(placement) - $tip .detach() .css({ top: 0, left: 0, display: 'block' }) - .insertAfter(this.$element) - pos = this.getPosition(inside) + this.options.container ? $tip.appendTo(this.options.container) : $tip.insertAfter(this.$element) + + pos = this.getPosition() actualWidth = $tip[0].offsetWidth actualHeight = $tip[0].offsetHeight - switch (inside ? placement.split(' ')[1] : placement) { + switch (placement) { case 'bottom': tp = {top: pos.top + pos.height, left: pos.left + pos.width / 2 - actualWidth / 2} break @@ -143,11 +160,56 @@ break } - $tip - .offset(tp) - .addClass(placement) - .addClass('in') + this.applyPlacement(tp, placement) + this.$element.trigger('shown') + } + } + + , applyPlacement: function(offset, placement){ + var $tip = this.tip() + , width = $tip[0].offsetWidth + , height = $tip[0].offsetHeight + , actualWidth + , actualHeight + , delta + , replace + + $tip + .offset(offset) + .addClass(placement) + .addClass('in') + + actualWidth = $tip[0].offsetWidth + actualHeight = $tip[0].offsetHeight + + if (placement == 'top' && actualHeight != height) { + offset.top = offset.top + height - actualHeight + replace = true + } + + if (placement == 'bottom' || placement == 'top') { + delta = 0 + + if (offset.left < 0){ + delta = offset.left * -2 + offset.left = 0 + $tip.offset(offset) + actualWidth = $tip[0].offsetWidth + actualHeight = $tip[0].offsetHeight + } + + this.replaceArrow(delta - width + actualWidth, actualWidth, 'left') + } else { + this.replaceArrow(actualHeight - height, actualHeight, 'top') } + + if (replace) $tip.offset(offset) + } + + , replaceArrow: function(delta, dimension, position){ + this + .arrow() + .css(position, delta ? (50 * (1 - delta / dimension) + "%") : '') } , setContent: function () { @@ -161,6 +223,10 @@ , hide: function () { var that = this , $tip = this.tip() + , e = $.Event('hide') + + this.$element.trigger(e) + if (e.isDefaultPrevented()) return $tip.removeClass('in') @@ -179,6 +245,8 @@ removeWithAnimation() : $tip.detach() + this.$element.trigger('hidden') + return this } @@ -193,11 +261,12 @@ return this.getTitle() } - , getPosition: function (inside) { - return $.extend({}, (inside ? {top: 0, left: 0} : this.$element.offset()), { - width: this.$element[0].offsetWidth - , height: this.$element[0].offsetHeight - }) + , getPosition: function () { + var el = this.$element[0] + return $.extend({}, (typeof el.getBoundingClientRect == 'function') ? el.getBoundingClientRect() : { + width: el.offsetWidth + , height: el.offsetHeight + }, this.$element.offset()) } , getTitle: function () { @@ -215,6 +284,10 @@ return this.$tip = this.$tip || $(this.options.template) } + , arrow: function(){ + return this.$arrow = this.$arrow || this.tip().find(".tooltip-arrow") + } + , validate: function () { if (!this.$element[0].parentNode) { this.hide() @@ -236,8 +309,8 @@ } , toggle: function (e) { - var self = $(e.currentTarget)[this.type](this._options).data(this.type) - self[self.tip().hasClass('in') ? 'hide' : 'show']() + var self = e ? $(e.currentTarget)[this.type](this._options).data(this.type) : this + self.tip().hasClass('in') ? self.hide() : self.show() } , destroy: function () { @@ -269,10 +342,11 @@ , placement: 'top' , selector: false , template: '
' - , trigger: 'hover' + , trigger: 'hover focus' , title: '' , delay: 0 , html: false + , container: false } @@ -285,4 +359,3 @@ } }(window.jQuery); - diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css index ca73d1592e1a5..efef31f8ea9e1 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css @@ -51,5 +51,5 @@ } .distribution { - width: 300px; + width: auto; } diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index 5b39ec6d198f0..e5f6f3eba3b76 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -45,6 +45,17 @@ function hideGraphTooltip() { graphTooltip.style("visibility", "hidden"); } +function showBootstrapTooltip(node, text) { + console.log(text); + $(node).tooltip({title: text, trigger: "manual", 'container': 'body'}); + $(node).tooltip("show"); + console.log($(node)); +} + +function hideBootstrapTooltip(node) { + $(node).tooltip("destroy"); +} + /** * @param id the `id` used in the html `div` tag * @param data the data for the timeline graph @@ -107,7 +118,8 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { .attr("r", function(d) { return 3; }) .on('mouseover', function(d) { var tip = d.y + " " + unitY + " at " + timeFormat[d.x]; - showGraphTooltip(tip, d3.event.pageX + 5, d3.event.pageY - 25); + showBootstrapTooltip(d3.select(this).node(), tip); + //showGraphTooltip(tip, d3.event.pageX + 5, d3.event.pageY - 25); // show the point d3.select(this) .attr("stroke", "steelblue") @@ -115,7 +127,8 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { .attr("opacity", "1"); }) .on('mouseout', function() { - hideGraphTooltip(); + hideBootstrapTooltip(d3.select(this).node()); + //hideGraphTooltip(); // hide the point d3.select(this) .attr("stroke", "white") @@ -172,7 +185,8 @@ function drawDistribution(id, values, minY, maxY, unitY) { .attr("width", function(d) { return x(d.y); }) .attr("height", function(d) { return height - y(d.dx); }) .on('mouseover', function(d) { - var tip = d.y + " between " + formatBinValue(d.x) + " " + unitY + " and " + formatBinValue(d.x + d.dx) + " " + unitY; + var tip = d.y + " between " + formatBinValue(d.x) + " and " + formatBinValue(d.x + d.dx) + " " + unitY; + showBootstrapTooltip(d3.select(this).node(), tip); // Calculate the location for tip var scrollTop = document.documentElement.scrollTop || document.body.scrollTop; @@ -184,13 +198,14 @@ function drawDistribution(id, values, minY, maxY, unitY) { point.x = targetBBox.x; point.y = targetBBox.y; var bbox = point.matrixTransform(matrix); - var tipX = bbox.x + scrollLeft + x(d.y) + 2; - var tipY = bbox.y + scrollTop - 6; + var tipX = bbox.x + scrollLeft; + var tipY = bbox.y + scrollTop + 15; - showGraphTooltip(tip, tipX, tipY); + //showGraphTooltip(tip, tipX, tipY); }) .on('mouseout', function() { - hideGraphTooltip(); + hideBootstrapTooltip(d3.select(this).node()); + //hideGraphTooltip(); }); } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index 6abdb98abb5dd..b7185a269d4c6 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -269,13 +269,13 @@ private[ui] class StreamingPage(parent: StreamingTab) val table = // scalastyle:off - +
- + - - + } @@ -535,6 +510,17 @@ private[ui] object StreamingPage { * DOM has finished loading. */ private[ui] class JsCollector { + + private var variableId = 0 + + /** + * Return the next unused JavaScript variable name + */ + def nextVariableName: String = { + variableId += 1 + "v" + variableId + } + /** * JavaScript statements that will execute before `statements` */ diff --git a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala index 8bef4b688d060..7d6352ed70eec 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala @@ -105,7 +105,6 @@ class UISeleniumSuite statTableHeaders should contain ("Histograms") val statTableCells = findAll(cssSelector("#stat-table td")).map(_.text).toSeq - println(statTableCells.toList) statTableCells.exists(_.contains("Input Rate")) should be (true) statTableCells.exists(_.contains("Streaming Scheduling Delay")) should be (true) statTableCells.exists(_.contains("Processing Time")) should be (true) From b50fa320889af1637b8013348e83806bf8861bf6 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Fri, 1 May 2015 20:30:19 -0700 Subject: [PATCH 22/31] Jump to the batch page when clicking a point in the timeline graphs --- .../resources/org/apache/spark/ui/static/streaming-page.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index acf5504ac4edf..43a803a6f5a88 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -164,6 +164,9 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) { .attr("stroke", "white") .attr("fill", "white") .attr("opacity", "0"); + }) + .on("click", function(d) { + window.location.href = "batch/?id=" + d.x; }); } From 3be4b7ab4d7fffc0f3917706d0fd7f4a61f19194 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Fri, 1 May 2015 20:29:59 -0700 Subject: [PATCH 23/31] Use InputInfo --- .../apache/spark/ui/static/streaming-page.js | 4 +- .../apache/spark/streaming/DStreamGraph.scala | 4 ++ .../streaming/dstream/InputDStream.scala | 5 ++ .../spark/streaming/ui/BatchUIData.scala | 4 +- .../ui/StreamingJobProgressListener.scala | 36 ++++++----- .../spark/streaming/ui/StreamingPage.scala | 62 ++++++++++--------- .../StreamingJobProgressListenerSuite.scala | 15 ++--- 7 files changed, 74 insertions(+), 56 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index 43a803a6f5a88..a4e03b156f13e 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -266,8 +266,8 @@ $(function() { } } - if (getParameterFromURL("show-receivers-detail") == "true") { - // Show the details for all receivers + if (getParameterFromURL("show-streams-detail") == "true") { + // Show the details for all InputDStream $('#inputs-table').toggle('collapsed'); $('#triangle').html('▼'); } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/DStreamGraph.scala b/streaming/src/main/scala/org/apache/spark/streaming/DStreamGraph.scala index 175140481e5ae..9c7f698840778 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/DStreamGraph.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/DStreamGraph.scala @@ -110,6 +110,10 @@ final private[streaming] class DStreamGraph extends Serializable with Logging { .toArray } + def getInputStreamName(streamId: Int): Option[String] = synchronized { + inputStreams.find(_.id == streamId).map(_.name) + } + def generateJobs(time: Time): Seq[Job] = { logDebug("Generating jobs for time " + time) val jobs = this.synchronized { diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/InputDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/InputDStream.scala index e4ad4b509d8d8..2e9308f25d7b6 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/InputDStream.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/InputDStream.scala @@ -44,6 +44,11 @@ abstract class InputDStream[T: ClassTag] (@transient ssc_ : StreamingContext) /** This is an unique identifier for the input stream. */ val id = ssc.getNewInputStreamId() + /** + * The name of this InputDStream. By default, it's the class name with its id. + */ + def name: String = s"${getClass.getSimpleName}-$id" + /** * Checks whether the 'time' is valid wrt slideDuration for generating RDD. * Additionally it also ensures valid times are in strictly increasing order. diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/BatchUIData.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/BatchUIData.scala index 99e10d2b0be12..a5514dfd71c9f 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/BatchUIData.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/BatchUIData.scala @@ -26,7 +26,7 @@ private[ui] case class OutputOpIdAndSparkJobId(outputOpId: OutputOpId, sparkJobI private[ui] case class BatchUIData( val batchTime: Time, - val receiverNumRecords: Map[Int, Long], + val streamIdToNumRecords: Map[Int, Long], val submissionTime: Long, val processingStartTime: Option[Long], val processingEndTime: Option[Long], @@ -58,7 +58,7 @@ private[ui] case class BatchUIData( /** * The number of recorders received by the receivers in this batch. */ - def numRecords: Long = receiverNumRecords.map(_._2).sum + def numRecords: Long = streamIdToNumRecords.values.sum } private[ui] object BatchUIData { diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala index 863fff415e386..0b6ca032bdd4f 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala @@ -144,7 +144,9 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) } } - def numReceivers: Int = ssc.graph.getReceiverInputStreams().size + def numReceivers: Int = synchronized { + receiverInfos.size + } def numTotalCompletedBatches: Long = synchronized { totalCompletedBatches @@ -174,35 +176,41 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) completedBatchUIData.toSeq } - def allReceivers: Seq[Int] = synchronized { - receiverInfos.keys.toSeq + def streamName(streamId: Int): Option[String] = { + ssc.graph.getInputStreamName(streamId) } /** - * Return all of the event rates for each receiver in each batch. + * Return all InputDStream Ids + */ + def streamIds: Seq[Int] = ssc.graph.getInputStreams().map(_.id) + + /** + * Return all of the event rates for each InputDStream in each batch. */ def receivedEventRateWithBatchTime: Map[Int, Seq[(Long, Double)]] = synchronized { - val latestBatches = retainedBatches.map { batchUIData => - (batchUIData.batchTime.milliseconds, batchUIData.receiverNumRecords) + val _retainedBatches = retainedBatches + val latestBatches = _retainedBatches.map { batchUIData => + (batchUIData.batchTime.milliseconds, batchUIData.streamIdToNumRecords) } - (0 until numReceivers).map { receiverId => + streamIds.map { streamId => val eventRates = latestBatches.map { - case (batchTime, receiverNumRecords) => - val numRecords = receiverNumRecords.getOrElse(receiverId, 0L) + case (batchTime, streamIdToNumRecords) => + val numRecords = streamIdToNumRecords.getOrElse(streamId, 0L) (batchTime, numRecords * 1000.0 / batchDuration) } - (receiverId, eventRates) + (streamId, eventRates) }.toMap } def lastReceivedBatchRecords: Map[Int, Long] = synchronized { - val lastReceivedBlockInfoOption = lastReceivedBatch.map(_.receiverNumRecords) + val lastReceivedBlockInfoOption = lastReceivedBatch.map(_.streamIdToNumRecords) lastReceivedBlockInfoOption.map { lastReceivedBlockInfo => - (0 until numReceivers).map { receiverId => - (receiverId, lastReceivedBlockInfo.getOrElse(receiverId, 0L)) + streamIds.map { streamId => + (streamId, lastReceivedBlockInfo.getOrElse(streamId, 0L)) }.toMap }.getOrElse { - (0 until numReceivers).map(receiverId => (receiverId, 0L)).toMap + streamIds.map(streamId => (streamId, 0L)).toMap } } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index 65292ed5cdade..5526c85c181c0 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -207,7 +207,7 @@ private[ui] class StreamingPage(parent: StreamingTab) val minBatchTime = if (batchTimes.isEmpty) startTime else batchTimes.min val maxBatchTime = if (batchTimes.isEmpty) startTime else batchTimes.max - val eventRateForAllReceivers = new EventRateUIData(batches.map { batchInfo => + val eventRateForAllStreams = new EventRateUIData(batches.map { batchInfo => (batchInfo.batchTime.milliseconds, batchInfo.numRecords * 1000.0 / listener.batchDuration) }) @@ -231,12 +231,12 @@ private[ui] class StreamingPage(parent: StreamingTab) val (maxTime, normalizedUnit) = UIUtils.normalizeDuration(_maxTime) val formattedUnit = UIUtils.shortTimeUnitString(normalizedUnit) - // Use the max input rate for all receivers' graphs to make the Y axis ranges same. + // Use the max input rate for all InputDStreams' graphs to make the Y axis ranges same. // If it's not an integral number, just use its ceil integral number. - val maxEventRate = eventRateForAllReceivers.max.map(_.ceil.toLong).getOrElse(0L) + val maxEventRate = eventRateForAllStreams.max.map(_.ceil.toLong).getOrElse(0L) val minEventRate = 0L - // JavaScript to show/hide the receiver sub table. + // JavaScript to show/hide the InputDStreams sub table. val triangleJs = s"""$$('#inputs-table').toggle('collapsed'); |var status = false; @@ -244,24 +244,24 @@ private[ui] class StreamingPage(parent: StreamingTab) |$$(this).html('$BLACK_DOWN_TRIANGLE_HTML');status = true;} |else {$$(this).html('$BLACK_RIGHT_TRIANGLE_HTML');status = false;} |window.history.pushState('', - | document.title, window.location.pathname + '?show-receivers-detail=' + status);""" + | document.title, window.location.pathname + '?show-streams-detail=' + status);""" .stripMargin.replaceAll("\\n", "") // it must be only one single line val batchInterval = StreamingPage.convertToTimeUnit(listener.batchDuration, normalizedUnit) val jsCollector = new JsCollector - val graphUIDataForEventRateOfAllReceivers = + val graphUIDataForEventRateOfAllStreams = new GraphUIData( - "all-receiver-events-timeline", - "all-receiver-events-histogram", - eventRateForAllReceivers.data, + "all-stream-events-timeline", + "all-stream-events-histogram", + eventRateForAllStreams.data, minBatchTime, maxBatchTime, minEventRate, maxEventRate, "events/sec") - graphUIDataForEventRateOfAllReceivers.generateDataJs(jsCollector) + graphUIDataForEventRateOfAllStreams.generateDataJs(jsCollector) val graphUIDataForSchedulingDelay = new GraphUIData( @@ -299,7 +299,8 @@ private[ui] class StreamingPage(parent: StreamingTab) formattedUnit) graphUIDataForTotalDelay.generateDataJs(jsCollector) - val hasReceiver = listener.allReceivers.nonEmpty + // It's false before the user registers the first InputDStream + val hasStream = listener.streamIds.nonEmpty val numCompletedBatches = listener.retainedCompletedBatches.size val numActiveBatches = batchTimes.length - numCompletedBatches @@ -317,21 +318,21 @@ private[ui] class StreamingPage(parent: StreamingTab) - - + + - {if (hasReceiver) { + {if (hasStream) { }} @@ -372,14 +373,14 @@ private[ui] class StreamingPage(parent: StreamingTab) generateTimeMap(batchTimes) ++ table ++ jsCollector.toHtml } - private def generateInputReceiversTable( + private def generateInputDStreamsTable( jsCollector: JsCollector, minX: Long, maxX: Long, minY: Double, maxY: Double): Seq[Node] = { - val content = listener.allReceivers.map { receiverId => - generateInputReceiverRow(jsCollector, receiverId, minX, maxX, minY, maxY) + val content = listener.receivedEventRateWithBatchTime.map { case (streamId, eventRates) => + generateInputDStreamRow(jsCollector, streamId, eventRates, minX, maxX, minY, maxY) }.foldLeft[Seq[Node]](Nil)(_ ++ _) // scalastyle:off @@ -400,33 +401,36 @@ private[ui] class StreamingPage(parent: StreamingTab) // scalastyle:on } - private def generateInputReceiverRow( + private def generateInputDStreamRow( jsCollector: JsCollector, - receiverId: Int, + streamId: Int, + eventRates: Seq[(Long, Double)], minX: Long, maxX: Long, minY: Double, maxY: Double): Seq[Node] = { - val receiverInfo = listener.receiverInfo(receiverId) - val receiverName = receiverInfo.map(_.name).getOrElse(s"Receiver-$receiverId") + // If this is a ReceiverInputDStream, we need to show the receiver info. Or we only need the + // InputDStream name. + val receiverInfo = listener.receiverInfo(streamId) + val receiverName = receiverInfo.map(_.name). + orElse(listener.streamName(streamId)).getOrElse(s"Stream-$streamId") val receiverActive = receiverInfo.map { info => if (info.active) "ACTIVE" else "INACTIVE" }.getOrElse(emptyCell) val receiverLocation = receiverInfo.map(_.location).getOrElse(emptyCell) - val receiverLastError = listener.receiverInfo(receiverId).map { info => + val receiverLastError = receiverInfo.map { info => val msg = s"${info.lastErrorMessage} - ${info.lastError}" if (msg.size > 100) msg.take(97) + "..." else msg }.getOrElse(emptyCell) val receiverLastErrorTime = receiverInfo.map { r => if (r.lastErrorTime < 0) "-" else UIUtils.formatDate(r.lastErrorTime) }.getOrElse(emptyCell) - val receivedRecords = - new EventRateUIData(listener.receivedEventRateWithBatchTime.get(receiverId).getOrElse(Seq())) + val receivedRecords = new EventRateUIData(eventRates) val graphUIDataForEventRate = new GraphUIData( - s"receiver-$receiverId-events-timeline", - s"receiver-$receiverId-events-histogram", + s"stream-$streamId-events-timeline", + s"stream-$streamId-events-histogram", receivedRecords.data, minX, maxX, diff --git a/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala index d72dcdc017fe4..2a0f45830e03c 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala @@ -94,7 +94,7 @@ class StreamingJobProgressListenerSuite extends TestSuiteBase with Matchers { batchUIData.get.schedulingDelay should be (batchInfoStarted.schedulingDelay) batchUIData.get.processingDelay should be (batchInfoStarted.processingDelay) batchUIData.get.totalDelay should be (batchInfoStarted.totalDelay) - batchUIData.get.receiverNumRecords should be (Map(0 -> 300L, 1 -> 300L)) + batchUIData.get.streamIdToNumRecords should be (Map(0 -> 300L, 1 -> 300L)) batchUIData.get.numRecords should be(600) batchUIData.get.outputOpIdSparkJobIdPairs should be Seq(OutputOpIdAndSparkJobId(0, 0), @@ -182,7 +182,7 @@ class StreamingJobProgressListenerSuite extends TestSuiteBase with Matchers { batchUIData.get.schedulingDelay should be (batchInfoSubmitted.schedulingDelay) batchUIData.get.processingDelay should be (batchInfoSubmitted.processingDelay) batchUIData.get.totalDelay should be (batchInfoSubmitted.totalDelay) - batchUIData.get.receiverNumRecords should be (Map.empty) + batchUIData.get.streamIdToNumRecords should be (Map.empty) batchUIData.get.numRecords should be (0) batchUIData.get.outputOpIdSparkJobIdPairs should be (Seq(OutputOpIdAndSparkJobId(0, 0))) @@ -211,17 +211,14 @@ class StreamingJobProgressListenerSuite extends TestSuiteBase with Matchers { val limit = ssc.conf.getInt("spark.streaming.ui.retainedBatches", 1000) for (_ <- 0 until 2 * limit) { - val receivedBlockInfo = Map( - 0 -> Array(ReceivedBlockInfo(0, 100, null), ReceivedBlockInfo(0, 200, null)), - 1 -> Array(ReceivedBlockInfo(1, 300, null)) - ) + val streamIdToNumRecords = Map(0 -> 300L, 1 -> 300L) // onBatchSubmitted - val batchInfoSubmitted = BatchInfo(Time(1000), receivedBlockInfo, 1000, None, None) + val batchInfoSubmitted = BatchInfo(Time(1000), streamIdToNumRecords, 1000, None, None) listener.onBatchSubmitted(StreamingListenerBatchSubmitted(batchInfoSubmitted)) // onBatchStarted - val batchInfoStarted = BatchInfo(Time(1000), receivedBlockInfo, 1000, Some(2000), None) + val batchInfoStarted = BatchInfo(Time(1000), streamIdToNumRecords, 1000, Some(2000), None) listener.onBatchStarted(StreamingListenerBatchStarted(batchInfoStarted)) // onJobStart @@ -238,7 +235,7 @@ class StreamingJobProgressListenerSuite extends TestSuiteBase with Matchers { listener.onJobStart(jobStart4) // onBatchCompleted - val batchInfoCompleted = BatchInfo(Time(1000), receivedBlockInfo, 1000, Some(2000), None) + val batchInfoCompleted = BatchInfo(Time(1000), streamIdToNumRecords, 1000, Some(2000), None) listener.onBatchCompleted(StreamingListenerBatchCompleted(batchInfoCompleted)) } From d5d86f642c3346d70d0c4ce60374871e923f5be7 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Mon, 4 May 2015 16:15:32 -0700 Subject: [PATCH 24/31] Fix incorrect lastErrorTime --- .../spark/streaming/scheduler/ReceiverTracker.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverTracker.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverTracker.scala index e1d80af9ac062..f6852bd26509d 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverTracker.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverTracker.scala @@ -154,12 +154,16 @@ class ReceiverTracker(ssc: StreamingContext, skipReceiverLaunch: Boolean = false private def deregisterReceiver(streamId: Int, message: String, error: String) { val newReceiverInfo = receiverInfo.get(streamId) match { case Some(oldInfo) => + val lastErrorTime = + if (error == null || error == "") -1 else ssc.scheduler.clock.getTimeMillis() oldInfo.copy(endpoint = null, active = false, lastErrorMessage = message, - lastError = error, lastErrorTime = ssc.scheduler.clock.getTimeMillis()) + lastError = error, lastErrorTime = lastErrorTime) case None => logWarning("No prior receiver info") + val lastErrorTime = + if (error == null || error == "") -1 else ssc.scheduler.clock.getTimeMillis() ReceiverInfo(streamId, "", null, false, "", lastErrorMessage = message, - lastError = error, lastErrorTime = ssc.scheduler.clock.getTimeMillis()) + lastError = error, lastErrorTime = lastErrorTime) } receiverInfo -= streamId listenerBus.post(StreamingListenerReceiverStopped(newReceiverInfo)) From e275e23266e49c29a27fcbf41f90e57c739f4405 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Mon, 4 May 2015 16:58:44 -0700 Subject: [PATCH 25/31] Move time related methods to Streaming's UIUtils --- .../scala/org/apache/spark/ui/UIUtils.scala | 37 ---------- .../spark/streaming/ui/StreamingPage.scala | 39 ++++------ .../apache/spark/streaming/ui/UIUtils.scala | 74 +++++++++++++++++++ .../spark/streaming}/ui/UIUtilsSuite.scala | 19 ++++- 4 files changed, 104 insertions(+), 65 deletions(-) create mode 100644 streaming/src/main/scala/org/apache/spark/streaming/ui/UIUtils.scala rename {core/src/test/scala/org/apache/spark => streaming/src/test/scala/org/apache/spark/streaming}/ui/UIUtilsSuite.scala (71%) diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 3a2dadb5661a9..2ddc55d56567f 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -35,43 +35,6 @@ private[spark] object UIUtils extends Logging { override def initialValue(): SimpleDateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss") } - /** - * Return the short string for a `TimeUnit`. - */ - def shortTimeUnitString(unit: TimeUnit): String = unit match { - case TimeUnit.NANOSECONDS => "ns" - case TimeUnit.MICROSECONDS => "us" - case TimeUnit.MILLISECONDS => "ms" - case TimeUnit.SECONDS => "sec" - case TimeUnit.MINUTES => "min" - case TimeUnit.HOURS => "hrs" - case TimeUnit.DAYS => "days" - } - - /** - * Find the best `TimeUnit` for converting milliseconds to a friendly string. Return the value - * after converting, also with its TimeUnit. - */ - def normalizeDuration(milliseconds: Long): (Double, TimeUnit) = { - if (milliseconds < 1000) { - return (milliseconds, TimeUnit.MILLISECONDS) - } - val seconds = milliseconds.toDouble / 1000 - if (seconds < 60) { - return (seconds, TimeUnit.SECONDS) - } - val minutes = seconds / 60 - if (minutes < 60) { - return (minutes, TimeUnit.MINUTES) - } - val hours = minutes / 60 - if (hours < 24) { - return (hours, TimeUnit.HOURS) - } - val days = hours / 24 - (days, TimeUnit.DAYS) - } - def formatDate(date: Date): String = dateFormat.get.format(date) def formatDate(timestamp: Long): String = dateFormat.get.format(new Date(timestamp)) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index 5526c85c181c0..f8ed28addabb0 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -27,7 +27,7 @@ import scala.xml.{Node, Unparsed} import org.apache.spark.Logging import org.apache.spark.ui._ -import org.apache.spark.ui.UIUtils._ +import org.apache.spark.ui.{UIUtils => SparkUIUtils} /** * @param timelineDivId the timeline `id` used in the html `div` tag @@ -103,13 +103,13 @@ private[ui] class MillisecondsStatUIData(data: Seq[(Long, Long)]) { * Converting the original data as per `unit`. */ def timelineData(unit: TimeUnit): Seq[(Long, Double)] = - data.map(x => x._1 -> StreamingPage.convertToTimeUnit(x._2, unit)) + data.map(x => x._1 -> UIUtils.convertToTimeUnit(x._2, unit)) /** * Converting the original data as per `unit`. */ def histogramData(unit: TimeUnit): Seq[Double] = - data.map(x => StreamingPage.convertToTimeUnit(x._2, unit)) + data.map(x => UIUtils.convertToTimeUnit(x._2, unit)) val avg: Option[Long] = if (data.isEmpty) None else Some(data.map(_._2).sum / data.size) @@ -149,7 +149,7 @@ private[ui] class StreamingPage(parent: StreamingTab) generateStatTable() ++ generateBatchListTables() } - UIUtils.headerSparkPage("Streaming Statistics", content, parent, Some(5000)) + SparkUIUtils.headerSparkPage("Streaming Statistics", content, parent, Some(5000)) } /** @@ -157,9 +157,9 @@ private[ui] class StreamingPage(parent: StreamingTab) */ private def generateLoadResources(): Seq[Node] = { // scalastyle:off - - - + + + // scalastyle:on } @@ -168,15 +168,15 @@ private[ui] class StreamingPage(parent: StreamingTab) val timeSinceStart = System.currentTimeMillis() - startTime
Running batches of - {formatDurationVerbose(listener.batchDuration)} + {SparkUIUtils.formatDurationVerbose(listener.batchDuration)} for - {formatDurationVerbose(timeSinceStart)} + {SparkUIUtils.formatDurationVerbose(timeSinceStart)} since - {UIUtils.formatDate(startTime)} + {SparkUIUtils.formatDate(startTime)}

@@ -247,7 +247,7 @@ private[ui] class StreamingPage(parent: StreamingTab) | document.title, window.location.pathname + '?show-streams-detail=' + status);""" .stripMargin.replaceAll("\\n", "") // it must be only one single line - val batchInterval = StreamingPage.convertToTimeUnit(listener.batchDuration, normalizedUnit) + val batchInterval = UIUtils.convertToTimeUnit(listener.batchDuration, normalizedUnit) val jsCollector = new JsCollector @@ -423,7 +423,7 @@ private[ui] class StreamingPage(parent: StreamingTab) if (msg.size > 100) msg.take(97) + "..." else msg }.getOrElse(emptyCell) val receiverLastErrorTime = receiverInfo.map { - r => if (r.lastErrorTime < 0) "-" else UIUtils.formatDate(r.lastErrorTime) + r => if (r.lastErrorTime < 0) "-" else SparkUIUtils.formatDate(r.lastErrorTime) }.getOrElse(emptyCell) val receivedRecords = new EventRateUIData(eventRates) @@ -491,22 +491,9 @@ private[ui] object StreamingPage { * Returns a human-readable string representing a duration such as "5 second 35 ms" */ def formatDurationOption(msOption: Option[Long]): String = { - msOption.map(formatDurationVerbose).getOrElse(emptyCell) + msOption.map(SparkUIUtils.formatDurationVerbose).getOrElse(emptyCell) } - /** - * Convert `milliseconds` to the specified `unit`. We cannot use `TimeUnit.convert` because it - * will discard the fractional part. - */ - def convertToTimeUnit(milliseconds: Long, unit: TimeUnit): Double = unit match { - case TimeUnit.NANOSECONDS => milliseconds * 1000 * 1000 - case TimeUnit.MICROSECONDS => milliseconds * 1000 - case TimeUnit.MILLISECONDS => milliseconds - case TimeUnit.SECONDS => milliseconds / 1000.0 - case TimeUnit.MINUTES => milliseconds / 1000.0 / 60.0 - case TimeUnit.HOURS => milliseconds / 1000.0 / 60.0 / 60.0 - case TimeUnit.DAYS => milliseconds / 1000.0 / 60.0 / 60.0 / 24.0 - } } /** diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/UIUtils.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/UIUtils.scala new file mode 100644 index 0000000000000..c206f973b2c66 --- /dev/null +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/UIUtils.scala @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.streaming.ui + +import java.util.concurrent.TimeUnit + +object UIUtils { + + /** + * Return the short string for a `TimeUnit`. + */ + def shortTimeUnitString(unit: TimeUnit): String = unit match { + case TimeUnit.NANOSECONDS => "ns" + case TimeUnit.MICROSECONDS => "us" + case TimeUnit.MILLISECONDS => "ms" + case TimeUnit.SECONDS => "sec" + case TimeUnit.MINUTES => "min" + case TimeUnit.HOURS => "hrs" + case TimeUnit.DAYS => "days" + } + + /** + * Find the best `TimeUnit` for converting milliseconds to a friendly string. Return the value + * after converting, also with its TimeUnit. + */ + def normalizeDuration(milliseconds: Long): (Double, TimeUnit) = { + if (milliseconds < 1000) { + return (milliseconds, TimeUnit.MILLISECONDS) + } + val seconds = milliseconds.toDouble / 1000 + if (seconds < 60) { + return (seconds, TimeUnit.SECONDS) + } + val minutes = seconds / 60 + if (minutes < 60) { + return (minutes, TimeUnit.MINUTES) + } + val hours = minutes / 60 + if (hours < 24) { + return (hours, TimeUnit.HOURS) + } + val days = hours / 24 + (days, TimeUnit.DAYS) + } + + /** + * Convert `milliseconds` to the specified `unit`. We cannot use `TimeUnit.convert` because it + * will discard the fractional part. + */ + def convertToTimeUnit(milliseconds: Long, unit: TimeUnit): Double = unit match { + case TimeUnit.NANOSECONDS => milliseconds * 1000 * 1000 + case TimeUnit.MICROSECONDS => milliseconds * 1000 + case TimeUnit.MILLISECONDS => milliseconds + case TimeUnit.SECONDS => milliseconds / 1000.0 + case TimeUnit.MINUTES => milliseconds / 1000.0 / 60.0 + case TimeUnit.HOURS => milliseconds / 1000.0 / 60.0 / 60.0 + case TimeUnit.DAYS => milliseconds / 1000.0 / 60.0 / 60.0 / 24.0 + } +} diff --git a/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ui/UIUtilsSuite.scala similarity index 71% rename from core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala rename to streaming/src/test/scala/org/apache/spark/streaming/ui/UIUtilsSuite.scala index 636b2c389e05e..6df1a63ab2e37 100644 --- a/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/ui/UIUtilsSuite.scala @@ -15,8 +15,7 @@ * limitations under the License. */ - -package org.apache.spark.ui +package org.apache.spark.streaming.ui import java.util.concurrent.TimeUnit @@ -49,4 +48,20 @@ class UIUtilsSuite extends FunSuite with Matchers{ time should be (expectedTime +- 1E-6) unit should be (expectedUnit) } + + test("convertToTimeUnit") { + verifyConvertToTimeUnit(60.0 * 1000 * 1000 * 1000, 60 * 1000, TimeUnit.NANOSECONDS) + verifyConvertToTimeUnit(60.0 * 1000 * 1000, 60 * 1000, TimeUnit.MICROSECONDS) + verifyConvertToTimeUnit(60 * 1000, 60 * 1000, TimeUnit.MILLISECONDS) + verifyConvertToTimeUnit(60, 60 * 1000, TimeUnit.SECONDS) + verifyConvertToTimeUnit(1, 60 * 1000, TimeUnit.MINUTES) + verifyConvertToTimeUnit(1.0 / 60, 60 * 1000, TimeUnit.HOURS) + verifyConvertToTimeUnit(1.0 / 60 / 24, 60 * 1000, TimeUnit.DAYS) + } + + private def verifyConvertToTimeUnit( + expectedTime: Double, milliseconds: Long, unit: TimeUnit): Unit = { + val convertedTime = UIUtils.convertToTimeUnit(milliseconds, unit) + convertedTime should be (expectedTime +- 1E-6) + } } From ff0bff81072b5acf04f75424a5d5c30d512b826b Mon Sep 17 00:00:00 2001 From: zsxwing Date: Mon, 4 May 2015 17:02:03 -0700 Subject: [PATCH 26/31] Make InputDStream.name private[streaming] --- .../scala/org/apache/spark/streaming/dstream/InputDStream.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/InputDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/InputDStream.scala index 2e9308f25d7b6..9716adb62817c 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/InputDStream.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/InputDStream.scala @@ -47,7 +47,7 @@ abstract class InputDStream[T: ClassTag] (@transient ssc_ : StreamingContext) /** * The name of this InputDStream. By default, it's the class name with its id. */ - def name: String = s"${getClass.getSimpleName}-$id" + private[streaming] def name: String = s"${getClass.getSimpleName}-$id" /** * Checks whether the 'time' is valid wrt slideDuration for generating RDD. From b3f303e45bef1e7b4b23a0d8773eb74b9acb6f2f Mon Sep 17 00:00:00 2001 From: zsxwing Date: Mon, 4 May 2015 17:14:20 -0700 Subject: [PATCH 27/31] Add comments for unclear classes and methods --- .../spark/streaming/ui/StreamingJobProgressListener.scala | 3 ++- .../org/apache/spark/streaming/ui/StreamingPage.scala | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala index 0b6ca032bdd4f..68e8ce98945e0 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala @@ -186,7 +186,8 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) def streamIds: Seq[Int] = ssc.graph.getInputStreams().map(_.id) /** - * Return all of the event rates for each InputDStream in each batch. + * Return all of the event rates for each InputDStream in each batch. The key of the return value + * is the stream id, and the value is a sequence of batch time with its event rate. */ def receivedEventRateWithBatchTime: Map[Int, Seq[(Long, Double)]] = synchronized { val _retainedBatches = retainedBatches diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index f8ed28addabb0..4c78f4fadc4d6 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -30,6 +30,8 @@ import org.apache.spark.ui._ import org.apache.spark.ui.{UIUtils => SparkUIUtils} /** + * A helper class to generate JavaScript and HTML for both timeline and histogram graphs. + * * @param timelineDivId the timeline `id` used in the html `div` tag * @param histogramDivId the timeline `id` used in the html `div` tag * @param data the data for the graph @@ -95,6 +97,9 @@ private[ui] class GraphUIData( } /** + * A helper class for "scheduling delay", "processing time" and "total delay" to generate data that + * will be used in the timeline and histogram graphs. + * * @param data (batchTime, milliseconds). "milliseconds" is something like "processing time". */ private[ui] class MillisecondsStatUIData(data: Seq[(Long, Long)]) { @@ -119,6 +124,9 @@ private[ui] class MillisecondsStatUIData(data: Seq[(Long, Long)]) { } /** + * A helper class for "input rate" to generate data that will be used in the timeline and histogram + * graphs. + * * @param data (batchTime, event-rate). */ private[ui] class EventRateUIData(val data: Seq[(Long, Double)]) { From d457277610e184da3da071875a0fe1a9c7e10607 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Mon, 4 May 2015 17:04:41 -0700 Subject: [PATCH 28/31] Fix UIUtils in BatchPage --- .../spark/streaming/ui/AllBatchesTable.scala | 10 ++++----- .../apache/spark/streaming/ui/BatchPage.scala | 22 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala index 6f8fbbdc36e6d..7975b0abbe4d8 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala @@ -19,7 +19,7 @@ package org.apache.spark.streaming.ui import scala.xml.Node -import org.apache.spark.ui.UIUtils +import org.apache.spark.ui.{UIUtils => SparkUIUtils} private[ui] abstract class BatchTableBase(tableId: String) { @@ -32,12 +32,12 @@ private[ui] abstract class BatchTableBase(tableId: String) { protected def baseRow(batch: BatchUIData): Seq[Node] = { val batchTime = batch.batchTime.milliseconds - val formattedBatchTime = UIUtils.formatDate(batch.batchTime.milliseconds) + val formattedBatchTime = SparkUIUtils.formatDate(batch.batchTime.milliseconds) val eventCount = batch.numRecords val schedulingDelay = batch.schedulingDelay - val formattedSchedulingDelay = schedulingDelay.map(UIUtils.formatDuration).getOrElse("-") + val formattedSchedulingDelay = schedulingDelay.map(SparkUIUtils.formatDuration).getOrElse("-") val processingTime = batch.processingDelay - val formattedProcessingTime = processingTime.map(UIUtils.formatDuration).getOrElse("-") + val formattedProcessingTime = processingTime.map(SparkUIUtils.formatDuration).getOrElse("-") - + } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index 4c78f4fadc4d6..ecbebe5c6c97b 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -347,7 +347,7 @@ private[ui] class StreamingPage(parent: StreamingTab) diff --git a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala index 7d6352ed70eec..2211f62383ce8 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala @@ -106,7 +106,7 @@ class UISeleniumSuite val statTableCells = findAll(cssSelector("#stat-table td")).map(_.text).toSeq statTableCells.exists(_.contains("Input Rate")) should be (true) - statTableCells.exists(_.contains("Streaming Scheduling Delay")) should be (true) + statTableCells.exists(_.contains("Scheduling Delay")) should be (true) statTableCells.exists(_.contains("Processing Time")) should be (true) statTableCells.exists(_.contains("Total Delay")) should be (true) @@ -116,11 +116,11 @@ class UISeleniumSuite h4Text.exists(_.matches("Completed Batches \\(last \\d+ out of \\d+\\)")) should be (true) findAll(cssSelector("""#active-batches-table th""")).map(_.text).toSeq should be { - List("Batch Time", "Input Size", "Streaming Scheduling Delay", "Processing Time", + List("Batch Time", "Input Size", "Scheduling Delay", "Processing Time", "Status") } findAll(cssSelector("""#completed-batches-table th""")).map(_.text).toSeq should be { - List("Batch Time", "Input Size", "Streaming Scheduling Delay", "Processing Time", + List("Batch Time", "Input Size", "Scheduling Delay", "Processing Time", "Total Delay") } From cd03424816af80732387ab1c4ed0c4d7d8752f60 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Mon, 4 May 2015 17:37:21 -0700 Subject: [PATCH 30/31] Fix .rat-excludes --- .rat-excludes | 1 - 1 file changed, 1 deletion(-) diff --git a/.rat-excludes b/.rat-excludes index eee6a2900eb81..dccf2db8055ce 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -34,7 +34,6 @@ d3.min.js dagre-d3.min.js graphlib-dot.min.js sorttable.js -d3.min.js vis.min.js vis.min.css vis.map From deacc3fccbc2c28ab02dae36152884c6904a8d03 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Mon, 4 May 2015 22:58:50 -0700 Subject: [PATCH 31/31] Remove unused import --- core/src/main/scala/org/apache/spark/ui/UIUtils.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 1a6318827f750..2f3fb181e4026 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -18,7 +18,6 @@ package org.apache.spark.ui import java.text.SimpleDateFormat -import java.util.concurrent.TimeUnit import java.util.{Locale, Date} import scala.xml.{Node, Text}
TimelinesDistributions
TimelinesHistograms
+
{Unparsed(BLACK_RIGHT_TRIANGLE_HTML)} Input Rate @@ -331,7 +331,7 @@ private[ui] class StreamingPage(parent: StreamingTab) generateInputReceiverRow(jsCollector, receiverId, distribution, minX, maxX, minY, maxY) }.foldLeft[Seq[Node]](Nil)(_ ++ _) - +
@@ -389,7 +389,7 @@ private[ui] class StreamingPage(parent: StreamingTab) "events/sec").toHtml(jsCollector) - - - - - - } - - /** - * Returns a human-readable string representing a duration such as "5 second 35 ms" - */ - private def formatDurationOption(msOption: Option[Long]): String = { - msOption.map(formatDurationVerbose).getOrElse(emptyCell) + + + + } private def generateBatchListTables(): Seq[Node] = { @@ -472,9 +509,13 @@ private[ui] object StreamingPage { msOption.map(formatDurationVerbose).getOrElse(emptyCell) } + /** + * Convert `milliseconds` to the specified `unit`. We cannot use `TimeUnit.convert` because it + * will discard the fractional part. + */ def convertToTimeUnit(milliseconds: Long, unit: TimeUnit): Double = unit match { - case TimeUnit.NANOSECONDS => milliseconds * 1000 * 1000 // not used yet - case TimeUnit.MICROSECONDS => milliseconds * 1000 // not used yet + case TimeUnit.NANOSECONDS => milliseconds * 1000 * 1000 + case TimeUnit.MICROSECONDS => milliseconds * 1000 case TimeUnit.MILLISECONDS => milliseconds case TimeUnit.SECONDS => milliseconds / 1000.0 case TimeUnit.MINUTES => milliseconds / 1000.0 / 60.0 diff --git a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala index a2f4f3fe19897..84c10ae6dc462 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala @@ -80,10 +80,12 @@ class UISeleniumSuite h4Text should contain("Completed Batches (last 0 out of 0)") findAll(cssSelector("""#active-batches-table th""")).map(_.text).toSeq should be { - List("Batch Time", "Input Size", "Scheduling Delay", "Processing Time", "Status") + List("Batch Time", "Input Size", "Streaming Scheduling Delay", "Processing Time", + "Status") } findAll(cssSelector("""#completed-batches-table th""")).map(_.text).toSeq should be { - List("Batch Time", "Input Size", "Scheduling Delay", "Processing Time", "Total Delay") + List("Batch Time", "Input Size", "Streaming Scheduling Delay", "Processing Time", + "Total Delay") } } From fd03ad0801ecc1e5229ce5f54b97299a76f55206 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Wed, 29 Apr 2015 21:54:51 -0700 Subject: [PATCH 17/31] Add a test to verify no memory leak --- .../StreamingJobProgressListenerSuite.scala | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala index 76bddfdb756f5..5103187b68aa9 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/ui/StreamingJobProgressListenerSuite.scala @@ -208,4 +208,51 @@ class StreamingJobProgressListenerSuite extends TestSuiteBase with Matchers { (listener.waitingBatches.size + listener.runningBatches.size + listener.retainedCompletedBatches.size + 10) } + + test("detect memory leak") { + val ssc = setupStreams(input, operation) + val listener = new StreamingJobProgressListener(ssc) + + val limit = ssc.conf.getInt("spark.streaming.ui.retainedBatches", 1000) + + for (_ <- 0 until 2 * limit) { + val receivedBlockInfo = Map( + 0 -> Array(ReceivedBlockInfo(0, 100, null), ReceivedBlockInfo(0, 200, null)), + 1 -> Array(ReceivedBlockInfo(1, 300, null)) + ) + + // onBatchSubmitted + val batchInfoSubmitted = BatchInfo(Time(1000), receivedBlockInfo, 1000, None, None) + listener.onBatchSubmitted(StreamingListenerBatchSubmitted(batchInfoSubmitted)) + + // onBatchStarted + val batchInfoStarted = BatchInfo(Time(1000), receivedBlockInfo, 1000, Some(2000), None) + listener.onBatchStarted(StreamingListenerBatchStarted(batchInfoStarted)) + + // onJobStart + val jobStart1 = createJobStart(Time(1000), outputOpId = 0, jobId = 0) + listener.onJobStart(jobStart1) + + val jobStart2 = createJobStart(Time(1000), outputOpId = 0, jobId = 1) + listener.onJobStart(jobStart2) + + val jobStart3 = createJobStart(Time(1000), outputOpId = 1, jobId = 0) + listener.onJobStart(jobStart3) + + val jobStart4 = createJobStart(Time(1000), outputOpId = 1, jobId = 1) + listener.onJobStart(jobStart4) + + // onBatchCompleted + val batchInfoCompleted = BatchInfo(Time(1000), receivedBlockInfo, 1000, Some(2000), None) + listener.onBatchCompleted(StreamingListenerBatchCompleted(batchInfoCompleted)) + } + + listener.waitingBatches.size should be (0) + listener.runningBatches.size should be (0) + listener.retainedCompletedBatches.size should be (limit) + listener.batchTimeToOutputOpIdSparkJobIdPair.size() should be <= + (listener.waitingBatches.size + listener.runningBatches.size + + listener.retainedCompletedBatches.size + 10) + } + } From a2972e988053168b7299f40d16cd2896a6eb8110 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Wed, 29 Apr 2015 22:53:08 -0700 Subject: [PATCH 18/31] Add some ui tests for StreamingPage --- .../apache/spark/streaming/ui/StreamingPage.scala | 2 +- .../apache/spark/streaming/UISeleniumSuite.scala | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index 8229b7417eac5..e0315da299129 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -322,7 +322,7 @@ private[ui] class StreamingPage(parent: StreamingTab) val numActiveBatches = batchTimes.length - numCompletedBatches val table = // scalastyle:off -
+
{receiverName}
From c81a1ee39b6ffe5e87c67d1d7dc87d57b5b197d1 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Tue, 28 Apr 2015 13:21:06 -0700 Subject: [PATCH 13/31] Change time unit in the graphs automatically --- .../apache/spark/ui/static/streaming-page.css | 6 +- .../apache/spark/ui/static/streaming-page.js | 10 +- .../scala/org/apache/spark/ui/UIUtils.scala | 34 ++++++ .../spark/streaming/ui/StreamingPage.scala | 110 +++++++++++------- 4 files changed, 107 insertions(+), 53 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css index efef31f8ea9e1..4ace2043d4129 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css @@ -32,18 +32,18 @@ .line { fill: none; - stroke: steelblue; + stroke: #0088cc; stroke-width: 1.5px; } .bar rect { - fill: steelblue; + fill: #0088cc; shape-rendering: crispEdges; } .bar rect:hover { //fill: rgb(49, 91, 125); - fill: rgb(100, 185, 255); + fill: #005580; } .timeline { diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index e5f6f3eba3b76..2c1b9c1809bd0 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -73,7 +73,8 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(function(d) { return timeFormat[d]; }); - var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5); + var formatYValue = d3.format(",.2f"); + var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5).tickFormat(formatYValue); var line = d3.svg.line() .x(function(d) { return x(d.x); }) @@ -117,7 +118,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { .attr("cy", function(d) { return y(d.y); }) .attr("r", function(d) { return 3; }) .on('mouseover', function(d) { - var tip = d.y + " " + unitY + " at " + timeFormat[d.x]; + var tip = formatYValue(d.y) + " " + unitY + " at " + timeFormat[d.x]; showBootstrapTooltip(d3.select(this).node(), tip); //showGraphTooltip(tip, d3.event.pageX + 5, d3.event.pageY - 25); // show the point @@ -210,9 +211,8 @@ function drawDistribution(id, values, minY, maxY, unitY) { } function prepareTimeline(minY, maxY) { - var y = d3.scale.linear().domain([0, maxY]).tickFormat(5); - console.log(y(maxY)); - var numOfChars = y(maxY).length; + var formatYValue = d3.format(",.2f"); + var numOfChars = formatYValue(maxY).length; var maxPx = numOfChars * 8 + 10; // Make sure we have enough space to show the ticks in the y axis of timeline timelineMarginLeft = maxPx > timelineMarginLeft? maxPx : timelineMarginLeft; diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index f07864141a21c..827b945ec2787 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -18,6 +18,7 @@ package org.apache.spark.ui import java.text.SimpleDateFormat +import java.util.concurrent.TimeUnit import java.util.{Locale, Date} import scala.xml.{Node, Text} @@ -34,6 +35,39 @@ private[spark] object UIUtils extends Logging { override def initialValue(): SimpleDateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss") } + /** + * Return the short string for a `TimeUnit`. + */ + def shortTimeUnitString(unit: TimeUnit): String = unit match { + case TimeUnit.NANOSECONDS => "ns" + case TimeUnit.MICROSECONDS => "us" + case TimeUnit.MILLISECONDS => "ms" + case TimeUnit.SECONDS => "s" + case TimeUnit.MINUTES => "min" + case TimeUnit.HOURS => "h" + case TimeUnit.DAYS => "d" + } + + /** + * Find the best `TimeUnit` for converting milliseconds to a friendly string. Return the value after converting with + * the `TimeUnit`. + */ + def normalizeDuration(milliseconds: Long): (Double, TimeUnit) = { + if (milliseconds < 1000) { + return (milliseconds, TimeUnit.MILLISECONDS) + } + val seconds = milliseconds.toDouble / 1000 + if (seconds < 60) { + return (seconds, TimeUnit.SECONDS) + } + val minutes = seconds / 60 + if (minutes < 60) { + return (minutes, TimeUnit.MINUTES) + } + val hours = minutes / 60 + (hours, TimeUnit.HOURS) + } + def formatDate(date: Date): String = dateFormat.get.format(date) def formatDate(timestamp: Long): String = dateFormat.get.format(new Date(timestamp)) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index b7185a269d4c6..2fe9fbd8a2811 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -19,6 +19,7 @@ package org.apache.spark.streaming.ui import java.text.SimpleDateFormat import java.util.Date +import java.util.concurrent.TimeUnit import javax.servlet.http.HttpServletRequest import scala.collection.mutable.ArrayBuffer @@ -38,8 +39,8 @@ import org.apache.spark.util.Distribution * @param maxY the max value of Y axis * @param unitY the unit of Y axis */ -private[ui] case class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: Long, maxX: Long, - minY: Long, maxY: Long, unitY: String) { +private[ui] class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: Long, maxX: Long, + minY: Double, maxY: Double, unitY: String) { def toHtml(jsCollector: JsCollector): Seq[Node] = { val jsForData = data.map { case (x, y) => @@ -60,8 +61,8 @@ private[ui] case class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: * @param maxY the max value of Y axis * @param unitY the unit of Y axis */ -private[ui] case class DistributionUIData( - divId: String, data: Seq[_], minY: Long, maxY: Long, unitY: String) { +private[ui] class DistributionUIData( + divId: String, data: Seq[_], minY: Double, maxY: Double, unitY: String) { def toHtml(jsCollector: JsCollector): Seq[Node] = { val jsForData = data.mkString("[", ",", "]") @@ -72,7 +73,11 @@ private[ui] case class DistributionUIData( } } -private[ui] case class MillisecondsStatUIData(data: Seq[(Long, Long)]) { +private[ui] class MillisecondsStatUIData(data: Seq[(Long, Long)]) { + + def timelineData(unit: TimeUnit) = data.map(x => x._1 -> StreamingPage.convertToTimeUnit(x._2, unit)) + + def distributionData(unit: TimeUnit) = data.map(x => StreamingPage.convertToTimeUnit(x._2, unit)) val avg: Option[Long] = if (data.isEmpty) None else Some(data.map(_._2).sum / data.size) @@ -81,7 +86,7 @@ private[ui] case class MillisecondsStatUIData(data: Seq[(Long, Long)]) { val max: Option[Long] = if (data.isEmpty) None else Some(data.map(_._2).max) } -private[ui] case class DoubleStatUIData(data: Seq[(Long, Double)]) { +private[ui] class DoubleStatUIData(val data: Seq[(Long, Double)]) { val avg: Option[Double] = if (data.isEmpty) None else Some(data.map(_._2).sum / data.size) @@ -158,17 +163,17 @@ private[ui] class StreamingPage(parent: StreamingTab) val minBatchTime = if (batchTimes.isEmpty) startTime else batchTimes.min val maxBatchTime = if (batchTimes.isEmpty) startTime else batchTimes.max - val eventRateForAllReceivers = DoubleStatUIData(batchInfos.map { batchInfo => + val eventRateForAllReceivers = new DoubleStatUIData(batchInfos.map { batchInfo => (batchInfo.batchTime.milliseconds, batchInfo.numRecords * 1000.0 / listener.batchDuration) }) - val schedulingDelay = MillisecondsStatUIData(batchInfos.flatMap { batchInfo => + val schedulingDelay = new MillisecondsStatUIData(batchInfos.flatMap { batchInfo => batchInfo.schedulingDelay.map(batchInfo.batchTime.milliseconds -> _) }) - val processingTime = MillisecondsStatUIData(batchInfos.flatMap { batchInfo => + val processingTime = new MillisecondsStatUIData(batchInfos.flatMap { batchInfo => batchInfo.processingDelay.map(batchInfo.batchTime.milliseconds -> _) }) - val totalDelay = MillisecondsStatUIData(batchInfos.flatMap { batchInfo => + val totalDelay = new MillisecondsStatUIData(batchInfos.flatMap { batchInfo => batchInfo.totalDelay.map(batchInfo.batchTime.milliseconds -> _) }) @@ -176,12 +181,13 @@ private[ui] class StreamingPage(parent: StreamingTab) // Use the max value of "schedulingDelay", "processingTime", and "totalDelay" to make the // Y axis ranges same. - val maxTime = + val _maxTime = (for (m1 <- schedulingDelay.max; m2 <- processingTime.max; m3 <- totalDelay.max) yield m1 max m2 max m3).getOrElse(0L) - List(1, 2, 3).sum // Should start at 0 val minTime = 0L + val (maxTime, unit) = UIUtils.normalizeDuration(_maxTime) + val formattedUnit = UIUtils.shortTimeUnitString(unit) // Use the max input rate for all receivers' graphs to make the Y axis ranges same. // If it's not an integral number, just use its ceil integral number. @@ -196,7 +202,7 @@ private[ui] class StreamingPage(parent: StreamingTab) |else $$(this).html('$BLACK_RIGHT_TRIANGLE_HTML');""".stripMargin.replaceAll("\\n", "") val timelineDataForEventRateOfAllReceivers = - TimelineUIData( + new TimelineUIData( "all-receiver-events-timeline", eventRateForAllReceivers.data, minBatchTime, @@ -206,7 +212,7 @@ private[ui] class StreamingPage(parent: StreamingTab) "events/sec").toHtml(jsCollector) val distributionDataForEventRateOfAllReceivers = - DistributionUIData( + new DistributionUIData( "all-receiver-events-distribution", eventRateForAllReceivers.data.map(_._2), minEventRate, @@ -214,73 +220,75 @@ private[ui] class StreamingPage(parent: StreamingTab) "events/sec").toHtml(jsCollector) val timelineDataForSchedulingDelay = - TimelineUIData( + new TimelineUIData( "scheduling-delay-timeline", - schedulingDelay.data, + schedulingDelay.timelineData(unit), minBatchTime, maxBatchTime, minTime, maxTime, - "ms").toHtml(jsCollector) + formattedUnit).toHtml(jsCollector) val distributionDataForSchedulingDelay = - DistributionUIData( + new DistributionUIData( "scheduling-delay-distribution", - schedulingDelay.data.map(_._2), + schedulingDelay.distributionData(unit), minTime, maxTime, - "ms").toHtml(jsCollector) + formattedUnit).toHtml(jsCollector) val timelineDataForProcessingTime = - TimelineUIData( + new TimelineUIData( "processing-time-timeline", - processingTime.data, + processingTime.timelineData(unit), minBatchTime, maxBatchTime, minTime, maxTime, - "ms").toHtml(jsCollector) + formattedUnit).toHtml(jsCollector) val distributionDataForProcessingTime = - DistributionUIData( + new DistributionUIData( "processing-time-distribution", - processingTime.data.map(_._2), + processingTime.distributionData(unit), minTime, maxTime, - "ms").toHtml(jsCollector) + formattedUnit).toHtml(jsCollector) val timelineDataForTotalDelay = - TimelineUIData( + new TimelineUIData( "total-delay-timeline", - totalDelay.data, + totalDelay.timelineData(unit), minBatchTime, maxBatchTime, minTime, maxTime, - "ms").toHtml(jsCollector) + formattedUnit).toHtml(jsCollector) val distributionDataForTotalDelay = - DistributionUIData( + new DistributionUIData( "total-delay-distribution", - totalDelay.data.map(_._2), + totalDelay.distributionData(unit), minTime, maxTime, - "ms").toHtml(jsCollector) + formattedUnit).toHtml(jsCollector) val table = // scalastyle:off - + - @@ -325,8 +333,8 @@ private[ui] class StreamingPage(parent: StreamingTab) jsCollector: JsCollector, minX: Long, maxX: Long, - minY: Long, - maxY: Long): Seq[Node] = { + minY: Double, + maxY: Double): Seq[Node] = { val content = listener.receivedRecordsDistributions.map { case (receiverId, distribution) => generateInputReceiverRow(jsCollector, receiverId, distribution, minX, maxX, minY, maxY) }.foldLeft[Seq[Node]](Nil)(_ ++ _) @@ -334,10 +342,10 @@ private[ui] class StreamingPage(parent: StreamingTab)
TimelinesHistograms
TimelinesHistograms
+ +
{Unparsed(BLACK_RIGHT_TRIANGLE_HTML)} Input Rate
Avg: {eventRateForAllReceivers.formattedAvg} events/sec
+
{timelineDataForEventRateOfAllReceivers} {distributionDataForEventRateOfAllReceivers}
- - - - + + + + @@ -353,8 +361,8 @@ private[ui] class StreamingPage(parent: StreamingTab) distribution: Option[Distribution], minX: Long, maxX: Long, - minY: Long, - maxY: Long): Seq[Node] = { + minY: Double, + maxY: Double): Seq[Node] = { val avgReceiverEvents = distribution.map(_.statCounter.mean.toLong) val receiverInfo = listener.receiverInfo(receiverId) val receiverName = receiverInfo.map(_.name).getOrElse(s"Receiver-$receiverId") @@ -371,7 +379,7 @@ private[ui] class StreamingPage(parent: StreamingTab) val receivedRecords = listener.receivedRecordsWithBatchTime.get(receiverId).getOrElse(Seq()) val timelineForEventRate = - TimelineUIData( + new TimelineUIData( s"receiver-$receiverId-events-timeline", receivedRecords, minX, @@ -381,7 +389,7 @@ private[ui] class StreamingPage(parent: StreamingTab) "events/sec").toHtml(jsCollector) val distributionForEventsRate = - DistributionUIData( + new DistributionUIData( s"receiver-$receiverId-events-distribution", receivedRecords.map(_._2), minY, @@ -389,11 +397,13 @@ private[ui] class StreamingPage(parent: StreamingTab) "events/sec").toHtml(jsCollector) - @@ -449,6 +459,16 @@ private[ui] object StreamingPage { def formatDurationOption(msOption: Option[Long]): String = { msOption.map(formatDurationVerbose).getOrElse(emptyCell) } + + def convertToTimeUnit(milliseconds: Long, unit: TimeUnit): Double = unit match { + case TimeUnit.NANOSECONDS => milliseconds * 1000 * 1000 // not used yet + case TimeUnit.MICROSECONDS => milliseconds * 1000 // not used yet + case TimeUnit.MILLISECONDS => milliseconds + case TimeUnit.SECONDS => milliseconds / 1000.0 + case TimeUnit.MINUTES => milliseconds / 1000.0 / 60.0 + case TimeUnit.HOURS => milliseconds / 1000.0 / 60.0 / 60.0 + case TimeUnit.DAYS => milliseconds / 1000.0 / 60.0 / 60.0 / 24.0 + } } /** From 8e4363c4f7ff8e7d3c056155a58a593903701834 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Tue, 28 Apr 2015 15:07:59 -0700 Subject: [PATCH 14/31] Prepare for the demo --- .../apache/spark/ui/static/streaming-page.css | 3 ++- .../apache/spark/ui/static/streaming-page.js | 21 ++++++++++++++++ .../spark/streaming/ui/StreamingPage.scala | 25 +++++++++++-------- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css index 4ace2043d4129..6b401c6dec76e 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css @@ -43,7 +43,8 @@ .bar rect:hover { //fill: rgb(49, 91, 125); - fill: #005580; + //fill: #005580; + fill: rgb(0, 194, 255); } .timeline { diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index 2c1b9c1809bd0..308ae0416a529 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -223,3 +223,24 @@ function prepareDistribution(values, minY, maxY) { var maxBarSize = d3.max(data, function(d) { return d.y; }); distributionMaxX = maxBarSize > distributionMaxX? maxBarSize : distributionMaxX; } + +function getUrlParameter(sParam) +{ + var sPageURL = window.location.search.substring(1); + var sURLVariables = sPageURL.split('&'); + for (var i = 0; i < sURLVariables.length; i++) + { + var sParameterName = sURLVariables[i].split('='); + if (sParameterName[0] == sParam) + { + return sParameterName[1]; + } + } +} + +$(function() { + if (getUrlParameter("show-receivers-detail") == "true") { + $('#inputs-table').toggle('collapsed'); + $('#triangle').html('▼'); + } +}); diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index 2fe9fbd8a2811..d61662500b83e 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -142,8 +142,9 @@ private[ui] class StreamingPage(parent: StreamingTab) since {UIUtils.formatDate(startTime)} - . + +
} private def generateTimeMap(times: Seq[Long]): Seq[Node] = { @@ -197,9 +198,11 @@ private[ui] class StreamingPage(parent: StreamingTab) // JavaScript to show/hide the receiver sub table. val triangleJs = s"""$$('#inputs-table').toggle('collapsed'); - |if ($$(this).html() == '$BLACK_RIGHT_TRIANGLE_HTML') - |$$(this).html('$BLACK_DOWN_TRIANGLE_HTML'); - |else $$(this).html('$BLACK_RIGHT_TRIANGLE_HTML');""".stripMargin.replaceAll("\\n", "") + |var status = false; + |if ($$(this).html() == '$BLACK_RIGHT_TRIANGLE_HTML') { + |$$(this).html('$BLACK_DOWN_TRIANGLE_HTML');status = true;} + |else {$$(this).html('$BLACK_RIGHT_TRIANGLE_HTML');status = false;} + |window.history.pushState('', document.title, window.location.pathname + '?show-receivers-detail=' + status);""".stripMargin.replaceAll("\\n", "") val timelineDataForEventRateOfAllReceivers = new TimelineUIData( @@ -282,9 +285,9 @@ private[ui] class StreamingPage(parent: StreamingTab) - + - + } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala index 0b5402a6d9fce..682a9e05fec6d 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala @@ -25,7 +25,6 @@ import org.apache.spark.streaming.scheduler.StreamingListenerReceiverStarted import org.apache.spark.streaming.scheduler.StreamingListenerBatchStarted import org.apache.spark.streaming.scheduler.BatchInfo import org.apache.spark.streaming.scheduler.StreamingListenerBatchSubmitted -import org.apache.spark.util.Distribution private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) @@ -39,7 +38,6 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) private var totalReceivedRecords = 0L private var totalProcessedRecords = 0L private val receiverInfos = new HashMap[Int, ReceiverInfo] - private val receiverLastErrorTime = new HashMap[Int, Long] val batchDuration = ssc.graph.batchDuration.milliseconds @@ -52,7 +50,6 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) override def onReceiverError(receiverError: StreamingListenerReceiverError) { synchronized { receiverInfos(receiverError.receiverInfo.streamId) = receiverError.receiverInfo - receiverLastErrorTime(receiverError.receiverInfo.streamId) = System.currentTimeMillis() } } @@ -123,32 +120,8 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) completedBatchInfos.toSeq } - def processingDelayDistribution: Option[Distribution] = synchronized { - extractDistribution(_.processingDelay) - } - - def schedulingDelayDistribution: Option[Distribution] = synchronized { - extractDistribution(_.schedulingDelay) - } - - def totalDelayDistribution: Option[Distribution] = synchronized { - extractDistribution(_.totalDelay) - } - - def receivedRecordsDistributions: Map[Int, Option[Distribution]] = synchronized { - val latestBatchInfos = retainedBatches.reverse.take(batchInfoLimit) - val latestBlockInfos = latestBatchInfos.map(_.receivedBlockInfo) - (0 until numReceivers).map { receiverId => - val blockInfoOfParticularReceiver = latestBlockInfos.map { batchInfo => - batchInfo.get(receiverId).getOrElse(Array.empty) - } - val recordsOfParticularReceiver = blockInfoOfParticularReceiver.map { blockInfo => - // calculate records per second for each batch - blockInfo.map(_.numRecords).sum.toDouble * 1000 / batchDuration - } - val distributionOption = Distribution(recordsOfParticularReceiver) - (receiverId, distributionOption) - }.toMap + def allReceivers: Seq[Int] = synchronized { + receiverInfos.keys.toSeq } def receivedRecordsWithBatchTime: Map[Int, Seq[(Long, Double)]] = synchronized { @@ -184,10 +157,6 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) receiverInfos.get(receiverId) } - def receiverLastErrorTime(receiverId: Int): Option[Long] = synchronized { - receiverLastErrorTime.get(receiverId) - } - def lastCompletedBatch: Option[BatchInfo] = synchronized { completedBatchInfos.sortBy(_.batchTime)(Time.ordering).lastOption } @@ -200,8 +169,4 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) (waitingBatchInfos.values.toSeq ++ runningBatchInfos.values.toSeq ++ completedBatchInfos).sortBy(_.batchTime)(Time.ordering) } - - private def extractDistribution(getMetric: BatchInfo => Option[Long]): Option[Distribution] = { - Distribution(completedBatchInfos.flatMap(getMetric(_)).map(_.toDouble)) - } } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index 4ff77e6c09141..f69fdb010c007 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -28,7 +28,6 @@ import scala.xml.{Node, Unparsed} import org.apache.spark.Logging import org.apache.spark.ui._ import org.apache.spark.ui.UIUtils._ -import org.apache.spark.util.Distribution /** * @param divId the `id` used in the html `div` tag @@ -38,18 +37,22 @@ import org.apache.spark.util.Distribution * @param minY the min value of Y axis * @param maxY the max value of Y axis * @param unitY the unit of Y axis + * @param batchInterval if `batchInterval` is not None, we will draw a line for `batchInterval` in + * the graph */ private[ui] class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: Long, maxX: Long, - minY: Double, maxY: Double, unitY: String, batchTime: Option[Double] = None) { + minY: Double, maxY: Double, unitY: String, batchInterval: Option[Double] = None) { def toHtml(jsCollector: JsCollector): Seq[Node] = { val jsForData = data.map { case (x, y) => s"""{"x": $x, "y": $y}""" }.mkString("[", ",", "]") - jsCollector.addPreparedStatement(s"prepareTimeline($minY, $maxY);") - if (batchTime.isDefined) { + jsCollector.addPreparedStatement(s"registerTimeline($minY, $maxY);") + if (batchInterval.isDefined) { jsCollector.addStatement( - s"drawTimeline('#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY', ${batchTime.get});") + "drawTimeline(" + + s"'#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY', ${batchInterval.get}" + + ");") } else { jsCollector.addStatement( s"drawTimeline('#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY');") @@ -60,31 +63,48 @@ private[ui] class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: Long /** * @param divId the `id` used in the html `div` tag - * @param data the data for the distribution graph + * @param data the data for the histogram graph * @param minY the min value of Y axis * @param maxY the max value of Y axis * @param unitY the unit of Y axis + * @param batchInterval if `batchInterval` is not None, we will draw a line for `batchInterval` in + * the graph */ -private[ui] class DistributionUIData( - divId: String, data: Seq[_], minY: Double, maxY: Double, unitY: String, batchTime: Option[Double] = None) { +private[ui] class HistogramUIData( + divId: String, data: Seq[_], minY: Double, maxY: Double, unitY: String, + batchInterval: Option[Double] = None) { def toHtml(jsCollector: JsCollector): Seq[Node] = { val jsForData = data.mkString("[", ",", "]") - jsCollector.addPreparedStatement(s"prepareDistribution($jsForData, $minY, $maxY);") - if (batchTime.isDefined) { - jsCollector.addStatement(s"drawDistribution('#$divId', $jsForData, $minY, $maxY, '$unitY', ${batchTime.get});") + jsCollector.addPreparedStatement(s"registerHistogram($jsForData, $minY, $maxY);") + if (batchInterval.isDefined) { + jsCollector.addStatement( + "drawHistogram(" + + s"'#$divId', $jsForData, $minY, $maxY, '$unitY', ${batchInterval.get}" + + ");") } else { - jsCollector.addStatement(s"drawDistribution('#$divId', $jsForData, $minY, $maxY, '$unitY');") + jsCollector.addStatement(s"drawHistogram('#$divId', $jsForData, $minY, $maxY, '$unitY');") }
} } +/** + * @param data (batchTime, milliseconds). "milliseconds" is something like "processing time". + */ private[ui] class MillisecondsStatUIData(data: Seq[(Long, Long)]) { - def timelineData(unit: TimeUnit) = data.map(x => x._1 -> StreamingPage.convertToTimeUnit(x._2, unit)) + /** + * Converting the original data as per `unit`. + */ + def timelineData(unit: TimeUnit): Seq[(Long, Double)] = + data.map(x => x._1 -> StreamingPage.convertToTimeUnit(x._2, unit)) - def distributionData(unit: TimeUnit) = data.map(x => StreamingPage.convertToTimeUnit(x._2, unit)) + /** + * Converting the original data as per `unit`. + */ + def histogramData(unit: TimeUnit): Seq[Double] = + data.map(x => StreamingPage.convertToTimeUnit(x._2, unit)) val avg: Option[Long] = if (data.isEmpty) None else Some(data.map(_._2).sum / data.size) @@ -93,7 +113,10 @@ private[ui] class MillisecondsStatUIData(data: Seq[(Long, Long)]) { val max: Option[Long] = if (data.isEmpty) None else Some(data.map(_._2).max) } -private[ui] class DoubleStatUIData(val data: Seq[(Long, Double)]) { +/** + * @param data (batchTime, event-rate). + */ +private[ui] class EventRateUIData(val data: Seq[(Long, Double)]) { val avg: Option[Double] = if (data.isEmpty) None else Some(data.map(_._2).sum / data.size) @@ -154,6 +177,14 @@ private[ui] class StreamingPage(parent: StreamingTab)
} + /** + * Generate a global "timeFormat" dictionary in the JavaScript to store the time and its formatted + * string. Because we cannot specify a timezone in JavaScript, to make sure the server and client + * use the same timezone, we use the "timeFormat" dictionary to format all time values used in the + * graphs. + * + * @param times all time values that will be used in the graphs. + */ private def generateTimeMap(times: Seq[Long]): Seq[Node] = { val dateFormat = new SimpleDateFormat("HH:mm:ss") val js = "var timeFormat = {};\n" + times.map { time => @@ -165,28 +196,26 @@ private[ui] class StreamingPage(parent: StreamingTab) } private def generateStatTable(): Seq[Node] = { - val batchInfos = listener.retainedBatches + val batches = listener.retainedBatches - val batchTimes = batchInfos.map(_.batchTime.milliseconds) + val batchTimes = batches.map(_.batchTime.milliseconds) val minBatchTime = if (batchTimes.isEmpty) startTime else batchTimes.min val maxBatchTime = if (batchTimes.isEmpty) startTime else batchTimes.max - val eventRateForAllReceivers = new DoubleStatUIData(batchInfos.map { batchInfo => + val eventRateForAllReceivers = new EventRateUIData(batches.map { batchInfo => (batchInfo.batchTime.milliseconds, batchInfo.numRecords * 1000.0 / listener.batchDuration) }) - val schedulingDelay = new MillisecondsStatUIData(batchInfos.flatMap { batchInfo => + val schedulingDelay = new MillisecondsStatUIData(batches.flatMap { batchInfo => batchInfo.schedulingDelay.map(batchInfo.batchTime.milliseconds -> _) }) - val processingTime = new MillisecondsStatUIData(batchInfos.flatMap { batchInfo => + val processingTime = new MillisecondsStatUIData(batches.flatMap { batchInfo => batchInfo.processingDelay.map(batchInfo.batchTime.milliseconds -> _) }) - val totalDelay = new MillisecondsStatUIData(batchInfos.flatMap { batchInfo => + val totalDelay = new MillisecondsStatUIData(batches.flatMap { batchInfo => batchInfo.totalDelay.map(batchInfo.batchTime.milliseconds -> _) }) - val jsCollector = new JsCollector - // Use the max value of "schedulingDelay", "processingTime", and "totalDelay" to make the // Y axis ranges same. val _maxTime = @@ -194,8 +223,8 @@ private[ui] class StreamingPage(parent: StreamingTab) m1 max m2 max m3).getOrElse(0L) // Should start at 0 val minTime = 0L - val (maxTime, unit) = UIUtils.normalizeDuration(_maxTime) - val formattedUnit = UIUtils.shortTimeUnitString(unit) + val (maxTime, normalizedUnit) = UIUtils.normalizeDuration(_maxTime) + val formattedUnit = UIUtils.shortTimeUnitString(normalizedUnit) // Use the max input rate for all receivers' graphs to make the Y axis ranges same. // If it's not an integral number, just use its ceil integral number. @@ -209,9 +238,13 @@ private[ui] class StreamingPage(parent: StreamingTab) |if ($$(this).html() == '$BLACK_RIGHT_TRIANGLE_HTML') { |$$(this).html('$BLACK_DOWN_TRIANGLE_HTML');status = true;} |else {$$(this).html('$BLACK_RIGHT_TRIANGLE_HTML');status = false;} - |window.history.pushState('', document.title, window.location.pathname + '?show-receivers-detail=' + status);""".stripMargin.replaceAll("\\n", "") + |window.history.pushState('', + | document.title, window.location.pathname + '?show-receivers-detail=' + status);""" + .stripMargin.replaceAll("\\n", "") // it must be only one single line - var batchTime = StreamingPage.convertToTimeUnit(listener.batchDuration, unit) + val batchInterval = StreamingPage.convertToTimeUnit(listener.batchDuration, normalizedUnit) + + val jsCollector = new JsCollector val timelineDataForEventRateOfAllReceivers = new TimelineUIData( @@ -223,9 +256,9 @@ private[ui] class StreamingPage(parent: StreamingTab) maxEventRate, "events/sec").toHtml(jsCollector) - val distributionDataForEventRateOfAllReceivers = - new DistributionUIData( - "all-receiver-events-distribution", + val histogramDataForEventRateOfAllReceivers = + new HistogramUIData( + "all-receiver-events-histogram", eventRateForAllReceivers.data.map(_._2), minEventRate, maxEventRate, @@ -234,17 +267,17 @@ private[ui] class StreamingPage(parent: StreamingTab) val timelineDataForSchedulingDelay = new TimelineUIData( "scheduling-delay-timeline", - schedulingDelay.timelineData(unit), + schedulingDelay.timelineData(normalizedUnit), minBatchTime, maxBatchTime, minTime, maxTime, formattedUnit).toHtml(jsCollector) - val distributionDataForSchedulingDelay = - new DistributionUIData( - "scheduling-delay-distribution", - schedulingDelay.distributionData(unit), + val histogramDataForSchedulingDelay = + new HistogramUIData( + "scheduling-delay-histogram", + schedulingDelay.histogramData(normalizedUnit), minTime, maxTime, formattedUnit).toHtml(jsCollector) @@ -252,58 +285,63 @@ private[ui] class StreamingPage(parent: StreamingTab) val timelineDataForProcessingTime = new TimelineUIData( "processing-time-timeline", - processingTime.timelineData(unit), + processingTime.timelineData(normalizedUnit), minBatchTime, maxBatchTime, minTime, maxTime, - formattedUnit, Some(batchTime)).toHtml(jsCollector) + formattedUnit, Some(batchInterval)).toHtml(jsCollector) - val distributionDataForProcessingTime = - new DistributionUIData( - "processing-time-distribution", - processingTime.distributionData(unit), + val histogramDataForProcessingTime = + new HistogramUIData( + "processing-time-histogram", + processingTime.histogramData(normalizedUnit), minTime, maxTime, - formattedUnit, Some(batchTime)).toHtml(jsCollector) + formattedUnit, Some(batchInterval)).toHtml(jsCollector) val timelineDataForTotalDelay = new TimelineUIData( "total-delay-timeline", - totalDelay.timelineData(unit), + totalDelay.timelineData(normalizedUnit), minBatchTime, maxBatchTime, minTime, maxTime, formattedUnit).toHtml(jsCollector) - val distributionDataForTotalDelay = - new DistributionUIData( - "total-delay-distribution", - totalDelay.distributionData(unit), + val histogramDataForTotalDelay = + new HistogramUIData( + "total-delay-histogram", + totalDelay.histogramData(normalizedUnit), minTime, maxTime, formattedUnit).toHtml(jsCollector) + val numCompletedBatches = listener.retainedCompletedBatches.size + val numActiveBatches = batchTimes.length - numCompletedBatches val table = // scalastyle:off
Status
Location
Last Error Time
Status
Location
Last Error Time
 Last Error Message
+ +
{receiverName}
Avg: {avgReceiverEvents.map(_.toString).getOrElse(emptyCell)} events/sec
+
{receiverActive} {receiverLocation}
-
+
- {Unparsed(BLACK_RIGHT_TRIANGLE_HTML)} + {Unparsed(BLACK_RIGHT_TRIANGLE_HTML)} Input Rate
Avg: {eventRateForAllReceivers.formattedAvg} events/sec
@@ -376,12 +379,12 @@ private[ui] class StreamingPage(parent: StreamingTab) }.getOrElse(emptyCell) val receiverLastErrorTime = listener.receiverLastErrorTime(receiverId).map(UIUtils.formatDate).getOrElse(emptyCell) - val receivedRecords = listener.receivedRecordsWithBatchTime.get(receiverId).getOrElse(Seq()) + val receivedRecords = new DoubleStatUIData(listener.receivedRecordsWithBatchTime.get(receiverId).getOrElse(Seq())) val timelineForEventRate = new TimelineUIData( s"receiver-$receiverId-events-timeline", - receivedRecords, + receivedRecords.data, minX, maxX, minY, @@ -391,7 +394,7 @@ private[ui] class StreamingPage(parent: StreamingTab) val distributionForEventsRate = new DistributionUIData( s"receiver-$receiverId-events-distribution", - receivedRecords.map(_._2), + receivedRecords.data.map(_._2), minY, maxY, "events/sec").toHtml(jsCollector) @@ -402,13 +405,13 @@ private[ui] class StreamingPage(parent: StreamingTab)
{receiverName}
-
Avg: {avgReceiverEvents.map(_.toString).getOrElse(emptyCell)} events/sec
+
Avg: {receivedRecords.formattedAvg} events/sec
 {receiverActive} {receiverLocation} {receiverLastErrorTime}{receiverLastError}
{receiverLastError}
From a459f4956879c3fbf2e1c52350ef9f667f70d135 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Tue, 28 Apr 2015 16:54:37 -0700 Subject: [PATCH 15/31] Add a dash line to processing time graphs --- .../apache/spark/ui/static/streaming-page.css | 8 +++ .../apache/spark/ui/static/streaming-page.js | 60 ++++++++++++++++++- .../scala/org/apache/spark/ui/UIUtils.scala | 6 +- .../spark/streaming/ui/StreamingPage.scala | 27 ++++++--- 4 files changed, 87 insertions(+), 14 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css index 6b401c6dec76e..357a0b61a3b9e 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css @@ -30,6 +30,10 @@ fill: gray; } +.tooltip-inner { + max-width: 500px !important; +} + .line { fill: none; stroke: #0088cc; @@ -47,6 +51,10 @@ fill: rgb(0, 194, 255); } +.stable-text text:hover { + fill: #0088cc; +} + .timeline { width: 500px; } diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index 308ae0416a529..d473d0dd16f59 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -63,7 +63,7 @@ function hideBootstrapTooltip(node) { * @param maxY the max value of Y axis * @param unitY the unit of Y axis */ -function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { +function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchTime) { d3.select(d3.select(id).node().parentNode).style("padding", "8px 0 8px 8px").style("border-right", "0px solid white"); var margin = {top: 20, right: 27, bottom: 30, left: timelineMarginLeft}; var width = 500 - margin.left - margin.right; @@ -101,6 +101,26 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { .attr("transform", "translate(0," + (-3) + ")") .text(unitY); + + if (batchTime) { + var batchTimeline = d3.svg.line() + .x(function(d) { return x(d.x); }) + .y(function(d) { return y(d.y); }); + + console.log(batchTime); + var batchTimeData = [ + {x: minX, y: batchTime}, {x: maxX, y: batchTime} + ]; + console.log(batchTimeData); + + svg.append("path") + .datum(batchTimeData) + .style("stroke-dasharray", ("6, 6")) + .style("stroke", "lightblue") + .attr("class", "line") + .attr("d", batchTimeline); + } + svg.append("path") .datum(data) .attr("class", "line") @@ -136,6 +156,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { .attr("fill", "white") .attr("opacity", "0"); }); + } /** @@ -145,7 +166,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY) { * @param maxY the max value of Y axis * @param unitY the unit of Y axis */ -function drawDistribution(id, values, minY, maxY, unitY) { +function drawDistribution(id, values, minY, maxY, unitY, batchTime) { d3.select(d3.select(id).node().parentNode).style("padding", "8px 8px 8px 0").style("border-left", "0px solid white"); var margin = {top: 20, right: 30, bottom: 30, left: 10}; var width = 300 - margin.left - margin.right; @@ -169,6 +190,25 @@ function drawDistribution(id, values, minY, maxY, unitY) { .append("g") .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + if (batchTime) { + var batchTimeline = d3.svg.line() + .x(function(d) { return x(d.x); }) + .y(function(d) { return y(d.y); }); + + console.log(batchTime); + var batchTimeData = [ + {x: distributionMinX, y: batchTime}, {x: distributionMaxX, y: batchTime} + ]; + console.log(batchTimeData); + + svg.append("path") + .datum(batchTimeData) + .style("stroke-dasharray", ("6, 6")) + .style("stroke", "lightblue") + .attr("class", "line") + .attr("d", batchTimeline); + } + svg.append("g") .attr("class", "x axis") .attr("transform", "translate(0," + 0 + ")") @@ -208,6 +248,22 @@ function drawDistribution(id, values, minY, maxY, unitY) { hideBootstrapTooltip(d3.select(this).node()); //hideGraphTooltip(); }); + + if (batchTime && batchTime <= maxY) { + svg.append("text") + .style("fill", "lightblue") + .attr("class", "stable-text") + .attr("text-anchor", "middle") + .attr("transform", "translate(" + (x(distributionMaxX)-20) +"," + (y(batchTime) + 15) + ")") + .text("stable") + .on('mouseover', function(d) { + var tip = "Processing Time <= Batch Interval (" + formatBinValue(batchTime) +" " + unitY +")"; + showBootstrapTooltip(d3.select(this).node(), tip); + }) + .on('mouseout', function() { + hideBootstrapTooltip(d3.select(this).node()); + }); + } } function prepareTimeline(minY, maxY) { diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 827b945ec2787..db0836533388b 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -42,10 +42,10 @@ private[spark] object UIUtils extends Logging { case TimeUnit.NANOSECONDS => "ns" case TimeUnit.MICROSECONDS => "us" case TimeUnit.MILLISECONDS => "ms" - case TimeUnit.SECONDS => "s" + case TimeUnit.SECONDS => "sec" case TimeUnit.MINUTES => "min" - case TimeUnit.HOURS => "h" - case TimeUnit.DAYS => "d" + case TimeUnit.HOURS => "hrs" + case TimeUnit.DAYS => "days" } /** diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index d61662500b83e..4ff77e6c09141 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -40,16 +40,20 @@ import org.apache.spark.util.Distribution * @param unitY the unit of Y axis */ private[ui] class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: Long, maxX: Long, - minY: Double, maxY: Double, unitY: String) { + minY: Double, maxY: Double, unitY: String, batchTime: Option[Double] = None) { def toHtml(jsCollector: JsCollector): Seq[Node] = { val jsForData = data.map { case (x, y) => s"""{"x": $x, "y": $y}""" }.mkString("[", ",", "]") jsCollector.addPreparedStatement(s"prepareTimeline($minY, $maxY);") - jsCollector.addStatement( - s"drawTimeline('#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY');") - + if (batchTime.isDefined) { + jsCollector.addStatement( + s"drawTimeline('#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY', ${batchTime.get});") + } else { + jsCollector.addStatement( + s"drawTimeline('#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY');") + }
} } @@ -62,13 +66,16 @@ private[ui] class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: Long * @param unitY the unit of Y axis */ private[ui] class DistributionUIData( - divId: String, data: Seq[_], minY: Double, maxY: Double, unitY: String) { + divId: String, data: Seq[_], minY: Double, maxY: Double, unitY: String, batchTime: Option[Double] = None) { def toHtml(jsCollector: JsCollector): Seq[Node] = { val jsForData = data.mkString("[", ",", "]") jsCollector.addPreparedStatement(s"prepareDistribution($jsForData, $minY, $maxY);") - jsCollector.addStatement(s"drawDistribution('#$divId', $jsForData, $minY, $maxY, '$unitY');") - + if (batchTime.isDefined) { + jsCollector.addStatement(s"drawDistribution('#$divId', $jsForData, $minY, $maxY, '$unitY', ${batchTime.get});") + } else { + jsCollector.addStatement(s"drawDistribution('#$divId', $jsForData, $minY, $maxY, '$unitY');") + }
} } @@ -204,6 +211,8 @@ private[ui] class StreamingPage(parent: StreamingTab) |else {$$(this).html('$BLACK_RIGHT_TRIANGLE_HTML');status = false;} |window.history.pushState('', document.title, window.location.pathname + '?show-receivers-detail=' + status);""".stripMargin.replaceAll("\\n", "") + var batchTime = StreamingPage.convertToTimeUnit(listener.batchDuration, unit) + val timelineDataForEventRateOfAllReceivers = new TimelineUIData( "all-receiver-events-timeline", @@ -248,7 +257,7 @@ private[ui] class StreamingPage(parent: StreamingTab) maxBatchTime, minTime, maxTime, - formattedUnit).toHtml(jsCollector) + formattedUnit, Some(batchTime)).toHtml(jsCollector) val distributionDataForProcessingTime = new DistributionUIData( @@ -256,7 +265,7 @@ private[ui] class StreamingPage(parent: StreamingTab) processingTime.distributionData(unit), minTime, maxTime, - formattedUnit).toHtml(jsCollector) + formattedUnit, Some(batchTime)).toHtml(jsCollector) val timelineDataForTotalDelay = new TimelineUIData( From d0b0aecf5819dc9d81f515510798983ea2beb627 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Wed, 29 Apr 2015 12:59:53 -0700 Subject: [PATCH 16/31] Clean up the codes --- .../apache/spark/ui/static/streaming-page.css | 12 +- .../apache/spark/ui/static/streaming-page.js | 309 ++++++++---------- .../scala/org/apache/spark/ui/UIUtils.scala | 10 +- .../org/apache/spark/ui/UIUtilsSuite.scala | 52 +++ .../streaming/scheduler/ReceiverInfo.scala | 3 +- .../streaming/scheduler/ReceiverTracker.scala | 9 +- .../spark/streaming/ui/AllBatchesTable.scala | 2 +- .../ui/StreamingJobProgressListener.scala | 39 +-- .../spark/streaming/ui/StreamingPage.scala | 229 +++++++------ .../spark/streaming/UISeleniumSuite.scala | 6 +- 10 files changed, 351 insertions(+), 320 deletions(-) create mode 100644 core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css index 357a0b61a3b9e..5da9d631ad124 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css @@ -31,7 +31,7 @@ } .tooltip-inner { - max-width: 500px !important; + max-width: 500px !important; // Make sure we only have one line tooltip } .line { @@ -46,19 +46,13 @@ } .bar rect:hover { - //fill: rgb(49, 91, 125); - //fill: #005580; - fill: rgb(0, 194, 255); -} - -.stable-text text:hover { - fill: #0088cc; + fill: #00c2ff; } .timeline { width: 500px; } -.distribution { +.histogram { width: auto; } diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index d473d0dd16f59..acf5504ac4edf 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -16,56 +16,82 @@ */ -var timelineMarginLeft = 50; -var distributionMinX = 0; -var distributionMaxX = 0; -var binCount = 10; - -// An invisible div to show details of a point in the graph -var graphTooltip = d3.select("body").append("div") - .attr("class", "label") - .style("display", "inline-block") - .style("position", "absolute") - .style("z-index", "10") - .style("visibility", "hidden") - .text(""); - -// Show "text" at location (x, y) -function showGraphTooltip(text, x, y) { - var left = x; - var top = y; - graphTooltip.style("visibility", "visible") - .style("top", top + "px") - .style("left", left + "px") - .text(text); -} +// timeFormat: StreamingPage.scala will generate a global "timeFormat" dictionary to store the time +// and its formatted string. Because we cannot specify a timezone in JavaScript, to make sure the +// server and client use the same timezone, we use the "timeFormat" dictionary to format all time +// values used in the graphs. -// Hide "graphTooltip" -function hideGraphTooltip() { - graphTooltip.style("visibility", "hidden"); -} +// A global margin left for all timeline graphs. It will be set in "registerTimeline". This will be +// used to align all timeline graphs. +var maxMarginLeftForTimeline = 0; + +// The max X values for all histograms. It will be set in "registerHistogram". +var maxXForHistogram = 0; +var histogramBinCount = 10; +var yValueFormat = d3.format(",.2f"); + +// Show a tooltip "text" for "node" function showBootstrapTooltip(node, text) { - console.log(text); - $(node).tooltip({title: text, trigger: "manual", 'container': 'body'}); + $(node).tooltip({title: text, trigger: "manual", container: "body"}); $(node).tooltip("show"); - console.log($(node)); } +// Hide the tooltip for "node" function hideBootstrapTooltip(node) { $(node).tooltip("destroy"); } +// Register a timeline graph. All timeline graphs should be register before calling any +// "drawTimeline" so that we can determine the max margin left for all timeline graphs. +function registerTimeline(minY, maxY) { + var numOfChars = yValueFormat(maxY).length; + // A least width for "maxY" in the graph + var pxForMaxY = numOfChars * 8 + 10; + // Make sure we have enough space to show the ticks in the y axis of timeline + maxMarginLeftForTimeline = pxForMaxY > maxMarginLeftForTimeline? pxForMaxY : maxMarginLeftForTimeline; +} + +// Register a histogram graph. All histogram graphs should be register before calling any +// "drawHistogram" so that we can determine the max X value for histograms. +function registerHistogram(values, minY, maxY) { + var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values); + // d.x is the y values while d.y is the x values + var maxX = d3.max(data, function(d) { return d.y; }); + maxXForHistogram = maxX > maxXForHistogram ? maxX : maxXForHistogram; +} + +// Draw a line between (x1, y1) and (x2, y2) +function drawLine(svg, xFunc, yFunc, x1, y1, x2, y2) { + var line = d3.svg.line() + .x(function(d) { return xFunc(d.x); }) + .y(function(d) { return yFunc(d.y); }); + var data = [{x: x1, y: y1}, {x: x2, y: y2}]; + svg.append("path") + .datum(data) + .style("stroke-dasharray", ("6, 6")) + .style("stroke", "lightblue") + .attr("class", "line") + .attr("d", line); +} + /** * @param id the `id` used in the html `div` tag * @param data the data for the timeline graph + * @param minX the min value of X axis + * @param maxX the max value of X axis * @param minY the min value of Y axis * @param maxY the max value of Y axis * @param unitY the unit of Y axis + * @param batchInterval if "batchInterval" is specified, we will draw a line for "batchInterval" in the graph */ -function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchTime) { - d3.select(d3.select(id).node().parentNode).style("padding", "8px 0 8px 8px").style("border-right", "0px solid white"); - var margin = {top: 20, right: 27, bottom: 30, left: timelineMarginLeft}; +function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) { + // Hide the right border of "		". We cannot use "css" directly, or "sorttable.js" will override them. + d3.select(d3.select(id).node().parentNode) + .style("padding", "8px 0 8px 8px") + .style("border-right", "0px solid white"); + + var margin = {top: 20, right: 27, bottom: 30, left: maxMarginLeftForTimeline}; var width = 500 - margin.left - margin.right; var height = 150 - margin.top - margin.bottom; @@ -102,23 +128,8 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchTime) { .text(unitY); - if (batchTime) { - var batchTimeline = d3.svg.line() - .x(function(d) { return x(d.x); }) - .y(function(d) { return y(d.y); }); - - console.log(batchTime); - var batchTimeData = [ - {x: minX, y: batchTime}, {x: maxX, y: batchTime} - ]; - console.log(batchTimeData); - - svg.append("path") - .datum(batchTimeData) - .style("stroke-dasharray", ("6, 6")) - .style("stroke", "lightblue") - .attr("class", "line") - .attr("d", batchTimeline); + if (batchInterval && batchInterval <= maxY) { + drawLine(svg, x, y, minX, batchInterval, maxX, batchInterval); } svg.append("path") @@ -131,87 +142,69 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchTime) { svg.selectAll(".point") .data(data) .enter().append("circle") - .attr("stroke", "white") // white and opacity = 0 make it invisible + .attr("stroke", "white") // white and opacity = 0 make it invisible + .attr("fill", "white") + .attr("opacity", "0") + .attr("cx", function(d) { return x(d.x); }) + .attr("cy", function(d) { return y(d.y); }) + .attr("r", function(d) { return 3; }) + .on('mouseover', function(d) { + var tip = formatYValue(d.y) + " " + unitY + " at " + timeFormat[d.x]; + showBootstrapTooltip(d3.select(this).node(), tip); + // show the point + d3.select(this) + .attr("stroke", "steelblue") + .attr("fill", "steelblue") + .attr("opacity", "1"); + }) + .on('mouseout', function() { + hideBootstrapTooltip(d3.select(this).node()); + // hide the point + d3.select(this) + .attr("stroke", "white") .attr("fill", "white") - .attr("opacity", "0") - .attr("cx", function(d) { return x(d.x); }) - .attr("cy", function(d) { return y(d.y); }) - .attr("r", function(d) { return 3; }) - .on('mouseover', function(d) { - var tip = formatYValue(d.y) + " " + unitY + " at " + timeFormat[d.x]; - showBootstrapTooltip(d3.select(this).node(), tip); - //showGraphTooltip(tip, d3.event.pageX + 5, d3.event.pageY - 25); - // show the point - d3.select(this) - .attr("stroke", "steelblue") - .attr("fill", "steelblue") - .attr("opacity", "1"); - }) - .on('mouseout', function() { - hideBootstrapTooltip(d3.select(this).node()); - //hideGraphTooltip(); - // hide the point - d3.select(this) - .attr("stroke", "white") - .attr("fill", "white") - .attr("opacity", "0"); - }); - + .attr("opacity", "0"); + }); } /** * @param id the `id` used in the html `div` tag - * @param data the data for the distribution graph + * @param values the data for the histogram graph * @param minY the min value of Y axis * @param maxY the max value of Y axis * @param unitY the unit of Y axis + * @param batchInterval if "batchInterval" is specified, we will draw a line for "batchInterval" in the graph */ -function drawDistribution(id, values, minY, maxY, unitY, batchTime) { - d3.select(d3.select(id).node().parentNode).style("padding", "8px 8px 8px 0").style("border-left", "0px solid white"); +function drawHistogram(id, values, minY, maxY, unitY, batchInterval) { + // Hide the left border of "". We cannot use "css" directly, or "sorttable.js" will override them. + d3.select(d3.select(id).node().parentNode) + .style("padding", "8px 8px 8px 0") + .style("border-left", "0px solid white"); + var margin = {top: 20, right: 30, bottom: 30, left: 10}; var width = 300 - margin.left - margin.right; var height = 150 - margin.top - margin.bottom; - var formatBinValue = d3.format(",.2f"); - + var x = d3.scale.linear().domain([0, maxXForHistogram]).range([0, width]); var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); - var data = d3.layout.histogram().range([minY, maxY]).bins(binCount)(values); - - var x = d3.scale.linear() - .domain([distributionMinX, distributionMaxX]) - .range([0, width]); var xAxis = d3.svg.axis().scale(x).orient("top").ticks(5); var yAxis = d3.svg.axis().scale(y).orient("left").ticks(0).tickFormat(function(d) { return ""; }); + var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values); + var svg = d3.select(id).append("svg") .attr("width", width + margin.left + margin.right) .attr("height", height + margin.top + margin.bottom) .append("g") .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); - if (batchTime) { - var batchTimeline = d3.svg.line() - .x(function(d) { return x(d.x); }) - .y(function(d) { return y(d.y); }); - - console.log(batchTime); - var batchTimeData = [ - {x: distributionMinX, y: batchTime}, {x: distributionMaxX, y: batchTime} - ]; - console.log(batchTimeData); - - svg.append("path") - .datum(batchTimeData) - .style("stroke-dasharray", ("6, 6")) - .style("stroke", "lightblue") - .attr("class", "line") - .attr("d", batchTimeline); + if (batchInterval && batchInterval <= maxY) { + drawLine(svg, x, y, 0, batchInterval, maxXForHistogram, batchInterval); } svg.append("g") .attr("class", "x axis") - .attr("transform", "translate(0," + 0 + ")") .call(xAxis) svg.append("g") @@ -220,82 +213,58 @@ function drawDistribution(id, values, minY, maxY, unitY, batchTime) { var bar = svg.selectAll(".bar") .data(data) - .enter().append("g") - .attr("transform", function(d) { return "translate(0," + (y(d.x) - height + y(d.dx)) + ")";}) - .attr("class", "bar").append("rect") - .attr("width", function(d) { return x(d.y); }) - .attr("height", function(d) { return height - y(d.dx); }) - .on('mouseover', function(d) { - var tip = d.y + " between " + formatBinValue(d.x) + " and " + formatBinValue(d.x + d.dx) + " " + unitY; - showBootstrapTooltip(d3.select(this).node(), tip); - - // Calculate the location for tip - var scrollTop = document.documentElement.scrollTop || document.body.scrollTop; - var scrollLeft = document.documentElement.scrollLeft || document.body.scrollLeft; - var target = d3.event.target; - var matrix = target.getScreenCTM(); - var targetBBox = target.getBBox(); - var point = svg.node().ownerSVGElement.createSVGPoint(); - point.x = targetBBox.x; - point.y = targetBBox.y; - var bbox = point.matrixTransform(matrix); - var tipX = bbox.x + scrollLeft; - var tipY = bbox.y + scrollTop + 15; - - //showGraphTooltip(tip, tipX, tipY); - }) - .on('mouseout', function() { - hideBootstrapTooltip(d3.select(this).node()); - //hideGraphTooltip(); - }); - - if (batchTime && batchTime <= maxY) { + .enter() + .append("g") + .attr("transform", function(d) { return "translate(0," + (y(d.x) - height + y(d.dx)) + ")";}) + .attr("class", "bar").append("rect") + .attr("width", function(d) { return x(d.y); }) + .attr("height", function(d) { return height - y(d.dx); }) + .on('mouseover', function(d) { + var percent = yValueFormat(d.y * 100.0 / values.length) + "%"; + var tip = d.y + " batches (" + percent + ") between " + yValueFormat(d.x) + " and " + yValueFormat(d.x + d.dx) + " " + unitY; + showBootstrapTooltip(d3.select(this).node(), tip); + }) + .on('mouseout', function() { + hideBootstrapTooltip(d3.select(this).node()); + }); + + if (batchInterval && batchInterval <= maxY) { + // Add the "stable" text to the graph below the batch interval line. + var stableXOffset = x(maxXForHistogram) - 20; + var stableYOffset = y(batchInterval) + 15; svg.append("text") - .style("fill", "lightblue") - .attr("class", "stable-text") - .attr("text-anchor", "middle") - .attr("transform", "translate(" + (x(distributionMaxX)-20) +"," + (y(batchTime) + 15) + ")") - .text("stable") - .on('mouseover', function(d) { - var tip = "Processing Time <= Batch Interval (" + formatBinValue(batchTime) +" " + unitY +")"; - showBootstrapTooltip(d3.select(this).node(), tip); - }) - .on('mouseout', function() { - hideBootstrapTooltip(d3.select(this).node()); - }); + .style("fill", "lightblue") + .attr("class", "stable-text") + .attr("text-anchor", "middle") + .attr("transform", "translate(" + stableXOffset + "," + stableYOffset + ")") + .text("stable") + .on('mouseover', function(d) { + var tip = "Processing Time <= Batch Interval (" + yValueFormat(batchInterval) +" " + unitY +")"; + showBootstrapTooltip(d3.select(this).node(), tip); + }) + .on('mouseout', function() { + hideBootstrapTooltip(d3.select(this).node()); + }); } } -function prepareTimeline(minY, maxY) { - var formatYValue = d3.format(",.2f"); - var numOfChars = formatYValue(maxY).length; - var maxPx = numOfChars * 8 + 10; - // Make sure we have enough space to show the ticks in the y axis of timeline - timelineMarginLeft = maxPx > timelineMarginLeft? maxPx : timelineMarginLeft; -} - -function prepareDistribution(values, minY, maxY) { - var data = d3.layout.histogram().range([minY, maxY]).bins(binCount)(values); - var maxBarSize = d3.max(data, function(d) { return d.y; }); - distributionMaxX = maxBarSize > distributionMaxX? maxBarSize : distributionMaxX; -} - -function getUrlParameter(sParam) -{ - var sPageURL = window.location.search.substring(1); - var sURLVariables = sPageURL.split('&'); - for (var i = 0; i < sURLVariables.length; i++) +$(function() { + function getParameterFromURL(param) { - var sParameterName = sURLVariables[i].split('='); - if (sParameterName[0] == sParam) + var parameters = window.location.search.substring(1); // Remove "?" + var keyValues = parameters.split('&'); + for (var i = 0; i < keyValues.length; i++) { - return sParameterName[1]; + var paramKeyValue = keyValues[i].split('='); + if (paramKeyValue[0] == param) + { + return paramKeyValue[1]; + } } } -} -$(function() { - if (getUrlParameter("show-receivers-detail") == "true") { + if (getParameterFromURL("show-receivers-detail") == "true") { + // Show the details for all receivers $('#inputs-table').toggle('collapsed'); $('#triangle').html('▼'); } diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index db0836533388b..1ad4f48506fda 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -49,8 +49,8 @@ private[spark] object UIUtils extends Logging { } /** - * Find the best `TimeUnit` for converting milliseconds to a friendly string. Return the value after converting with - * the `TimeUnit`. + * Find the best `TimeUnit` for converting milliseconds to a friendly string. Return the value + * after converting, also with its TimeUnit. */ def normalizeDuration(milliseconds: Long): (Double, TimeUnit) = { if (milliseconds < 1000) { @@ -65,7 +65,11 @@ private[spark] object UIUtils extends Logging { return (minutes, TimeUnit.MINUTES) } val hours = minutes / 60 - (hours, TimeUnit.HOURS) + if (hours < 24) { + return (hours, TimeUnit.HOURS) + } + val days = hours / 24 + (days, TimeUnit.DAYS) } def formatDate(date: Date): String = dateFormat.get.format(date) diff --git a/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala b/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala new file mode 100644 index 0000000000000..636b2c389e05e --- /dev/null +++ b/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.spark.ui + +import java.util.concurrent.TimeUnit + +import org.scalatest.FunSuite +import org.scalatest.Matchers + +class UIUtilsSuite extends FunSuite with Matchers{ + + test("shortTimeUnitString") { + assert("ns" === UIUtils.shortTimeUnitString(TimeUnit.NANOSECONDS)) + assert("us" === UIUtils.shortTimeUnitString(TimeUnit.MICROSECONDS)) + assert("ms" === UIUtils.shortTimeUnitString(TimeUnit.MILLISECONDS)) + assert("sec" === UIUtils.shortTimeUnitString(TimeUnit.SECONDS)) + assert("min" === UIUtils.shortTimeUnitString(TimeUnit.MINUTES)) + assert("hrs" === UIUtils.shortTimeUnitString(TimeUnit.HOURS)) + assert("days" === UIUtils.shortTimeUnitString(TimeUnit.DAYS)) + } + + test("normalizeDuration") { + verifyNormalizedTime(900, TimeUnit.MILLISECONDS, 900) + verifyNormalizedTime(1.0, TimeUnit.SECONDS, 1000) + verifyNormalizedTime(1.0, TimeUnit.MINUTES, 60 * 1000) + verifyNormalizedTime(1.0, TimeUnit.HOURS, 60 * 60 * 1000) + verifyNormalizedTime(1.0, TimeUnit.DAYS, 24 * 60 * 60 * 1000) + } + + private def verifyNormalizedTime( + expectedTime: Double, expectedUnit: TimeUnit, input: Long): Unit = { + val (time, unit) = UIUtils.normalizeDuration(input) + time should be (expectedTime +- 1E-6) + unit should be (expectedUnit) + } +} diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverInfo.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverInfo.scala index d7e39c528c519..5bf2097333fbe 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverInfo.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverInfo.scala @@ -32,6 +32,7 @@ case class ReceiverInfo( active: Boolean, location: String, lastErrorMessage: String = "", - lastError: String = "" + lastError: String = "", + lastErrorTime: Long = -1L ) { } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverTracker.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverTracker.scala index 98900473138fe..a462298fe3345 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverTracker.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceiverTracker.scala @@ -155,10 +155,12 @@ class ReceiverTracker(ssc: StreamingContext, skipReceiverLaunch: Boolean = false private def deregisterReceiver(streamId: Int, message: String, error: String) { val newReceiverInfo = receiverInfo.get(streamId) match { case Some(oldInfo) => - oldInfo.copy(actor = null, active = false, lastErrorMessage = message, lastError = error) + oldInfo.copy(actor = null, active = false, lastErrorMessage = message, + lastError = error, lastErrorTime = ssc.scheduler.clock.getTimeMillis()) case None => logWarning("No prior receiver info") - ReceiverInfo(streamId, "", null, false, "", lastErrorMessage = message, lastError = error) + ReceiverInfo(streamId, "", null, false, "", lastErrorMessage = message, + lastError = error, lastErrorTime = ssc.scheduler.clock.getTimeMillis()) } receiverInfo -= streamId listenerBus.post(StreamingListenerReceiverStopped(newReceiverInfo)) @@ -182,7 +184,8 @@ class ReceiverTracker(ssc: StreamingContext, skipReceiverLaunch: Boolean = false oldInfo.copy(lastErrorMessage = message, lastError = error) case None => logWarning("No prior receiver info") - ReceiverInfo(streamId, "", null, false, "", lastErrorMessage = message, lastError = error) + ReceiverInfo(streamId, "", null, false, "", lastErrorMessage = message, + lastError = error, lastErrorTime = ssc.scheduler.clock.getTimeMillis()) } receiverInfo(streamId) = newReceiverInfo listenerBus.post(StreamingListenerReceiverError(receiverInfo(streamId))) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala index df1c0a10704c3..61fa86838861c 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala @@ -27,7 +27,7 @@ private[ui] abstract class BatchTableBase(tableId: String) { protected def columns: Seq[Node] = { Batch Time Input SizeScheduling DelayStreaming Scheduling Delay Processing Time
- + + + + - - + - + - + - +
TimelinesHistograms
Timelines (Last {batchTimes.length} batches, {numActiveBatches} active, {numCompletedBatches} completed)Histograms
+
-
- {Unparsed(BLACK_RIGHT_TRIANGLE_HTML)} - Input Rate -
-
Avg: {eventRateForAllReceivers.formattedAvg} events/sec
+
+ {Unparsed(BLACK_RIGHT_TRIANGLE_HTML)} + Input Rate +
+
Avg: {eventRateForAllReceivers.formattedAvg} events/sec
 {timelineDataForEventRateOfAllReceivers}{distributionDataForEventRateOfAllReceivers}{histogramDataForEventRateOfAllReceivers}
-
Scheduling Delay
-
Avg: {schedulingDelay.formattedAvg}
+
+
Streaming Scheduling Delay
+
Avg: {schedulingDelay.formattedAvg}
+
{timelineDataForSchedulingDelay}{distributionDataForSchedulingDelay}{histogramDataForSchedulingDelay}
-
Processing Time
-
Avg: {processingTime.formattedAvg}
+
+
Processing Time
+
Avg: {processingTime.formattedAvg}
+
{timelineDataForProcessingTime}{distributionDataForProcessingTime}{histogramDataForProcessingTime}
-
Total Delay
-
Avg: {totalDelay.formattedAvg}
+
+
Total Delay
+
Avg: {totalDelay.formattedAvg}
+
{timelineDataForTotalDelay}{distributionDataForTotalDelay}{histogramDataForTotalDelay}
@@ -347,10 +391,11 @@ private[ui] class StreamingPage(parent: StreamingTab) maxX: Long, minY: Double, maxY: Double): Seq[Node] = { - val content = listener.receivedRecordsDistributions.map { case (receiverId, distribution) => - generateInputReceiverRow(jsCollector, receiverId, distribution, minX, maxX, minY, maxY) + val content = listener.allReceivers.map { receiverId => + generateInputReceiverRow(jsCollector, receiverId, minX, maxX, minY, maxY) }.foldLeft[Seq[Node]](Nil)(_ ++ _) + // scalastyle:off @@ -365,17 +410,16 @@ private[ui] class StreamingPage(parent: StreamingTab) {content}
+ // scalastyle:on } private def generateInputReceiverRow( jsCollector: JsCollector, receiverId: Int, - distribution: Option[Distribution], minX: Long, maxX: Long, minY: Double, maxY: Double): Seq[Node] = { - val avgReceiverEvents = distribution.map(_.statCounter.mean.toLong) val receiverInfo = listener.receiverInfo(receiverId) val receiverName = receiverInfo.map(_.name).getOrElse(s"Receiver-$receiverId") val receiverActive = receiverInfo.map { info => @@ -386,9 +430,11 @@ private[ui] class StreamingPage(parent: StreamingTab) val msg = s"${info.lastErrorMessage} - ${info.lastError}" if (msg.size > 100) msg.take(97) + "..." else msg }.getOrElse(emptyCell) - val receiverLastErrorTime = - listener.receiverLastErrorTime(receiverId).map(UIUtils.formatDate).getOrElse(emptyCell) - val receivedRecords = new DoubleStatUIData(listener.receivedRecordsWithBatchTime.get(receiverId).getOrElse(Seq())) + val receiverLastErrorTime = receiverInfo.map { + r => if (r.lastErrorTime < 0) "-" else UIUtils.formatDate(r.lastErrorTime) + }.getOrElse(emptyCell) + val receivedRecords = + new EventRateUIData(listener.receivedRecordsWithBatchTime.get(receiverId).getOrElse(Seq())) val timelineForEventRate = new TimelineUIData( @@ -400,9 +446,9 @@ private[ui] class StreamingPage(parent: StreamingTab) maxY, "events/sec").toHtml(jsCollector) - val distributionForEventsRate = - new DistributionUIData( - s"receiver-$receiverId-events-distribution", + val histogramForEventsRate = + new HistogramUIData( + s"receiver-$receiverId-events-histogram", receivedRecords.data.map(_._2), minY, maxY, @@ -410,31 +456,22 @@ private[ui] class StreamingPage(parent: StreamingTab)
-
-
- {receiverName} +
+
{receiverName}
+
Avg: {receivedRecords.formattedAvg} events/sec
-
Avg: {receivedRecords.formattedAvg} events/sec
-
{receiverActive} {receiverLocation} {receiverLastErrorTime}
{receiverLastError}
- {timelineForEventRate} - {distributionForEventsRate}
+ {timelineForEventRate} + {histogramForEventsRate}
+
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala index bbf7dd0fb7720..8bef4b688d060 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala @@ -97,6 +97,21 @@ class UISeleniumSuite val h3Text = findAll(cssSelector("h3")).map(_.text).toSeq h3Text should contain("Streaming Statistics") + // Check stat table + val statTableHeaders = findAll(cssSelector("#stat-table th")).map(_.text).toSeq + statTableHeaders.exists( + _.matches("Timelines \\(Last \\d+ batches, \\d+ active, \\d+ completed\\)")) should be + (true) + statTableHeaders should contain ("Histograms") + + val statTableCells = findAll(cssSelector("#stat-table td")).map(_.text).toSeq + println(statTableCells.toList) + statTableCells.exists(_.contains("Input Rate")) should be (true) + statTableCells.exists(_.contains("Streaming Scheduling Delay")) should be (true) + statTableCells.exists(_.contains("Processing Time")) should be (true) + statTableCells.exists(_.contains("Total Delay")) should be (true) + + // Check batch tables val h4Text = findAll(cssSelector("h4")).map(_.text).toSeq h4Text.exists(_.matches("Active Batches \\(\\d+\\)")) should be (true) h4Text.exists(_.matches("Completed Batches \\(last \\d+ out of \\d+\\)")) should be (true) From 7bbdc0a9a9c8138dfb546d74d40f9ed665bc3823 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Thu, 30 Apr 2015 12:40:29 -0700 Subject: [PATCH 19/31] Hide the receiver sub table if no receiver --- .../org/apache/spark/streaming/ui/StreamingPage.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index e0315da299129..01eda4dac2be2 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -318,6 +318,8 @@ private[ui] class StreamingPage(parent: StreamingTab) maxTime, formattedUnit).toHtml(jsCollector) + val hasReceiver = listener.allReceivers.nonEmpty + val numCompletedBatches = listener.retainedCompletedBatches.size val numActiveBatches = batchTimes.length - numCompletedBatches val table = @@ -334,7 +336,9 @@ private[ui] class StreamingPage(parent: StreamingTab) + {if (hasReceiver) { + }} - - + + {if (hasReceiver) { @@ -361,8 +342,8 @@ private[ui] class StreamingPage(parent: StreamingTab)
Avg: {schedulingDelay.formattedAvg}
- - + + - - + + - - + +
+ {if (hasReceiver) { {Unparsed(BLACK_RIGHT_TRIANGLE_HTML)} + }} Input Rate
Avg: {eventRateForAllReceivers.formattedAvg} events/sec
@@ -343,11 +347,13 @@ private[ui] class StreamingPage(parent: StreamingTab)
{timelineDataForEventRateOfAllReceivers} {histogramDataForEventRateOfAllReceivers}
From 70d8533b3bcc63c472d6405c6dc440203c230338 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Thu, 30 Apr 2015 15:56:52 -0700 Subject: [PATCH 20/31] Remove BatchInfo.numRecords and a few renames --- .../org/apache/spark/streaming/scheduler/BatchInfo.scala | 7 ------- .../spark/streaming/ui/StreamingJobProgressListener.scala | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/BatchInfo.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/BatchInfo.scala index 9d21be50a566f..92dc113f397ca 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/BatchInfo.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/BatchInfo.scala @@ -58,11 +58,4 @@ case class BatchInfo( */ def totalDelay: Option[Long] = schedulingDelay.zip(processingDelay) .map(x => x._1 + x._2).headOption - - /** - * The number of recorders received by the receivers in this batch. - */ - def numRecords: Long = receivedBlockInfo.map { case (_, infos) => - infos.map(_.numRecords).sum - }.sum } diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala index 7ccc0dfd5cbe6..863fff415e386 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala @@ -182,8 +182,8 @@ private[streaming] class StreamingJobProgressListener(ssc: StreamingContext) * Return all of the event rates for each receiver in each batch. */ def receivedEventRateWithBatchTime: Map[Int, Seq[(Long, Double)]] = synchronized { - val latestBatches = retainedBatches.map { batchInfo => - (batchInfo.batchTime.milliseconds, batchInfo.receiverNumRecords) + val latestBatches = retainedBatches.map { batchUIData => + (batchUIData.batchTime.milliseconds, batchUIData.receiverNumRecords) } (0 until numReceivers).map { receiverId => val eventRates = latestBatches.map { From 74307cf930bce09e6edb145aaf4df5ca139563ac Mon Sep 17 00:00:00 2001 From: zsxwing Date: Fri, 1 May 2015 12:28:14 -0700 Subject: [PATCH 21/31] Reuse the data for histogram graphs to reduce the page size --- .../spark/streaming/ui/StreamingPage.scala | 172 ++++++++---------- .../spark/streaming/UISeleniumSuite.scala | 1 - 2 files changed, 79 insertions(+), 94 deletions(-) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala index 01eda4dac2be2..65292ed5cdade 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala @@ -30,8 +30,9 @@ import org.apache.spark.ui._ import org.apache.spark.ui.UIUtils._ /** - * @param divId the `id` used in the html `div` tag - * @param data the data for the timeline graph + * @param timelineDivId the timeline `id` used in the html `div` tag + * @param histogramDivId the timeline `id` used in the html `div` tag + * @param data the data for the graph * @param minX the min value of X axis * @param maxX the max value of X axis * @param minY the min value of Y axis @@ -40,52 +41,56 @@ import org.apache.spark.ui.UIUtils._ * @param batchInterval if `batchInterval` is not None, we will draw a line for `batchInterval` in * the graph */ -private[ui] class TimelineUIData(divId: String, data: Seq[(Long, _)], minX: Long, maxX: Long, - minY: Double, maxY: Double, unitY: String, batchInterval: Option[Double] = None) { +private[ui] class GraphUIData( + timelineDivId: String, + histogramDivId: String, + data: Seq[(Long, Double)], + minX: Long, + maxX: Long, + minY: Double, + maxY: Double, + unitY: String, + batchInterval: Option[Double] = None) { + + private var dataJavaScriptName: String = _ - def toHtml(jsCollector: JsCollector): Seq[Node] = { + def generateDataJs(jsCollector: JsCollector): Unit = { val jsForData = data.map { case (x, y) => s"""{"x": $x, "y": $y}""" }.mkString("[", ",", "]") + dataJavaScriptName = jsCollector.nextVariableName + jsCollector.addPreparedStatement(s"var $dataJavaScriptName = $jsForData;") + } + + def generateTimelineHtml(jsCollector: JsCollector): Seq[Node] = { jsCollector.addPreparedStatement(s"registerTimeline($minY, $maxY);") if (batchInterval.isDefined) { jsCollector.addStatement( "drawTimeline(" + - s"'#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY', ${batchInterval.get}" + + s"'#$timelineDivId', $dataJavaScriptName, $minX, $maxX, $minY, $maxY, '$unitY'," + + s" ${batchInterval.get}" + ");") } else { jsCollector.addStatement( - s"drawTimeline('#$divId', $jsForData, $minX, $maxX, $minY, $maxY, '$unitY');") + s"drawTimeline('#$timelineDivId', $dataJavaScriptName, $minX, $maxX, $minY, $maxY," + + s" '$unitY');") } -
+
} -} -/** - * @param divId the `id` used in the html `div` tag - * @param data the data for the histogram graph - * @param minY the min value of Y axis - * @param maxY the max value of Y axis - * @param unitY the unit of Y axis - * @param batchInterval if `batchInterval` is not None, we will draw a line for `batchInterval` in - * the graph - */ -private[ui] class HistogramUIData( - divId: String, data: Seq[_], minY: Double, maxY: Double, unitY: String, - batchInterval: Option[Double] = None) { - - def toHtml(jsCollector: JsCollector): Seq[Node] = { - val jsForData = data.mkString("[", ",", "]") - jsCollector.addPreparedStatement(s"registerHistogram($jsForData, $minY, $maxY);") + def generateHistogramHtml(jsCollector: JsCollector): Seq[Node] = { + val histogramData = s"$dataJavaScriptName.map(function(d) { return d.y; })" + jsCollector.addPreparedStatement(s"registerHistogram($histogramData, $minY, $maxY);") if (batchInterval.isDefined) { jsCollector.addStatement( "drawHistogram(" + - s"'#$divId', $jsForData, $minY, $maxY, '$unitY', ${batchInterval.get}" + + s"'#$histogramDivId', $histogramData, $minY, $maxY, '$unitY', ${batchInterval.get}" + ");") } else { - jsCollector.addStatement(s"drawHistogram('#$divId', $jsForData, $minY, $maxY, '$unitY');") + jsCollector.addStatement( + s"drawHistogram('#$histogramDivId', $histogramData, $minY, $maxY, '$unitY');") } -
+
} } @@ -246,77 +251,53 @@ private[ui] class StreamingPage(parent: StreamingTab) val jsCollector = new JsCollector - val timelineDataForEventRateOfAllReceivers = - new TimelineUIData( + val graphUIDataForEventRateOfAllReceivers = + new GraphUIData( "all-receiver-events-timeline", + "all-receiver-events-histogram", eventRateForAllReceivers.data, minBatchTime, maxBatchTime, minEventRate, maxEventRate, - "events/sec").toHtml(jsCollector) + "events/sec") + graphUIDataForEventRateOfAllReceivers.generateDataJs(jsCollector) - val histogramDataForEventRateOfAllReceivers = - new HistogramUIData( - "all-receiver-events-histogram", - eventRateForAllReceivers.data.map(_._2), - minEventRate, - maxEventRate, - "events/sec").toHtml(jsCollector) - - val timelineDataForSchedulingDelay = - new TimelineUIData( + val graphUIDataForSchedulingDelay = + new GraphUIData( "scheduling-delay-timeline", + "scheduling-delay-histogram", schedulingDelay.timelineData(normalizedUnit), minBatchTime, maxBatchTime, minTime, maxTime, - formattedUnit).toHtml(jsCollector) - - val histogramDataForSchedulingDelay = - new HistogramUIData( - "scheduling-delay-histogram", - schedulingDelay.histogramData(normalizedUnit), - minTime, - maxTime, - formattedUnit).toHtml(jsCollector) + formattedUnit) + graphUIDataForSchedulingDelay.generateDataJs(jsCollector) - val timelineDataForProcessingTime = - new TimelineUIData( + val graphUIDataForProcessingTime = + new GraphUIData( "processing-time-timeline", + "processing-time-histogram", processingTime.timelineData(normalizedUnit), minBatchTime, maxBatchTime, minTime, maxTime, - formattedUnit, Some(batchInterval)).toHtml(jsCollector) - - val histogramDataForProcessingTime = - new HistogramUIData( - "processing-time-histogram", - processingTime.histogramData(normalizedUnit), - minTime, - maxTime, - formattedUnit, Some(batchInterval)).toHtml(jsCollector) + formattedUnit, Some(batchInterval)) + graphUIDataForProcessingTime.generateDataJs(jsCollector) - val timelineDataForTotalDelay = - new TimelineUIData( + val graphUIDataForTotalDelay = + new GraphUIData( "total-delay-timeline", + "total-delay-histogram", totalDelay.timelineData(normalizedUnit), minBatchTime, maxBatchTime, minTime, maxTime, - formattedUnit).toHtml(jsCollector) - - val histogramDataForTotalDelay = - new HistogramUIData( - "total-delay-histogram", - totalDelay.histogramData(normalizedUnit), - minTime, - maxTime, - formattedUnit).toHtml(jsCollector) + formattedUnit) + graphUIDataForTotalDelay.generateDataJs(jsCollector) val hasReceiver = listener.allReceivers.nonEmpty @@ -344,8 +325,8 @@ private[ui] class StreamingPage(parent: StreamingTab)
Avg: {eventRateForAllReceivers.formattedAvg} events/sec
{timelineDataForEventRateOfAllReceivers}{histogramDataForEventRateOfAllReceivers}{graphUIDataForEventRateOfAllReceivers.generateTimelineHtml(jsCollector)}{graphUIDataForEventRateOfAllReceivers.generateHistogramHtml(jsCollector)}
@@ -371,8 +352,8 @@ private[ui] class StreamingPage(parent: StreamingTab)
Avg: {processingTime.formattedAvg}
{timelineDataForProcessingTime}{histogramDataForProcessingTime}{graphUIDataForProcessingTime.generateTimelineHtml(jsCollector)}{graphUIDataForProcessingTime.generateHistogramHtml(jsCollector)}
@@ -381,8 +362,8 @@ private[ui] class StreamingPage(parent: StreamingTab)
Avg: {totalDelay.formattedAvg}
{timelineDataForTotalDelay}{histogramDataForTotalDelay}{graphUIDataForTotalDelay.generateTimelineHtml(jsCollector)}{graphUIDataForTotalDelay.generateHistogramHtml(jsCollector)}
@@ -442,23 +423,17 @@ private[ui] class StreamingPage(parent: StreamingTab) val receivedRecords = new EventRateUIData(listener.receivedEventRateWithBatchTime.get(receiverId).getOrElse(Seq())) - val timelineForEventRate = - new TimelineUIData( + val graphUIDataForEventRate = + new GraphUIData( s"receiver-$receiverId-events-timeline", + s"receiver-$receiverId-events-histogram", receivedRecords.data, minX, maxX, minY, maxY, - "events/sec").toHtml(jsCollector) - - val histogramForEventsRate = - new HistogramUIData( - s"receiver-$receiverId-events-histogram", - receivedRecords.data.map(_._2), - minY, - maxY, - "events/sec").toHtml(jsCollector) + "events/sec") + graphUIDataForEventRate.generateDataJs(jsCollector)
@@ -474,9 +449,9 @@ private[ui] class StreamingPage(parent: StreamingTab)
- {timelineForEventRate} + {graphUIDataForEventRate.generateTimelineHtml(jsCollector)} {histogramForEventsRate}{graphUIDataForEventRate.generateHistogramHtml(jsCollector)}
- {if (hasReceiver) { + {if (hasStream) { {Unparsed(BLACK_RIGHT_TRIANGLE_HTML)} }} Input Rate
-
Avg: {eventRateForAllReceivers.formattedAvg} events/sec
+
Avg: {eventRateForAllStreams.formattedAvg} events/sec
{graphUIDataForEventRateOfAllReceivers.generateTimelineHtml(jsCollector)}{graphUIDataForEventRateOfAllReceivers.generateHistogramHtml(jsCollector)}{graphUIDataForEventRateOfAllStreams.generateTimelineHtml(jsCollector)}{graphUIDataForEventRateOfAllStreams.generateHistogramHtml(jsCollector)}
@@ -107,7 +107,7 @@ private[ui] class CompletedBatchTable(batches: Seq[BatchUIData]) private def completedBatchRow(batch: BatchUIData): Seq[Node] = { val totalDelay = batch.totalDelay - val formattedTotalDelay = totalDelay.map(UIUtils.formatDuration).getOrElse("-") + val formattedTotalDelay = totalDelay.map(SparkUIUtils.formatDuration).getOrElse("-") baseRow(batch) ++ {formattedTotalDelay} diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/BatchPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/BatchPage.scala index 2da9a29e2529e..3f1cab69068dc 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/BatchPage.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/BatchPage.scala @@ -24,7 +24,7 @@ import scala.xml.{NodeSeq, Node} import org.apache.commons.lang3.StringEscapeUtils import org.apache.spark.streaming.Time -import org.apache.spark.ui.{UIUtils, WebUIPage} +import org.apache.spark.ui.{UIUtils => SparkUIUtils, WebUIPage} import org.apache.spark.streaming.ui.StreamingJobProgressListener.{SparkJobId, OutputOpId} import org.apache.spark.ui.jobs.UIData.JobUIData @@ -73,8 +73,8 @@ private[ui] class BatchPage(parent: StreamingTab) extends WebUIPage("batch") { sparkJob.stageIds.sorted.reverse.flatMap(sparkListener.stageIdToInfo.get). dropWhile(_.failureReason == None).take(1). // get the first info that contains failure flatMap(info => info.failureReason).headOption.getOrElse("") - val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("-") - val detailUrl = s"${UIUtils.prependBaseUri(parent.basePath)}/jobs/job?id=${sparkJob.jobId}" + val formattedDuration = duration.map(d => SparkUIUtils.formatDuration(d)).getOrElse("-") + val detailUrl = s"${SparkUIUtils.prependBaseUri(parent.basePath)}/jobs/job?id=${sparkJob.jobId}" // In the first row, output op id and its information needs to be shown. In other rows, these // cells will be taken up due to "rowspan". @@ -110,7 +110,7 @@ private[ui] class BatchPage(parent: StreamingTab) extends WebUIPage("batch") { { - UIUtils.makeProgressBar( + SparkUIUtils.makeProgressBar( started = sparkJob.numActiveTasks, completed = sparkJob.numCompletedTasks, failed = sparkJob.numFailedTasks, @@ -135,7 +135,7 @@ private[ui] class BatchPage(parent: StreamingTab) extends WebUIPage("batch") { // If any job does not finish, set "formattedOutputOpDuration" to "-" "-" } else { - UIUtils.formatDuration(sparkjobDurations.flatMap(x => x).sum) + SparkUIUtils.formatDuration(sparkjobDurations.flatMap(x => x).sum) } generateJobRow(outputOpId, formattedOutputOpDuration, sparkJobs.size, true, sparkJobs.head) ++ sparkJobs.tail.map { sparkJob => @@ -212,24 +212,24 @@ private[ui] class BatchPage(parent: StreamingTab) extends WebUIPage("batch") { val batchTime = Option(request.getParameter("id")).map(id => Time(id.toLong)).getOrElse { throw new IllegalArgumentException(s"Missing id parameter") } - val formattedBatchTime = UIUtils.formatDate(batchTime.milliseconds) + val formattedBatchTime = SparkUIUtils.formatDate(batchTime.milliseconds) val batchUIData = streamingListener.getBatchUIData(batchTime).getOrElse { throw new IllegalArgumentException(s"Batch $formattedBatchTime does not exist") } val formattedSchedulingDelay = - batchUIData.schedulingDelay.map(UIUtils.formatDuration).getOrElse("-") + batchUIData.schedulingDelay.map(SparkUIUtils.formatDuration).getOrElse("-") val formattedProcessingTime = - batchUIData.processingDelay.map(UIUtils.formatDuration).getOrElse("-") - val formattedTotalDelay = batchUIData.totalDelay.map(UIUtils.formatDuration).getOrElse("-") + batchUIData.processingDelay.map(SparkUIUtils.formatDuration).getOrElse("-") + val formattedTotalDelay = batchUIData.totalDelay.map(SparkUIUtils.formatDuration).getOrElse("-") val summary: NodeSeq =
  • Batch Duration: - {UIUtils.formatDuration(streamingListener.batchDuration)} + {SparkUIUtils.formatDuration(streamingListener.batchDuration)}
  • Input data size: @@ -259,6 +259,6 @@ private[ui] class BatchPage(parent: StreamingTab) extends WebUIPage("batch") { val content = summary ++ jobTable - UIUtils.headerSparkPage(s"Details of batch at $formattedBatchTime", content, parent) + SparkUIUtils.headerSparkPage(s"Details of batch at $formattedBatchTime", content, parent) } } From 70cc87d9fa70a4df4ed1744320b6ae4814cd558e Mon Sep 17 00:00:00 2001 From: zsxwing Date: Mon, 4 May 2015 17:19:54 -0700 Subject: [PATCH 29/31] Streaming Scheduling Delay => Scheduling Delay --- .../org/apache/spark/streaming/ui/AllBatchesTable.scala | 2 +- .../scala/org/apache/spark/streaming/ui/StreamingPage.scala | 2 +- .../scala/org/apache/spark/streaming/UISeleniumSuite.scala | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala index 7975b0abbe4d8..2960b528d4c5e 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/AllBatchesTable.scala @@ -26,7 +26,7 @@ private[ui] abstract class BatchTableBase(tableId: String) { protected def columns: Seq[Node] = {
Batch Time Input SizeStreaming Scheduling DelayScheduling Delay Processing Time
-
Streaming Scheduling Delay
+
Scheduling Delay
Avg: {schedulingDelay.formattedAvg}