@@ -132,7 +133,7 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
val totalTasksNumStr = if (totalTasks == storedTasks) {
s"$totalTasks"
} else {
- s"$storedTasks, showing ${totalTasks}"
+ s"$totalTasks, showing $storedTasks"
}
val summary =
@@ -685,7 +686,7 @@ private[ui] class TaskDataSource(
private var _tasksToShow: Seq[TaskData] = null
- override def dataSize: Int = taskCount(stage)
+ override def dataSize: Int = store.taskCount(stage.stageId, stage.attemptId).toInt
override def sliceData(from: Int, to: Int): Seq[TaskData] = {
if (_tasksToShow == null) {
@@ -847,7 +848,7 @@ private[ui] class TaskPagedTable(
spark.blacklist.enabled
diff --git a/docs/graphx-programming-guide.md b/docs/graphx-programming-guide.md
index 35293348e3f3d..cb96fd773aa5a 100644
--- a/docs/graphx-programming-guide.md
+++ b/docs/graphx-programming-guide.md
@@ -726,7 +726,7 @@ class GraphOps[VD, ED] {
var g = mapVertices( (vid, vdata) => vprog(vid, vdata, initialMsg) ).cache()
// compute the messages
- var messages = g.mapReduceTriplets(sendMsg, mergeMsg)
+ var messages = GraphXUtils.mapReduceTriplets(g, sendMsg, mergeMsg)
var activeMessages = messages.count()
// Loop until no messages remain or maxIterations is achieved
var i = 0
diff --git a/docs/index.md b/docs/index.md
index 40f628b794c01..9a59097bb4efb 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -10,6 +10,11 @@ It provides high-level APIs in Java, Scala, Python and R,
and an optimized engine that supports general execution graphs.
It also supports a rich set of higher-level tools including [Spark SQL](sql-programming-guide.html) for SQL and structured data processing, [MLlib](ml-guide.html) for machine learning, [GraphX](graphx-programming-guide.html) for graph processing, and [Spark Streaming](streaming-programming-guide.html).
+# Security
+
+Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
+Please see [Spark Security](security.html) before downloading and running Spark.
+
# Downloading
Get Spark from the [downloads page](https://spark.apache.org/downloads.html) of the project website. This documentation is for Spark version {{site.SPARK_VERSION}}. Spark uses Hadoop's client libraries for HDFS and YARN. Downloads are pre-packaged for a handful of popular Hadoop versions.
@@ -26,12 +31,13 @@ Spark runs on both Windows and UNIX-like systems (e.g. Linux, Mac OS). It's easy
locally on one machine --- all you need is to have `java` installed on your system `PATH`,
or the `JAVA_HOME` environment variable pointing to a Java installation.
-Spark runs on Java 8+, Python 2.7+/3.4+ and R 3.1+. For the Scala API, Spark {{site.SPARK_VERSION}}
+Spark runs on Java 8, Python 2.7+/3.4+ and R 3.1+. For the Scala API, Spark {{site.SPARK_VERSION}}
uses Scala {{site.SCALA_BINARY_VERSION}}. You will need to use a compatible Scala version
({{site.SCALA_BINARY_VERSION}}.x).
Note that support for Java 7, Python 2.6 and old Hadoop versions before 2.6.5 were removed as of Spark 2.2.0.
-Support for Scala 2.10 was removed as of 2.3.0.
+Support for Scala 2.10 was removed as of 2.3.0. Support for Scala 2.11 is deprecated as of Spark 2.4.1
+and will be removed in Spark 3.0.
# Running the Examples and Shell
diff --git a/docs/js/api-docs.js b/docs/js/api-docs.js
index 13514e11b9332..287fce61540e9 100644
--- a/docs/js/api-docs.js
+++ b/docs/js/api-docs.js
@@ -58,6 +58,7 @@ $(document).ready(function() {
});
};
script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') +
- 'cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML';
+ 'cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js' +
+ '?config=TeX-AMS-MML_HTMLorMML';
document.getElementsByTagName('head')[0].appendChild(script);
});
diff --git a/docs/js/vendor/jquery-1.12.4.min.js b/docs/js/vendor/jquery-1.12.4.min.js
new file mode 100755
index 0000000000000..e836475870da6
--- /dev/null
+++ b/docs/js/vendor/jquery-1.12.4.min.js
@@ -0,0 +1,5 @@
+/*! jQuery v1.12.4 | (c) jQuery Foundation | jquery.org/license */
+!function(a,b){"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var c=[],d=a.document,e=c.slice,f=c.concat,g=c.push,h=c.indexOf,i={},j=i.toString,k=i.hasOwnProperty,l={},m="1.12.4",n=function(a,b){return new n.fn.init(a,b)},o=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,p=/^-ms-/,q=/-([\da-z])/gi,r=function(a,b){return b.toUpperCase()};n.fn=n.prototype={jquery:m,constructor:n,selector:"",length:0,toArray:function(){return e.call(this)},get:function(a){return null!=a?0>a?this[a+this.length]:this[a]:e.call(this)},pushStack:function(a){var b=n.merge(this.constructor(),a);return b.prevObject=this,b.context=this.context,b},each:function(a){return n.each(this,a)},map:function(a){return this.pushStack(n.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(e.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return this.prevObject||this.constructor()},push:g,sort:c.sort,splice:c.splice},n.extend=n.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||n.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(e=arguments[h]))for(d in e)a=g[d],c=e[d],g!==c&&(j&&c&&(n.isPlainObject(c)||(b=n.isArray(c)))?(b?(b=!1,f=a&&n.isArray(a)?a:[]):f=a&&n.isPlainObject(a)?a:{},g[d]=n.extend(j,f,c)):void 0!==c&&(g[d]=c));return g},n.extend({expando:"jQuery"+(m+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===n.type(a)},isArray:Array.isArray||function(a){return"array"===n.type(a)},isWindow:function(a){return null!=a&&a==a.window},isNumeric:function(a){var b=a&&a.toString();return!n.isArray(a)&&b-parseFloat(b)+1>=0},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},isPlainObject:function(a){var b;if(!a||"object"!==n.type(a)||a.nodeType||n.isWindow(a))return!1;try{if(a.constructor&&!k.call(a,"constructor")&&!k.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}if(!l.ownFirst)for(b in a)return k.call(a,b);for(b in a);return void 0===b||k.call(a,b)},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?i[j.call(a)]||"object":typeof a},globalEval:function(b){b&&n.trim(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return a.replace(p,"ms-").replace(q,r)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b){var c,d=0;if(s(a)){for(c=a.length;c>d;d++)if(b.call(a[d],d,a[d])===!1)break}else for(d in a)if(b.call(a[d],d,a[d])===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(o,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(s(Object(a))?n.merge(c,"string"==typeof a?[a]:a):g.call(c,a)),c},inArray:function(a,b,c){var d;if(b){if(h)return h.call(b,a,c);for(d=b.length,c=c?0>c?Math.max(0,d+c):c:0;d>c;c++)if(c in b&&b[c]===a)return c}return-1},merge:function(a,b){var c=+b.length,d=0,e=a.length;while(c>d)a[e++]=b[d++];if(c!==c)while(void 0!==b[d])a[e++]=b[d++];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,e,g=0,h=[];if(s(a))for(d=a.length;d>g;g++)e=b(a[g],g,c),null!=e&&h.push(e);else for(g in a)e=b(a[g],g,c),null!=e&&h.push(e);return f.apply([],h)},guid:1,proxy:function(a,b){var c,d,f;return"string"==typeof b&&(f=a[b],b=a,a=f),n.isFunction(a)?(c=e.call(arguments,2),d=function(){return a.apply(b||this,c.concat(e.call(arguments)))},d.guid=a.guid=a.guid||n.guid++,d):void 0},now:function(){return+new Date},support:l}),"function"==typeof Symbol&&(n.fn[Symbol.iterator]=c[Symbol.iterator]),n.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(a,b){i["[object "+b+"]"]=b.toLowerCase()});function s(a){var b=!!a&&"length"in a&&a.length,c=n.type(a);return"function"===c||n.isWindow(a)?!1:"array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a}var t=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=ga(),z=ga(),A=ga(),B=function(a,b){return a===b&&(l=!0),0},C=1<<31,D={}.hasOwnProperty,E=[],F=E.pop,G=E.push,H=E.push,I=E.slice,J=function(a,b){for(var c=0,d=a.length;d>c;c++)if(a[c]===b)return c;return-1},K="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",L="[\\x20\\t\\r\\n\\f]",M="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",N="\\["+L+"*("+M+")(?:"+L+"*([*^$|!~]?=)"+L+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+M+"))|)"+L+"*\\]",O=":("+M+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+N+")*)|.*)\\)|)",P=new RegExp(L+"+","g"),Q=new RegExp("^"+L+"+|((?:^|[^\\\\])(?:\\\\.)*)"+L+"+$","g"),R=new RegExp("^"+L+"*,"+L+"*"),S=new RegExp("^"+L+"*([>+~]|"+L+")"+L+"*"),T=new RegExp("="+L+"*([^\\]'\"]*?)"+L+"*\\]","g"),U=new RegExp(O),V=new RegExp("^"+M+"$"),W={ID:new RegExp("^#("+M+")"),CLASS:new RegExp("^\\.("+M+")"),TAG:new RegExp("^("+M+"|[*])"),ATTR:new RegExp("^"+N),PSEUDO:new RegExp("^"+O),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+L+"*(even|odd|(([+-]|)(\\d*)n|)"+L+"*(?:([+-]|)"+L+"*(\\d+)|))"+L+"*\\)|)","i"),bool:new RegExp("^(?:"+K+")$","i"),needsContext:new RegExp("^"+L+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+L+"*((?:-\\d)?\\d*)"+L+"*\\)|)(?=[^-]|$)","i")},X=/^(?:input|select|textarea|button)$/i,Y=/^h\d$/i,Z=/^[^{]+\{\s*\[native \w/,$=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,_=/[+~]/,aa=/'|\\/g,ba=new RegExp("\\\\([\\da-f]{1,6}"+L+"?|("+L+")|.)","ig"),ca=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:0>d?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)},da=function(){m()};try{H.apply(E=I.call(v.childNodes),v.childNodes),E[v.childNodes.length].nodeType}catch(ea){H={apply:E.length?function(a,b){G.apply(a,I.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function fa(a,b,d,e){var f,h,j,k,l,o,r,s,w=b&&b.ownerDocument,x=b?b.nodeType:9;if(d=d||[],"string"!=typeof a||!a||1!==x&&9!==x&&11!==x)return d;if(!e&&((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,p)){if(11!==x&&(o=$.exec(a)))if(f=o[1]){if(9===x){if(!(j=b.getElementById(f)))return d;if(j.id===f)return d.push(j),d}else if(w&&(j=w.getElementById(f))&&t(b,j)&&j.id===f)return d.push(j),d}else{if(o[2])return H.apply(d,b.getElementsByTagName(a)),d;if((f=o[3])&&c.getElementsByClassName&&b.getElementsByClassName)return H.apply(d,b.getElementsByClassName(f)),d}if(c.qsa&&!A[a+" "]&&(!q||!q.test(a))){if(1!==x)w=b,s=a;else if("object"!==b.nodeName.toLowerCase()){(k=b.getAttribute("id"))?k=k.replace(aa,"\\$&"):b.setAttribute("id",k=u),r=g(a),h=r.length,l=V.test(k)?"#"+k:"[id='"+k+"']";while(h--)r[h]=l+" "+qa(r[h]);s=r.join(","),w=_.test(a)&&oa(b.parentNode)||b}if(s)try{return H.apply(d,w.querySelectorAll(s)),d}catch(y){}finally{k===u&&b.removeAttribute("id")}}}return i(a.replace(Q,"$1"),b,d,e)}function ga(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function ha(a){return a[u]=!0,a}function ia(a){var b=n.createElement("div");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function ja(a,b){var c=a.split("|"),e=c.length;while(e--)d.attrHandle[c[e]]=b}function ka(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&(~b.sourceIndex||C)-(~a.sourceIndex||C);if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function la(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function ma(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function na(a){return ha(function(b){return b=+b,ha(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function oa(a){return a&&"undefined"!=typeof a.getElementsByTagName&&a}c=fa.support={},f=fa.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?"HTML"!==b.nodeName:!1},m=fa.setDocument=function(a){var b,e,g=a?a.ownerDocument||a:v;return g!==n&&9===g.nodeType&&g.documentElement?(n=g,o=n.documentElement,p=!f(n),(e=n.defaultView)&&e.top!==e&&(e.addEventListener?e.addEventListener("unload",da,!1):e.attachEvent&&e.attachEvent("onunload",da)),c.attributes=ia(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=ia(function(a){return a.appendChild(n.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=Z.test(n.getElementsByClassName),c.getById=ia(function(a){return o.appendChild(a).id=u,!n.getElementsByName||!n.getElementsByName(u).length}),c.getById?(d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c=b.getElementById(a);return c?[c]:[]}},d.filter.ID=function(a){var b=a.replace(ba,ca);return function(a){return a.getAttribute("id")===b}}):(delete d.find.ID,d.filter.ID=function(a){var b=a.replace(ba,ca);return function(a){var c="undefined"!=typeof a.getAttributeNode&&a.getAttributeNode("id");return c&&c.value===b}}),d.find.TAG=c.getElementsByTagName?function(a,b){return"undefined"!=typeof b.getElementsByTagName?b.getElementsByTagName(a):c.qsa?b.querySelectorAll(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){return"undefined"!=typeof b.getElementsByClassName&&p?b.getElementsByClassName(a):void 0},r=[],q=[],(c.qsa=Z.test(n.querySelectorAll))&&(ia(function(a){o.appendChild(a).innerHTML="b;b++)d+=a[b].value;return d}function ra(a,b,c){var d=b.dir,e=c&&"parentNode"===d,f=x++;return b.first?function(b,c,f){while(b=b[d])if(1===b.nodeType||e)return a(b,c,f)}:function(b,c,g){var h,i,j,k=[w,f];if(g){while(b=b[d])if((1===b.nodeType||e)&&a(b,c,g))return!0}else while(b=b[d])if(1===b.nodeType||e){if(j=b[u]||(b[u]={}),i=j[b.uniqueID]||(j[b.uniqueID]={}),(h=i[d])&&h[0]===w&&h[1]===f)return k[2]=h[2];if(i[d]=k,k[2]=a(b,c,g))return!0}}}function sa(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function ta(a,b,c){for(var d=0,e=b.length;e>d;d++)fa(a,b[d],c);return c}function ua(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;i>h;h++)(f=a[h])&&(c&&!c(f,d,e)||(g.push(f),j&&b.push(h)));return g}function va(a,b,c,d,e,f){return d&&!d[u]&&(d=va(d)),e&&!e[u]&&(e=va(e,f)),ha(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||ta(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:ua(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=ua(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?J(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=ua(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):H.apply(g,r)})}function wa(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=ra(function(a){return a===b},h,!0),l=ra(function(a){return J(b,a)>-1},h,!0),m=[function(a,c,d){var e=!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d));return b=null,e}];f>i;i++)if(c=d.relative[a[i].type])m=[ra(sa(m),c)];else{if(c=d.filter[a[i].type].apply(null,a[i].matches),c[u]){for(e=++i;f>e;e++)if(d.relative[a[e].type])break;return va(i>1&&sa(m),i>1&&qa(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(Q,"$1"),c,e>i&&wa(a.slice(i,e)),f>e&&wa(a=a.slice(e)),f>e&&qa(a))}m.push(c)}return sa(m)}function xa(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,h,i,k){var l,o,q,r=0,s="0",t=f&&[],u=[],v=j,x=f||e&&d.find.TAG("*",k),y=w+=null==v?1:Math.random()||.1,z=x.length;for(k&&(j=g===n||g||k);s!==z&&null!=(l=x[s]);s++){if(e&&l){o=0,g||l.ownerDocument===n||(m(l),h=!p);while(q=a[o++])if(q(l,g||n,h)){i.push(l);break}k&&(w=y)}c&&((l=!q&&l)&&r--,f&&t.push(l))}if(r+=s,c&&s!==r){o=0;while(q=b[o++])q(t,u,g,h);if(f){if(r>0)while(s--)t[s]||u[s]||(u[s]=F.call(i));u=ua(u)}H.apply(i,u),k&&!f&&u.length>0&&r+b.length>1&&fa.uniqueSort(i)}return k&&(w=y,j=v),t};return c?ha(f):f}return h=fa.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=wa(b[c]),f[u]?d.push(f):e.push(f);f=A(a,xa(e,d)),f.selector=a}return f},i=fa.select=function(a,b,e,f){var i,j,k,l,m,n="function"==typeof a&&a,o=!f&&g(a=n.selector||a);if(e=e||[],1===o.length){if(j=o[0]=o[0].slice(0),j.length>2&&"ID"===(k=j[0]).type&&c.getById&&9===b.nodeType&&p&&d.relative[j[1].type]){if(b=(d.find.ID(k.matches[0].replace(ba,ca),b)||[])[0],!b)return e;n&&(b=b.parentNode),a=a.slice(j.shift().value.length)}i=W.needsContext.test(a)?0:j.length;while(i--){if(k=j[i],d.relative[l=k.type])break;if((m=d.find[l])&&(f=m(k.matches[0].replace(ba,ca),_.test(j[0].type)&&oa(b.parentNode)||b))){if(j.splice(i,1),a=f.length&&qa(j),!a)return H.apply(e,f),e;break}}}return(n||h(a,o))(f,b,!p,e,!b||_.test(a)&&oa(b.parentNode)||b),e},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=ia(function(a){return 1&a.compareDocumentPosition(n.createElement("div"))}),ia(function(a){return a.innerHTML="-1)f.splice(c,1),h>=c&&h--}),this},has:function(a){return a?n.inArray(a,f)>-1:f.length>0},empty:function(){return f&&(f=[]),this},disable:function(){return e=g=[],f=c="",this},disabled:function(){return!f},lock:function(){return e=!0,c||j.disable(),this},locked:function(){return!!e},fireWith:function(a,c){return e||(c=c||[],c=[a,c.slice?c.slice():c],g.push(c),b||i()),this},fire:function(){return j.fireWith(this,arguments),this},fired:function(){return!!d}};return j},n.extend({Deferred:function(a){var b=[["resolve","done",n.Callbacks("once memory"),"resolved"],["reject","fail",n.Callbacks("once memory"),"rejected"],["notify","progress",n.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return n.Deferred(function(c){n.each(b,function(b,f){var g=n.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&n.isFunction(a.promise)?a.promise().progress(c.notify).done(c.resolve).fail(c.reject):c[f[0]+"With"](this===d?c.promise():this,g?[a]:arguments)})}),a=null}).promise()},promise:function(a){return null!=a?n.extend(a,d):d}},e={};return d.pipe=d.then,n.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[1^a][2].disable,b[2][2].lock),e[f[0]]=function(){return e[f[0]+"With"](this===e?d:this,arguments),this},e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=e.call(arguments),d=c.length,f=1!==d||a&&n.isFunction(a.promise)?d:0,g=1===f?a:n.Deferred(),h=function(a,b,c){return function(d){b[a]=this,c[a]=arguments.length>1?e.call(arguments):d,c===i?g.notifyWith(b,c):--f||g.resolveWith(b,c)}},i,j,k;if(d>1)for(i=new Array(d),j=new Array(d),k=new Array(d);d>b;b++)c[b]&&n.isFunction(c[b].promise)?c[b].promise().progress(h(b,j,i)).done(h(b,k,c)).fail(g.reject):--f;return f||g.resolveWith(k,c),g.promise()}});var I;n.fn.ready=function(a){return n.ready.promise().done(a),this},n.extend({isReady:!1,readyWait:1,holdReady:function(a){a?n.readyWait++:n.ready(!0)},ready:function(a){(a===!0?--n.readyWait:n.isReady)||(n.isReady=!0,a!==!0&&--n.readyWait>0||(I.resolveWith(d,[n]),n.fn.triggerHandler&&(n(d).triggerHandler("ready"),n(d).off("ready"))))}});function J(){d.addEventListener?(d.removeEventListener("DOMContentLoaded",K),a.removeEventListener("load",K)):(d.detachEvent("onreadystatechange",K),a.detachEvent("onload",K))}function K(){(d.addEventListener||"load"===a.event.type||"complete"===d.readyState)&&(J(),n.ready())}n.ready.promise=function(b){if(!I)if(I=n.Deferred(),"complete"===d.readyState||"loading"!==d.readyState&&!d.documentElement.doScroll)a.setTimeout(n.ready);else if(d.addEventListener)d.addEventListener("DOMContentLoaded",K),a.addEventListener("load",K);else{d.attachEvent("onreadystatechange",K),a.attachEvent("onload",K);var c=!1;try{c=null==a.frameElement&&d.documentElement}catch(e){}c&&c.doScroll&&!function f(){if(!n.isReady){try{c.doScroll("left")}catch(b){return a.setTimeout(f,50)}J(),n.ready()}}()}return I.promise(b)},n.ready.promise();var L;for(L in n(l))break;l.ownFirst="0"===L,l.inlineBlockNeedsLayout=!1,n(function(){var a,b,c,e;c=d.getElementsByTagName("body")[0],c&&c.style&&(b=d.createElement("div"),e=d.createElement("div"),e.style.cssText="position:absolute;border:0;width:0;height:0;top:0;left:-9999px",c.appendChild(e).appendChild(b),"undefined"!=typeof b.style.zoom&&(b.style.cssText="display:inline;margin:0;border:0;padding:1px;width:1px;zoom:1",l.inlineBlockNeedsLayout=a=3===b.offsetWidth,a&&(c.style.zoom=1)),c.removeChild(e))}),function(){var a=d.createElement("div");l.deleteExpando=!0;try{delete a.test}catch(b){l.deleteExpando=!1}a=null}();var M=function(a){var b=n.noData[(a.nodeName+" ").toLowerCase()],c=+a.nodeType||1;return 1!==c&&9!==c?!1:!b||b!==!0&&a.getAttribute("classid")===b},N=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,O=/([A-Z])/g;function P(a,b,c){if(void 0===c&&1===a.nodeType){var d="data-"+b.replace(O,"-$1").toLowerCase();if(c=a.getAttribute(d),"string"==typeof c){try{c="true"===c?!0:"false"===c?!1:"null"===c?null:+c+""===c?+c:N.test(c)?n.parseJSON(c):c}catch(e){}n.data(a,b,c)}else c=void 0;
+}return c}function Q(a){var b;for(b in a)if(("data"!==b||!n.isEmptyObject(a[b]))&&"toJSON"!==b)return!1;return!0}function R(a,b,d,e){if(M(a)){var f,g,h=n.expando,i=a.nodeType,j=i?n.cache:a,k=i?a[h]:a[h]&&h;if(k&&j[k]&&(e||j[k].data)||void 0!==d||"string"!=typeof b)return k||(k=i?a[h]=c.pop()||n.guid++:h),j[k]||(j[k]=i?{}:{toJSON:n.noop}),"object"!=typeof b&&"function"!=typeof b||(e?j[k]=n.extend(j[k],b):j[k].data=n.extend(j[k].data,b)),g=j[k],e||(g.data||(g.data={}),g=g.data),void 0!==d&&(g[n.camelCase(b)]=d),"string"==typeof b?(f=g[b],null==f&&(f=g[n.camelCase(b)])):f=g,f}}function S(a,b,c){if(M(a)){var d,e,f=a.nodeType,g=f?n.cache:a,h=f?a[n.expando]:n.expando;if(g[h]){if(b&&(d=c?g[h]:g[h].data)){n.isArray(b)?b=b.concat(n.map(b,n.camelCase)):b in d?b=[b]:(b=n.camelCase(b),b=b in d?[b]:b.split(" ")),e=b.length;while(e--)delete d[b[e]];if(c?!Q(d):!n.isEmptyObject(d))return}(c||(delete g[h].data,Q(g[h])))&&(f?n.cleanData([a],!0):l.deleteExpando||g!=g.window?delete g[h]:g[h]=void 0)}}}n.extend({cache:{},noData:{"applet ":!0,"embed ":!0,"object ":"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(a){return a=a.nodeType?n.cache[a[n.expando]]:a[n.expando],!!a&&!Q(a)},data:function(a,b,c){return R(a,b,c)},removeData:function(a,b){return S(a,b)},_data:function(a,b,c){return R(a,b,c,!0)},_removeData:function(a,b){return S(a,b,!0)}}),n.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=n.data(f),1===f.nodeType&&!n._data(f,"parsedAttrs"))){c=g.length;while(c--)g[c]&&(d=g[c].name,0===d.indexOf("data-")&&(d=n.camelCase(d.slice(5)),P(f,d,e[d])));n._data(f,"parsedAttrs",!0)}return e}return"object"==typeof a?this.each(function(){n.data(this,a)}):arguments.length>1?this.each(function(){n.data(this,a,b)}):f?P(f,a,n.data(f,a)):void 0},removeData:function(a){return this.each(function(){n.removeData(this,a)})}}),n.extend({queue:function(a,b,c){var d;return a?(b=(b||"fx")+"queue",d=n._data(a,b),c&&(!d||n.isArray(c)?d=n._data(a,b,n.makeArray(c)):d.push(c)),d||[]):void 0},dequeue:function(a,b){b=b||"fx";var c=n.queue(a,b),d=c.length,e=c.shift(),f=n._queueHooks(a,b),g=function(){n.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return n._data(a,c)||n._data(a,c,{empty:n.Callbacks("once memory").add(function(){n._removeData(a,b+"queue"),n._removeData(a,c)})})}}),n.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.lengthh;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f},Z=/^(?:checkbox|radio)$/i,$=/<([\w:-]+)/,_=/^$|\/(?:java|ecma)script/i,aa=/^\s+/,ba="abbr|article|aside|audio|bdi|canvas|data|datalist|details|dialog|figcaption|figure|footer|header|hgroup|main|mark|meter|nav|output|picture|progress|section|summary|template|time|video";function ca(a){var b=ba.split("|"),c=a.createDocumentFragment();if(c.createElement)while(b.length)c.createElement(b.pop());return c}!function(){var a=d.createElement("div"),b=d.createDocumentFragment(),c=d.createElement("input");a.innerHTML=" a "," "],legend:[1,""," "],area:[1,""," "],param:[1,""," "],thead:[1,""],tr:[2,""],col:[2,""],td:[3,""],_default:l.htmlSerialize?[0,"",""]:[1,"X","
"]};da.optgroup=da.option,da.tbody=da.tfoot=da.colgroup=da.caption=da.thead,da.th=da.td;function ea(a,b){var c,d,e=0,f="undefined"!=typeof a.getElementsByTagName?a.getElementsByTagName(b||"*"):"undefined"!=typeof a.querySelectorAll?a.querySelectorAll(b||"*"):void 0;if(!f)for(f=[],c=a.childNodes||a;null!=(d=c[e]);e++)!b||n.nodeName(d,b)?f.push(d):n.merge(f,ea(d,b));return void 0===b||b&&n.nodeName(a,b)?n.merge([a],f):f}function fa(a,b){for(var c,d=0;null!=(c=a[d]);d++)n._data(c,"globalEval",!b||n._data(b[d],"globalEval"))}var ga=/<|&#?\w+;/,ha=/r;r++)if(g=a[r],g||0===g)if("object"===n.type(g))n.merge(q,g.nodeType?[g]:g);else if(ga.test(g)){i=i||p.appendChild(b.createElement("div")),j=($.exec(g)||["",""])[1].toLowerCase(),m=da[j]||da._default,i.innerHTML=m[1]+n.htmlPrefilter(g)+m[2],f=m[0];while(f--)i=i.lastChild;if(!l.leadingWhitespace&&aa.test(g)&&q.push(b.createTextNode(aa.exec(g)[0])),!l.tbody){g="table"!==j||ha.test(g)?""!==m[1]||ha.test(g)?0:i:i.firstChild,f=g&&g.childNodes.length;while(f--)n.nodeName(k=g.childNodes[f],"tbody")&&!k.childNodes.length&&g.removeChild(k)}n.merge(q,i.childNodes),i.textContent="";while(i.firstChild)i.removeChild(i.firstChild);i=p.lastChild}else q.push(b.createTextNode(g));i&&p.removeChild(i),l.appendChecked||n.grep(ea(q,"input"),ia),r=0;while(g=q[r++])if(d&&n.inArray(g,d)>-1)e&&e.push(g);else if(h=n.contains(g.ownerDocument,g),i=ea(p.appendChild(g),"script"),h&&fa(i),c){f=0;while(g=i[f++])_.test(g.type||"")&&c.push(g)}return i=null,p}!function(){var b,c,e=d.createElement("div");for(b in{submit:!0,change:!0,focusin:!0})c="on"+b,(l[b]=c in a)||(e.setAttribute(c,"t"),l[b]=e.attributes[c].expando===!1);e=null}();var ka=/^(?:input|select|textarea)$/i,la=/^key/,ma=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,na=/^(?:focusinfocus|focusoutblur)$/,oa=/^([^.]*)(?:\.(.+)|)/;function pa(){return!0}function qa(){return!1}function ra(){try{return d.activeElement}catch(a){}}function sa(a,b,c,d,e,f){var g,h;if("object"==typeof b){"string"!=typeof c&&(d=d||c,c=void 0);for(h in b)sa(a,h,c,d,b[h],f);return a}if(null==d&&null==e?(e=c,d=c=void 0):null==e&&("string"==typeof c?(e=d,d=void 0):(e=d,d=c,c=void 0)),e===!1)e=qa;else if(!e)return a;return 1===f&&(g=e,e=function(a){return n().off(a),g.apply(this,arguments)},e.guid=g.guid||(g.guid=n.guid++)),a.each(function(){n.event.add(this,b,e,d,c)})}n.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=n._data(a);if(r){c.handler&&(i=c,c=i.handler,e=i.selector),c.guid||(c.guid=n.guid++),(g=r.events)||(g=r.events={}),(k=r.handle)||(k=r.handle=function(a){return"undefined"==typeof n||a&&n.event.triggered===a.type?void 0:n.event.dispatch.apply(k.elem,arguments)},k.elem=a),b=(b||"").match(G)||[""],h=b.length;while(h--)f=oa.exec(b[h])||[],o=q=f[1],p=(f[2]||"").split(".").sort(),o&&(j=n.event.special[o]||{},o=(e?j.delegateType:j.bindType)||o,j=n.event.special[o]||{},l=n.extend({type:o,origType:q,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&n.expr.match.needsContext.test(e),namespace:p.join(".")},i),(m=g[o])||(m=g[o]=[],m.delegateCount=0,j.setup&&j.setup.call(a,d,p,k)!==!1||(a.addEventListener?a.addEventListener(o,k,!1):a.attachEvent&&a.attachEvent("on"+o,k))),j.add&&(j.add.call(a,l),l.handler.guid||(l.handler.guid=c.guid)),e?m.splice(m.delegateCount++,0,l):m.push(l),n.event.global[o]=!0);a=null}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=n.hasData(a)&&n._data(a);if(r&&(k=r.events)){b=(b||"").match(G)||[""],j=b.length;while(j--)if(h=oa.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o){l=n.event.special[o]||{},o=(d?l.delegateType:l.bindType)||o,m=k[o]||[],h=h[2]&&new RegExp("(^|\\.)"+p.join("\\.(?:.*\\.|)")+"(\\.|$)"),i=f=m.length;while(f--)g=m[f],!e&&q!==g.origType||c&&c.guid!==g.guid||h&&!h.test(g.namespace)||d&&d!==g.selector&&("**"!==d||!g.selector)||(m.splice(f,1),g.selector&&m.delegateCount--,l.remove&&l.remove.call(a,g));i&&!m.length&&(l.teardown&&l.teardown.call(a,p,r.handle)!==!1||n.removeEvent(a,o,r.handle),delete k[o])}else for(o in k)n.event.remove(a,o+b[j],c,d,!0);n.isEmptyObject(k)&&(delete r.handle,n._removeData(a,"events"))}},trigger:function(b,c,e,f){var g,h,i,j,l,m,o,p=[e||d],q=k.call(b,"type")?b.type:b,r=k.call(b,"namespace")?b.namespace.split("."):[];if(i=m=e=e||d,3!==e.nodeType&&8!==e.nodeType&&!na.test(q+n.event.triggered)&&(q.indexOf(".")>-1&&(r=q.split("."),q=r.shift(),r.sort()),h=q.indexOf(":")<0&&"on"+q,b=b[n.expando]?b:new n.Event(q,"object"==typeof b&&b),b.isTrigger=f?2:3,b.namespace=r.join("."),b.rnamespace=b.namespace?new RegExp("(^|\\.)"+r.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=e),c=null==c?[b]:n.makeArray(c,[b]),l=n.event.special[q]||{},f||!l.trigger||l.trigger.apply(e,c)!==!1)){if(!f&&!l.noBubble&&!n.isWindow(e)){for(j=l.delegateType||q,na.test(j+q)||(i=i.parentNode);i;i=i.parentNode)p.push(i),m=i;m===(e.ownerDocument||d)&&p.push(m.defaultView||m.parentWindow||a)}o=0;while((i=p[o++])&&!b.isPropagationStopped())b.type=o>1?j:l.bindType||q,g=(n._data(i,"events")||{})[b.type]&&n._data(i,"handle"),g&&g.apply(i,c),g=h&&i[h],g&&g.apply&&M(i)&&(b.result=g.apply(i,c),b.result===!1&&b.preventDefault());if(b.type=q,!f&&!b.isDefaultPrevented()&&(!l._default||l._default.apply(p.pop(),c)===!1)&&M(e)&&h&&e[q]&&!n.isWindow(e)){m=e[h],m&&(e[h]=null),n.event.triggered=q;try{e[q]()}catch(s){}n.event.triggered=void 0,m&&(e[h]=m)}return b.result}},dispatch:function(a){a=n.event.fix(a);var b,c,d,f,g,h=[],i=e.call(arguments),j=(n._data(this,"events")||{})[a.type]||[],k=n.event.special[a.type]||{};if(i[0]=a,a.delegateTarget=this,!k.preDispatch||k.preDispatch.call(this,a)!==!1){h=n.event.handlers.call(this,a,j),b=0;while((f=h[b++])&&!a.isPropagationStopped()){a.currentTarget=f.elem,c=0;while((g=f.handlers[c++])&&!a.isImmediatePropagationStopped())a.rnamespace&&!a.rnamespace.test(g.namespace)||(a.handleObj=g,a.data=g.data,d=((n.event.special[g.origType]||{}).handle||g.handler).apply(f.elem,i),void 0!==d&&(a.result=d)===!1&&(a.preventDefault(),a.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,a),a.result}},handlers:function(a,b){var c,d,e,f,g=[],h=b.delegateCount,i=a.target;if(h&&i.nodeType&&("click"!==a.type||isNaN(a.button)||a.button<1))for(;i!=this;i=i.parentNode||this)if(1===i.nodeType&&(i.disabled!==!0||"click"!==a.type)){for(d=[],c=0;h>c;c++)f=b[c],e=f.selector+" ",void 0===d[e]&&(d[e]=f.needsContext?n(e,this).index(i)>-1:n.find(e,this,null,[i]).length),d[e]&&d.push(f);d.length&&g.push({elem:i,handlers:d})}return h\s*$/g,Aa=ca(d),Ba=Aa.appendChild(d.createElement("div"));function Ca(a,b){return n.nodeName(a,"table")&&n.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function Da(a){return a.type=(null!==n.find.attr(a,"type"))+"/"+a.type,a}function Ea(a){var b=ya.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function Fa(a,b){if(1===b.nodeType&&n.hasData(a)){var c,d,e,f=n._data(a),g=n._data(b,f),h=f.events;if(h){delete g.handle,g.events={};for(c in h)for(d=0,e=h[c].length;e>d;d++)n.event.add(b,c,h[c][d])}g.data&&(g.data=n.extend({},g.data))}}function Ga(a,b){var c,d,e;if(1===b.nodeType){if(c=b.nodeName.toLowerCase(),!l.noCloneEvent&&b[n.expando]){e=n._data(b);for(d in e.events)n.removeEvent(b,d,e.handle);b.removeAttribute(n.expando)}"script"===c&&b.text!==a.text?(Da(b).text=a.text,Ea(b)):"object"===c?(b.parentNode&&(b.outerHTML=a.outerHTML),l.html5Clone&&a.innerHTML&&!n.trim(b.innerHTML)&&(b.innerHTML=a.innerHTML)):"input"===c&&Z.test(a.type)?(b.defaultChecked=b.checked=a.checked,b.value!==a.value&&(b.value=a.value)):"option"===c?b.defaultSelected=b.selected=a.defaultSelected:"input"!==c&&"textarea"!==c||(b.defaultValue=a.defaultValue)}}function Ha(a,b,c,d){b=f.apply([],b);var e,g,h,i,j,k,m=0,o=a.length,p=o-1,q=b[0],r=n.isFunction(q);if(r||o>1&&"string"==typeof q&&!l.checkClone&&xa.test(q))return a.each(function(e){var f=a.eq(e);r&&(b[0]=q.call(this,e,f.html())),Ha(f,b,c,d)});if(o&&(k=ja(b,a[0].ownerDocument,!1,a,d),e=k.firstChild,1===k.childNodes.length&&(k=e),e||d)){for(i=n.map(ea(k,"script"),Da),h=i.length;o>m;m++)g=k,m!==p&&(g=n.clone(g,!0,!0),h&&n.merge(i,ea(g,"script"))),c.call(a[m],g,m);if(h)for(j=i[i.length-1].ownerDocument,n.map(i,Ea),m=0;h>m;m++)g=i[m],_.test(g.type||"")&&!n._data(g,"globalEval")&&n.contains(j,g)&&(g.src?n._evalUrl&&n._evalUrl(g.src):n.globalEval((g.text||g.textContent||g.innerHTML||"").replace(za,"")));k=e=null}return a}function Ia(a,b,c){for(var d,e=b?n.filter(b,a):a,f=0;null!=(d=e[f]);f++)c||1!==d.nodeType||n.cleanData(ea(d)),d.parentNode&&(c&&n.contains(d.ownerDocument,d)&&fa(ea(d,"script")),d.parentNode.removeChild(d));return a}n.extend({htmlPrefilter:function(a){return a.replace(va,"<$1>")},clone:function(a,b,c){var d,e,f,g,h,i=n.contains(a.ownerDocument,a);if(l.html5Clone||n.isXMLDoc(a)||!ua.test("<"+a.nodeName+">")?f=a.cloneNode(!0):(Ba.innerHTML=a.outerHTML,Ba.removeChild(f=Ba.firstChild)),!(l.noCloneEvent&&l.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||n.isXMLDoc(a)))for(d=ea(f),h=ea(a),g=0;null!=(e=h[g]);++g)d[g]&&Ga(e,d[g]);if(b)if(c)for(h=h||ea(a),d=d||ea(f),g=0;null!=(e=h[g]);g++)Fa(e,d[g]);else Fa(a,f);return d=ea(f,"script"),d.length>0&&fa(d,!i&&ea(a,"script")),d=h=e=null,f},cleanData:function(a,b){for(var d,e,f,g,h=0,i=n.expando,j=n.cache,k=l.attributes,m=n.event.special;null!=(d=a[h]);h++)if((b||M(d))&&(f=d[i],g=f&&j[f])){if(g.events)for(e in g.events)m[e]?n.event.remove(d,e):n.removeEvent(d,e,g.handle);j[f]&&(delete j[f],k||"undefined"==typeof d.removeAttribute?d[i]=void 0:d.removeAttribute(i),c.push(f))}}}),n.fn.extend({domManip:Ha,detach:function(a){return Ia(this,a,!0)},remove:function(a){return Ia(this,a)},text:function(a){return Y(this,function(a){return void 0===a?n.text(this):this.empty().append((this[0]&&this[0].ownerDocument||d).createTextNode(a))},null,a,arguments.length)},append:function(){return Ha(this,arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=Ca(this,a);b.appendChild(a)}})},prepend:function(){return Ha(this,arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=Ca(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return Ha(this,arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return Ha(this,arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},empty:function(){for(var a,b=0;null!=(a=this[b]);b++){1===a.nodeType&&n.cleanData(ea(a,!1));while(a.firstChild)a.removeChild(a.firstChild);a.options&&n.nodeName(a,"select")&&(a.options.length=0)}return this},clone:function(a,b){return a=null==a?!1:a,b=null==b?a:b,this.map(function(){return n.clone(this,a,b)})},html:function(a){return Y(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a)return 1===b.nodeType?b.innerHTML.replace(ta,""):void 0;if("string"==typeof a&&!wa.test(a)&&(l.htmlSerialize||!ua.test(a))&&(l.leadingWhitespace||!aa.test(a))&&!da[($.exec(a)||["",""])[1].toLowerCase()]){a=n.htmlPrefilter(a);try{for(;d>c;c++)b=this[c]||{},1===b.nodeType&&(n.cleanData(ea(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=[];return Ha(this,arguments,function(b){var c=this.parentNode;n.inArray(this,a)<0&&(n.cleanData(ea(this)),c&&c.replaceChild(b,this))},a)}}),n.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){n.fn[a]=function(a){for(var c,d=0,e=[],f=n(a),h=f.length-1;h>=d;d++)c=d===h?this:this.clone(!0),n(f[d])[b](c),g.apply(e,c.get());return this.pushStack(e)}});var Ja,Ka={HTML:"block",BODY:"block"};function La(a,b){var c=n(b.createElement(a)).appendTo(b.body),d=n.css(c[0],"display");return c.detach(),d}function Ma(a){var b=d,c=Ka[a];return c||(c=La(a,b),"none"!==c&&c||(Ja=(Ja||n("")).appendTo(b.documentElement),b=(Ja[0].contentWindow||Ja[0].contentDocument).document,b.write(),b.close(),c=La(a,b),Ja.detach()),Ka[a]=c),c}var Na=/^margin/,Oa=new RegExp("^("+T+")(?!px)[a-z%]+$","i"),Pa=function(a,b,c,d){var e,f,g={};for(f in b)g[f]=a.style[f],a.style[f]=b[f];e=c.apply(a,d||[]);for(f in b)a.style[f]=g[f];return e},Qa=d.documentElement;!function(){var b,c,e,f,g,h,i=d.createElement("div"),j=d.createElement("div");if(j.style){j.style.cssText="float:left;opacity:.5",l.opacity="0.5"===j.style.opacity,l.cssFloat=!!j.style.cssFloat,j.style.backgroundClip="content-box",j.cloneNode(!0).style.backgroundClip="",l.clearCloneStyle="content-box"===j.style.backgroundClip,i=d.createElement("div"),i.style.cssText="border:0;width:8px;height:0;top:0;left:-9999px;padding:0;margin-top:1px;position:absolute",j.innerHTML="",i.appendChild(j),l.boxSizing=""===j.style.boxSizing||""===j.style.MozBoxSizing||""===j.style.WebkitBoxSizing,n.extend(l,{reliableHiddenOffsets:function(){return null==b&&k(),f},boxSizingReliable:function(){return null==b&&k(),e},pixelMarginRight:function(){return null==b&&k(),c},pixelPosition:function(){return null==b&&k(),b},reliableMarginRight:function(){return null==b&&k(),g},reliableMarginLeft:function(){return null==b&&k(),h}});function k(){var k,l,m=d.documentElement;m.appendChild(i),j.style.cssText="-webkit-box-sizing:border-box;box-sizing:border-box;position:relative;display:block;margin:auto;border:1px;padding:1px;top:1%;width:50%",b=e=h=!1,c=g=!0,a.getComputedStyle&&(l=a.getComputedStyle(j),b="1%"!==(l||{}).top,h="2px"===(l||{}).marginLeft,e="4px"===(l||{width:"4px"}).width,j.style.marginRight="50%",c="4px"===(l||{marginRight:"4px"}).marginRight,k=j.appendChild(d.createElement("div")),k.style.cssText=j.style.cssText="-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;display:block;margin:0;border:0;padding:0",k.style.marginRight=k.style.width="0",j.style.width="1px",g=!parseFloat((a.getComputedStyle(k)||{}).marginRight),j.removeChild(k)),j.style.display="none",f=0===j.getClientRects().length,f&&(j.style.display="",j.innerHTML="",j.childNodes[0].style.borderCollapse="separate",k=j.getElementsByTagName("td"),k[0].style.cssText="margin:0;border:0;padding:0;display:none",f=0===k[0].offsetHeight,f&&(k[0].style.display="",k[1].style.display="none",f=0===k[0].offsetHeight)),m.removeChild(i)}}}();var Ra,Sa,Ta=/^(top|right|bottom|left)$/;a.getComputedStyle?(Ra=function(b){var c=b.ownerDocument.defaultView;return c&&c.opener||(c=a),c.getComputedStyle(b)},Sa=function(a,b,c){var d,e,f,g,h=a.style;return c=c||Ra(a),g=c?c.getPropertyValue(b)||c[b]:void 0,""!==g&&void 0!==g||n.contains(a.ownerDocument,a)||(g=n.style(a,b)),c&&!l.pixelMarginRight()&&Oa.test(g)&&Na.test(b)&&(d=h.width,e=h.minWidth,f=h.maxWidth,h.minWidth=h.maxWidth=h.width=g,g=c.width,h.width=d,h.minWidth=e,h.maxWidth=f),void 0===g?g:g+""}):Qa.currentStyle&&(Ra=function(a){return a.currentStyle},Sa=function(a,b,c){var d,e,f,g,h=a.style;return c=c||Ra(a),g=c?c[b]:void 0,null==g&&h&&h[b]&&(g=h[b]),Oa.test(g)&&!Ta.test(b)&&(d=h.left,e=a.runtimeStyle,f=e&&e.left,f&&(e.left=a.currentStyle.left),h.left="fontSize"===b?"1em":g,g=h.pixelLeft+"px",h.left=d,f&&(e.left=f)),void 0===g?g:g+""||"auto"});function Ua(a,b){return{get:function(){return a()?void delete this.get:(this.get=b).apply(this,arguments)}}}var Va=/alpha\([^)]*\)/i,Wa=/opacity\s*=\s*([^)]*)/i,Xa=/^(none|table(?!-c[ea]).+)/,Ya=new RegExp("^("+T+")(.*)$","i"),Za={position:"absolute",visibility:"hidden",display:"block"},$a={letterSpacing:"0",fontWeight:"400"},_a=["Webkit","O","Moz","ms"],ab=d.createElement("div").style;function bb(a){if(a in ab)return a;var b=a.charAt(0).toUpperCase()+a.slice(1),c=_a.length;while(c--)if(a=_a[c]+b,a in ab)return a}function cb(a,b){for(var c,d,e,f=[],g=0,h=a.length;h>g;g++)d=a[g],d.style&&(f[g]=n._data(d,"olddisplay"),c=d.style.display,b?(f[g]||"none"!==c||(d.style.display=""),""===d.style.display&&W(d)&&(f[g]=n._data(d,"olddisplay",Ma(d.nodeName)))):(e=W(d),(c&&"none"!==c||!e)&&n._data(d,"olddisplay",e?c:n.css(d,"display"))));for(g=0;h>g;g++)d=a[g],d.style&&(b&&"none"!==d.style.display&&""!==d.style.display||(d.style.display=b?f[g]||"":"none"));return a}function db(a,b,c){var d=Ya.exec(b);return d?Math.max(0,d[1]-(c||0))+(d[2]||"px"):b}function eb(a,b,c,d,e){for(var f=c===(d?"border":"content")?4:"width"===b?1:0,g=0;4>f;f+=2)"margin"===c&&(g+=n.css(a,c+V[f],!0,e)),d?("content"===c&&(g-=n.css(a,"padding"+V[f],!0,e)),"margin"!==c&&(g-=n.css(a,"border"+V[f]+"Width",!0,e))):(g+=n.css(a,"padding"+V[f],!0,e),"padding"!==c&&(g+=n.css(a,"border"+V[f]+"Width",!0,e)));return g}function fb(a,b,c){var d=!0,e="width"===b?a.offsetWidth:a.offsetHeight,f=Ra(a),g=l.boxSizing&&"border-box"===n.css(a,"boxSizing",!1,f);if(0>=e||null==e){if(e=Sa(a,b,f),(0>e||null==e)&&(e=a.style[b]),Oa.test(e))return e;d=g&&(l.boxSizingReliable()||e===a.style[b]),e=parseFloat(e)||0}return e+eb(a,b,c||(g?"border":"content"),d,f)+"px"}n.extend({cssHooks:{opacity:{get:function(a,b){if(b){var c=Sa(a,"opacity");return""===c?"1":c}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{"float":l.cssFloat?"cssFloat":"styleFloat"},style:function(a,b,c,d){if(a&&3!==a.nodeType&&8!==a.nodeType&&a.style){var e,f,g,h=n.camelCase(b),i=a.style;if(b=n.cssProps[h]||(n.cssProps[h]=bb(h)||h),g=n.cssHooks[b]||n.cssHooks[h],void 0===c)return g&&"get"in g&&void 0!==(e=g.get(a,!1,d))?e:i[b];if(f=typeof c,"string"===f&&(e=U.exec(c))&&e[1]&&(c=X(a,b,e),f="number"),null!=c&&c===c&&("number"===f&&(c+=e&&e[3]||(n.cssNumber[h]?"":"px")),l.clearCloneStyle||""!==c||0!==b.indexOf("background")||(i[b]="inherit"),!(g&&"set"in g&&void 0===(c=g.set(a,c,d)))))try{i[b]=c}catch(j){}}},css:function(a,b,c,d){var e,f,g,h=n.camelCase(b);return b=n.cssProps[h]||(n.cssProps[h]=bb(h)||h),g=n.cssHooks[b]||n.cssHooks[h],g&&"get"in g&&(f=g.get(a,!0,c)),void 0===f&&(f=Sa(a,b,d)),"normal"===f&&b in $a&&(f=$a[b]),""===c||c?(e=parseFloat(f),c===!0||isFinite(e)?e||0:f):f}}),n.each(["height","width"],function(a,b){n.cssHooks[b]={get:function(a,c,d){return c?Xa.test(n.css(a,"display"))&&0===a.offsetWidth?Pa(a,Za,function(){return fb(a,b,d)}):fb(a,b,d):void 0},set:function(a,c,d){var e=d&&Ra(a);return db(a,c,d?eb(a,b,d,l.boxSizing&&"border-box"===n.css(a,"boxSizing",!1,e),e):0)}}}),l.opacity||(n.cssHooks.opacity={get:function(a,b){return Wa.test((b&&a.currentStyle?a.currentStyle.filter:a.style.filter)||"")?.01*parseFloat(RegExp.$1)+"":b?"1":""},set:function(a,b){var c=a.style,d=a.currentStyle,e=n.isNumeric(b)?"alpha(opacity="+100*b+")":"",f=d&&d.filter||c.filter||"";c.zoom=1,(b>=1||""===b)&&""===n.trim(f.replace(Va,""))&&c.removeAttribute&&(c.removeAttribute("filter"),""===b||d&&!d.filter)||(c.filter=Va.test(f)?f.replace(Va,e):f+" "+e)}}),n.cssHooks.marginRight=Ua(l.reliableMarginRight,function(a,b){return b?Pa(a,{display:"inline-block"},Sa,[a,"marginRight"]):void 0}),n.cssHooks.marginLeft=Ua(l.reliableMarginLeft,function(a,b){return b?(parseFloat(Sa(a,"marginLeft"))||(n.contains(a.ownerDocument,a)?a.getBoundingClientRect().left-Pa(a,{
+marginLeft:0},function(){return a.getBoundingClientRect().left}):0))+"px":void 0}),n.each({margin:"",padding:"",border:"Width"},function(a,b){n.cssHooks[a+b]={expand:function(c){for(var d=0,e={},f="string"==typeof c?c.split(" "):[c];4>d;d++)e[a+V[d]+b]=f[d]||f[d-2]||f[0];return e}},Na.test(a)||(n.cssHooks[a+b].set=db)}),n.fn.extend({css:function(a,b){return Y(this,function(a,b,c){var d,e,f={},g=0;if(n.isArray(b)){for(d=Ra(a),e=b.length;e>g;g++)f[b[g]]=n.css(a,b[g],!1,d);return f}return void 0!==c?n.style(a,b,c):n.css(a,b)},a,b,arguments.length>1)},show:function(){return cb(this,!0)},hide:function(){return cb(this)},toggle:function(a){return"boolean"==typeof a?a?this.show():this.hide():this.each(function(){W(this)?n(this).show():n(this).hide()})}});function gb(a,b,c,d,e){return new gb.prototype.init(a,b,c,d,e)}n.Tween=gb,gb.prototype={constructor:gb,init:function(a,b,c,d,e,f){this.elem=a,this.prop=c,this.easing=e||n.easing._default,this.options=b,this.start=this.now=this.cur(),this.end=d,this.unit=f||(n.cssNumber[c]?"":"px")},cur:function(){var a=gb.propHooks[this.prop];return a&&a.get?a.get(this):gb.propHooks._default.get(this)},run:function(a){var b,c=gb.propHooks[this.prop];return this.options.duration?this.pos=b=n.easing[this.easing](a,this.options.duration*a,0,1,this.options.duration):this.pos=b=a,this.now=(this.end-this.start)*b+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),c&&c.set?c.set(this):gb.propHooks._default.set(this),this}},gb.prototype.init.prototype=gb.prototype,gb.propHooks={_default:{get:function(a){var b;return 1!==a.elem.nodeType||null!=a.elem[a.prop]&&null==a.elem.style[a.prop]?a.elem[a.prop]:(b=n.css(a.elem,a.prop,""),b&&"auto"!==b?b:0)},set:function(a){n.fx.step[a.prop]?n.fx.step[a.prop](a):1!==a.elem.nodeType||null==a.elem.style[n.cssProps[a.prop]]&&!n.cssHooks[a.prop]?a.elem[a.prop]=a.now:n.style(a.elem,a.prop,a.now+a.unit)}}},gb.propHooks.scrollTop=gb.propHooks.scrollLeft={set:function(a){a.elem.nodeType&&a.elem.parentNode&&(a.elem[a.prop]=a.now)}},n.easing={linear:function(a){return a},swing:function(a){return.5-Math.cos(a*Math.PI)/2},_default:"swing"},n.fx=gb.prototype.init,n.fx.step={};var hb,ib,jb=/^(?:toggle|show|hide)$/,kb=/queueHooks$/;function lb(){return a.setTimeout(function(){hb=void 0}),hb=n.now()}function mb(a,b){var c,d={height:a},e=0;for(b=b?1:0;4>e;e+=2-b)c=V[e],d["margin"+c]=d["padding"+c]=a;return b&&(d.opacity=d.width=a),d}function nb(a,b,c){for(var d,e=(qb.tweeners[b]||[]).concat(qb.tweeners["*"]),f=0,g=e.length;g>f;f++)if(d=e[f].call(c,b,a))return d}function ob(a,b,c){var d,e,f,g,h,i,j,k,m=this,o={},p=a.style,q=a.nodeType&&W(a),r=n._data(a,"fxshow");c.queue||(h=n._queueHooks(a,"fx"),null==h.unqueued&&(h.unqueued=0,i=h.empty.fire,h.empty.fire=function(){h.unqueued||i()}),h.unqueued++,m.always(function(){m.always(function(){h.unqueued--,n.queue(a,"fx").length||h.empty.fire()})})),1===a.nodeType&&("height"in b||"width"in b)&&(c.overflow=[p.overflow,p.overflowX,p.overflowY],j=n.css(a,"display"),k="none"===j?n._data(a,"olddisplay")||Ma(a.nodeName):j,"inline"===k&&"none"===n.css(a,"float")&&(l.inlineBlockNeedsLayout&&"inline"!==Ma(a.nodeName)?p.zoom=1:p.display="inline-block")),c.overflow&&(p.overflow="hidden",l.shrinkWrapBlocks()||m.always(function(){p.overflow=c.overflow[0],p.overflowX=c.overflow[1],p.overflowY=c.overflow[2]}));for(d in b)if(e=b[d],jb.exec(e)){if(delete b[d],f=f||"toggle"===e,e===(q?"hide":"show")){if("show"!==e||!r||void 0===r[d])continue;q=!0}o[d]=r&&r[d]||n.style(a,d)}else j=void 0;if(n.isEmptyObject(o))"inline"===("none"===j?Ma(a.nodeName):j)&&(p.display=j);else{r?"hidden"in r&&(q=r.hidden):r=n._data(a,"fxshow",{}),f&&(r.hidden=!q),q?n(a).show():m.done(function(){n(a).hide()}),m.done(function(){var b;n._removeData(a,"fxshow");for(b in o)n.style(a,b,o[b])});for(d in o)g=nb(q?r[d]:0,d,m),d in r||(r[d]=g.start,q&&(g.end=g.start,g.start="width"===d||"height"===d?1:0))}}function pb(a,b){var c,d,e,f,g;for(c in a)if(d=n.camelCase(c),e=b[d],f=a[c],n.isArray(f)&&(e=f[1],f=a[c]=f[0]),c!==d&&(a[d]=f,delete a[c]),g=n.cssHooks[d],g&&"expand"in g){f=g.expand(f),delete a[d];for(c in f)c in a||(a[c]=f[c],b[c]=e)}else b[d]=e}function qb(a,b,c){var d,e,f=0,g=qb.prefilters.length,h=n.Deferred().always(function(){delete i.elem}),i=function(){if(e)return!1;for(var b=hb||lb(),c=Math.max(0,j.startTime+j.duration-b),d=c/j.duration||0,f=1-d,g=0,i=j.tweens.length;i>g;g++)j.tweens[g].run(f);return h.notifyWith(a,[j,f,c]),1>f&&i?c:(h.resolveWith(a,[j]),!1)},j=h.promise({elem:a,props:n.extend({},b),opts:n.extend(!0,{specialEasing:{},easing:n.easing._default},c),originalProperties:b,originalOptions:c,startTime:hb||lb(),duration:c.duration,tweens:[],createTween:function(b,c){var d=n.Tween(a,j.opts,b,c,j.opts.specialEasing[b]||j.opts.easing);return j.tweens.push(d),d},stop:function(b){var c=0,d=b?j.tweens.length:0;if(e)return this;for(e=!0;d>c;c++)j.tweens[c].run(1);return b?(h.notifyWith(a,[j,1,0]),h.resolveWith(a,[j,b])):h.rejectWith(a,[j,b]),this}}),k=j.props;for(pb(k,j.opts.specialEasing);g>f;f++)if(d=qb.prefilters[f].call(j,a,k,j.opts))return n.isFunction(d.stop)&&(n._queueHooks(j.elem,j.opts.queue).stop=n.proxy(d.stop,d)),d;return n.map(k,nb,j),n.isFunction(j.opts.start)&&j.opts.start.call(a,j),n.fx.timer(n.extend(i,{elem:a,anim:j,queue:j.opts.queue})),j.progress(j.opts.progress).done(j.opts.done,j.opts.complete).fail(j.opts.fail).always(j.opts.always)}n.Animation=n.extend(qb,{tweeners:{"*":[function(a,b){var c=this.createTween(a,b);return X(c.elem,a,U.exec(b),c),c}]},tweener:function(a,b){n.isFunction(a)?(b=a,a=["*"]):a=a.match(G);for(var c,d=0,e=a.length;e>d;d++)c=a[d],qb.tweeners[c]=qb.tweeners[c]||[],qb.tweeners[c].unshift(b)},prefilters:[ob],prefilter:function(a,b){b?qb.prefilters.unshift(a):qb.prefilters.push(a)}}),n.speed=function(a,b,c){var d=a&&"object"==typeof a?n.extend({},a):{complete:c||!c&&b||n.isFunction(a)&&a,duration:a,easing:c&&b||b&&!n.isFunction(b)&&b};return d.duration=n.fx.off?0:"number"==typeof d.duration?d.duration:d.duration in n.fx.speeds?n.fx.speeds[d.duration]:n.fx.speeds._default,null!=d.queue&&d.queue!==!0||(d.queue="fx"),d.old=d.complete,d.complete=function(){n.isFunction(d.old)&&d.old.call(this),d.queue&&n.dequeue(this,d.queue)},d},n.fn.extend({fadeTo:function(a,b,c,d){return this.filter(W).css("opacity",0).show().end().animate({opacity:b},a,c,d)},animate:function(a,b,c,d){var e=n.isEmptyObject(a),f=n.speed(b,c,d),g=function(){var b=qb(this,n.extend({},a),f);(e||n._data(this,"finish"))&&b.stop(!0)};return g.finish=g,e||f.queue===!1?this.each(g):this.queue(f.queue,g)},stop:function(a,b,c){var d=function(a){var b=a.stop;delete a.stop,b(c)};return"string"!=typeof a&&(c=b,b=a,a=void 0),b&&a!==!1&&this.queue(a||"fx",[]),this.each(function(){var b=!0,e=null!=a&&a+"queueHooks",f=n.timers,g=n._data(this);if(e)g[e]&&g[e].stop&&d(g[e]);else for(e in g)g[e]&&g[e].stop&&kb.test(e)&&d(g[e]);for(e=f.length;e--;)f[e].elem!==this||null!=a&&f[e].queue!==a||(f[e].anim.stop(c),b=!1,f.splice(e,1));!b&&c||n.dequeue(this,a)})},finish:function(a){return a!==!1&&(a=a||"fx"),this.each(function(){var b,c=n._data(this),d=c[a+"queue"],e=c[a+"queueHooks"],f=n.timers,g=d?d.length:0;for(c.finish=!0,n.queue(this,a,[]),e&&e.stop&&e.stop.call(this,!0),b=f.length;b--;)f[b].elem===this&&f[b].queue===a&&(f[b].anim.stop(!0),f.splice(b,1));for(b=0;g>b;b++)d[b]&&d[b].finish&&d[b].finish.call(this);delete c.finish})}}),n.each(["toggle","show","hide"],function(a,b){var c=n.fn[b];n.fn[b]=function(a,d,e){return null==a||"boolean"==typeof a?c.apply(this,arguments):this.animate(mb(b,!0),a,d,e)}}),n.each({slideDown:mb("show"),slideUp:mb("hide"),slideToggle:mb("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(a,b){n.fn[a]=function(a,c,d){return this.animate(b,a,c,d)}}),n.timers=[],n.fx.tick=function(){var a,b=n.timers,c=0;for(hb=n.now();ca ").append(n.parseHTML(a)).find(d):a)}).always(c&&function(a,b){g.each(function(){c.apply(this,f||[a.responseText,b,a])})}),this},n.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(a,b){n.fn[b]=function(a){return this.on(b,a)}}),n.expr.filters.animated=function(a){return n.grep(n.timers,function(b){return a===b.elem}).length};function mc(a){return n.isWindow(a)?a:9===a.nodeType?a.defaultView||a.parentWindow:!1}n.offset={setOffset:function(a,b,c){var d,e,f,g,h,i,j,k=n.css(a,"position"),l=n(a),m={};"static"===k&&(a.style.position="relative"),h=l.offset(),f=n.css(a,"top"),i=n.css(a,"left"),j=("absolute"===k||"fixed"===k)&&n.inArray("auto",[f,i])>-1,j?(d=l.position(),g=d.top,e=d.left):(g=parseFloat(f)||0,e=parseFloat(i)||0),n.isFunction(b)&&(b=b.call(a,c,n.extend({},h))),null!=b.top&&(m.top=b.top-h.top+g),null!=b.left&&(m.left=b.left-h.left+e),"using"in b?b.using.call(a,m):l.css(m)}},n.fn.extend({offset:function(a){if(arguments.length)return void 0===a?this:this.each(function(b){n.offset.setOffset(this,a,b)});var b,c,d={top:0,left:0},e=this[0],f=e&&e.ownerDocument;if(f)return b=f.documentElement,n.contains(b,e)?("undefined"!=typeof e.getBoundingClientRect&&(d=e.getBoundingClientRect()),c=mc(f),{top:d.top+(c.pageYOffset||b.scrollTop)-(b.clientTop||0),left:d.left+(c.pageXOffset||b.scrollLeft)-(b.clientLeft||0)}):d},position:function(){if(this[0]){var a,b,c={top:0,left:0},d=this[0];return"fixed"===n.css(d,"position")?b=d.getBoundingClientRect():(a=this.offsetParent(),b=this.offset(),n.nodeName(a[0],"html")||(c=a.offset()),c.top+=n.css(a[0],"borderTopWidth",!0),c.left+=n.css(a[0],"borderLeftWidth",!0)),{top:b.top-c.top-n.css(d,"marginTop",!0),left:b.left-c.left-n.css(d,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var a=this.offsetParent;while(a&&!n.nodeName(a,"html")&&"static"===n.css(a,"position"))a=a.offsetParent;return a||Qa})}}),n.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(a,b){var c=/Y/.test(b);n.fn[a]=function(d){return Y(this,function(a,d,e){var f=mc(a);return void 0===e?f?b in f?f[b]:f.document.documentElement[d]:a[d]:void(f?f.scrollTo(c?n(f).scrollLeft():e,c?e:n(f).scrollTop()):a[d]=e)},a,d,arguments.length,null)}}),n.each(["top","left"],function(a,b){n.cssHooks[b]=Ua(l.pixelPosition,function(a,c){return c?(c=Sa(a,b),Oa.test(c)?n(a).position()[b]+"px":c):void 0})}),n.each({Height:"height",Width:"width"},function(a,b){n.each({
+padding:"inner"+a,content:b,"":"outer"+a},function(c,d){n.fn[d]=function(d,e){var f=arguments.length&&(c||"boolean"!=typeof d),g=c||(d===!0||e===!0?"margin":"border");return Y(this,function(b,c,d){var e;return n.isWindow(b)?b.document.documentElement["client"+a]:9===b.nodeType?(e=b.documentElement,Math.max(b.body["scroll"+a],e["scroll"+a],b.body["offset"+a],e["offset"+a],e["client"+a])):void 0===d?n.css(b,c,g):n.style(b,c,d,g)},b,f?d:void 0,f,null)}})}),n.fn.extend({bind:function(a,b,c){return this.on(a,null,b,c)},unbind:function(a,b){return this.off(a,null,b)},delegate:function(a,b,c,d){return this.on(b,a,c,d)},undelegate:function(a,b,c){return 1===arguments.length?this.off(a,"**"):this.off(b,a||"**",c)}}),n.fn.size=function(){return this.length},n.fn.andSelf=n.fn.addBack,"function"==typeof define&&define.amd&&define("jquery",[],function(){return n});var nc=a.jQuery,oc=a.$;return n.noConflict=function(b){return a.$===n&&(a.$=oc),b&&a.jQuery===n&&(a.jQuery=nc),n},b||(a.jQuery=a.$=n),n});
diff --git a/docs/js/vendor/jquery-1.8.0.min.js b/docs/js/vendor/jquery-1.8.0.min.js
deleted file mode 100755
index 066d72c7e3a2e..0000000000000
--- a/docs/js/vendor/jquery-1.8.0.min.js
+++ /dev/null
@@ -1,2 +0,0 @@
-/*! jQuery v@1.8.0 jquery.com | jquery.org/license */
-(function(a,b){function G(a){var b=F[a]={};return p.each(a.split(s),function(a,c){b[c]=!0}),b}function J(a,c,d){if(d===b&&a.nodeType===1){var e="data-"+c.replace(I,"-$1").toLowerCase();d=a.getAttribute(e);if(typeof d=="string"){try{d=d==="true"?!0:d==="false"?!1:d==="null"?null:+d+""===d?+d:H.test(d)?p.parseJSON(d):d}catch(f){}p.data(a,c,d)}else d=b}return d}function K(a){var b;for(b in a){if(b==="data"&&p.isEmptyObject(a[b]))continue;if(b!=="toJSON")return!1}return!0}function ba(){return!1}function bb(){return!0}function bh(a){return!a||!a.parentNode||a.parentNode.nodeType===11}function bi(a,b){do a=a[b];while(a&&a.nodeType!==1);return a}function bj(a,b,c){b=b||0;if(p.isFunction(b))return p.grep(a,function(a,d){var e=!!b.call(a,d,a);return e===c});if(b.nodeType)return p.grep(a,function(a,d){return a===b===c});if(typeof b=="string"){var d=p.grep(a,function(a){return a.nodeType===1});if(be.test(b))return p.filter(b,d,!c);b=p.filter(b,d)}return p.grep(a,function(a,d){return p.inArray(a,b)>=0===c})}function bk(a){var b=bl.split("|"),c=a.createDocumentFragment();if(c.createElement)while(b.length)c.createElement(b.pop());return c}function bC(a,b){return a.getElementsByTagName(b)[0]||a.appendChild(a.ownerDocument.createElement(b))}function bD(a,b){if(b.nodeType!==1||!p.hasData(a))return;var c,d,e,f=p._data(a),g=p._data(b,f),h=f.events;if(h){delete g.handle,g.events={};for(c in h)for(d=0,e=h[c].length;d
").appendTo(e.body),c=b.css("display");b.remove();if(c==="none"||c===""){bI=e.body.appendChild(bI||p.extend(e.createElement("iframe"),{frameBorder:0,width:0,height:0}));if(!bJ||!bI.createElement)bJ=(bI.contentWindow||bI.contentDocument).document,bJ.write(""),bJ.close();b=bJ.body.appendChild(bJ.createElement(a)),c=bH(b,"display"),e.body.removeChild(bI)}return bR[a]=c,c}function ch(a,b,c,d){var e;if(p.isArray(b))p.each(b,function(b,e){c||cd.test(a)?d(a,e):ch(a+"["+(typeof e=="object"?b:"")+"]",e,c,d)});else if(!c&&p.type(b)==="object")for(e in b)ch(a+"["+e+"]",b[e],c,d);else d(a,b)}function cy(a){return function(b,c){typeof b!="string"&&(c=b,b="*");var d,e,f,g=b.toLowerCase().split(s),h=0,i=g.length;if(p.isFunction(c))for(;h)[^>]*$|#([\w\-]*)$)/,v=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,w=/^[\],:{}\s]*$/,x=/(?:^|:|,)(?:\s*\[)+/g,y=/\\(?:["\\\/bfnrt]|u[\da-fA-F]{4})/g,z=/"[^"\\\r\n]*"|true|false|null|-?(?:\d\d*\.|)\d+(?:[eE][\-+]?\d+|)/g,A=/^-ms-/,B=/-([\da-z])/gi,C=function(a,b){return(b+"").toUpperCase()},D=function(){e.addEventListener?(e.removeEventListener("DOMContentLoaded",D,!1),p.ready()):e.readyState==="complete"&&(e.detachEvent("onreadystatechange",D),p.ready())},E={};p.fn=p.prototype={constructor:p,init:function(a,c,d){var f,g,h,i;if(!a)return this;if(a.nodeType)return this.context=this[0]=a,this.length=1,this;if(typeof a=="string"){a.charAt(0)==="<"&&a.charAt(a.length-1)===">"&&a.length>=3?f=[null,a,null]:f=u.exec(a);if(f&&(f[1]||!c)){if(f[1])return c=c instanceof p?c[0]:c,i=c&&c.nodeType?c.ownerDocument||c:e,a=p.parseHTML(f[1],i,!0),v.test(f[1])&&p.isPlainObject(c)&&this.attr.call(a,c,!0),p.merge(this,a);g=e.getElementById(f[2]);if(g&&g.parentNode){if(g.id!==f[2])return d.find(a);this.length=1,this[0]=g}return this.context=e,this.selector=a,this}return!c||c.jquery?(c||d).find(a):this.constructor(c).find(a)}return p.isFunction(a)?d.ready(a):(a.selector!==b&&(this.selector=a.selector,this.context=a.context),p.makeArray(a,this))},selector:"",jquery:"1.8.0",length:0,size:function(){return this.length},toArray:function(){return k.call(this)},get:function(a){return a==null?this.toArray():a<0?this[this.length+a]:this[a]},pushStack:function(a,b,c){var d=p.merge(this.constructor(),a);return d.prevObject=this,d.context=this.context,b==="find"?d.selector=this.selector+(this.selector?" ":"")+c:b&&(d.selector=this.selector+"."+b+"("+c+")"),d},each:function(a,b){return p.each(this,a,b)},ready:function(a){return p.ready.promise().done(a),this},eq:function(a){return a=+a,a===-1?this.slice(a):this.slice(a,a+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(k.apply(this,arguments),"slice",k.call(arguments).join(","))},map:function(a){return this.pushStack(p.map(this,function(b,c){return a.call(b,c,b)}))},end:function(){return this.prevObject||this.constructor(null)},push:j,sort:[].sort,splice:[].splice},p.fn.init.prototype=p.fn,p.extend=p.fn.extend=function(){var a,c,d,e,f,g,h=arguments[0]||{},i=1,j=arguments.length,k=!1;typeof h=="boolean"&&(k=h,h=arguments[1]||{},i=2),typeof h!="object"&&!p.isFunction(h)&&(h={}),j===i&&(h=this,--i);for(;i0)return;d.resolveWith(e,[p]),p.fn.trigger&&p(e).trigger("ready").off("ready")},isFunction:function(a){return p.type(a)==="function"},isArray:Array.isArray||function(a){return p.type(a)==="array"},isWindow:function(a){return a!=null&&a==a.window},isNumeric:function(a){return!isNaN(parseFloat(a))&&isFinite(a)},type:function(a){return a==null?String(a):E[m.call(a)]||"object"},isPlainObject:function(a){if(!a||p.type(a)!=="object"||a.nodeType||p.isWindow(a))return!1;try{if(a.constructor&&!n.call(a,"constructor")&&!n.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}var d;for(d in a);return d===b||n.call(a,d)},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},error:function(a){throw new Error(a)},parseHTML:function(a,b,c){var d;return!a||typeof a!="string"?null:(typeof b=="boolean"&&(c=b,b=0),b=b||e,(d=v.exec(a))?[b.createElement(d[1])]:(d=p.buildFragment([a],b,c?null:[]),p.merge([],(d.cacheable?p.clone(d.fragment):d.fragment).childNodes)))},parseJSON:function(b){if(!b||typeof b!="string")return null;b=p.trim(b);if(a.JSON&&a.JSON.parse)return a.JSON.parse(b);if(w.test(b.replace(y,"@").replace(z,"]").replace(x,"")))return(new Function("return "+b))();p.error("Invalid JSON: "+b)},parseXML:function(c){var d,e;if(!c||typeof c!="string")return null;try{a.DOMParser?(e=new DOMParser,d=e.parseFromString(c,"text/xml")):(d=new ActiveXObject("Microsoft.XMLDOM"),d.async="false",d.loadXML(c))}catch(f){d=b}return(!d||!d.documentElement||d.getElementsByTagName("parsererror").length)&&p.error("Invalid XML: "+c),d},noop:function(){},globalEval:function(b){b&&r.test(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return a.replace(A,"ms-").replace(B,C)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toUpperCase()===b.toUpperCase()},each:function(a,c,d){var e,f=0,g=a.length,h=g===b||p.isFunction(a);if(d){if(h){for(e in a)if(c.apply(a[e],d)===!1)break}else for(;f0&&a[0]&&a[i-1]||i===0||p.isArray(a));if(j)for(;h-1)i.splice(c,1),e&&(c<=g&&g--,c<=h&&h--)}),this},has:function(a){return p.inArray(a,i)>-1},empty:function(){return i=[],this},disable:function(){return i=j=c=b,this},disabled:function(){return!i},lock:function(){return j=b,c||l.disable(),this},locked:function(){return!j},fireWith:function(a,b){return b=b||[],b=[a,b.slice?b.slice():b],i&&(!d||j)&&(e?j.push(b):k(b)),this},fire:function(){return l.fireWith(this,arguments),this},fired:function(){return!!d}};return l},p.extend({Deferred:function(a){var b=[["resolve","done",p.Callbacks("once memory"),"resolved"],["reject","fail",p.Callbacks("once memory"),"rejected"],["notify","progress",p.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return p.Deferred(function(c){p.each(b,function(b,d){var f=d[0],g=a[b];e[d[1]](p.isFunction(g)?function(){var a=g.apply(this,arguments);a&&p.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f+"With"](this===e?c:this,[a])}:c[f])}),a=null}).promise()},promise:function(a){return typeof a=="object"?p.extend(a,d):d}},e={};return d.pipe=d.then,p.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[a^1][2].disable,b[2][2].lock),e[f[0]]=g.fire,e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=k.call(arguments),d=c.length,e=d!==1||a&&p.isFunction(a.promise)?d:0,f=e===1?a:p.Deferred(),g=function(a,b,c){return function(d){b[a]=this,c[a]=arguments.length>1?k.call(arguments):d,c===h?f.notifyWith(b,c):--e||f.resolveWith(b,c)}},h,i,j;if(d>1){h=new Array(d),i=new Array(d),j=new Array(d);for(;ba
",d.firstChild.style.width="5px",b.shrinkWrapBlocks=d.offsetWidth!==3,c.style.zoom=1),i.removeChild(c),c=d=f=g=null}),i.removeChild(n),c=d=f=g=h=i=n=null,b}();var H=/^(?:\{.*\}|\[.*\])$/,I=/([A-Z])/g;p.extend({cache:{},deletedIds:[],uuid:0,expando:"jQuery"+(p.fn.jquery+Math.random()).replace(/\D/g,""),noData:{embed:!0,object:"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000",applet:!0},hasData:function(a){return a=a.nodeType?p.cache[a[p.expando]]:a[p.expando],!!a&&!K(a)},data:function(a,c,d,e){if(!p.acceptData(a))return;var f,g,h=p.expando,i=typeof c=="string",j=a.nodeType,k=j?p.cache:a,l=j?a[h]:a[h]&&h;if((!l||!k[l]||!e&&!k[l].data)&&i&&d===b)return;l||(j?a[h]=l=p.deletedIds.pop()||++p.uuid:l=h),k[l]||(k[l]={},j||(k[l].toJSON=p.noop));if(typeof c=="object"||typeof c=="function")e?k[l]=p.extend(k[l],c):k[l].data=p.extend(k[l].data,c);return f=k[l],e||(f.data||(f.data={}),f=f.data),d!==b&&(f[p.camelCase(c)]=d),i?(g=f[c],g==null&&(g=f[p.camelCase(c)])):g=f,g},removeData:function(a,b,c){if(!p.acceptData(a))return;var d,e,f,g=a.nodeType,h=g?p.cache:a,i=g?a[p.expando]:p.expando;if(!h[i])return;if(b){d=c?h[i]:h[i].data;if(d){p.isArray(b)||(b in d?b=[b]:(b=p.camelCase(b),b in d?b=[b]:b=b.split(" ")));for(e=0,f=b.length;e1,null,!1))},removeData:function(a){return this.each(function(){p.removeData(this,a)})}}),p.extend({queue:function(a,b,c){var d;if(a)return b=(b||"fx")+"queue",d=p._data(a,b),c&&(!d||p.isArray(c)?d=p._data(a,b,p.makeArray(c)):d.push(c)),d||[]},dequeue:function(a,b){b=b||"fx";var c=p.queue(a,b),d=c.shift(),e=p._queueHooks(a,b),f=function(){p.dequeue(a,b)};d==="inprogress"&&(d=c.shift()),d&&(b==="fx"&&c.unshift("inprogress"),delete e.stop,d.call(a,f,e)),!c.length&&e&&e.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return p._data(a,c)||p._data(a,c,{empty:p.Callbacks("once memory").add(function(){p.removeData(a,b+"queue",!0),p.removeData(a,c,!0)})})}}),p.fn.extend({queue:function(a,c){var d=2;return typeof a!="string"&&(c=a,a="fx",d--),arguments.length1)},removeAttr:function(a){return this.each(function(){p.removeAttr(this,a)})},prop:function(a,b){return p.access(this,p.prop,a,b,arguments.length>1)},removeProp:function(a){return a=p.propFix[a]||a,this.each(function(){try{this[a]=b,delete this[a]}catch(c){}})},addClass:function(a){var b,c,d,e,f,g,h;if(p.isFunction(a))return this.each(function(b){p(this).addClass(a.call(this,b,this.className))});if(a&&typeof a=="string"){b=a.split(s);for(c=0,d=this.length;c-1)d=d.replace(" "+c[f]+" "," ");e.className=a?p.trim(d):""}}}return this},toggleClass:function(a,b){var c=typeof a,d=typeof b=="boolean";return p.isFunction(a)?this.each(function(c){p(this).toggleClass(a.call(this,c,this.className,b),b)}):this.each(function(){if(c==="string"){var e,f=0,g=p(this),h=b,i=a.split(s);while(e=i[f++])h=d?h:!g.hasClass(e),g[h?"addClass":"removeClass"](e)}else if(c==="undefined"||c==="boolean")this.className&&p._data(this,"__className__",this.className),this.className=this.className||a===!1?"":p._data(this,"__className__")||""})},hasClass:function(a){var b=" "+a+" ",c=0,d=this.length;for(;c-1)return!0;return!1},val:function(a){var c,d,e,f=this[0];if(!arguments.length){if(f)return c=p.valHooks[f.type]||p.valHooks[f.nodeName.toLowerCase()],c&&"get"in c&&(d=c.get(f,"value"))!==b?d:(d=f.value,typeof d=="string"?d.replace(P,""):d==null?"":d);return}return e=p.isFunction(a),this.each(function(d){var f,g=p(this);if(this.nodeType!==1)return;e?f=a.call(this,d,g.val()):f=a,f==null?f="":typeof f=="number"?f+="":p.isArray(f)&&(f=p.map(f,function(a){return a==null?"":a+""})),c=p.valHooks[this.type]||p.valHooks[this.nodeName.toLowerCase()];if(!c||!("set"in c)||c.set(this,f,"value")===b)this.value=f})}}),p.extend({valHooks:{option:{get:function(a){var b=a.attributes.value;return!b||b.specified?a.value:a.text}},select:{get:function(a){var b,c,d,e,f=a.selectedIndex,g=[],h=a.options,i=a.type==="select-one";if(f<0)return null;c=i?f:0,d=i?f+1:h.length;for(;c=0}),c.length||(a.selectedIndex=-1),c}}},attrFn:{},attr:function(a,c,d,e){var f,g,h,i=a.nodeType;if(!a||i===3||i===8||i===2)return;if(e&&p.isFunction(p.fn[c]))return p(a)[c](d);if(typeof a.getAttribute=="undefined")return p.prop(a,c,d);h=i!==1||!p.isXMLDoc(a),h&&(c=c.toLowerCase(),g=p.attrHooks[c]||(T.test(c)?M:L));if(d!==b){if(d===null){p.removeAttr(a,c);return}return g&&"set"in g&&h&&(f=g.set(a,d,c))!==b?f:(a.setAttribute(c,""+d),d)}return g&&"get"in g&&h&&(f=g.get(a,c))!==null?f:(f=a.getAttribute(c),f===null?b:f)},removeAttr:function(a,b){var c,d,e,f,g=0;if(b&&a.nodeType===1){d=b.split(s);for(;g=0}})});var V=/^(?:textarea|input|select)$/i,W=/^([^\.]*|)(?:\.(.+)|)$/,X=/(?:^|\s)hover(\.\S+|)\b/,Y=/^key/,Z=/^(?:mouse|contextmenu)|click/,$=/^(?:focusinfocus|focusoutblur)$/,_=function(a){return p.event.special.hover?a:a.replace(X,"mouseenter$1 mouseleave$1")};p.event={add:function(a,c,d,e,f){var g,h,i,j,k,l,m,n,o,q,r;if(a.nodeType===3||a.nodeType===8||!c||!d||!(g=p._data(a)))return;d.handler&&(o=d,d=o.handler,f=o.selector),d.guid||(d.guid=p.guid++),i=g.events,i||(g.events=i={}),h=g.handle,h||(g.handle=h=function(a){return typeof p!="undefined"&&(!a||p.event.triggered!==a.type)?p.event.dispatch.apply(h.elem,arguments):b},h.elem=a),c=p.trim(_(c)).split(" ");for(j=0;j=0&&(s=s.slice(0,-1),i=!0),s.indexOf(".")>=0&&(t=s.split("."),s=t.shift(),t.sort());if((!f||p.event.customEvent[s])&&!p.event.global[s])return;c=typeof c=="object"?c[p.expando]?c:new p.Event(s,c):new p.Event(s),c.type=s,c.isTrigger=!0,c.exclusive=i,c.namespace=t.join("."),c.namespace_re=c.namespace?new RegExp("(^|\\.)"+t.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,m=s.indexOf(":")<0?"on"+s:"";if(!f){h=p.cache;for(j in h)h[j].events&&h[j].events[s]&&p.event.trigger(c,d,h[j].handle.elem,!0);return}c.result=b,c.target||(c.target=f),d=d!=null?p.makeArray(d):[],d.unshift(c),n=p.event.special[s]||{};if(n.trigger&&n.trigger.apply(f,d)===!1)return;q=[[f,n.bindType||s]];if(!g&&!n.noBubble&&!p.isWindow(f)){r=n.delegateType||s,k=$.test(r+s)?f:f.parentNode;for(l=f;k;k=k.parentNode)q.push([k,r]),l=k;l===(f.ownerDocument||e)&&q.push([l.defaultView||l.parentWindow||a,r])}for(j=0;jq&&u.push({elem:this,matches:o.slice(q)});for(d=0;d0?this.on(b,null,a,c):this.trigger(b)},Y.test(b)&&(p.event.fixHooks[b]=p.event.keyHooks),Z.test(b)&&(p.event.fixHooks[b]=p.event.mouseHooks)}),function(a,b){function bd(a,b,c,d){var e=0,f=b.length;for(;e0?h(g,c,f):[]}function bf(a,c,d,e,f){var g,h,i,j,k,l,m,n,p=0,q=f.length,s=L.POS,t=new RegExp("^"+s.source+"(?!"+r+")","i"),u=function(){var a=1,c=arguments.length-2;for(;ai){m=a.slice(i,g.index),i=n,l=[c],B.test(m)&&(k&&(l=k),k=e);if(h=H.test(m))m=m.slice(0,-5).replace(B,"$&*");g.length>1&&g[0].replace(t,u),k=be(m,g[1],g[2],l,k,h)}}k?(j=j.concat(k),(m=a.slice(i))&&m!==")"?B.test(m)?bd(m,j,d,e):Z(m,c,d,e?e.concat(k):k):o.apply(d,j)):Z(a,c,d,e)}return q===1?d:Z.uniqueSort(d)}function bg(a,b,c){var d,e,f,g=[],i=0,j=D.exec(a),k=!j.pop()&&!j.pop(),l=k&&a.match(C)||[""],m=$.preFilter,n=$.filter,o=!c&&b!==h;for(;(e=l[i])!=null&&k;i++){g.push(d=[]),o&&(e=" "+e);while(e){k=!1;if(j=B.exec(e))e=e.slice(j[0].length),k=d.push({part:j.pop().replace(A," "),captures:j});for(f in n)(j=L[f].exec(e))&&(!m[f]||(j=m[f](j,b,c)))&&(e=e.slice(j.shift().length),k=d.push({part:f,captures:j}));if(!k)break}}return k||Z.error(a),g}function bh(a,b,e){var f=b.dir,g=m++;return a||(a=function(a){return a===e}),b.first?function(b,c){while(b=b[f])if(b.nodeType===1)return a(b,c)&&b}:function(b,e){var h,i=g+"."+d,j=i+"."+c;while(b=b[f])if(b.nodeType===1){if((h=b[q])===j)return b.sizset;if(typeof h=="string"&&h.indexOf(i)===0){if(b.sizset)return b}else{b[q]=j;if(a(b,e))return b.sizset=!0,b;b.sizset=!1}}}}function bi(a,b){return a?function(c,d){var e=b(c,d);return e&&a(e===!0?c:e,d)}:b}function bj(a,b,c){var d,e,f=0;for(;d=a[f];f++)$.relative[d.part]?e=bh(e,$.relative[d.part],b):(d.captures.push(b,c),e=bi(e,$.filter[d.part].apply(null,d.captures)));return e}function bk(a){return function(b,c){var d,e=0;for(;d=a[e];e++)if(d(b,c))return!0;return!1}}var c,d,e,f,g,h=a.document,i=h.documentElement,j="undefined",k=!1,l=!0,m=0,n=[].slice,o=[].push,q=("sizcache"+Math.random()).replace(".",""),r="[\\x20\\t\\r\\n\\f]",s="(?:\\\\.|[-\\w]|[^\\x00-\\xa0])+",t=s.replace("w","w#"),u="([*^$|!~]?=)",v="\\["+r+"*("+s+")"+r+"*(?:"+u+r+"*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|("+t+")|)|)"+r+"*\\]",w=":("+s+")(?:\\((?:(['\"])((?:\\\\.|[^\\\\])*?)\\2|((?:[^,]|\\\\,|(?:,(?=[^\\[]*\\]))|(?:,(?=[^\\(]*\\))))*))\\)|)",x=":(nth|eq|gt|lt|first|last|even|odd)(?:\\((\\d*)\\)|)(?=[^-]|$)",y=r+"*([\\x20\\t\\r\\n\\f>+~])"+r+"*",z="(?=[^\\x20\\t\\r\\n\\f])(?:\\\\.|"+v+"|"+w.replace(2,7)+"|[^\\\\(),])+",A=new RegExp("^"+r+"+|((?:^|[^\\\\])(?:\\\\.)*)"+r+"+$","g"),B=new RegExp("^"+y),C=new RegExp(z+"?(?="+r+"*,|$)","g"),D=new RegExp("^(?:(?!,)(?:(?:^|,)"+r+"*"+z+")*?|"+r+"*(.*?))(\\)|$)"),E=new RegExp(z.slice(19,-6)+"\\x20\\t\\r\\n\\f>+~])+|"+y,"g"),F=/^(?:#([\w\-]+)|(\w+)|\.([\w\-]+))$/,G=/[\x20\t\r\n\f]*[+~]/,H=/:not\($/,I=/h\d/i,J=/input|select|textarea|button/i,K=/\\(?!\\)/g,L={ID:new RegExp("^#("+s+")"),CLASS:new RegExp("^\\.("+s+")"),NAME:new RegExp("^\\[name=['\"]?("+s+")['\"]?\\]"),TAG:new RegExp("^("+s.replace("[-","[-\\*")+")"),ATTR:new RegExp("^"+v),PSEUDO:new RegExp("^"+w),CHILD:new RegExp("^:(only|nth|last|first)-child(?:\\("+r+"*(even|odd|(([+-]|)(\\d*)n|)"+r+"*(?:([+-]|)"+r+"*(\\d+)|))"+r+"*\\)|)","i"),POS:new RegExp(x,"ig"),needsContext:new RegExp("^"+r+"*[>+~]|"+x,"i")},M={},N=[],O={},P=[],Q=function(a){return a.sizzleFilter=!0,a},R=function(a){return function(b){return b.nodeName.toLowerCase()==="input"&&b.type===a}},S=function(a){return function(b){var c=b.nodeName.toLowerCase();return(c==="input"||c==="button")&&b.type===a}},T=function(a){var b=!1,c=h.createElement("div");try{b=a(c)}catch(d){}return c=null,b},U=T(function(a){a.innerHTML="
",i.insertBefore(a,i.firstChild);var b=h.getElementsByName&&h.getElementsByName(q).length===2+h.getElementsByName(q+0).length;return g=!h.getElementById(q),i.removeChild(a),b}),W=T(function(a){return a.appendChild(h.createComment("")),a.getElementsByTagName("*").length===0}),X=T(function(a){return a.innerHTML="
",!a.getElementsByClassName||a.getElementsByClassName("e").length===0?!1:(a.lastChild.className="e",a.getElementsByClassName("e").length!==1)}),Z=function(a,b,c,d){c=c||[],b=b||h;var e,f,g,i,j=b.nodeType;if(j!==1&&j!==9)return[];if(!a||typeof a!="string")return c;g=ba(b);if(!g&&!d)if(e=F.exec(a))if(i=e[1]){if(j===9){f=b.getElementById(i);if(!f||!f.parentNode)return c;if(f.id===i)return c.push(f),c}else if(b.ownerDocument&&(f=b.ownerDocument.getElementById(i))&&bb(b,f)&&f.id===i)return c.push(f),c}else{if(e[2])return o.apply(c,n.call(b.getElementsByTagName(a),0)),c;if((i=e[3])&&Y&&b.getElementsByClassName)return o.apply(c,n.call(b.getElementsByClassName(i),0)),c}return bm(a,b,c,d,g)},$=Z.selectors={cacheLength:50,match:L,order:["ID","TAG"],attrHandle:{},createPseudo:Q,find:{ID:g?function(a,b,c){if(typeof b.getElementById!==j&&!c){var d=b.getElementById(a);return d&&d.parentNode?[d]:[]}}:function(a,c,d){if(typeof c.getElementById!==j&&!d){var e=c.getElementById(a);return e?e.id===a||typeof e.getAttributeNode!==j&&e.getAttributeNode("id").value===a?[e]:b:[]}},TAG:W?function(a,b){if(typeof b.getElementsByTagName!==j)return b.getElementsByTagName(a)}:function(a,b){var c=b.getElementsByTagName(a);if(a==="*"){var d,e=[],f=0;for(;d=c[f];f++)d.nodeType===1&&e.push(d);return e}return c}},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(K,""),a[3]=(a[4]||a[5]||"").replace(K,""),a[2]==="~="&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),a[1]==="nth"?(a[2]||Z.error(a[0]),a[3]=+(a[3]?a[4]+(a[5]||1):2*(a[2]==="even"||a[2]==="odd")),a[4]=+(a[6]+a[7]||a[2]==="odd")):a[2]&&Z.error(a[0]),a},PSEUDO:function(a){var b,c=a[4];return L.CHILD.test(a[0])?null:(c&&(b=D.exec(c))&&b.pop()&&(a[0]=a[0].slice(0,b[0].length-c.length-1),c=b[0].slice(0,-1)),a.splice(2,3,c||a[3]),a)}},filter:{ID:g?function(a){return a=a.replace(K,""),function(b){return b.getAttribute("id")===a}}:function(a){return a=a.replace(K,""),function(b){var c=typeof b.getAttributeNode!==j&&b.getAttributeNode("id");return c&&c.value===a}},TAG:function(a){return a==="*"?function(){return!0}:(a=a.replace(K,"").toLowerCase(),function(b){return b.nodeName&&b.nodeName.toLowerCase()===a})},CLASS:function(a){var b=M[a];return b||(b=M[a]=new RegExp("(^|"+r+")"+a+"("+r+"|$)"),N.push(a),N.length>$.cacheLength&&delete M[N.shift()]),function(a){return b.test(a.className||typeof a.getAttribute!==j&&a.getAttribute("class")||"")}},ATTR:function(a,b,c){return b?function(d){var e=Z.attr(d,a),f=e+"";if(e==null)return b==="!=";switch(b){case"=":return f===c;case"!=":return f!==c;case"^=":return c&&f.indexOf(c)===0;case"*=":return c&&f.indexOf(c)>-1;case"$=":return c&&f.substr(f.length-c.length)===c;case"~=":return(" "+f+" ").indexOf(c)>-1;case"|=":return f===c||f.substr(0,c.length+1)===c+"-"}}:function(b){return Z.attr(b,a)!=null}},CHILD:function(a,b,c,d){if(a==="nth"){var e=m++;return function(a){var b,f,g=0,h=a;if(c===1&&d===0)return!0;b=a.parentNode;if(b&&(b[q]!==e||!a.sizset)){for(h=b.firstChild;h;h=h.nextSibling)if(h.nodeType===1){h.sizset=++g;if(h===a)break}b[q]=e}return f=a.sizset-d,c===0?f===0:f%c===0&&f/c>=0}}return function(b){var c=b;switch(a){case"only":case"first":while(c=c.previousSibling)if(c.nodeType===1)return!1;if(a==="first")return!0;c=b;case"last":while(c=c.nextSibling)if(c.nodeType===1)return!1;return!0}}},PSEUDO:function(a,b,c,d){var e=$.pseudos[a]||$.pseudos[a.toLowerCase()];return e||Z.error("unsupported pseudo: "+a),e.sizzleFilter?e(b,c,d):e}},pseudos:{not:Q(function(a,b,c){var d=bl(a.replace(A,"$1"),b,c);return function(a){return!d(a)}}),enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return b==="input"&&!!a.checked||b==="option"&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},parent:function(a){return!$.pseudos.empty(a)},empty:function(a){var b;a=a.firstChild;while(a){if(a.nodeName>"@"||(b=a.nodeType)===3||b===4)return!1;a=a.nextSibling}return!0},contains:Q(function(a){return function(b){return(b.textContent||b.innerText||bc(b)).indexOf(a)>-1}}),has:Q(function(a){return function(b){return Z(a,b).length>0}}),header:function(a){return I.test(a.nodeName)},text:function(a){var b,c;return a.nodeName.toLowerCase()==="input"&&(b=a.type)==="text"&&((c=a.getAttribute("type"))==null||c.toLowerCase()===b)},radio:R("radio"),checkbox:R("checkbox"),file:R("file"),password:R("password"),image:R("image"),submit:S("submit"),reset:S("reset"),button:function(a){var b=a.nodeName.toLowerCase();return b==="input"&&a.type==="button"||b==="button"},input:function(a){return J.test(a.nodeName)},focus:function(a){var b=a.ownerDocument;return a===b.activeElement&&(!b.hasFocus||b.hasFocus())&&(!!a.type||!!a.href)},active:function(a){return a===a.ownerDocument.activeElement}},setFilters:{first:function(a,b,c){return c?a.slice(1):[a[0]]},last:function(a,b,c){var d=a.pop();return c?a:[d]},even:function(a,b,c){var d=[],e=c?1:0,f=a.length;for(;e$.cacheLength&&delete O[P.shift()],g};Z.matches=function(a,b){return Z(a,null,null,b)},Z.matchesSelector=function(a,b){return Z(b,null,null,[a]).length>0};var bm=function(a,b,e,f,g){a=a.replace(A,"$1");var h,i,j,k,l,m,p,q,r,s=a.match(C),t=a.match(E),u=b.nodeType;if(L.POS.test(a))return bf(a,b,e,f,s);if(f)h=n.call(f,0);else if(s&&s.length===1){if(t.length>1&&u===9&&!g&&(s=L.ID.exec(t[0]))){b=$.find.ID(s[1],b,g)[0];if(!b)return e;a=a.slice(t.shift().length)}q=(s=G.exec(t[0]))&&!s.index&&b.parentNode||b,r=t.pop(),m=r.split(":not")[0];for(j=0,k=$.order.length;j
",a.querySelectorAll("[test^='']").length&&e.push("[*^$]="+r+"*(?:\"\"|'')"),a.innerHTML="0)for(e=d;e=0:p.filter(a,this).length>0:this.filter(a).length>0)},closest:function(a,b){var c,d=0,e=this.length,f=[],g=bf.test(a)||typeof a!="string"?p(a,b||this.context):0;for(;d-1:p.find.matchesSelector(c,a)){f.push(c);break}c=c.parentNode}}return f=f.length>1?p.unique(f):f,this.pushStack(f,"closest",a)},index:function(a){return a?typeof a=="string"?p.inArray(this[0],p(a)):p.inArray(a.jquery?a[0]:a,this):this[0]&&this[0].parentNode?this.prevAll().length:-1},add:function(a,b){var c=typeof a=="string"?p(a,b):p.makeArray(a&&a.nodeType?[a]:a),d=p.merge(this.get(),c);return this.pushStack(bh(c[0])||bh(d[0])?d:p.unique(d))},addBack:function(a){return this.add(a==null?this.prevObject:this.prevObject.filter(a))}}),p.fn.andSelf=p.fn.addBack,p.each({parent:function(a){var b=a.parentNode;return b&&b.nodeType!==11?b:null},parents:function(a){return p.dir(a,"parentNode")},parentsUntil:function(a,b,c){return p.dir(a,"parentNode",c)},next:function(a){return bi(a,"nextSibling")},prev:function(a){return bi(a,"previousSibling")},nextAll:function(a){return p.dir(a,"nextSibling")},prevAll:function(a){return p.dir(a,"previousSibling")},nextUntil:function(a,b,c){return p.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return p.dir(a,"previousSibling",c)},siblings:function(a){return p.sibling((a.parentNode||{}).firstChild,a)},children:function(a){return p.sibling(a.firstChild)},contents:function(a){return p.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:p.merge([],a.childNodes)}},function(a,b){p.fn[a]=function(c,d){var e=p.map(this,b,c);return bc.test(a)||(d=c),d&&typeof d=="string"&&(e=p.filter(d,e)),e=this.length>1&&!bg[a]?p.unique(e):e,this.length>1&&bd.test(a)&&(e=e.reverse()),this.pushStack(e,a,k.call(arguments).join(","))}}),p.extend({filter:function(a,b,c){return c&&(a=":not("+a+")"),b.length===1?p.find.matchesSelector(b[0],a)?[b[0]]:[]:p.find.matches(a,b)},dir:function(a,c,d){var e=[],f=a[c];while(f&&f.nodeType!==9&&(d===b||f.nodeType!==1||!p(f).is(d)))f.nodeType===1&&e.push(f),f=f[c];return e},sibling:function(a,b){var c=[];for(;a;a=a.nextSibling)a.nodeType===1&&a!==b&&c.push(a);return c}});var bl="abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|header|hgroup|mark|meter|nav|output|progress|section|summary|time|video",bm=/ jQuery\d+="(?:null|\d+)"/g,bn=/^\s+/,bo=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,bp=/<([\w:]+)/,bq=/"," "],legend:[1,""," "],thead:[1,""],tr:[2,""],td:[3,""],col:[2,""],area:[1,""," "],_default:[0,"",""]},bA=bk(e),bB=bA.appendChild(e.createElement("div"));bz.optgroup=bz.option,bz.tbody=bz.tfoot=bz.colgroup=bz.caption=bz.thead,bz.th=bz.td,p.support.htmlSerialize||(bz._default=[1,"X","
"]),p.fn.extend({text:function(a){return p.access(this,function(a){return a===b?p.text(this):this.empty().append((this[0]&&this[0].ownerDocument||e).createTextNode(a))},null,a,arguments.length)},wrapAll:function(a){if(p.isFunction(a))return this.each(function(b){p(this).wrapAll(a.call(this,b))});if(this[0]){var b=p(a,this[0].ownerDocument).eq(0).clone(!0);this[0].parentNode&&b.insertBefore(this[0]),b.map(function(){var a=this;while(a.firstChild&&a.firstChild.nodeType===1)a=a.firstChild;return a}).append(this)}return this},wrapInner:function(a){return p.isFunction(a)?this.each(function(b){p(this).wrapInner(a.call(this,b))}):this.each(function(){var b=p(this),c=b.contents();c.length?c.wrapAll(a):b.append(a)})},wrap:function(a){var b=p.isFunction(a);return this.each(function(c){p(this).wrapAll(b?a.call(this,c):a)})},unwrap:function(){return this.parent().each(function(){p.nodeName(this,"body")||p(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,!0,function(a){(this.nodeType===1||this.nodeType===11)&&this.appendChild(a)})},prepend:function(){return this.domManip(arguments,!0,function(a){(this.nodeType===1||this.nodeType===11)&&this.insertBefore(a,this.firstChild)})},before:function(){if(!bh(this[0]))return this.domManip(arguments,!1,function(a){this.parentNode.insertBefore(a,this)});if(arguments.length){var a=p.clean(arguments);return this.pushStack(p.merge(a,this),"before",this.selector)}},after:function(){if(!bh(this[0]))return this.domManip(arguments,!1,function(a){this.parentNode.insertBefore(a,this.nextSibling)});if(arguments.length){var a=p.clean(arguments);return this.pushStack(p.merge(this,a),"after",this.selector)}},remove:function(a,b){var c,d=0;for(;(c=this[d])!=null;d++)if(!a||p.filter(a,[c]).length)!b&&c.nodeType===1&&(p.cleanData(c.getElementsByTagName("*")),p.cleanData([c])),c.parentNode&&c.parentNode.removeChild(c);return this},empty:function(){var a,b=0;for(;(a=this[b])!=null;b++){a.nodeType===1&&p.cleanData(a.getElementsByTagName("*"));while(a.firstChild)a.removeChild(a.firstChild)}return this},clone:function(a,b){return a=a==null?!1:a,b=b==null?a:b,this.map(function(){return p.clone(this,a,b)})},html:function(a){return p.access(this,function(a){var c=this[0]||{},d=0,e=this.length;if(a===b)return c.nodeType===1?c.innerHTML.replace(bm,""):b;if(typeof a=="string"&&!bs.test(a)&&(p.support.htmlSerialize||!bu.test(a))&&(p.support.leadingWhitespace||!bn.test(a))&&!bz[(bp.exec(a)||["",""])[1].toLowerCase()]){a=a.replace(bo,"<$1>");try{for(;d1&&typeof j=="string"&&bw.test(j))return this.each(function(){p(this).domManip(a,c,d)});if(p.isFunction(j))return this.each(function(e){var f=p(this);a[0]=j.call(this,e,c?f.html():b),f.domManip(a,c,d)});if(this[0]){e=p.buildFragment(a,this,k),g=e.fragment,f=g.firstChild,g.childNodes.length===1&&(g=f);if(f){c=c&&p.nodeName(f,"tr");for(h=e.cacheable||l-1;i0?this.clone(!0):this).get(),p(g[e])[b](d),f=f.concat(d);return this.pushStack(f,a,g.selector)}}),p.extend({clone:function(a,b,c){var d,e,f,g;p.support.html5Clone||p.isXMLDoc(a)||!bu.test("<"+a.nodeName+">")?g=a.cloneNode(!0):(bB.innerHTML=a.outerHTML,bB.removeChild(g=bB.firstChild));if((!p.support.noCloneEvent||!p.support.noCloneChecked)&&(a.nodeType===1||a.nodeType===11)&&!p.isXMLDoc(a)){bE(a,g),d=bF(a),e=bF(g);for(f=0;d[f];++f)e[f]&&bE(d[f],e[f])}if(b){bD(a,g);if(c){d=bF(a),e=bF(g);for(f=0;d[f];++f)bD(d[f],e[f])}}return d=e=null,g},clean:function(a,b,c,d){var f,g,h,i,j,k,l,m,n,o,q,r,s=0,t=[];if(!b||typeof b.createDocumentFragment=="undefined")b=e;for(g=b===e&&bA;(h=a[s])!=null;s++){typeof h=="number"&&(h+="");if(!h)continue;if(typeof h=="string")if(!br.test(h))h=b.createTextNode(h);else{g=g||bk(b),l=l||g.appendChild(b.createElement("div")),h=h.replace(bo,"<$1>"),i=(bp.exec(h)||["",""])[1].toLowerCase(),j=bz[i]||bz._default,k=j[0],l.innerHTML=j[1]+h+j[2];while(k--)l=l.lastChild;if(!p.support.tbody){m=bq.test(h),n=i==="table"&&!m?l.firstChild&&l.firstChild.childNodes:j[1]===""&&!m?l.childNodes:[];for(f=n.length-1;f>=0;--f)p.nodeName(n[f],"tbody")&&!n[f].childNodes.length&&n[f].parentNode.removeChild(n[f])}!p.support.leadingWhitespace&&bn.test(h)&&l.insertBefore(b.createTextNode(bn.exec(h)[0]),l.firstChild),h=l.childNodes,l=g.lastChild}h.nodeType?t.push(h):t=p.merge(t,h)}l&&(g.removeChild(l),h=l=g=null);if(!p.support.appendChecked)for(s=0;(h=t[s])!=null;s++)p.nodeName(h,"input")?bG(h):typeof h.getElementsByTagName!="undefined"&&p.grep(h.getElementsByTagName("input"),bG);if(c){q=function(a){if(!a.type||bx.test(a.type))return d?d.push(a.parentNode?a.parentNode.removeChild(a):a):c.appendChild(a)};for(s=0;(h=t[s])!=null;s++)if(!p.nodeName(h,"script")||!q(h))c.appendChild(h),typeof h.getElementsByTagName!="undefined"&&(r=p.grep(p.merge([],h.getElementsByTagName("script")),q),t.splice.apply(t,[s+1,0].concat(r)),s+=r.length)}return t},cleanData:function(a,b){var c,d,e,f,g=0,h=p.expando,i=p.cache,j=p.support.deleteExpando,k=p.event.special;for(;(e=a[g])!=null;g++)if(b||p.acceptData(e)){d=e[h],c=d&&i[d];if(c){if(c.events)for(f in c.events)k[f]?p.event.remove(e,f):p.removeEvent(e,f,c.handle);i[d]&&(delete i[d],j?delete e[h]:e.removeAttribute?e.removeAttribute(h):e[h]=null,p.deletedIds.push(d))}}}}),function(){var a,b;p.uaMatch=function(a){a=a.toLowerCase();var b=/(chrome)[ \/]([\w.]+)/.exec(a)||/(webkit)[ \/]([\w.]+)/.exec(a)||/(opera)(?:.*version|)[ \/]([\w.]+)/.exec(a)||/(msie) ([\w.]+)/.exec(a)||a.indexOf("compatible")<0&&/(mozilla)(?:.*? rv:([\w.]+)|)/.exec(a)||[];return{browser:b[1]||"",version:b[2]||"0"}},a=p.uaMatch(g.userAgent),b={},a.browser&&(b[a.browser]=!0,b.version=a.version),b.webkit&&(b.safari=!0),p.browser=b,p.sub=function(){function a(b,c){return new a.fn.init(b,c)}p.extend(!0,a,this),a.superclass=this,a.fn=a.prototype=this(),a.fn.constructor=a,a.sub=this.sub,a.fn.init=function c(c,d){return d&&d instanceof p&&!(d instanceof a)&&(d=a(d)),p.fn.init.call(this,c,d,b)},a.fn.init.prototype=a.fn;var b=a(e);return a}}();var bH,bI,bJ,bK=/alpha\([^)]*\)/i,bL=/opacity=([^)]*)/,bM=/^(top|right|bottom|left)$/,bN=/^margin/,bO=new RegExp("^("+q+")(.*)$","i"),bP=new RegExp("^("+q+")(?!px)[a-z%]+$","i"),bQ=new RegExp("^([-+])=("+q+")","i"),bR={},bS={position:"absolute",visibility:"hidden",display:"block"},bT={letterSpacing:0,fontWeight:400,lineHeight:1},bU=["Top","Right","Bottom","Left"],bV=["Webkit","O","Moz","ms"],bW=p.fn.toggle;p.fn.extend({css:function(a,c){return p.access(this,function(a,c,d){return d!==b?p.style(a,c,d):p.css(a,c)},a,c,arguments.length>1)},show:function(){return bZ(this,!0)},hide:function(){return bZ(this)},toggle:function(a,b){var c=typeof a=="boolean";return p.isFunction(a)&&p.isFunction(b)?bW.apply(this,arguments):this.each(function(){(c?a:bY(this))?p(this).show():p(this).hide()})}}),p.extend({cssHooks:{opacity:{get:function(a,b){if(b){var c=bH(a,"opacity");return c===""?"1":c}}}},cssNumber:{fillOpacity:!0,fontWeight:!0,lineHeight:!0,opacity:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{"float":p.support.cssFloat?"cssFloat":"styleFloat"},style:function(a,c,d,e){if(!a||a.nodeType===3||a.nodeType===8||!a.style)return;var f,g,h,i=p.camelCase(c),j=a.style;c=p.cssProps[i]||(p.cssProps[i]=bX(j,i)),h=p.cssHooks[c]||p.cssHooks[i];if(d===b)return h&&"get"in h&&(f=h.get(a,!1,e))!==b?f:j[c];g=typeof d,g==="string"&&(f=bQ.exec(d))&&(d=(f[1]+1)*f[2]+parseFloat(p.css(a,c)),g="number");if(d==null||g==="number"&&isNaN(d))return;g==="number"&&!p.cssNumber[i]&&(d+="px");if(!h||!("set"in h)||(d=h.set(a,d,e))!==b)try{j[c]=d}catch(k){}},css:function(a,c,d,e){var f,g,h,i=p.camelCase(c);return c=p.cssProps[i]||(p.cssProps[i]=bX(a.style,i)),h=p.cssHooks[c]||p.cssHooks[i],h&&"get"in h&&(f=h.get(a,!0,e)),f===b&&(f=bH(a,c)),f==="normal"&&c in bT&&(f=bT[c]),d||e!==b?(g=parseFloat(f),d||p.isNumeric(g)?g||0:f):f},swap:function(a,b,c){var d,e,f={};for(e in b)f[e]=a.style[e],a.style[e]=b[e];d=c.call(a);for(e in b)a.style[e]=f[e];return d}}),a.getComputedStyle?bH=function(a,b){var c,d,e,f,g=getComputedStyle(a,null),h=a.style;return g&&(c=g[b],c===""&&!p.contains(a.ownerDocument.documentElement,a)&&(c=p.style(a,b)),bP.test(c)&&bN.test(b)&&(d=h.width,e=h.minWidth,f=h.maxWidth,h.minWidth=h.maxWidth=h.width=c,c=g.width,h.width=d,h.minWidth=e,h.maxWidth=f)),c}:e.documentElement.currentStyle&&(bH=function(a,b){var c,d,e=a.currentStyle&&a.currentStyle[b],f=a.style;return e==null&&f&&f[b]&&(e=f[b]),bP.test(e)&&!bM.test(b)&&(c=f.left,d=a.runtimeStyle&&a.runtimeStyle.left,d&&(a.runtimeStyle.left=a.currentStyle.left),f.left=b==="fontSize"?"1em":e,e=f.pixelLeft+"px",f.left=c,d&&(a.runtimeStyle.left=d)),e===""?"auto":e}),p.each(["height","width"],function(a,b){p.cssHooks[b]={get:function(a,c,d){if(c)return a.offsetWidth!==0||bH(a,"display")!=="none"?ca(a,b,d):p.swap(a,bS,function(){return ca(a,b,d)})},set:function(a,c,d){return b$(a,c,d?b_(a,b,d,p.support.boxSizing&&p.css(a,"boxSizing")==="border-box"):0)}}}),p.support.opacity||(p.cssHooks.opacity={get:function(a,b){return bL.test((b&&a.currentStyle?a.currentStyle.filter:a.style.filter)||"")?.01*parseFloat(RegExp.$1)+"":b?"1":""},set:function(a,b){var c=a.style,d=a.currentStyle,e=p.isNumeric(b)?"alpha(opacity="+b*100+")":"",f=d&&d.filter||c.filter||"";c.zoom=1;if(b>=1&&p.trim(f.replace(bK,""))===""&&c.removeAttribute){c.removeAttribute("filter");if(d&&!d.filter)return}c.filter=bK.test(f)?f.replace(bK,e):f+" "+e}}),p(function(){p.support.reliableMarginRight||(p.cssHooks.marginRight={get:function(a,b){return p.swap(a,{display:"inline-block"},function(){if(b)return bH(a,"marginRight")})}}),!p.support.pixelPosition&&p.fn.position&&p.each(["top","left"],function(a,b){p.cssHooks[b]={get:function(a,c){if(c){var d=bH(a,b);return bP.test(d)?p(a).position()[b]+"px":d}}}})}),p.expr&&p.expr.filters&&(p.expr.filters.hidden=function(a){return a.offsetWidth===0&&a.offsetHeight===0||!p.support.reliableHiddenOffsets&&(a.style&&a.style.display||bH(a,"display"))==="none"},p.expr.filters.visible=function(a){return!p.expr.filters.hidden(a)}),p.each({margin:"",padding:"",border:"Width"},function(a,b){p.cssHooks[a+b]={expand:function(c){var d,e=typeof c=="string"?c.split(" "):[c],f={};for(d=0;d<4;d++)f[a+bU[d]+b]=e[d]||e[d-2]||e[0];return f}},bN.test(a)||(p.cssHooks[a+b].set=b$)});var cc=/%20/g,cd=/\[\]$/,ce=/\r?\n/g,cf=/^(?:color|date|datetime|datetime-local|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i,cg=/^(?:select|textarea)/i;p.fn.extend({serialize:function(){return p.param(this.serializeArray())},serializeArray:function(){return this.map(function(){return this.elements?p.makeArray(this.elements):this}).filter(function(){return this.name&&!this.disabled&&(this.checked||cg.test(this.nodeName)||cf.test(this.type))}).map(function(a,b){var c=p(this).val();return c==null?null:p.isArray(c)?p.map(c,function(a,c){return{name:b.name,value:a.replace(ce,"\r\n")}}):{name:b.name,value:c.replace(ce,"\r\n")}}).get()}}),p.param=function(a,c){var d,e=[],f=function(a,b){b=p.isFunction(b)?b():b==null?"":b,e[e.length]=encodeURIComponent(a)+"="+encodeURIComponent(b)};c===b&&(c=p.ajaxSettings&&p.ajaxSettings.traditional);if(p.isArray(a)||a.jquery&&!p.isPlainObject(a))p.each(a,function(){f(this.name,this.value)});else for(d in a)ch(d,a[d],c,f);return e.join("&").replace(cc,"+")};var ci,cj,ck=/#.*$/,cl=/^(.*?):[ \t]*([^\r\n]*)\r?$/mg,cm=/^(?:about|app|app\-storage|.+\-extension|file|res|widget):$/,cn=/^(?:GET|HEAD)$/,co=/^\/\//,cp=/\?/,cq=/)<[^<]*)*<\/script>/gi,cr=/([?&])_=[^&]*/,cs=/^([\w\+\.\-]+:)(?:\/\/([^\/?#:]*)(?::(\d+)|)|)/,ct=p.fn.load,cu={},cv={},cw=["*/"]+["*"];try{ci=f.href}catch(cx){ci=e.createElement("a"),ci.href="",ci=ci.href}cj=cs.exec(ci.toLowerCase())||[],p.fn.load=function(a,c,d){if(typeof a!="string"&&ct)return ct.apply(this,arguments);if(!this.length)return this;var e,f,g,h=this,i=a.indexOf(" ");return i>=0&&(e=a.slice(i,a.length),a=a.slice(0,i)),p.isFunction(c)?(d=c,c=b):typeof c=="object"&&(f="POST"),p.ajax({url:a,type:f,dataType:"html",data:c,complete:function(a,b){d&&h.each(d,g||[a.responseText,b,a])}}).done(function(a){g=arguments,h.html(e?p("").append(a.replace(cq,"")).find(e):a)}),this},p.each("ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split(" "),function(a,b){p.fn[b]=function(a){return this.on(b,a)}}),p.each(["get","post"],function(a,c){p[c]=function(a,d,e,f){return p.isFunction(d)&&(f=f||e,e=d,d=b),p.ajax({type:c,url:a,data:d,success:e,dataType:f})}}),p.extend({getScript:function(a,c){return p.get(a,b,c,"script")},getJSON:function(a,b,c){return p.get(a,b,c,"json")},ajaxSetup:function(a,b){return b?cA(a,p.ajaxSettings):(b=a,a=p.ajaxSettings),cA(a,b),a},ajaxSettings:{url:ci,isLocal:cm.test(cj[1]),global:!0,type:"GET",contentType:"application/x-www-form-urlencoded; charset=UTF-8",processData:!0,async:!0,accepts:{xml:"application/xml, text/xml",html:"text/html",text:"text/plain",json:"application/json, text/javascript","*":cw},contents:{xml:/xml/,html:/html/,json:/json/},responseFields:{xml:"responseXML",text:"responseText"},converters:{"* text":a.String,"text html":!0,"text json":p.parseJSON,"text xml":p.parseXML},flatOptions:{context:!0,url:!0}},ajaxPrefilter:cy(cu),ajaxTransport:cy(cv),ajax:function(a,c){function y(a,c,f,i){var k,s,t,u,w,y=c;if(v===2)return;v=2,h&&clearTimeout(h),g=b,e=i||"",x.readyState=a>0?4:0,f&&(u=cB(l,x,f));if(a>=200&&a<300||a===304)l.ifModified&&(w=x.getResponseHeader("Last-Modified"),w&&(p.lastModified[d]=w),w=x.getResponseHeader("Etag"),w&&(p.etag[d]=w)),a===304?(y="notmodified",k=!0):(k=cC(l,u),y=k.state,s=k.data,t=k.error,k=!t);else{t=y;if(!y||a)y="error",a<0&&(a=0)}x.status=a,x.statusText=""+(c||y),k?o.resolveWith(m,[s,y,x]):o.rejectWith(m,[x,y,t]),x.statusCode(r),r=b,j&&n.trigger("ajax"+(k?"Success":"Error"),[x,l,k?s:t]),q.fireWith(m,[x,y]),j&&(n.trigger("ajaxComplete",[x,l]),--p.active||p.event.trigger("ajaxStop"))}typeof a=="object"&&(c=a,a=b),c=c||{};var d,e,f,g,h,i,j,k,l=p.ajaxSetup({},c),m=l.context||l,n=m!==l&&(m.nodeType||m instanceof p)?p(m):p.event,o=p.Deferred(),q=p.Callbacks("once memory"),r=l.statusCode||{},t={},u={},v=0,w="canceled",x={readyState:0,setRequestHeader:function(a,b){if(!v){var c=a.toLowerCase();a=u[c]=u[c]||a,t[a]=b}return this},getAllResponseHeaders:function(){return v===2?e:null},getResponseHeader:function(a){var c;if(v===2){if(!f){f={};while(c=cl.exec(e))f[c[1].toLowerCase()]=c[2]}c=f[a.toLowerCase()]}return c===b?null:c},overrideMimeType:function(a){return v||(l.mimeType=a),this},abort:function(a){return a=a||w,g&&g.abort(a),y(0,a),this}};o.promise(x),x.success=x.done,x.error=x.fail,x.complete=q.add,x.statusCode=function(a){if(a){var b;if(v<2)for(b in a)r[b]=[r[b],a[b]];else b=a[x.status],x.always(b)}return this},l.url=((a||l.url)+"").replace(ck,"").replace(co,cj[1]+"//"),l.dataTypes=p.trim(l.dataType||"*").toLowerCase().split(s),l.crossDomain==null&&(i=cs.exec(l.url.toLowerCase()),l.crossDomain=!(!i||i[1]==cj[1]&&i[2]==cj[2]&&(i[3]||(i[1]==="http:"?80:443))==(cj[3]||(cj[1]==="http:"?80:443)))),l.data&&l.processData&&typeof l.data!="string"&&(l.data=p.param(l.data,l.traditional)),cz(cu,l,c,x);if(v===2)return x;j=l.global,l.type=l.type.toUpperCase(),l.hasContent=!cn.test(l.type),j&&p.active++===0&&p.event.trigger("ajaxStart");if(!l.hasContent){l.data&&(l.url+=(cp.test(l.url)?"&":"?")+l.data,delete l.data),d=l.url;if(l.cache===!1){var z=p.now(),A=l.url.replace(cr,"$1_="+z);l.url=A+(A===l.url?(cp.test(l.url)?"&":"?")+"_="+z:"")}}(l.data&&l.hasContent&&l.contentType!==!1||c.contentType)&&x.setRequestHeader("Content-Type",l.contentType),l.ifModified&&(d=d||l.url,p.lastModified[d]&&x.setRequestHeader("If-Modified-Since",p.lastModified[d]),p.etag[d]&&x.setRequestHeader("If-None-Match",p.etag[d])),x.setRequestHeader("Accept",l.dataTypes[0]&&l.accepts[l.dataTypes[0]]?l.accepts[l.dataTypes[0]]+(l.dataTypes[0]!=="*"?", "+cw+"; q=0.01":""):l.accepts["*"]);for(k in l.headers)x.setRequestHeader(k,l.headers[k]);if(!l.beforeSend||l.beforeSend.call(m,x,l)!==!1&&v!==2){w="abort";for(k in{success:1,error:1,complete:1})x[k](l[k]);g=cz(cv,l,c,x);if(!g)y(-1,"No Transport");else{x.readyState=1,j&&n.trigger("ajaxSend",[x,l]),l.async&&l.timeout>0&&(h=setTimeout(function(){x.abort("timeout")},l.timeout));try{v=1,g.send(t,y)}catch(B){if(v<2)y(-1,B);else throw B}}return x}return x.abort()},active:0,lastModified:{},etag:{}});var cD=[],cE=/\?/,cF=/(=)\?(?=&|$)|\?\?/,cG=p.now();p.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var a=cD.pop()||p.expando+"_"+cG++;return this[a]=!0,a}}),p.ajaxPrefilter("json jsonp",function(c,d,e){var f,g,h,i=c.data,j=c.url,k=c.jsonp!==!1,l=k&&cF.test(j),m=k&&!l&&typeof i=="string"&&!(c.contentType||"").indexOf("application/x-www-form-urlencoded")&&cF.test(i);if(c.dataTypes[0]==="jsonp"||l||m)return f=c.jsonpCallback=p.isFunction(c.jsonpCallback)?c.jsonpCallback():c.jsonpCallback,g=a[f],l?c.url=j.replace(cF,"$1"+f):m?c.data=i.replace(cF,"$1"+f):k&&(c.url+=(cE.test(j)?"&":"?")+c.jsonp+"="+f),c.converters["script json"]=function(){return h||p.error(f+" was not called"),h[0]},c.dataTypes[0]="json",a[f]=function(){h=arguments},e.always(function(){a[f]=g,c[f]&&(c.jsonpCallback=d.jsonpCallback,cD.push(f)),h&&p.isFunction(g)&&g(h[0]),h=g=b}),"script"}),p.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/javascript|ecmascript/},converters:{"text script":function(a){return p.globalEval(a),a}}}),p.ajaxPrefilter("script",function(a){a.cache===b&&(a.cache=!1),a.crossDomain&&(a.type="GET",a.global=!1)}),p.ajaxTransport("script",function(a){if(a.crossDomain){var c,d=e.head||e.getElementsByTagName("head")[0]||e.documentElement;return{send:function(f,g){c=e.createElement("script"),c.async="async",a.scriptCharset&&(c.charset=a.scriptCharset),c.src=a.url,c.onload=c.onreadystatechange=function(a,e){if(e||!c.readyState||/loaded|complete/.test(c.readyState))c.onload=c.onreadystatechange=null,d&&c.parentNode&&d.removeChild(c),c=b,e||g(200,"success")},d.insertBefore(c,d.firstChild)},abort:function(){c&&c.onload(0,1)}}}});var cH,cI=a.ActiveXObject?function(){for(var a in cH)cH[a](0,1)}:!1,cJ=0;p.ajaxSettings.xhr=a.ActiveXObject?function(){return!this.isLocal&&cK()||cL()}:cK,function(a){p.extend(p.support,{ajax:!!a,cors:!!a&&"withCredentials"in a})}(p.ajaxSettings.xhr()),p.support.ajax&&p.ajaxTransport(function(c){if(!c.crossDomain||p.support.cors){var d;return{send:function(e,f){var g,h,i=c.xhr();c.username?i.open(c.type,c.url,c.async,c.username,c.password):i.open(c.type,c.url,c.async);if(c.xhrFields)for(h in c.xhrFields)i[h]=c.xhrFields[h];c.mimeType&&i.overrideMimeType&&i.overrideMimeType(c.mimeType),!c.crossDomain&&!e["X-Requested-With"]&&(e["X-Requested-With"]="XMLHttpRequest");try{for(h in e)i.setRequestHeader(h,e[h])}catch(j){}i.send(c.hasContent&&c.data||null),d=function(a,e){var h,j,k,l,m;try{if(d&&(e||i.readyState===4)){d=b,g&&(i.onreadystatechange=p.noop,cI&&delete cH[g]);if(e)i.readyState!==4&&i.abort();else{h=i.status,k=i.getAllResponseHeaders(),l={},m=i.responseXML,m&&m.documentElement&&(l.xml=m);try{l.text=i.responseText}catch(a){}try{j=i.statusText}catch(n){j=""}!h&&c.isLocal&&!c.crossDomain?h=l.text?200:404:h===1223&&(h=204)}}}catch(o){e||f(-1,o)}l&&f(h,j,l,k)},c.async?i.readyState===4?setTimeout(d,0):(g=++cJ,cI&&(cH||(cH={},p(a).unload(cI)),cH[g]=d),i.onreadystatechange=d):d()},abort:function(){d&&d(0,1)}}}});var cM,cN,cO=/^(?:toggle|show|hide)$/,cP=new RegExp("^(?:([-+])=|)("+q+")([a-z%]*)$","i"),cQ=/queueHooks$/,cR=[cX],cS={"*":[function(a,b){var c,d,e,f=this.createTween(a,b),g=cP.exec(b),h=f.cur(),i=+h||0,j=1;if(g){c=+g[2],d=g[3]||(p.cssNumber[a]?"":"px");if(d!=="px"&&i){i=p.css(f.elem,a,!0)||c||1;do e=j=j||".5",i=i/j,p.style(f.elem,a,i+d),j=f.cur()/h;while(j!==1&&j!==e)}f.unit=d,f.start=i,f.end=g[1]?i+(g[1]+1)*c:c}return f}]};p.Animation=p.extend(cV,{tweener:function(a,b){p.isFunction(a)?(b=a,a=["*"]):a=a.split(" ");var c,d=0,e=a.length;for(;d
-1,j={},k={},l,m;i?(k=e.position(),l=k.top,m=k.left):(l=parseFloat(g)||0,m=parseFloat(h)||0),p.isFunction(b)&&(b=b.call(a,c,f)),b.top!=null&&(j.top=b.top-f.top+l),b.left!=null&&(j.left=b.left-f.left+m),"using"in b?b.using.call(a,j):e.css(j)}},p.fn.extend({position:function(){if(!this[0])return;var a=this[0],b=this.offsetParent(),c=this.offset(),d=c$.test(b[0].nodeName)?{top:0,left:0}:b.offset();return c.top-=parseFloat(p.css(a,"marginTop"))||0,c.left-=parseFloat(p.css(a,"marginLeft"))||0,d.top+=parseFloat(p.css(b[0],"borderTopWidth"))||0,d.left+=parseFloat(p.css(b[0],"borderLeftWidth"))||0,{top:c.top-d.top,left:c.left-d.left}},offsetParent:function(){return this.map(function(){var a=this.offsetParent||e.body;while(a&&!c$.test(a.nodeName)&&p.css(a,"position")==="static")a=a.offsetParent;return a||e.body})}}),p.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(a,c){var d=/Y/.test(c);p.fn[a]=function(e){return p.access(this,function(a,e,f){var g=c_(a);if(f===b)return g?c in g?g[c]:g.document.documentElement[e]:a[e];g?g.scrollTo(d?p(g).scrollLeft():f,d?f:p(g).scrollTop()):a[e]=f},a,e,arguments.length,null)}}),p.each({Height:"height",Width:"width"},function(a,c){p.each({padding:"inner"+a,content:c,"":"outer"+a},function(d,e){p.fn[e]=function(e,f){var g=arguments.length&&(d||typeof e!="boolean"),h=d||(e===!0||f===!0?"margin":"border");return p.access(this,function(c,d,e){var f;return p.isWindow(c)?c.document.documentElement["client"+a]:c.nodeType===9?(f=c.documentElement,Math.max(c.body["scroll"+a],f["scroll"+a],c.body["offset"+a],f["offset"+a],f["client"+a])):e===b?p.css(c,d,e,h):p.style(c,d,e,h)},c,g?e:b,g)}})}),a.jQuery=a.$=p,typeof define=="function"&&define.amd&&define.amd.jQuery&&define("jquery",[],function(){return p})})(window);
\ No newline at end of file
diff --git a/docs/ml-datasource.md b/docs/ml-datasource.md
new file mode 100644
index 0000000000000..15083326240ac
--- /dev/null
+++ b/docs/ml-datasource.md
@@ -0,0 +1,108 @@
+---
+layout: global
+title: Data sources
+displayTitle: Data sources
+---
+
+In this section, we introduce how to use data source in ML to load data.
+Beside some general data sources such as Parquet, CSV, JSON and JDBC, we also provide some specific data sources for ML.
+
+**Table of Contents**
+
+* This will become a table of contents (this text will be scraped).
+{:toc}
+
+## Image data source
+
+This image data source is used to load image files from a directory, it can load compressed image (jpeg, png, etc.) into raw image representation via `ImageIO` in Java library.
+The loaded DataFrame has one `StructType` column: "image", containing image data stored as image schema.
+The schema of the `image` column is:
+ - origin: `StringType` (represents the file path of the image)
+ - height: `IntegerType` (height of the image)
+ - width: `IntegerType` (width of the image)
+ - nChannels: `IntegerType` (number of image channels)
+ - mode: `IntegerType` (OpenCV-compatible type)
+ - data: `BinaryType` (Image bytes in OpenCV-compatible order: row-wise BGR in most cases)
+
+
+
+
+[`ImageDataSource`](api/scala/index.html#org.apache.spark.ml.source.image.ImageDataSource)
+implements a Spark SQL data source API for loading image data as a DataFrame.
+
+{% highlight scala %}
+scala> val df = spark.read.format("image").option("dropInvalid", true).load("data/mllib/images/origin/kittens")
+df: org.apache.spark.sql.DataFrame = [image: struct]
+
+scala> df.select("image.origin", "image.width", "image.height").show(truncate=false)
++-----------------------------------------------------------------------+-----+------+
+|origin |width|height|
++-----------------------------------------------------------------------+-----+------+
+|file:///spark/data/mllib/images/origin/kittens/54893.jpg |300 |311 |
+|file:///spark/data/mllib/images/origin/kittens/DP802813.jpg |199 |313 |
+|file:///spark/data/mllib/images/origin/kittens/29.5.a_b_EGDP022204.jpg |300 |200 |
+|file:///spark/data/mllib/images/origin/kittens/DP153539.jpg |300 |296 |
++-----------------------------------------------------------------------+-----+------+
+{% endhighlight %}
+
+
+
+[`ImageDataSource`](api/java/org/apache/spark/ml/source/image/ImageDataSource.html)
+implements Spark SQL data source API for loading image data as DataFrame.
+
+{% highlight java %}
+Dataset imagesDF = spark.read().format("image").option("dropInvalid", true).load("data/mllib/images/origin/kittens");
+imageDF.select("image.origin", "image.width", "image.height").show(false);
+/*
+Will output:
++-----------------------------------------------------------------------+-----+------+
+|origin |width|height|
++-----------------------------------------------------------------------+-----+------+
+|file:///spark/data/mllib/images/origin/kittens/54893.jpg |300 |311 |
+|file:///spark/data/mllib/images/origin/kittens/DP802813.jpg |199 |313 |
+|file:///spark/data/mllib/images/origin/kittens/29.5.a_b_EGDP022204.jpg |300 |200 |
+|file:///spark/data/mllib/images/origin/kittens/DP153539.jpg |300 |296 |
++-----------------------------------------------------------------------+-----+------+
+*/
+{% endhighlight %}
+
+
+
+In PySpark we provide Spark SQL data source API for loading image data as DataFrame.
+
+{% highlight python %}
+>>> df = spark.read.format("image").option("dropInvalid", true).load("data/mllib/images/origin/kittens")
+>>> df.select("image.origin", "image.width", "image.height").show(truncate=False)
++-----------------------------------------------------------------------+-----+------+
+|origin |width|height|
++-----------------------------------------------------------------------+-----+------+
+|file:///spark/data/mllib/images/origin/kittens/54893.jpg |300 |311 |
+|file:///spark/data/mllib/images/origin/kittens/DP802813.jpg |199 |313 |
+|file:///spark/data/mllib/images/origin/kittens/29.5.a_b_EGDP022204.jpg |300 |200 |
+|file:///spark/data/mllib/images/origin/kittens/DP153539.jpg |300 |296 |
++-----------------------------------------------------------------------+-----+------+
+{% endhighlight %}
+
+
+
+In SparkR we provide Spark SQL data source API for loading image data as DataFrame.
+
+{% highlight r %}
+> df = read.df("data/mllib/images/origin/kittens", "image")
+> head(select(df, df$image.origin, df$image.width, df$image.height))
+
+1 file:///spark/data/mllib/images/origin/kittens/54893.jpg
+2 file:///spark/data/mllib/images/origin/kittens/DP802813.jpg
+3 file:///spark/data/mllib/images/origin/kittens/29.5.a_b_EGDP022204.jpg
+4 file:///spark/data/mllib/images/origin/kittens/DP153539.jpg
+ width height
+1 300 311
+2 199 313
+3 300 200
+4 300 296
+
+{% endhighlight %}
+
+
+
+
+
+
+Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.fpm.PrefixSpan) for more details.
+
+{% include_example scala/org/apache/spark/examples/ml/PrefixSpanExample.scala %}
+
+
+
+Refer to the [Java API docs](api/java/org/apache/spark/ml/fpm/PrefixSpan.html) for more details.
+
+{% include_example java/org/apache/spark/examples/ml/JavaPrefixSpanExample.java %}
+
+
+
+Refer to the [Python API docs](api/python/pyspark.ml.html#pyspark.ml.fpm.PrefixSpan) for more details.
+
+{% include_example python/ml/prefixspan_example.py %}
+
+
+
- $p(k)=\frac{1}{M} \sum_{i=0}^{M-1} {\frac{1}{k} \sum_{j=0}^{\text{min}(\left|D\right|, k) - 1} rel_{D_i}(R_i(j))}$
+ $p(k)=\frac{1}{M} \sum_{i=0}^{M-1} {\frac{1}{k} \sum_{j=0}^{\text{min}(Q_i, k) - 1} rel_{D_i}(R_i(j))}$
- Precision at k is a measure of
+ Precision at k is a measure of
how many of the first k recommended documents are in the set of true relevant documents averaged across all
users. In this metric, the order of the recommendations is not taken into account.
@@ -450,10 +450,10 @@ $$rel_D(r) = \begin{cases}1 & \text{if $r \in D$}, \\ 0 & \text{otherwise}.\end{
Mean Average Precision
- $MAP=\frac{1}{M} \sum_{i=0}^{M-1} {\frac{1}{\left|D_i\right|} \sum_{j=0}^{Q-1} \frac{rel_{D_i}(R_i(j))}{j + 1}}$
+ $MAP=\frac{1}{M} \sum_{i=0}^{M-1} {\frac{1}{N_i} \sum_{j=0}^{Q_i-1} \frac{rel_{D_i}(R_i(j))}{j + 1}}$
- MAP is a measure of how
+ MAP is a measure of how
many of the recommended documents are in the set of true relevant documents, where the
order of the recommendations is taken into account (i.e. penalty for highly relevant documents is higher).
@@ -462,10 +462,10 @@ $$rel_D(r) = \begin{cases}1 & \text{if $r \in D$}, \\ 0 & \text{otherwise}.\end{
Normalized Discounted Cumulative Gain
$NDCG(k)=\frac{1}{M} \sum_{i=0}^{M-1} {\frac{1}{IDCG(D_i, k)}\sum_{j=0}^{n-1}
- \frac{rel_{D_i}(R_i(j))}{\text{ln}(j+2)}} \\
+ \frac{rel_{D_i}(R_i(j))}{\text{log}(j+2)}} \\
\text{Where} \\
- \hspace{5 mm} n = \text{min}\left(\text{max}\left(|R_i|,|D_i|\right),k\right) \\
- \hspace{5 mm} IDCG(D, k) = \sum_{j=0}^{\text{min}(\left|D\right|, k) - 1} \frac{1}{\text{ln}(j+2)}$
+ \hspace{5 mm} n = \text{min}\left(\text{max}\left(Q_i, N_i\right),k\right) \\
+ \hspace{5 mm} IDCG(D, k) = \sum_{j=0}^{\text{min}(\left|D\right|, k) - 1} \frac{1}{\text{log}(j+2)}$
NDCG at k is a
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 2717dd091c751..69bf3082f0f27 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -185,6 +185,23 @@ Security options for the Spark History Server are covered more detail in the
Job history files older than this will be deleted when the filesystem history cleaner runs.
+
+ spark.history.fs.endEventReparseChunkSize
+ 1m
+
+ How many bytes to parse at the end of log files looking for the end event.
+ This is used to speed up generation of application listings by skipping unnecessary
+ parts of event log files. It can be disabled by setting this config to 0.
+
+
+
+ spark.history.fs.inProgressOptimization.enabled
+ true
+
+ Enable optimized handling of in-progress logs. This option may leave finished
+ applications that fail to rename their event logs listed as in-progress.
+
+
spark.history.fs.numReplayThreads
25% of available cores
@@ -192,6 +209,14 @@ Security options for the Spark History Server are covered more detail in the
Number of threads that will be used by history server to process event logs.
+
+ spark.history.store.maxDiskUsage
+ 10g
+
+ Maximum disk usage for the local directory where the cache application history information
+ are stored.
+
+
spark.history.store.path
(none)
@@ -388,6 +413,158 @@ value triggering garbage collection on jobs, and `spark.ui.retainedStages` that
Note that the garbage collection takes place on playback: it is possible to retrieve
more entries by increasing these values and restarting the history server.
+### Executor Task Metrics
+
+The REST API exposes the values of the Task Metrics collected by Spark executors with the granularity
+of task execution. The metrics can be used for performance troubleshooting and workload characterization.
+A list of the available metrics, with a short description:
+
+
+ Spark Executor Task Metric name
+ Short description
+
+ executorRunTime
+ Elapsed time the executor spent running this task. This includes time fetching shuffle data.
+ The value is expressed in milliseconds.
+
+
+ executorCpuTime
+ CPU time the executor spent running this task. This includes time fetching shuffle data.
+ The value is expressed in nanoseconds.
+
+
+ executorDeserializeTime
+ Elapsed time spent to deserialize this task. The value is expressed in milliseconds.
+
+
+ executorDeserializeCpuTime
+ CPU time taken on the executor to deserialize this task. The value is expressed
+ in nanoseconds.
+
+
+ resultSize
+ The number of bytes this task transmitted back to the driver as the TaskResult.
+
+
+ jvmGCTime
+ Elapsed time the JVM spent in garbage collection while executing this task.
+ The value is expressed in milliseconds.
+
+
+ resultSerializationTime
+ Elapsed time spent serializing the task result. The value is expressed in milliseconds.
+
+
+ memoryBytesSpilled
+ The number of in-memory bytes spilled by this task.
+
+
+ diskBytesSpilled
+ The number of on-disk bytes spilled by this task.
+
+
+ peakExecutionMemory
+ Peak memory used by internal data structures created during shuffles, aggregations and
+ joins. The value of this accumulator should be approximately the sum of the peak sizes
+ across all such data structures created in this task. For SQL jobs, this only tracks all
+ unsafe operators and ExternalSort.
+
+
+ inputMetrics.*
+ Metrics related to reading data from [[org.apache.spark.rdd.HadoopRDD]]
+ or from persisted data.
+
+
+ .bytesRead
+ Total number of bytes read.
+
+
+ .recordsRead
+ Total number of records read.
+
+
+ outputMetrics.*
+ Metrics related to writing data externally (e.g. to a distributed filesystem),
+ defined only in tasks with output.
+
+
+ .bytesWritten
+ Total number of bytes written
+
+
+ .recordsWritten
+ Total number of records written
+
+
+ shuffleReadMetrics.*
+ Metrics related to shuffle read operations.
+
+
+ .recordsRead
+ Number of records read in shuffle operations
+
+
+ .remoteBlocksFetched
+ Number of remote blocks fetched in shuffle operations
+
+
+ .localBlocksFetched
+ Number of local (as opposed to read from a remote executor) blocks fetched
+ in shuffle operations
+
+
+ .totalBlocksFetched
+ Number of blocks fetched in shuffle operations (both local and remote)
+
+
+ .remoteBytesRead
+ Number of remote bytes read in shuffle operations
+
+
+ .localBytesRead
+ Number of bytes read in shuffle operations from local disk (as opposed to
+ read from a remote executor)
+
+
+ .totalBytesRead
+ Number of bytes read in shuffle operations (both local and remote)
+
+
+ .remoteBytesReadToDisk
+ Number of remote bytes read to disk in shuffle operations.
+ Large blocks are fetched to disk in shuffle read operations, as opposed to
+ being read into memory, which is the default behavior.
+
+
+ .fetchWaitTime
+ Time the task spent waiting for remote shuffle blocks.
+ This only includes the time blocking on shuffle input data.
+ For instance if block B is being fetched while the task is still not finished
+ processing block A, it is not considered to be blocking on block B.
+ The value is expressed in milliseconds.
+
+
+ shuffleWriteMetrics.*
+ Metrics related to operations writing shuffle data.
+
+
+ .bytesWritten
+ Number of bytes written in shuffle operations
+
+
+ .recordsWritten
+ Number of records written in shuffle operations
+
+
+ .writeTime
+ Time spent blocking on writes to disk or buffer cache. The value is expressed
+ in nanoseconds.
+
+
+
+
+
### API Versioning Policy
These endpoints have been strongly versioned to make it easier to develop applications on top.
diff --git a/docs/quick-start.md b/docs/quick-start.md
index ef7af6c3f6cec..8bf0d6ee6bba8 100644
--- a/docs/quick-start.md
+++ b/docs/quick-start.md
@@ -17,6 +17,11 @@ you can download a package for any version of Hadoop.
Note that, before Spark 2.0, the main programming interface of Spark was the Resilient Distributed Dataset (RDD). After Spark 2.0, RDDs are replaced by Dataset, which is strongly-typed like an RDD, but with richer optimizations under the hood. The RDD interface is still supported, and you can get a more detailed reference at the [RDD programming guide](rdd-programming-guide.html). However, we highly recommend you to switch to use Dataset, which has better performance than RDD. See the [SQL programming guide](sql-programming-guide.html) to get more information about Dataset.
+# Security
+
+Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
+Please see [Spark Security](security.html) before running Spark.
+
# Interactive Analysis with the Spark Shell
## Basics
@@ -336,6 +341,7 @@ Note that Spark artifacts are tagged with a Scala version.
org.apache.spark
spark-sql_{{site.SCALA_BINARY_VERSION}}
{{site.SPARK_VERSION}}
+ provided
diff --git a/docs/rdd-programming-guide.md b/docs/rdd-programming-guide.md
index d95b757f36859..8655056851999 100644
--- a/docs/rdd-programming-guide.md
+++ b/docs/rdd-programming-guide.md
@@ -839,7 +839,7 @@ The most common ones are distributed "shuffle" operations, such as grouping or a
by a key.
In Scala, these operations are automatically available on RDDs containing
-[Tuple2](http://www.scala-lang.org/api/{{site.SCALA_VERSION}}/index.html#scala.Tuple2) objects
+[Tuple2](https://www.scala-lang.org/api/{{site.SCALA_VERSION}}/scala/Tuple2.html) objects
(the built-in tuples in the language, created by simply writing `(a, b)`). The key-value pair operations are available in the
[PairRDDFunctions](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions) class,
which automatically wraps around an RDD of tuples.
@@ -859,7 +859,7 @@ We could also use `counts.sortByKey()`, for example, to sort the pairs alphabeti
**Note:** when using custom objects as the key in key-value pair operations, you must be sure that a
custom `equals()` method is accompanied with a matching `hashCode()` method. For full details, see
the contract outlined in the [Object.hashCode()
-documentation](http://docs.oracle.com/javase/7/docs/api/java/lang/Object.html#hashCode()).
+documentation](https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html#hashCode--).
@@ -871,7 +871,7 @@ The most common ones are distributed "shuffle" operations, such as grouping or a
by a key.
In Java, key-value pairs are represented using the
-[scala.Tuple2](http://www.scala-lang.org/api/{{site.SCALA_VERSION}}/index.html#scala.Tuple2) class
+[scala.Tuple2](https://www.scala-lang.org/api/{{site.SCALA_VERSION}}/scala/Tuple2.html) class
from the Scala standard library. You can simply call `new Tuple2(a, b)` to create a tuple, and access
its fields later with `tuple._1()` and `tuple._2()`.
@@ -896,7 +896,7 @@ We could also use `counts.sortByKey()`, for example, to sort the pairs alphabeti
**Note:** when using custom objects as the key in key-value pair operations, you must be sure that a
custom `equals()` method is accompanied with a matching `hashCode()` method. For full details, see
the contract outlined in the [Object.hashCode()
-documentation](http://docs.oracle.com/javase/7/docs/api/java/lang/Object.html#hashCode()).
+documentation](https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html#hashCode--).
@@ -1465,6 +1465,10 @@ jsc.sc().register(myVectorAcc, "MyVectorAcc1");
Note that, when programmers define their own type of AccumulatorV2, the resulting type can be different than that of the elements added.
+*Warning*: When a Spark task finishes, Spark will try to merge the accumulated updates in this task to an accumulator.
+If it fails, Spark will ignore the failure and still mark the task successful and continue to run other tasks. Hence,
+a buggy accumulator will not impact a Spark job, but it may not get updated correctly although a Spark job is successful.
+
diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index c83dad6df1e7b..037e1d54661e3 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -12,6 +12,23 @@ Kubernetes scheduler that has been added to Spark.
In future versions, there may be behavioral changes around configuration,
container images and entrypoints.**
+# Security
+
+Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
+Please see [Spark Security](security.html) and the specific advice below before running Spark.
+
+## User Identity
+
+Images built from the project provided Dockerfiles do not contain any [`USER`](https://docs.docker.com/engine/reference/builder/#user) directives. This means that the resulting images will be running the Spark processes as `root` inside the container. On unsecured clusters this may provide an attack vector for privilege escalation and container breakout. Therefore security conscious deployments should consider providing custom images with `USER` directives specifying an unprivileged UID and GID.
+
+Alternatively the [Pod Template](#pod-template) feature can be used to add a [Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#volumes-and-file-systems) with a `runAsUser` to the pods that Spark submits. Please bear in mind that this requires cooperation from your users and as such may not be a suitable solution for shared environments. Cluster administrators should use [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/#users-and-groups) if they wish to limit the users that pods may run as.
+
+## Volume Mounts
+
+As described later in this document under [Using Kubernetes Volumes](#using-kubernetes-volumes) Spark on K8S provides configuration options that allow for mounting certain volume types into the driver and executor pods. In particular it allows for [`hostPath`](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) volumes which as described in the Kubernetes documentation have known security vulnerabilities.
+
+Cluster administrators should use [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) to limit the ability to mount `hostPath` volumes appropriately for their environments.
+
# Prerequisites
* A runnable distribution of Spark 2.3 or above.
@@ -86,7 +103,7 @@ $ bin/spark-submit \
```
The Spark master, specified either via passing the `--master` command line argument to `spark-submit` or by setting
-`spark.master` in the application's configuration, must be a URL with the format `k8s://
`. Prefixing the
+`spark.master` in the application's configuration, must be a URL with the format `k8s://:`. The port must always be specified, even if it's the HTTPS port 443. Prefixing the
master string with `k8s://` will cause the Spark application to launch on the Kubernetes cluster, with the API server
being contacted at `api_server_url`. If no HTTP protocol is specified in the URL, it defaults to `https`. For example,
setting the master to `k8s://example.com:443` is equivalent to setting it to `k8s://https://example.com:443`, but to
@@ -136,7 +153,7 @@ hostname via `spark.driver.host` and your spark driver's port to `spark.driver.p
### Client Mode Executor Pod Garbage Collection
-If you run your Spark driver in a pod, it is highly recommended to set `spark.driver.pod.name` to the name of that pod.
+If you run your Spark driver in a pod, it is highly recommended to set `spark.kubernetes.driver.pod.name` to the name of that pod.
When this property is set, the Spark scheduler will deploy the executor pods with an
[OwnerReference](https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/), which in turn will
ensure that once the driver pod is deleted from the cluster, all of the application's executor pods will also be deleted.
@@ -145,7 +162,7 @@ an OwnerReference pointing to that pod will be added to each executor pod's Owne
setting the OwnerReference to a pod that is not actually that driver pod, or else the executors may be terminated
prematurely when the wrong pod is deleted.
-If your application is not running inside a pod, or if `spark.driver.pod.name` is not set when your application is
+If your application is not running inside a pod, or if `spark.kubernetes.driver.pod.name` is not set when your application is
actually running in a pod, keep in mind that the executor pods may not be properly deleted from the cluster when the
application exits. The Spark scheduler attempts to delete these pods, but if the network request to the API server fails
for any reason, these pods will remain in the cluster. The executor processes should exit when they cannot reach the
@@ -192,6 +209,8 @@ Starting with Spark 2.4.0, users can mount the following types of Kubernetes [vo
* [emptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir): an initially empty volume created when a pod is assigned to a node.
* [persistentVolumeClaim](https://kubernetes.io/docs/concepts/storage/volumes/#persistentvolumeclaim): used to mount a `PersistentVolume` into a pod.
+**NB:** Please see the [Security](#security) section of this document for security issues related to volume mounts.
+
To mount a volume of any of the types above into the driver pod, use the following configuration property:
```
@@ -667,23 +686,24 @@ specific to Spark on Kubernetes.
spark.kubernetes.driver.limit.cores(none)
- Specify a hard cpu [limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) for the driver pod.
+ Specify a hard cpu limit for the driver pod.
spark.kubernetes.executor.request.cores(none)
- Specify the cpu request for each executor pod. Values conform to the Kubernetes [convention](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu).
- Example values include 0.1, 500m, 1.5, 5, etc., with the definition of cpu units documented in [CPU units](https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units).
+ Specify the cpu request for each executor pod. Values conform to the Kubernetes convention .
+ Example values include 0.1, 500m, 1.5, 5, etc., with the definition of cpu units documented in CPU units .
This is distinct from spark.executor.cores: it is only used and takes precedence over spark.executor.cores for specifying the executor pod cpu request if set. Task
parallelism, e.g., number of tasks an executor can run concurrently is not affected by this.
+
spark.kubernetes.executor.limit.cores(none)
- Specify a hard cpu [limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) for each executor pod launched for the Spark Application.
+ Specify a hard cpu limit for each executor pod launched for the Spark Application.
diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md
index b473e654563d6..2502cd4ca86f4 100644
--- a/docs/running-on-mesos.md
+++ b/docs/running-on-mesos.md
@@ -13,6 +13,11 @@ The advantages of deploying Spark with Mesos include:
[frameworks](https://mesos.apache.org/documentation/latest/frameworks/)
- scalable partitioning between multiple instances of Spark
+# Security
+
+Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
+Please see [Spark Security](security.html) and the specific security sections in this doc before running Spark.
+
# How it Works
In a standalone cluster deployment, the cluster manager in the below diagram is a Spark master
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index e3d67c34d53eb..f265075e351e0 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -9,6 +9,11 @@ Support for running on [YARN (Hadoop
NextGen)](http://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/YARN.html)
was added to Spark in version 0.6.0, and improved in subsequent releases.
+# Security
+
+Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
+Please see [Spark Security](security.html) and the specific security sections in this doc before running Spark.
+
# Launching Spark on YARN
Ensure that `HADOOP_CONF_DIR` or `YARN_CONF_DIR` points to the directory which contains the (client side) configuration files for the Hadoop cluster.
diff --git a/docs/security.md b/docs/security.md
index 7fb3e17de94c9..2948fbcb7d9d4 100644
--- a/docs/security.md
+++ b/docs/security.md
@@ -6,7 +6,20 @@ title: Security
* This will become a table of contents (this text will be scraped).
{:toc}
-# Spark RPC
+# Spark Security: Things You Need To Know
+
+Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
+Spark supports multiple deployments types and each one supports different levels of security. Not
+all deployment types will be secure in all environments and none are secure by default. Be
+sure to evaluate your environment, what Spark supports, and take the appropriate measure to secure
+your Spark deployment.
+
+There are many different types of security concerns. Spark does not necessarily protect against
+all things. Listed below are some of the things Spark supports. Also check the deployment
+documentation for the type of deployment you are using for deployment specific settings. Anything
+not documented, Spark does not support.
+
+# Spark RPC (Communication protocol between Spark processes)
## Authentication
@@ -123,7 +136,7 @@ The following table describes the different options available for configuring th
Spark supports encrypting temporary data written to local disks. This covers shuffle files, shuffle
spills and data blocks stored on disk (for both caching and broadcast variables). It does not cover
encrypting output data generated by applications with APIs such as `saveAsHadoopFile` or
-`saveAsTable`.
+`saveAsTable`. It also may not cover temporary files created explicitly by the user.
The following settings cover enabling encryption for data written to disk:
diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index 7975b0c8b11ca..1511c66d0fa66 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -8,6 +8,11 @@ title: Spark Standalone Mode
In addition to running on the Mesos or YARN cluster managers, Spark also provides a simple standalone deploy mode. You can launch a standalone cluster either manually, by starting a master and workers by hand, or use our provided [launch scripts](#cluster-launch-scripts). It is also possible to run these daemons on a single machine for testing.
+# Security
+
+Security in Spark is OFF by default. This could mean you are vulnerable to attack by default.
+Please see [Spark Security](security.html) and the specific security sections in this doc before running Spark.
+
# Installing Spark Standalone to a Cluster
To install Spark Standalone mode, you simply place a compiled version of Spark on each node on the cluster. You can obtain pre-built versions of Spark with each release or [build it yourself](building-spark.html).
@@ -357,7 +362,7 @@ In addition, detailed log output for each job is also written to the work direct
# Running Alongside Hadoop
-You can run Spark alongside your existing Hadoop cluster by just launching it as a separate service on the same machines. To access Hadoop data from Spark, just use a hdfs:// URL (typically `hdfs://:9000/path`, but you can find the right URL on your Hadoop Namenode's web UI). Alternatively, you can set up a separate cluster for Spark, and still have it access HDFS over the network; this will be slower than disk-local access, but may not be a concern if you are still running in the same local area network (e.g. you place a few Spark machines on each rack that you have Hadoop on).
+You can run Spark alongside your existing Hadoop cluster by just launching it as a separate service on the same machines. To access Hadoop data from Spark, just use an hdfs:// URL (typically `hdfs://:9000/path`, but you can find the right URL on your Hadoop Namenode's web UI). Alternatively, you can set up a separate cluster for Spark, and still have it access HDFS over the network; this will be slower than disk-local access, but may not be a concern if you are still running in the same local area network (e.g. you place a few Spark machines on each rack that you have Hadoop on).
# Configuring Ports for Network Security
diff --git a/docs/sparkr.md b/docs/sparkr.md
index b4248e8bb21de..d9963b751f37a 100644
--- a/docs/sparkr.md
+++ b/docs/sparkr.md
@@ -104,7 +104,7 @@ The following Spark driver properties can be set in `sparkConfig` with `sparkR.s
## Creating SparkDataFrames
-With a `SparkSession`, applications can create `SparkDataFrame`s from a local R data frame, from a [Hive table](sql-programming-guide.html#hive-tables), or from other [data sources](sql-programming-guide.html#data-sources).
+With a `SparkSession`, applications can create `SparkDataFrame`s from a local R data frame, from a [Hive table](sql-data-sources-hive-tables.html), or from other [data sources](sql-data-sources.html).
### From local data frames
The simplest way to create a data frame is to convert a local R data frame into a SparkDataFrame. Specifically, we can use `as.DataFrame` or `createDataFrame` and pass in the local R data frame to create a SparkDataFrame. As an example, the following creates a `SparkDataFrame` based using the `faithful` dataset from R.
@@ -125,7 +125,7 @@ head(df)
### From Data Sources
-SparkR supports operating on a variety of data sources through the `SparkDataFrame` interface. This section describes the general methods for loading and saving data using Data Sources. You can check the Spark SQL programming guide for more [specific options](sql-programming-guide.html#manually-specifying-options) that are available for the built-in data sources.
+SparkR supports operating on a variety of data sources through the `SparkDataFrame` interface. This section describes the general methods for loading and saving data using Data Sources. You can check the Spark SQL programming guide for more [specific options](sql-data-sources-load-save-functions.html#manually-specifying-options) that are available for the built-in data sources.
The general method for creating SparkDataFrames from data sources is `read.df`. This method takes in the path for the file to load and the type of data source, and the currently active SparkSession will be used automatically.
SparkR supports reading JSON, CSV and Parquet files natively, and through packages available from sources like [Third Party Projects](https://spark.apache.org/third-party-projects.html), you can find data source connectors for popular file formats like Avro. These packages can either be added by
@@ -180,7 +180,7 @@ write.df(people, path = "people.parquet", source = "parquet", mode = "overwrite"
### From Hive tables
-You can also create SparkDataFrames from Hive tables. To do this we will need to create a SparkSession with Hive support which can access tables in the Hive MetaStore. Note that Spark should have been built with [Hive support](building-spark.html#building-with-hive-and-jdbc-support) and more details can be found in the [SQL programming guide](sql-programming-guide.html#starting-point-sparksession). In SparkR, by default it will attempt to create a SparkSession with Hive support enabled (`enableHiveSupport = TRUE`).
+You can also create SparkDataFrames from Hive tables. To do this we will need to create a SparkSession with Hive support which can access tables in the Hive MetaStore. Note that Spark should have been built with [Hive support](building-spark.html#building-with-hive-and-jdbc-support) and more details can be found in the [SQL programming guide](sql-getting-started.html#starting-point-sparksession). In SparkR, by default it will attempt to create a SparkSession with Hive support enabled (`enableHiveSupport = TRUE`).
{% highlight r %}
diff --git a/docs/avro-data-source-guide.md b/docs/sql-data-sources-avro.md
similarity index 97%
rename from docs/avro-data-source-guide.md
rename to docs/sql-data-sources-avro.md
index d3b81f029d377..285b3aaf268d9 100644
--- a/docs/avro-data-source-guide.md
+++ b/docs/sql-data-sources-avro.md
@@ -148,8 +148,8 @@ Data source options of Avro can be set using the `.option` method on `DataFrameR
avroSchemaNone
- Optional Avro schema provided by an user in JSON format. The date type and naming of record fields
- should match the input Avro data or Catalyst data, otherwise the read/write action will fail.
+ Optional Avro schema provided by a user in JSON format. The data type and naming of record fields
+ should match the Avro data type when reading from Avro or match the Spark's internal data type (e.g., StringType, IntegerType) when writing to Avro files; otherwise, the read/write action will fail.
read and write
diff --git a/docs/sql-data-sources-hive-tables.md b/docs/sql-data-sources-hive-tables.md
new file mode 100644
index 0000000000000..687e6f8e0a7cc
--- /dev/null
+++ b/docs/sql-data-sources-hive-tables.md
@@ -0,0 +1,166 @@
+---
+layout: global
+title: Hive Tables
+displayTitle: Hive Tables
+---
+
+* Table of contents
+{:toc}
+
+Spark SQL also supports reading and writing data stored in [Apache Hive](http://hive.apache.org/).
+However, since Hive has a large number of dependencies, these dependencies are not included in the
+default Spark distribution. If Hive dependencies can be found on the classpath, Spark will load them
+automatically. Note that these Hive dependencies must also be present on all of the worker nodes, as
+they will need access to the Hive serialization and deserialization libraries (SerDes) in order to
+access data stored in Hive.
+
+Configuration of Hive is done by placing your `hive-site.xml`, `core-site.xml` (for security configuration),
+and `hdfs-site.xml` (for HDFS configuration) file in `conf/`.
+
+When working with Hive, one must instantiate `SparkSession` with Hive support, including
+connectivity to a persistent Hive metastore, support for Hive serdes, and Hive user-defined functions.
+Users who do not have an existing Hive deployment can still enable Hive support. When not configured
+by the `hive-site.xml`, the context automatically creates `metastore_db` in the current directory and
+creates a directory configured by `spark.sql.warehouse.dir`, which defaults to the directory
+`spark-warehouse` in the current directory that the Spark application is started. Note that
+the `hive.metastore.warehouse.dir` property in `hive-site.xml` is deprecated since Spark 2.0.0.
+Instead, use `spark.sql.warehouse.dir` to specify the default location of database in warehouse.
+You may need to grant write privilege to the user who starts the Spark application.
+
+
+
+
+{% include_example spark_hive scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala %}
+
+
+
+{% include_example spark_hive java/org/apache/spark/examples/sql/hive/JavaSparkHiveExample.java %}
+
+
+
+{% include_example spark_hive python/sql/hive.py %}
+
+
+
+
+When working with Hive one must instantiate `SparkSession` with Hive support. This
+adds support for finding tables in the MetaStore and writing queries using HiveQL.
+
+{% include_example spark_hive r/RSparkSQLExample.R %}
+
+
+
+ Property Name Meaning
+ fileFormat
+ A fileFormat is kind of a package of storage format specifications, including "serde", "input format" and
+ "output format". Currently we support 6 fileFormats: 'sequencefile', 'rcfile', 'orc', 'parquet', 'textfile' and 'avro'.
+
+
+
+
+ inputFormat, outputFormat
+ These 2 options specify the name of a corresponding `InputFormat` and `OutputFormat` class as a string literal,
+ e.g. `org.apache.hadoop.hive.ql.io.orc.OrcInputFormat`. These 2 options must be appeared in pair, and you can not
+ specify them if you already specified the `fileFormat` option.
+
+
+
+
+ serde
+ This option specifies the name of a serde class. When the `fileFormat` option is specified, do not specify this option
+ if the given `fileFormat` already include the information of serde. Currently "sequencefile", "textfile" and "rcfile"
+ don't include the serde information and you can use this option with these 3 fileFormats.
+
+
+
+
+ fieldDelim, escapeDelim, collectionDelim, mapkeyDelim, lineDelim
+ These options can only be used with "textfile" fileFormat. They define how to read delimited files into rows.
+
+
+
+
+All other properties defined with `OPTIONS` will be regarded as Hive serde properties.
+
+### Interacting with Different Versions of Hive Metastore
+
+One of the most important pieces of Spark SQL's Hive support is interaction with Hive metastore,
+which enables Spark SQL to access metadata of Hive tables. Starting from Spark 1.4.0, a single binary
+build of Spark SQL can be used to query different versions of Hive metastores, using the configuration described below.
+Note that independent of the version of Hive that is being used to talk to the metastore, internally Spark SQL
+will compile against Hive 1.2.1 and use those classes for internal execution (serdes, UDFs, UDAFs, etc).
+
+The following options can be used to configure the version of Hive that is used to retrieve metadata:
+
+
+ Property Name Default Meaning
+ spark.sql.hive.metastore.version1.2.1
+ Version of the Hive metastore. Available
+ options are 0.12.0 through 2.3.3.
+
+
+
+ spark.sql.hive.metastore.jarsbuiltin
+ Location of the jars that should be used to instantiate the HiveMetastoreClient. This
+ property can be one of three options:
+
+ builtin-Phive is
+ enabled. When this option is chosen, spark.sql.hive.metastore.version must be
+ either 1.2.1 or not defined.
+ mavenA classpath in the standard format for the JVM. This classpath must include all of Hive
+ and its dependencies, including the correct version of Hadoop. These jars only need to be
+ present on the driver, but if you are running in yarn cluster mode then you must ensure
+ they are packaged with your application.
+
+
+
+
+ spark.sql.hive.metastore.sharedPrefixescom.mysql.jdbc,
+
+ A comma-separated list of class prefixes that should be loaded using the classloader that is
+ shared between Spark SQL and a specific version of Hive. An example of classes that should
+ be shared is JDBC drivers that are needed to talk to the metastore. Other classes that need
+ to be shared are those that interact with classes that are already shared. For example,
+ custom appenders that are used by log4j.
+
+
+
+
+ spark.sql.hive.metastore.barrierPrefixes(empty)
+
+ A comma separated list of class prefixes that should explicitly be reloaded for each version
+ of Hive that Spark SQL is communicating with. For example, Hive UDFs that are declared in a
+ prefix that typically would be shared (i.e. org.apache.spark.*).
+
+
+
+
diff --git a/docs/sql-data-sources-jdbc.md b/docs/sql-data-sources-jdbc.md
new file mode 100644
index 0000000000000..0f2bc4941744e
--- /dev/null
+++ b/docs/sql-data-sources-jdbc.md
@@ -0,0 +1,223 @@
+---
+layout: global
+title: JDBC To Other Databases
+displayTitle: JDBC To Other Databases
+---
+
+* Table of contents
+{:toc}
+
+Spark SQL also includes a data source that can read data from other databases using JDBC. This
+functionality should be preferred over using [JdbcRDD](api/scala/index.html#org.apache.spark.rdd.JdbcRDD).
+This is because the results are returned
+as a DataFrame and they can easily be processed in Spark SQL or joined with other data sources.
+The JDBC data source is also easier to use from Java or Python as it does not require the user to
+provide a ClassTag.
+(Note that this is different than the Spark SQL JDBC server, which allows other applications to
+run queries using Spark SQL).
+
+To get started you will need to include the JDBC driver for your particular database on the
+spark classpath. For example, to connect to postgres from the Spark Shell you would run the
+following command:
+
+{% highlight bash %}
+bin/spark-shell --driver-class-path postgresql-9.4.1207.jar --jars postgresql-9.4.1207.jar
+{% endhighlight %}
+
+Tables from the remote database can be loaded as a DataFrame or Spark SQL temporary view using
+the Data Sources API. Users can specify the JDBC connection properties in the data source options.
+user and password are normally provided as connection properties for
+logging into the data sources. In addition to the connection properties, Spark also supports
+the following case-insensitive options:
+
+
+ Property Name Meaning
+ url
+ The JDBC URL to connect to. The source-specific connection properties may be specified in the URL. e.g., jdbc:postgresql://localhost/test?user=fred&password=secret
+
+
+
+
+ dbtable
+ The JDBC table that should be read from or written into. Note that when using it in the read
+ path anything that is valid in a FROM clause of a SQL query can be used.
+ For example, instead of a full table you could also use a subquery in parentheses. It is not
+ allowed to specify `dbtable` and `query` options at the same time.
+
+
+
+ query
+ A query that will be used to read data into Spark. The specified query will be parenthesized and used
+ as a subquery in the FROM clause. Spark will also assign an alias to the subquery clause.
+ As an example, spark will issue a query of the following form to the JDBC Source. SELECT <columns> FROM (<user_specified_query>) spark_gen_alias
+ It is not allowed to specify `dbtable` and `query` options at the same time.
+ It is not allowed to specify `query` and `partitionColumn` options at the same time. When specifying
+ `partitionColumn` option is required, the subquery can be specified using `dbtable` option instead and
+ partition columns can be qualified using the subquery alias provided as part of `dbtable`.
+ spark.read.format("jdbc")
+
+
+
+
+
+ driver
+ The class name of the JDBC driver to use to connect to this URL.
+
+
+
+
+ partitionColumn, lowerBound, upperBound
+ These options must all be specified if any of them is specified. In addition,
+ numPartitions must be specified. They describe how to partition the table when
+ reading in parallel from multiple workers.
+ partitionColumn must be a numeric, date, or timestamp column from the table in question.
+ Notice that lowerBound and upperBound are just used to decide the
+ partition stride, not for filtering the rows in table. So all rows in the table will be
+ partitioned and returned. This option applies only to reading.
+
+
+
+
+ numPartitions
+ The maximum number of partitions that can be used for parallelism in table reading and
+ writing. This also determines the maximum number of concurrent JDBC connections.
+ If the number of partitions to write exceeds this limit, we decrease it to this limit by
+ calling coalesce(numPartitions) before writing.
+
+
+
+
+ queryTimeout
+ The number of seconds the driver will wait for a Statement object to execute to the given
+ number of seconds. Zero means there is no limit. In the write path, this option depends on
+ how JDBC drivers implement the API setQueryTimeout, e.g., the h2 JDBC driver
+ checks the timeout of each query instead of an entire JDBC batch.
+ It defaults to 0.
+
+
+
+
+ fetchsize
+ The JDBC fetch size, which determines how many rows to fetch per round trip. This can help performance on JDBC drivers which default to low fetch size (eg. Oracle with 10 rows). This option applies only to reading.
+
+
+
+
+ batchsize
+ The JDBC batch size, which determines how many rows to insert per round trip. This can help performance on JDBC drivers. This option applies only to writing. It defaults to 1000.
+
+
+
+
+ isolationLevel
+ The transaction isolation level, which applies to current connection. It can be one of NONE, READ_COMMITTED, READ_UNCOMMITTED, REPEATABLE_READ, or SERIALIZABLE, corresponding to standard transaction isolation levels defined by JDBC's Connection object, with default of READ_UNCOMMITTED. This option applies only to writing. Please refer the documentation in java.sql.Connection.
+
+
+
+
+ sessionInitStatement
+ After each database session is opened to the remote DB and before starting to read data, this option executes a custom SQL statement (or a PL/SQL block). Use this to implement session initialization code. Example: option("sessionInitStatement", """BEGIN execute immediate 'alter session set "_serial_direct_read"=true'; END;""")
+
+
+
+
+ truncate
+ This is a JDBC writer related option. When SaveMode.Overwrite is enabled, this option causes Spark to truncate an existing table instead of dropping and recreating it. This can be more efficient, and prevents the table metadata (e.g., indices) from being removed. However, it will not work in some cases, such as when the new data has a different schema. It defaults to false. This option applies only to writing.
+
+
+
+
+ cascadeTruncate
+ This is a JDBC writer related option. If enabled and supported by the JDBC database (PostgreSQL and Oracle at the moment), this options allows execution of a TRUNCATE TABLE t CASCADE (in the case of PostgreSQL a TRUNCATE TABLE ONLY t CASCADE is executed to prevent inadvertently truncating descendant tables). This will affect other tables, and thus should be used with care. This option applies only to writing. It defaults to the default cascading truncate behaviour of the JDBC database in question, specified in the isCascadeTruncate in each JDBCDialect.
+
+
+
+
+ createTableOptions
+ This is a JDBC writer related option. If specified, this option allows setting of database-specific table and partition options when creating a table (e.g., CREATE TABLE t (name string) ENGINE=InnoDB.). This option applies only to writing.
+
+
+
+
+ createTableColumnTypes
+ The database column data types to use instead of the defaults, when creating the table. Data type information should be specified in the same format as CREATE TABLE columns syntax (e.g: "name CHAR(64), comments VARCHAR(1024)"). The specified types should be valid spark sql data types. This option applies only to writing.
+
+
+
+
+ customSchema
+ The custom schema to use for reading data from JDBC connectors. For example, "id DECIMAL(38, 0), name STRING". You can also specify partial fields, and the others use the default type mapping. For example, "id DECIMAL(38, 0)". The column names should be identical to the corresponding column names of JDBC table. Users can specify the corresponding data types of Spark SQL instead of using the defaults. This option applies only to reading.
+
+
+
+
+ pushDownPredicate
+ The option to enable or disable predicate push-down into the JDBC data source. The default value is true, in which case Spark will push down filters to the JDBC data source as much as possible. Otherwise, if set to false, no filter will be pushed down to the JDBC data source and thus all filters will be handled by Spark. Predicate push-down is usually turned off when the predicate filtering is performed faster by Spark than by the JDBC data source.
+
+
+
+
+
+
+
+{% include_example jdbc_dataset scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+
+
+
+{% include_example jdbc_dataset java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+
+
+
+{% include_example jdbc_dataset python/sql/datasource.py %}
+
+
+
+{% include_example jdbc_dataset r/RSparkSQLExample.R %}
+
+
+
+
+{% highlight sql %}
+
+CREATE TEMPORARY VIEW jdbcTable
+USING org.apache.spark.sql.jdbc
+OPTIONS (
+ url "jdbc:postgresql:dbserver",
+ dbtable "schema.tablename",
+ user 'username',
+ password 'password'
+)
+
+INSERT INTO TABLE jdbcTable
+SELECT * FROM resultTable
+{% endhighlight %}
+
+
+
+
+
+Spark SQL can automatically infer the schema of a JSON dataset and load it as a `Dataset[Row]`.
+This conversion can be done using `SparkSession.read.json()` on either a `Dataset[String]`,
+or a JSON file.
+
+Note that the file that is offered as _a json file_ is not a typical JSON file. Each
+line must contain a separate, self-contained valid JSON object. For more information, please see
+[JSON Lines text format, also called newline-delimited JSON](http://jsonlines.org/).
+
+For a regular multi-line JSON file, set the `multiLine` option to `true`.
+
+{% include_example json_dataset scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+
+
+
+Spark SQL can automatically infer the schema of a JSON dataset and load it as a `Dataset`.
+This conversion can be done using `SparkSession.read().json()` on either a `Dataset`,
+or a JSON file.
+
+Note that the file that is offered as _a json file_ is not a typical JSON file. Each
+line must contain a separate, self-contained valid JSON object. For more information, please see
+[JSON Lines text format, also called newline-delimited JSON](http://jsonlines.org/).
+
+For a regular multi-line JSON file, set the `multiLine` option to `true`.
+
+{% include_example json_dataset java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+
+
+
+Spark SQL can automatically infer the schema of a JSON dataset and load it as a DataFrame.
+This conversion can be done using `SparkSession.read.json` on a JSON file.
+
+Note that the file that is offered as _a json file_ is not a typical JSON file. Each
+line must contain a separate, self-contained valid JSON object. For more information, please see
+[JSON Lines text format, also called newline-delimited JSON](http://jsonlines.org/).
+
+For a regular multi-line JSON file, set the `multiLine` parameter to `True`.
+
+{% include_example json_dataset python/sql/datasource.py %}
+
+
+
+Spark SQL can automatically infer the schema of a JSON dataset and load it as a DataFrame. using
+the `read.json()` function, which loads data from a directory of JSON files where each line of the
+files is a JSON object.
+
+Note that the file that is offered as _a json file_ is not a typical JSON file. Each
+line must contain a separate, self-contained valid JSON object. For more information, please see
+[JSON Lines text format, also called newline-delimited JSON](http://jsonlines.org/).
+
+For a regular multi-line JSON file, set a named parameter `multiLine` to `TRUE`.
+
+{% include_example json_dataset r/RSparkSQLExample.R %}
+
+
+
+
+
+{% highlight sql %}
+
+CREATE TEMPORARY VIEW jsonTable
+USING org.apache.spark.sql.json
+OPTIONS (
+ path "examples/src/main/resources/people.json"
+)
+
+SELECT * FROM jsonTable
+
+{% endhighlight %}
+
+
+
+
+
+{% include_example generic_load_save_functions scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+
+
+
+{% include_example generic_load_save_functions java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+
+
+
+
+{% include_example generic_load_save_functions python/sql/datasource.py %}
+
+
+
+
+{% include_example generic_load_save_functions r/RSparkSQLExample.R %}
+
+
+
+
+{% include_example manual_load_options scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+
+
+
+{% include_example manual_load_options java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+
+
+
+{% include_example manual_load_options python/sql/datasource.py %}
+
+
+
+{% include_example manual_load_options r/RSparkSQLExample.R %}
+
+
+
+{% include_example manual_load_options_csv scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+
+
+
+{% include_example manual_load_options_csv java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+
+
+
+{% include_example manual_load_options_csv python/sql/datasource.py %}
+
+
+
+{% include_example manual_load_options_csv r/RSparkSQLExample.R %}
+
+
+
+
+
+{% include_example manual_save_options_orc scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+
+
+
+{% include_example manual_save_options_orc java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+
+
+
+{% include_example manual_save_options_orc python/sql/datasource.py %}
+
+
+
+{% include_example manual_save_options_orc r/RSparkSQLExample.R %}
+
+
+
+
+{% highlight sql %}
+CREATE TABLE users_with_options (
+ name STRING,
+ favorite_color STRING,
+ favorite_numbers array
+) USING ORC
+OPTIONS (
+ orc.bloom.filter.columns 'favorite_color',
+ orc.dictionary.key.threshold '1.0'
+)
+{% endhighlight %}
+
+
+
+
+
+{% include_example direct_sql scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+
+
+
+{% include_example direct_sql java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+
+
+
+{% include_example direct_sql python/sql/datasource.py %}
+
+
+
+{% include_example direct_sql r/RSparkSQLExample.R %}
+
+
+
+Scala/Java Any Language Meaning
+ SaveMode.ErrorIfExists (default)"error" or "errorifexists" (default)
+ When saving a DataFrame to a data source, if data already exists,
+ an exception is expected to be thrown.
+
+
+
+ SaveMode.Append"append"
+ When saving a DataFrame to a data source, if data/table already exists,
+ contents of the DataFrame are expected to be appended to existing data.
+
+
+
+ SaveMode.Overwrite"overwrite"
+ Overwrite mode means that when saving a DataFrame to a data source,
+ if data/table already exists, existing data is expected to be overwritten by the contents of
+ the DataFrame.
+
+
+
+ SaveMode.Ignore"ignore"
+ Ignore mode means that when saving a DataFrame to a data source, if data already exists,
+ the save operation is expected not to save the contents of the DataFrame and not to
+ change the existing data. This is similar to a CREATE TABLE IF NOT EXISTS in SQL.
+
+
+
+
+### Saving to Persistent Tables
+
+`DataFrames` can also be saved as persistent tables into Hive metastore using the `saveAsTable`
+command. Notice that an existing Hive deployment is not necessary to use this feature. Spark will create a
+default local Hive metastore (using Derby) for you. Unlike the `createOrReplaceTempView` command,
+`saveAsTable` will materialize the contents of the DataFrame and create a pointer to the data in the
+Hive metastore. Persistent tables will still exist even after your Spark program has restarted, as
+long as you maintain your connection to the same metastore. A DataFrame for a persistent table can
+be created by calling the `table` method on a `SparkSession` with the name of the table.
+
+For file-based data source, e.g. text, parquet, json, etc. you can specify a custom table path via the
+`path` option, e.g. `df.write.option("path", "/some/path").saveAsTable("t")`. When the table is dropped,
+the custom table path will not be removed and the table data is still there. If no custom table path is
+specified, Spark will write data to a default table path under the warehouse directory. When the table is
+dropped, the default table path will be removed too.
+
+Starting from Spark 2.1, persistent datasource tables have per-partition metadata stored in the Hive metastore. This brings several benefits:
+
+- Since the metastore can return only necessary partitions for a query, discovering all the partitions on the first query to the table is no longer needed.
+- Hive DDLs such as `ALTER TABLE PARTITION ... SET LOCATION` are now available for tables created with the Datasource API.
+
+Note that partition information is not gathered by default when creating external datasource tables (those with a `path` option). To sync the partition information in the metastore, you can invoke `MSCK REPAIR TABLE`.
+
+### Bucketing, Sorting and Partitioning
+
+For file-based data source, it is also possible to bucket and sort or partition the output.
+Bucketing and sorting are applicable only to persistent tables:
+
+
+
+
+{% include_example write_sorting_and_bucketing scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+
+
+
+{% include_example write_sorting_and_bucketing java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+
+
+
+{% include_example write_sorting_and_bucketing python/sql/datasource.py %}
+
+
+
+
+{% highlight sql %}
+
+CREATE TABLE users_bucketed_by_name(
+ name STRING,
+ favorite_color STRING,
+ favorite_numbers array
+) USING parquet
+CLUSTERED BY(name) INTO 42 BUCKETS;
+
+{% endhighlight %}
+
+
+
+
+
+
+{% include_example write_partitioning scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+
+
+
+{% include_example write_partitioning java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+
+
+
+{% include_example write_partitioning python/sql/datasource.py %}
+
+
+
+
+{% highlight sql %}
+
+CREATE TABLE users_by_favorite_color(
+ name STRING,
+ favorite_color STRING,
+ favorite_numbers array
+) USING csv PARTITIONED BY(favorite_color);
+
+{% endhighlight %}
+
+
+
+
+
+
+{% include_example write_partition_and_bucket scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+
+
+
+{% include_example write_partition_and_bucket java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+
+
+
+{% include_example write_partition_and_bucket python/sql/datasource.py %}
+
+
+
+
+{% highlight sql %}
+
+CREATE TABLE users_bucketed_and_partitioned(
+ name STRING,
+ favorite_color STRING,
+ favorite_numbers array
+) USING parquet
+PARTITIONED BY (favorite_color)
+CLUSTERED BY(name) SORTED BY (favorite_numbers) INTO 42 BUCKETS;
+
+{% endhighlight %}
+
+
+
+
+ Property Name Default Meaning
+ spark.sql.orc.implnativeThe name of ORC implementation. It can be one of native and hive. native means the native ORC support that is built on Apache ORC 1.4. `hive` means the ORC library in Hive 1.2.1.
+
+
+ spark.sql.orc.enableVectorizedReadertrueEnables vectorized orc decoding in native implementation. If false, a new non-vectorized ORC reader is used in native implementation. For hive implementation, this is ignored.
+
+
diff --git a/docs/sql-data-sources-parquet.md b/docs/sql-data-sources-parquet.md
new file mode 100644
index 0000000000000..71fa57d9d8095
--- /dev/null
+++ b/docs/sql-data-sources-parquet.md
@@ -0,0 +1,309 @@
+---
+layout: global
+title: Parquet Files
+displayTitle: Parquet Files
+---
+
+* Table of contents
+{:toc}
+
+[Parquet](http://parquet.io) is a columnar format that is supported by many other data processing systems.
+Spark SQL provides support for both reading and writing Parquet files that automatically preserves the schema
+of the original data. When writing Parquet files, all columns are automatically converted to be nullable for
+compatibility reasons.
+
+### Loading Data Programmatically
+
+Using the data from the above example:
+
+
+
+
+{% include_example basic_parquet_example scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+
+
+
+{% include_example basic_parquet_example java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+
+
+
+
+{% include_example basic_parquet_example python/sql/datasource.py %}
+
+
+
+
+{% include_example basic_parquet_example r/RSparkSQLExample.R %}
+
+
+
+
+
+{% highlight sql %}
+
+CREATE TEMPORARY VIEW parquetTable
+USING org.apache.spark.sql.parquet
+OPTIONS (
+ path "examples/src/main/resources/people.parquet"
+)
+
+SELECT * FROM parquetTable
+
+{% endhighlight %}
+
+
+
+
+
+
+{% include_example schema_merging scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+
+
+
+{% include_example schema_merging java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+
+
+
+
+{% include_example schema_merging python/sql/datasource.py %}
+
+
+
+
+{% include_example schema_merging r/RSparkSQLExample.R %}
+
+
+
+
+
+
+
+{% highlight scala %}
+// spark is an existing SparkSession
+spark.catalog.refreshTable("my_table")
+{% endhighlight %}
+
+
+
+
+
+{% highlight java %}
+// spark is an existing SparkSession
+spark.catalog().refreshTable("my_table");
+{% endhighlight %}
+
+
+
+
+
+{% highlight python %}
+# spark is an existing SparkSession
+spark.catalog.refreshTable("my_table")
+{% endhighlight %}
+
+
+
+
+
+{% highlight r %}
+refreshTable("my_table")
+{% endhighlight %}
+
+
+
+
+
+{% highlight sql %}
+REFRESH TABLE my_table;
+{% endhighlight %}
+
+
+
+
+Property Name Default Meaning
+ spark.sql.parquet.binaryAsStringfalse
+
+ Some other Parquet-producing systems, in particular Impala, Hive, and older versions of Spark SQL, do
+ not differentiate between binary data and strings when writing out the Parquet schema. This
+ flag tells Spark SQL to interpret binary data as a string to provide compatibility with these systems.
+
+
+
+ spark.sql.parquet.int96AsTimestamptrue
+
+ Some Parquet-producing systems, in particular Impala and Hive, store Timestamp into INT96. This
+ flag tells Spark SQL to interpret INT96 data as a timestamp to provide compatibility with these systems.
+
+
+
+ spark.sql.parquet.compression.codecsnappy
+
+ Sets the compression codec used when writing Parquet files. If either `compression` or
+ `parquet.compression` is specified in the table-specific options/properties, the precedence would be
+ `compression`, `parquet.compression`, `spark.sql.parquet.compression.codec`. Acceptable values include:
+ none, uncompressed, snappy, gzip, lzo, brotli, lz4, zstd.
+ Note that `zstd` requires `ZStandardCodec` to be installed before Hadoop 2.9.0, `brotli` requires
+ `BrotliCodec` to be installed.
+
+
+
+ spark.sql.parquet.filterPushdowntrue
+ Enables Parquet filter push-down optimization when set to true.
+
+
+ spark.sql.hive.convertMetastoreParquettrue
+
+ When set to false, Spark SQL will use the Hive SerDe for parquet tables instead of the built in
+ support.
+
+
+
+ spark.sql.parquet.mergeSchemafalse
+
+
+ When true, the Parquet data source merges schemas collected from all data files, otherwise the
+ schema is picked from the summary file or a random data file if no summary file is available.
+
+
+
+
+ spark.sql.parquet.writeLegacyFormatfalse
+
+ If true, data will be written in a way of Spark 1.4 and earlier. For example, decimal values
+ will be written in Apache Parquet's fixed-length byte array format, which other systems such as
+ Apache Hive and Apache Impala use. If false, the newer format in Parquet will be used. For
+ example, decimals will be written in int-based format. If Parquet output is intended for use
+ with systems that do not support this newer format, set to true.
+
+
+
diff --git a/docs/sql-data-sources-troubleshooting.md b/docs/sql-data-sources-troubleshooting.md
new file mode 100644
index 0000000000000..5775eb8b5c956
--- /dev/null
+++ b/docs/sql-data-sources-troubleshooting.md
@@ -0,0 +1,9 @@
+---
+layout: global
+title: Troubleshooting
+displayTitle: Troubleshooting
+---
+
+ * The JDBC driver class must be visible to the primordial class loader on the client session and on all executors. This is because Java's DriverManager class does a security check that results in it ignoring all drivers not visible to the primordial class loader when one goes to open a connection. One convenient way to do this is to modify compute_classpath.sh on all worker nodes to include your driver JARs.
+ * Some databases, such as H2, convert all names to upper case. You'll need to use upper case to refer to those names in Spark SQL.
+ * Users can specify vendor-specific JDBC connection properties in the data source options to do special treatment. For example, `spark.read.format("jdbc").option("url", oracleJdbcUrl).option("oracle.jdbc.mapDateToTimestamp", "false")`. `oracle.jdbc.mapDateToTimestamp` defaults to true, users often need to disable this flag to avoid Oracle date being resolved as timestamp.
diff --git a/docs/sql-data-sources.md b/docs/sql-data-sources.md
new file mode 100644
index 0000000000000..636636af6263c
--- /dev/null
+++ b/docs/sql-data-sources.md
@@ -0,0 +1,42 @@
+---
+layout: global
+title: Data Sources
+displayTitle: Data Sources
+---
+
+
+Spark SQL supports operating on a variety of data sources through the DataFrame interface.
+A DataFrame can be operated on using relational transformations and can also be used to create a temporary view.
+Registering a DataFrame as a temporary view allows you to run SQL queries over its data. This section
+describes the general methods for loading and saving data using the Spark Data Sources and then
+goes into specific options that are available for the built-in data sources.
+
+
+* [Generic Load/Save Functions](sql-data-sources-load-save-functions.html)
+ * [Manually Specifying Options](sql-data-sources-load-save-functions.html#manually-specifying-options)
+ * [Run SQL on files directly](sql-data-sources-load-save-functions.html#run-sql-on-files-directly)
+ * [Save Modes](sql-data-sources-load-save-functions.html#save-modes)
+ * [Saving to Persistent Tables](sql-data-sources-load-save-functions.html#saving-to-persistent-tables)
+ * [Bucketing, Sorting and Partitioning](sql-data-sources-load-save-functions.html#bucketing-sorting-and-partitioning)
+* [Parquet Files](sql-data-sources-parquet.html)
+ * [Loading Data Programmatically](sql-data-sources-parquet.html#loading-data-programmatically)
+ * [Partition Discovery](sql-data-sources-parquet.html#partition-discovery)
+ * [Schema Merging](sql-data-sources-parquet.html#schema-merging)
+ * [Hive metastore Parquet table conversion](sql-data-sources-parquet.html#hive-metastore-parquet-table-conversion)
+ * [Configuration](sql-data-sources-parquet.html#configuration)
+* [ORC Files](sql-data-sources-orc.html)
+* [JSON Files](sql-data-sources-json.html)
+* [Hive Tables](sql-data-sources-hive-tables.html)
+ * [Specifying storage format for Hive tables](sql-data-sources-hive-tables.html#specifying-storage-format-for-hive-tables)
+ * [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)
+* [JDBC To Other Databases](sql-data-sources-jdbc.html)
+* [Avro Files](sql-data-sources-avro.html)
+ * [Deploying](sql-data-sources-avro.html#deploying)
+ * [Load and Save Functions](sql-data-sources-avro.html#load-and-save-functions)
+ * [to_avro() and from_avro()](sql-data-sources-avro.html#to_avro-and-from_avro)
+ * [Data Source Option](sql-data-sources-avro.html#data-source-option)
+ * [Configuration](sql-data-sources-avro.html#configuration)
+ * [Compatibility with Databricks spark-avro](sql-data-sources-avro.html#compatibility-with-databricks-spark-avro)
+ * [Supported types for Avro -> Spark SQL conversion](sql-data-sources-avro.html#supported-types-for-avro---spark-sql-conversion)
+ * [Supported types for Spark SQL -> Avro conversion](sql-data-sources-avro.html#supported-types-for-spark-sql---avro-conversion)
+* [Troubleshooting](sql-data-sources-troubleshooting.html)
diff --git a/docs/sql-distributed-sql-engine.md b/docs/sql-distributed-sql-engine.md
new file mode 100644
index 0000000000000..66d6fdaf90a08
--- /dev/null
+++ b/docs/sql-distributed-sql-engine.md
@@ -0,0 +1,84 @@
+---
+layout: global
+title: Distributed SQL Engine
+displayTitle: Distributed SQL Engine
+---
+
+* Table of contents
+{:toc}
+
+Spark SQL can also act as a distributed query engine using its JDBC/ODBC or command-line interface.
+In this mode, end-users or applications can interact with Spark SQL directly to run SQL queries,
+without the need to write any code.
+
+## Running the Thrift JDBC/ODBC server
+
+The Thrift JDBC/ODBC server implemented here corresponds to the [`HiveServer2`](https://cwiki.apache.org/confluence/display/Hive/Setting+Up+HiveServer2)
+in Hive 1.2.1. You can test the JDBC server with the beeline script that comes with either Spark or Hive 1.2.1.
+
+To start the JDBC/ODBC server, run the following in the Spark directory:
+
+ ./sbin/start-thriftserver.sh
+
+This script accepts all `bin/spark-submit` command line options, plus a `--hiveconf` option to
+specify Hive properties. You may run `./sbin/start-thriftserver.sh --help` for a complete list of
+all available options. By default, the server listens on localhost:10000. You may override this
+behaviour via either environment variables, i.e.:
+
+{% highlight bash %}
+export HIVE_SERVER2_THRIFT_PORT=
+export HIVE_SERVER2_THRIFT_BIND_HOST=
+./sbin/start-thriftserver.sh \
+ --master \
+ ...
+{% endhighlight %}
+
+or system properties:
+
+{% highlight bash %}
+./sbin/start-thriftserver.sh \
+ --hiveconf hive.server2.thrift.port= \
+ --hiveconf hive.server2.thrift.bind.host= \
+ --master
+ ...
+{% endhighlight %}
+
+Now you can use beeline to test the Thrift JDBC/ODBC server:
+
+ ./bin/beeline
+
+Connect to the JDBC/ODBC server in beeline with:
+
+ beeline> !connect jdbc:hive2://localhost:10000
+
+Beeline will ask you for a username and password. In non-secure mode, simply enter the username on
+your machine and a blank password. For secure mode, please follow the instructions given in the
+[beeline documentation](https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Clients).
+
+Configuration of Hive is done by placing your `hive-site.xml`, `core-site.xml` and `hdfs-site.xml` files in `conf/`.
+
+You may also use the beeline script that comes with Hive.
+
+Thrift JDBC server also supports sending thrift RPC messages over HTTP transport.
+Use the following setting to enable HTTP mode as system property or in `hive-site.xml` file in `conf/`:
+
+ hive.server2.transport.mode - Set this to value: http
+ hive.server2.thrift.http.port - HTTP port number to listen on; default is 10001
+ hive.server2.http.endpoint - HTTP endpoint; default is cliservice
+
+To test, use beeline to connect to the JDBC/ODBC server in http mode with:
+
+ beeline> !connect jdbc:hive2://:/?hive.server2.transport.mode=http;hive.server2.thrift.http.path=
+
+
+## Running the Spark SQL CLI
+
+The Spark SQL CLI is a convenient tool to run the Hive metastore service in local mode and execute
+queries input from the command line. Note that the Spark SQL CLI cannot talk to the Thrift JDBC server.
+
+To start the Spark SQL CLI, run the following in the Spark directory:
+
+ ./bin/spark-sql
+
+Configuration of Hive is done by placing your `hive-site.xml`, `core-site.xml` and `hdfs-site.xml` files in `conf/`.
+You may run `./bin/spark-sql --help` for a complete list of all available options.
diff --git a/docs/sql-getting-started.md b/docs/sql-getting-started.md
new file mode 100644
index 0000000000000..88512205894ab
--- /dev/null
+++ b/docs/sql-getting-started.md
@@ -0,0 +1,369 @@
+---
+layout: global
+title: Getting Started
+displayTitle: Getting Started
+---
+
+* Table of contents
+{:toc}
+
+## Starting Point: SparkSession
+
+
+
+
+The entry point into all functionality in Spark is the [`SparkSession`](api/scala/index.html#org.apache.spark.sql.SparkSession) class. To create a basic `SparkSession`, just use `SparkSession.builder()`:
+
+{% include_example init_session scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
+
+
+
+
+The entry point into all functionality in Spark is the [`SparkSession`](api/java/index.html#org.apache.spark.sql.SparkSession) class. To create a basic `SparkSession`, just use `SparkSession.builder()`:
+
+{% include_example init_session java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
+
+
+
+
+The entry point into all functionality in Spark is the [`SparkSession`](api/python/pyspark.sql.html#pyspark.sql.SparkSession) class. To create a basic `SparkSession`, just use `SparkSession.builder`:
+
+{% include_example init_session python/sql/basic.py %}
+
+
+
+
+The entry point into all functionality in Spark is the [`SparkSession`](api/R/sparkR.session.html) class. To initialize a basic `SparkSession`, just call `sparkR.session()`:
+
+{% include_example init_session r/RSparkSQLExample.R %}
+
+Note that when invoked for the first time, `sparkR.session()` initializes a global `SparkSession` singleton instance, and always returns a reference to this instance for successive invocations. In this way, users only need to initialize the `SparkSession` once, then SparkR functions like `read.df` will be able to access this global instance implicitly, and users don't need to pass the `SparkSession` instance around.
+
+
+
+With a `SparkSession`, applications can create DataFrames from an [existing `RDD`](#interoperating-with-rdds),
+from a Hive table, or from [Spark data sources](sql-data-sources.html).
+
+As an example, the following creates a DataFrame based on the content of a JSON file:
+
+{% include_example create_df scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
+
+
+
+With a `SparkSession`, applications can create DataFrames from an [existing `RDD`](#interoperating-with-rdds),
+from a Hive table, or from [Spark data sources](sql-data-sources.html).
+
+As an example, the following creates a DataFrame based on the content of a JSON file:
+
+{% include_example create_df java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
+
+
+
+With a `SparkSession`, applications can create DataFrames from an [existing `RDD`](#interoperating-with-rdds),
+from a Hive table, or from [Spark data sources](sql-data-sources.html).
+
+As an example, the following creates a DataFrame based on the content of a JSON file:
+
+{% include_example create_df python/sql/basic.py %}
+
+
+
+With a `SparkSession`, applications can create DataFrames from a local R data.frame,
+from a Hive table, or from [Spark data sources](sql-data-sources.html).
+
+As an example, the following creates a DataFrame based on the content of a JSON file:
+
+{% include_example create_df r/RSparkSQLExample.R %}
+
+
+
+
+{% include_example untyped_ops scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
+
+For a complete list of the types of operations that can be performed on a Dataset refer to the [API Documentation](api/scala/index.html#org.apache.spark.sql.Dataset).
+
+In addition to simple column references and expressions, Datasets also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/scala/index.html#org.apache.spark.sql.functions$).
+
+
+
+
+{% include_example untyped_ops java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
+
+For a complete list of the types of operations that can be performed on a Dataset refer to the [API Documentation](api/java/org/apache/spark/sql/Dataset.html).
+
+In addition to simple column references and expressions, Datasets also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/java/org/apache/spark/sql/functions.html).
+
+
+
+In Python, it's possible to access a DataFrame's columns either by attribute
+(`df.age`) or by indexing (`df['age']`). While the former is convenient for
+interactive data exploration, users are highly encouraged to use the
+latter form, which is future proof and won't break with column names that
+are also attributes on the DataFrame class.
+
+{% include_example untyped_ops python/sql/basic.py %}
+For a complete list of the types of operations that can be performed on a DataFrame refer to the [API Documentation](api/python/pyspark.sql.html#pyspark.sql.DataFrame).
+
+In addition to simple column references and expressions, DataFrames also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/python/pyspark.sql.html#module-pyspark.sql.functions).
+
+
+
+
+
+{% include_example untyped_ops r/RSparkSQLExample.R %}
+
+For a complete list of the types of operations that can be performed on a DataFrame refer to the [API Documentation](api/R/index.html).
+
+In addition to simple column references and expressions, DataFrames also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/R/SparkDataFrame.html).
+
+
+
+
+
+The `sql` function on a `SparkSession` enables applications to run SQL queries programmatically and returns the result as a `DataFrame`.
+
+{% include_example run_sql scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
+
+
+
+The `sql` function on a `SparkSession` enables applications to run SQL queries programmatically and returns the result as a `Dataset`.
+
+{% include_example run_sql java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
+
+
+
+The `sql` function on a `SparkSession` enables applications to run SQL queries programmatically and returns the result as a `DataFrame`.
+
+{% include_example run_sql python/sql/basic.py %}
+
+
+
+The `sql` function enables applications to run SQL queries programmatically and returns the result as a `SparkDataFrame`.
+
+{% include_example run_sql r/RSparkSQLExample.R %}
+
+
+
+
+{% include_example global_temp_view scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
+
+
+
+{% include_example global_temp_view java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
+
+
+
+{% include_example global_temp_view python/sql/basic.py %}
+
+
+
+
+{% highlight sql %}
+
+CREATE GLOBAL TEMPORARY VIEW temp_view AS SELECT a + 1, b * 2 FROM tbl
+
+SELECT * FROM global_temp.temp_view
+
+{% endhighlight %}
+
+
+
+
+{% include_example create_ds scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
+
+
+
+{% include_example create_ds java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
+
+
+
+
+
+The Scala interface for Spark SQL supports automatically converting an RDD containing case classes
+to a DataFrame. The case class
+defines the schema of the table. The names of the arguments to the case class are read using
+reflection and become the names of the columns. Case classes can also be nested or contain complex
+types such as `Seq`s or `Array`s. This RDD can be implicitly converted to a DataFrame and then be
+registered as a table. Tables can be used in subsequent SQL statements.
+
+{% include_example schema_inferring scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
+
+
+
+
+Spark SQL supports automatically converting an RDD of
+[JavaBeans](http://stackoverflow.com/questions/3295496/what-is-a-javabean-exactly) into a DataFrame.
+The `BeanInfo`, obtained using reflection, defines the schema of the table. Currently, Spark SQL
+does not support JavaBeans that contain `Map` field(s). Nested JavaBeans and `List` or `Array`
+fields are supported though. You can create a JavaBean by creating a class that implements
+Serializable and has getters and setters for all of its fields.
+
+{% include_example schema_inferring java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
+
+
+
+
+Spark SQL can convert an RDD of Row objects to a DataFrame, inferring the datatypes. Rows are constructed by passing a list of
+key/value pairs as kwargs to the Row class. The keys of this list define the column names of the table,
+and the types are inferred by sampling the whole dataset, similar to the inference that is performed on JSON files.
+
+{% include_example schema_inferring python/sql/basic.py %}
+
+
+
+
+
+
+When case classes cannot be defined ahead of time (for example,
+the structure of records is encoded in a string, or a text dataset will be parsed
+and fields will be projected differently for different users),
+a `DataFrame` can be created programmatically with three steps.
+
+1. Create an RDD of `Row`s from the original RDD;
+2. Create the schema represented by a `StructType` matching the structure of
+`Row`s in the RDD created in Step 1.
+3. Apply the schema to the RDD of `Row`s via `createDataFrame` method provided
+by `SparkSession`.
+
+For example:
+
+{% include_example programmatic_schema scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
+
+
+
+
+When JavaBean classes cannot be defined ahead of time (for example,
+the structure of records is encoded in a string, or a text dataset will be parsed and
+fields will be projected differently for different users),
+a `Dataset` can be created programmatically with three steps.
+
+1. Create an RDD of `Row`s from the original RDD;
+2. Create the schema represented by a `StructType` matching the structure of
+`Row`s in the RDD created in Step 1.
+3. Apply the schema to the RDD of `Row`s via `createDataFrame` method provided
+by `SparkSession`.
+
+For example:
+
+{% include_example programmatic_schema java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
+
+
+
+
+When a dictionary of kwargs cannot be defined ahead of time (for example,
+the structure of records is encoded in a string, or a text dataset will be parsed and
+fields will be projected differently for different users),
+a `DataFrame` can be created programmatically with three steps.
+
+1. Create an RDD of tuples or lists from the original RDD;
+2. Create the schema represented by a `StructType` matching the structure of
+tuples or lists in the RDD created in the step 1.
+3. Apply the schema to the RDD via `createDataFrame` method provided by `SparkSession`.
+
+For example:
+
+{% include_example programmatic_schema python/sql/basic.py %}
+
+
+
+
+{% include_example untyped_custom_aggregation scala/org/apache/spark/examples/sql/UserDefinedUntypedAggregation.scala%}
+
+
+{% include_example untyped_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedUntypedAggregation.java%}
+
+
+
+{% include_example typed_custom_aggregation scala/org/apache/spark/examples/sql/UserDefinedTypedAggregation.scala%}
+
+
+{% include_example typed_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedTypedAggregation.java%}
+
+
+
+
+ Query
+
+
+ Result Spark 2.3 or Prior
+
+
+ Result Spark 2.4
+
+
+ Remarks
+
+
+
+
+ SELECT
+
+
+ true
+
+
+ false
+
+
+ In Spark 2.4, left and right parameters are promoted to array(double) and double type respectively.
+
+
+
+
+ SELECT
+
+
+ true
+
+
+ AnalysisException is thrown since integer type can not be promoted to string type in a loss-less manner.
+
+
+ Users can use explicit cast
+
+
+
+
+ SELECT
+
+
+ null
+
+
+ AnalysisException is thrown since integer type can not be promoted to string type in a loss-less manner.
+
+
+ Users can use explicit cast
+
+
+
+
+ - Since Spark 2.4, when there is a struct field in front of the IN operator before a subquery, the inner query must contain a struct field as well. In previous versions, instead, the fields of the struct were compared to the output of the inner query. Eg. if `a` is a `struct(a string, b int)`, in Spark 2.4 `a in (select (1 as a, 'a' as b) from range(1))` is a valid query, while `a in (select 1, 'a' from range(1))` is not. In previous version it was the opposite.
+
+ - In versions 2.2.1+ and 2.3, if `spark.sql.caseSensitive` is set to true, then the `CURRENT_DATE` and `CURRENT_TIMESTAMP` functions incorrectly became case-sensitive and would resolve to columns (unless typed in lower case). In Spark 2.4 this has been fixed and the functions are no longer case-sensitive.
+
+ - Since Spark 2.4, Spark will evaluate the set operations referenced in a query by following a precedence rule as per the SQL standard. If the order is not specified by parentheses, set operations are performed from left to right with the exception that all INTERSECT operations are performed before any UNION, EXCEPT or MINUS operations. The old behaviour of giving equal precedence to all the set operations are preserved under a newly added configuration `spark.sql.legacy.setopsPrecedence.enabled` with a default value of `false`. When this property is set to `true`, spark will evaluate the set operators from left to right as they appear in the query given no explicit ordering is enforced by usage of parenthesis.
+
+ - Since Spark 2.4, Spark will display table description column Last Access value as UNKNOWN when the value was Jan 01 1970.
+
+ - Since Spark 2.4, Spark maximizes the usage of a vectorized ORC reader for ORC files by default. To do that, `spark.sql.orc.impl` and `spark.sql.orc.filterPushdown` change their default values to `native` and `true` respectively. ORC files created by native ORC writer cannot be read by some old Apache Hive releases. Use `spark.sql.orc.impl=hive` to create the files shared with Hive 2.1.1 and older.
+
+ - In PySpark, when Arrow optimization is enabled, previously `toPandas` just failed when Arrow optimization is unable to be used whereas `createDataFrame` from Pandas DataFrame allowed the fallback to non-optimization. Now, both `toPandas` and `createDataFrame` from Pandas DataFrame allow the fallback by default, which can be switched off by `spark.sql.execution.arrow.fallback.enabled`.
+
+ - Since Spark 2.4, writing an empty dataframe to a directory launches at least one write task, even if physically the dataframe has no partition. This introduces a small behavior change that for self-describing file formats like Parquet and Orc, Spark creates a metadata-only file in the target directory when writing a 0-partition dataframe, so that schema inference can still work if users read that directory later. The new behavior is more reasonable and more consistent regarding writing empty dataframe.
+
+ - Since Spark 2.4, expression IDs in UDF arguments do not appear in column names. For example, a column name in Spark 2.4 is not `UDF:f(col0 AS colA#28)` but ``UDF:f(col0 AS `colA`)``.
+
+ - Since Spark 2.4, writing a dataframe with an empty or nested empty schema using any file formats (parquet, orc, json, text, csv etc.) is not allowed. An exception is thrown when attempting to write dataframes with empty schema.
+
+ - Since Spark 2.4, Spark compares a DATE type with a TIMESTAMP type after promotes both sides to TIMESTAMP. To set `false` to `spark.sql.legacy.compareDateTimestampInTimestamp` restores the previous behavior. This option will be removed in Spark 3.0.
+
+ - Since Spark 2.4, creating a managed table with nonempty location is not allowed. An exception is thrown when attempting to create a managed table with nonempty location. To set `true` to `spark.sql.legacy.allowCreatingManagedTableUsingNonemptyLocation` restores the previous behavior. This option will be removed in Spark 3.0.
+
+ - Since Spark 2.4, renaming a managed table to existing location is not allowed. An exception is thrown when attempting to rename a managed table to existing location.
+
+ - Since Spark 2.4, the type coercion rules can automatically promote the argument types of the variadic SQL functions (e.g., IN/COALESCE) to the widest common type, no matter how the input arguments order. In prior Spark versions, the promotion could fail in some specific orders (e.g., TimestampType, IntegerType and StringType) and throw an exception.
+
+ - Since Spark 2.4, Spark has enabled non-cascading SQL cache invalidation in addition to the traditional cache invalidation mechanism. The non-cascading cache invalidation mechanism allows users to remove a cache without impacting its dependent caches. This new cache invalidation mechanism is used in scenarios where the data of the cache to be removed is still valid, e.g., calling unpersist() on a Dataset, or dropping a temporary view. This allows users to free up memory and keep the desired caches valid at the same time.
+
+ - In version 2.3 and earlier, Spark converts Parquet Hive tables by default but ignores table properties like `TBLPROPERTIES (parquet.compression 'NONE')`. This happens for ORC Hive table properties like `TBLPROPERTIES (orc.compress 'NONE')` in case of `spark.sql.hive.convertMetastoreOrc=true`, too. Since Spark 2.4, Spark respects Parquet/ORC specific table properties while converting Parquet/ORC Hive tables. As an example, `CREATE TABLE t(id int) STORED AS PARQUET TBLPROPERTIES (parquet.compression 'NONE')` would generate Snappy parquet files during insertion in Spark 2.3, and in Spark 2.4, the result would be uncompressed parquet files.
+
+ - Since Spark 2.0, Spark converts Parquet Hive tables by default for better performance. Since Spark 2.4, Spark converts ORC Hive tables by default, too. It means Spark uses its own ORC support by default instead of Hive SerDe. As an example, `CREATE TABLE t(id int) STORED AS ORC` would be handled with Hive SerDe in Spark 2.3, and in Spark 2.4, it would be converted into Spark's ORC data source table and ORC vectorization would be applied. In addition, this makes Spark's Hive table read behavior more consistent over different formats and with the behavior of `spark.read.load`. For example, for both ORC/Parquet Hive tables, `LOCATION '/table/*'` is required instead of `LOCATION '/table/'` to create an external table reading its direct sub-directories like `'/table/dir'`. To set `false` to `spark.sql.hive.convertMetastoreOrc` restores the previous behavior.
+
+ - In version 2.3 and earlier, CSV rows are considered as malformed if at least one column value in the row is malformed. CSV parser dropped such rows in the DROPMALFORMED mode or outputs an error in the FAILFAST mode. Since Spark 2.4, CSV row is considered as malformed only when it contains malformed column values requested from CSV datasource, other values can be ignored. As an example, CSV file contains the "id,name" header and one row "1234". In Spark 2.4, selection of the id column consists of a row with one column value 1234 but in Spark 2.3 and earlier it is empty in the DROPMALFORMED mode. To restore the previous behavior, set `spark.sql.csv.parser.columnPruning.enabled` to `false`.
+
+ - Since Spark 2.4, File listing for compute statistics is done in parallel by default. This can be disabled by setting `spark.sql.statistics.parallelFileListingInStatsComputation.enabled` to `False`.
+
+ - Since Spark 2.4, Metadata files (e.g. Parquet summary files) and temporary files are not counted as data files when calculating table size during Statistics computation.
+
+ - Since Spark 2.4, empty strings are saved as quoted empty strings `""`. In version 2.3 and earlier, empty strings are equal to `null` values and do not reflect to any characters in saved CSV files. For example, the row of `"a", null, "", 1` was written as `a,,,1`. Since Spark 2.4, the same row is saved as `a,,"",1`. To restore the previous behavior, set the CSV option `emptyValue` to empty (not quoted) string.
+
+ - Since Spark 2.4, The LOAD DATA command supports wildcard `?` and `*`, which match any one character, and zero or more characters, respectively. Example: `LOAD DATA INPATH '/tmp/folder*/'` or `LOAD DATA INPATH '/tmp/part-?'`. Special Characters like `space` also now work in paths. Example: `LOAD DATA INPATH '/tmp/folder name/'`.
+
+ - In Spark version 2.3 and earlier, HAVING without GROUP BY is treated as WHERE. This means, `SELECT 1 FROM range(10) HAVING true` is executed as `SELECT 1 FROM range(10) WHERE true` and returns 10 rows. This violates SQL standard, and has been fixed in Spark 2.4. Since Spark 2.4, HAVING without GROUP BY is treated as a global aggregate, which means `SELECT 1 FROM range(10) HAVING true` will return only one row. To restore the previous behavior, set `spark.sql.legacy.parser.havingWithoutGroupByAsWhere` to `true`.
+
+ - In version 2.3 and earlier, when reading from a Parquet data source table, Spark always returns null for any column whose column names in Hive metastore schema and Parquet schema are in different letter cases, no matter whether `spark.sql.caseSensitive` is set to `true` or `false`. Since 2.4, when `spark.sql.caseSensitive` is set to `false`, Spark does case insensitive column name resolution between Hive metastore schema and Parquet schema, so even column names are in different letter cases, Spark returns corresponding column values. An exception is thrown if there is ambiguity, i.e. more than one Parquet column is matched. This change also applies to Parquet Hive tables when `spark.sql.hive.convertMetastoreParquet` is set to `true`.
+
+## Upgrading From Spark SQL 2.3.0 to 2.3.1 and above
+
+ - As of version 2.3.1 Arrow functionality, including `pandas_udf` and `toPandas()`/`createDataFrame()` with `spark.sql.execution.arrow.enabled` set to `True`, has been marked as experimental. These are still evolving and not currently recommended for use in production.
+
+## Upgrading From Spark SQL 2.2 to 2.3
+
+ - Since Spark 2.3, the queries from raw JSON/CSV files are disallowed when the referenced columns only include the internal corrupt record column (named `_corrupt_record` by default). For example, `spark.read.schema(schema).json(file).filter($"_corrupt_record".isNotNull).count()` and `spark.read.schema(schema).json(file).select("_corrupt_record").show()`. Instead, you can cache or save the parsed results and then send the same query. For example, `val df = spark.read.schema(schema).json(file).cache()` and then `df.filter($"_corrupt_record".isNotNull).count()`.
+
+ - The `percentile_approx` function previously accepted numeric type input and output double type results. Now it supports date type, timestamp type and numeric types as input types. The result type is also changed to be the same as the input type, which is more reasonable for percentiles.
+
+ - Since Spark 2.3, the Join/Filter's deterministic predicates that are after the first non-deterministic predicates are also pushed down/through the child operators, if possible. In prior Spark versions, these filters are not eligible for predicate pushdown.
+
+ - Partition column inference previously found incorrect common type for different inferred types, for example, previously it ended up with double type as the common type for double type and date type. Now it finds the correct common type for such conflicts. The conflict resolution follows the table below:
+
+
+
+ InputA \ InputB
+
+
+ NullType
+
+
+ IntegerType
+
+
+ LongType
+
+
+ DecimalType(38,0)*
+
+
+ DoubleType
+
+
+ DateType
+
+
+ TimestampType
+
+
+ StringType
+
+
+
+
+ NullType
+
+ NullType
+ IntegerType
+ LongType
+ DecimalType(38,0)
+ DoubleType
+ DateType
+ TimestampType
+ StringType
+
+
+
+ IntegerType
+
+ IntegerType
+ IntegerType
+ LongType
+ DecimalType(38,0)
+ DoubleType
+ StringType
+ StringType
+ StringType
+
+
+
+ LongType
+
+ LongType
+ LongType
+ LongType
+ DecimalType(38,0)
+ StringType
+ StringType
+ StringType
+ StringType
+
+
+
+ DecimalType(38,0)*
+
+ DecimalType(38,0)
+ DecimalType(38,0)
+ DecimalType(38,0)
+ DecimalType(38,0)
+ StringType
+ StringType
+ StringType
+ StringType
+
+
+
+ DoubleType
+
+ DoubleType
+ DoubleType
+ StringType
+ StringType
+ DoubleType
+ StringType
+ StringType
+ StringType
+
+
+
+ DateType
+
+ DateType
+ StringType
+ StringType
+ StringType
+ StringType
+ DateType
+ TimestampType
+ StringType
+
+
+
+ TimestampType
+
+ TimestampType
+ StringType
+ StringType
+ StringType
+ StringType
+ TimestampType
+ TimestampType
+ StringType
+
+
+
+ StringType
+
+ StringType
+ StringType
+ StringType
+ StringType
+ StringType
+ StringType
+ StringType
+ StringType
+
+
+
+ Note that, for DecimalType(38,0)* , the table above intentionally does not cover all other combinations of scales and precisions because currently we only infer decimal type like `BigInteger`/`BigInt`. For example, 1.1 is inferred as double type.
+
+ - In PySpark, now we need Pandas 0.19.2 or upper if you want to use Pandas related functionalities, such as `toPandas`, `createDataFrame` from Pandas DataFrame, etc.
+
+ - In PySpark, the behavior of timestamp values for Pandas related functionalities was changed to respect session timezone. If you want to use the old behavior, you need to set a configuration `spark.sql.execution.pandas.respectSessionTimeZone` to `False`. See [SPARK-22395](https://issues.apache.org/jira/browse/SPARK-22395) for details.
+
+ - In PySpark, `na.fill()` or `fillna` also accepts boolean and replaces nulls with booleans. In prior Spark versions, PySpark just ignores it and returns the original Dataset/DataFrame.
+
+ - Since Spark 2.3, when either broadcast hash join or broadcast nested loop join is applicable, we prefer to broadcasting the table that is explicitly specified in a broadcast hint. For details, see the section [Broadcast Hint](sql-performance-tuning.html#broadcast-hint-for-sql-queries) and [SPARK-22489](https://issues.apache.org/jira/browse/SPARK-22489).
+
+ - Since Spark 2.3, when all inputs are binary, `functions.concat()` returns an output as binary. Otherwise, it returns as a string. Until Spark 2.3, it always returns as a string despite of input types. To keep the old behavior, set `spark.sql.function.concatBinaryAsString` to `true`.
+
+ - Since Spark 2.3, when all inputs are binary, SQL `elt()` returns an output as binary. Otherwise, it returns as a string. Until Spark 2.3, it always returns as a string despite of input types. To keep the old behavior, set `spark.sql.function.eltOutputAsString` to `true`.
+
+ - Since Spark 2.3, by default arithmetic operations between decimals return a rounded value if an exact representation is not possible (instead of returning NULL). This is compliant with SQL ANSI 2011 specification and Hive's new behavior introduced in Hive 2.2 (HIVE-15331). This involves the following changes
+
+ - The rules to determine the result type of an arithmetic operation have been updated. In particular, if the precision / scale needed are out of the range of available values, the scale is reduced up to 6, in order to prevent the truncation of the integer part of the decimals. All the arithmetic operations are affected by the change, ie. addition (`+`), subtraction (`-`), multiplication (`*`), division (`/`), remainder (`%`) and positive module (`pmod`).
+
+ - Literal values used in SQL operations are converted to DECIMAL with the exact precision and scale needed by them.
+
+ - The configuration `spark.sql.decimalOperations.allowPrecisionLoss` has been introduced. It defaults to `true`, which means the new behavior described here; if set to `false`, Spark uses previous rules, ie. it doesn't adjust the needed scale to represent the values and it returns NULL if an exact representation of the value is not possible.
+
+ - In PySpark, `df.replace` does not allow to omit `value` when `to_replace` is not a dictionary. Previously, `value` could be omitted in the other cases and had `None` by default, which is counterintuitive and error-prone.
+
+ - Un-aliased subquery's semantic has not been well defined with confusing behaviors. Since Spark 2.3, we invalidate such confusing cases, for example: `SELECT v.i from (SELECT i FROM v)`, Spark will throw an analysis exception in this case because users should not be able to use the qualifier inside a subquery. See [SPARK-20690](https://issues.apache.org/jira/browse/SPARK-20690) and [SPARK-21335](https://issues.apache.org/jira/browse/SPARK-21335) for more details.
+
+ - When creating a `SparkSession` with `SparkSession.builder.getOrCreate()`, if there is an existing `SparkContext`, the builder was trying to update the `SparkConf` of the existing `SparkContext` with configurations specified to the builder, but the `SparkContext` is shared by all `SparkSession`s, so we should not update them. Since 2.3, the builder comes to not update the configurations. If you want to update them, you need to update them prior to creating a `SparkSession`.
+
+## Upgrading From Spark SQL 2.1 to 2.2
+
+ - Spark 2.1.1 introduced a new configuration key: `spark.sql.hive.caseSensitiveInferenceMode`. It had a default setting of `NEVER_INFER`, which kept behavior identical to 2.1.0. However, Spark 2.2.0 changes this setting's default value to `INFER_AND_SAVE` to restore compatibility with reading Hive metastore tables whose underlying file schema have mixed-case column names. With the `INFER_AND_SAVE` configuration value, on first access Spark will perform schema inference on any Hive metastore table for which it has not already saved an inferred schema. Note that schema inference can be a very time-consuming operation for tables with thousands of partitions. If compatibility with mixed-case column names is not a concern, you can safely set `spark.sql.hive.caseSensitiveInferenceMode` to `NEVER_INFER` to avoid the initial overhead of schema inference. Note that with the new default `INFER_AND_SAVE` setting, the results of the schema inference are saved as a metastore key for future use. Therefore, the initial schema inference occurs only at a table's first access.
+
+ - Since Spark 2.2.1 and 2.3.0, the schema is always inferred at runtime when the data source tables have the columns that exist in both partition schema and data schema. The inferred schema does not have the partitioned columns. When reading the table, Spark respects the partition values of these overlapping columns instead of the values stored in the data source files. In 2.2.0 and 2.1.x release, the inferred schema is partitioned but the data of the table is invisible to users (i.e., the result set is empty).
+
+ - Since Spark 2.2, view definitions are stored in a different way from prior versions. This may cause Spark unable to read views created by prior versions. In such cases, you need to recreate the views using `ALTER VIEW AS` or `CREATE OR REPLACE VIEW AS` with newer Spark versions.
+
+## Upgrading From Spark SQL 2.0 to 2.1
+
+ - Datasource tables now store partition metadata in the Hive metastore. This means that Hive DDLs such as `ALTER TABLE PARTITION ... SET LOCATION` are now available for tables created with the Datasource API.
+
+ - Legacy datasource tables can be migrated to this format via the `MSCK REPAIR TABLE` command. Migrating legacy tables is recommended to take advantage of Hive DDL support and improved planning performance.
+
+ - To determine if a table has been migrated, look for the `PartitionProvider: Catalog` attribute when issuing `DESCRIBE FORMATTED` on the table.
+ - Changes to `INSERT OVERWRITE TABLE ... PARTITION ...` behavior for Datasource tables.
+
+ - In prior Spark versions `INSERT OVERWRITE` overwrote the entire Datasource table, even when given a partition specification. Now only partitions matching the specification are overwritten.
+
+ - Note that this still differs from the behavior of Hive tables, which is to overwrite only partitions overlapping with newly inserted data.
+
+## Upgrading From Spark SQL 1.6 to 2.0
+
+ - `SparkSession` is now the new entry point of Spark that replaces the old `SQLContext` and
+
+ `HiveContext`. Note that the old SQLContext and HiveContext are kept for backward compatibility. A new `catalog` interface is accessible from `SparkSession` - existing API on databases and tables access such as `listTables`, `createExternalTable`, `dropTempView`, `cacheTable` are moved here.
+
+ - Dataset API and DataFrame API are unified. In Scala, `DataFrame` becomes a type alias for
+ `Dataset[Row]`, while Java API users must replace `DataFrame` with `Dataset`. Both the typed
+ transformations (e.g., `map`, `filter`, and `groupByKey`) and untyped transformations (e.g.,
+ `select` and `groupBy`) are available on the Dataset class. Since compile-time type-safety in
+ Python and R is not a language feature, the concept of Dataset does not apply to these languages’
+ APIs. Instead, `DataFrame` remains the primary programming abstraction, which is analogous to the
+ single-node data frame notion in these languages.
+
+ - Dataset and DataFrame API `unionAll` has been deprecated and replaced by `union`
+
+ - Dataset and DataFrame API `explode` has been deprecated, alternatively, use `functions.explode()` with `select` or `flatMap`
+
+ - Dataset and DataFrame API `registerTempTable` has been deprecated and replaced by `createOrReplaceTempView`
+
+ - Changes to `CREATE TABLE ... LOCATION` behavior for Hive tables.
+
+ - From Spark 2.0, `CREATE TABLE ... LOCATION` is equivalent to `CREATE EXTERNAL TABLE ... LOCATION`
+ in order to prevent accidental dropping the existing data in the user-provided locations.
+ That means, a Hive table created in Spark SQL with the user-specified location is always a Hive external table.
+ Dropping external tables will not remove the data. Users are not allowed to specify the location for Hive managed tables.
+ Note that this is different from the Hive behavior.
+
+ - As a result, `DROP TABLE` statements on those tables will not remove the data.
+
+ - `spark.sql.parquet.cacheMetadata` is no longer used.
+ See [SPARK-13664](https://issues.apache.org/jira/browse/SPARK-13664) for details.
+
+## Upgrading From Spark SQL 1.5 to 1.6
+
+ - From Spark 1.6, by default, the Thrift server runs in multi-session mode. Which means each JDBC/ODBC
+ connection owns a copy of their own SQL configuration and temporary function registry. Cached
+ tables are still shared though. If you prefer to run the Thrift server in the old single-session
+ mode, please set option `spark.sql.hive.thriftServer.singleSession` to `true`. You may either add
+ this option to `spark-defaults.conf`, or pass it to `start-thriftserver.sh` via `--conf`:
+
+ {% highlight bash %}
+ ./sbin/start-thriftserver.sh \
+ --conf spark.sql.hive.thriftServer.singleSession=true \
+ ...
+ {% endhighlight %}
+
+ - Since 1.6.1, withColumn method in sparkR supports adding a new column to or replacing existing columns
+ of the same name of a DataFrame.
+
+ - From Spark 1.6, LongType casts to TimestampType expect seconds instead of microseconds. This
+ change was made to match the behavior of Hive 1.2 for more consistent type casting to TimestampType
+ from numeric types. See [SPARK-11724](https://issues.apache.org/jira/browse/SPARK-11724) for
+ details.
+
+## Upgrading From Spark SQL 1.4 to 1.5
+
+ - Optimized execution using manually managed memory (Tungsten) is now enabled by default, along with
+ code generation for expression evaluation. These features can both be disabled by setting
+ `spark.sql.tungsten.enabled` to `false`.
+
+ - Parquet schema merging is no longer enabled by default. It can be re-enabled by setting
+ `spark.sql.parquet.mergeSchema` to `true`.
+
+ - Resolution of strings to columns in python now supports using dots (`.`) to qualify the column or
+ access nested values. For example `df['table.column.nestedField']`. However, this means that if
+ your column name contains any dots you must now escape them using backticks (e.g., ``table.`column.with.dots`.nested``).
+
+ - In-memory columnar storage partition pruning is on by default. It can be disabled by setting
+ `spark.sql.inMemoryColumnarStorage.partitionPruning` to `false`.
+
+ - Unlimited precision decimal columns are no longer supported, instead Spark SQL enforces a maximum
+ precision of 38. When inferring schema from `BigDecimal` objects, a precision of (38, 18) is now
+ used. When no precision is specified in DDL then the default remains `Decimal(10, 0)`.
+
+ - Timestamps are now stored at a precision of 1us, rather than 1ns
+
+ - In the `sql` dialect, floating point numbers are now parsed as decimal. HiveQL parsing remains
+ unchanged.
+
+ - The canonical name of SQL/DataFrame functions are now lower case (e.g., sum vs SUM).
+
+ - JSON data source will not automatically load new files that are created by other applications
+ (i.e. files that are not inserted to the dataset through Spark SQL).
+ For a JSON persistent table (i.e. the metadata of the table is stored in Hive Metastore),
+ users can use `REFRESH TABLE` SQL command or `HiveContext`'s `refreshTable` method
+ to include those new files to the table. For a DataFrame representing a JSON dataset, users need to recreate
+ the DataFrame and the new DataFrame will include new files.
+
+ - DataFrame.withColumn method in pySpark supports adding a new column or replacing existing columns of the same name.
+
+## Upgrading from Spark SQL 1.3 to 1.4
+
+#### DataFrame data reader/writer interface
+
+Based on user feedback, we created a new, more fluid API for reading data in (`SQLContext.read`)
+and writing data out (`DataFrame.write`),
+and deprecated the old APIs (e.g., `SQLContext.parquetFile`, `SQLContext.jsonFile`).
+
+See the API docs for `SQLContext.read` (
+ Scala ,
+ Java ,
+ Python
+) and `DataFrame.write` (
+ Scala ,
+ Java ,
+ Python
+) more information.
+
+
+#### DataFrame.groupBy retains grouping columns
+
+Based on user feedback, we changed the default behavior of `DataFrame.groupBy().agg()` to retain the
+grouping columns in the resulting `DataFrame`. To keep the behavior in 1.3, set `spark.sql.retainGroupColumns` to `false`.
+
+
+
+{% highlight scala %}
+
+// In 1.3.x, in order for the grouping column "department" to show up,
+// it must be included explicitly as part of the agg function call.
+df.groupBy("department").agg($"department", max("age"), sum("expense"))
+
+// In 1.4+, grouping column "department" is included automatically.
+df.groupBy("department").agg(max("age"), sum("expense"))
+
+// Revert to 1.3 behavior (not retaining grouping column) by:
+sqlContext.setConf("spark.sql.retainGroupColumns", "false")
+
+{% endhighlight %}
+
+
+
+{% highlight java %}
+
+// In 1.3.x, in order for the grouping column "department" to show up,
+// it must be included explicitly as part of the agg function call.
+df.groupBy("department").agg(col("department"), max("age"), sum("expense"));
+
+// In 1.4+, grouping column "department" is included automatically.
+df.groupBy("department").agg(max("age"), sum("expense"));
+
+// Revert to 1.3 behavior (not retaining grouping column) by:
+sqlContext.setConf("spark.sql.retainGroupColumns", "false");
+
+{% endhighlight %}
+
+
+
+{% highlight python %}
+
+import pyspark.sql.functions as func
+
+# In 1.3.x, in order for the grouping column "department" to show up,
+# it must be included explicitly as part of the agg function call.
+df.groupBy("department").agg(df["department"], func.max("age"), func.sum("expense"))
+
+# In 1.4+, grouping column "department" is included automatically.
+df.groupBy("department").agg(func.max("age"), func.sum("expense"))
+
+# Revert to 1.3.x behavior (not retaining grouping column) by:
+sqlContext.setConf("spark.sql.retainGroupColumns", "false")
+
+{% endhighlight %}
+
+
+
+
+{% highlight scala %}
+
+sqlContext.udf.register("strLen", (s: String) => s.length())
+
+{% endhighlight %}
+
+
+
+{% highlight java %}
+
+sqlContext.udf().register("strLen", (String s) -> s.length(), DataTypes.IntegerType);
+
+{% endhighlight %}
+
+
+
+Property Name Default Meaning
+ spark.sql.inMemoryColumnarStorage.compressedtrue
+
+ When set to true Spark SQL will automatically select a compression codec for each column based
+ on statistics of the data.
+
+
+
+ spark.sql.inMemoryColumnarStorage.batchSize10000
+
+ Controls the size of batches for columnar caching. Larger batch sizes can improve memory utilization
+ and compression, but risk OOMs when caching data.
+
+
+
+
+
+## Other Configuration Options
+
+The following options can also be used to tune the performance of query execution. It is possible
+that these options will be deprecated in future release as more optimizations are performed automatically.
+
+
+ Property Name Default Meaning
+ spark.sql.files.maxPartitionBytes134217728 (128 MB)
+
+ The maximum number of bytes to pack into a single partition when reading files.
+
+
+
+ spark.sql.files.openCostInBytes4194304 (4 MB)
+
+ The estimated cost to open a file, measured by the number of bytes could be scanned in the same
+ time. This is used when putting multiple files into a partition. It is better to over-estimated,
+ then the partitions with small files will be faster than partitions with bigger files (which is
+ scheduled first).
+
+
+
+ spark.sql.broadcastTimeout300
+
+
+ Timeout in seconds for the broadcast wait time in broadcast joins
+
+
+
+
+ spark.sql.autoBroadcastJoinThreshold10485760 (10 MB)
+
+ Configures the maximum size in bytes for a table that will be broadcast to all worker nodes when
+ performing a join. By setting this value to -1 broadcasting can be disabled. Note that currently
+ statistics are only supported for Hive Metastore tables where the command
+ ANALYZE TABLE <tableName> COMPUTE STATISTICS noscan has been run.
+
+
+
+ spark.sql.shuffle.partitions200
+
+ Configures the number of partitions to use when shuffling data for joins or aggregations.
+
+
+
+
+## Broadcast Hint for SQL Queries
+
+The `BROADCAST` hint guides Spark to broadcast each specified table when joining them with another table or view.
+When Spark deciding the join methods, the broadcast hash join (i.e., BHJ) is preferred,
+even if the statistics is above the configuration `spark.sql.autoBroadcastJoinThreshold`.
+When both sides of a join are specified, Spark broadcasts the one having the lower statistics.
+Note Spark does not guarantee BHJ is always chosen, since not all cases (e.g. full outer join)
+support BHJ. When the broadcast nested loop join is selected, we still respect the hint.
+
+
+
+
+
+{% highlight scala %}
+import org.apache.spark.sql.functions.broadcast
+broadcast(spark.table("src")).join(spark.table("records"), "key").show()
+{% endhighlight %}
+
+
+
+
+
+{% highlight java %}
+import static org.apache.spark.sql.functions.broadcast;
+broadcast(spark.table("src")).join(spark.table("records"), "key").show();
+{% endhighlight %}
+
+
+
+
+
+{% highlight python %}
+from pyspark.sql.functions import broadcast
+broadcast(spark.table("src")).join(spark.table("records"), "key").show()
+{% endhighlight %}
+
+
+
+
+
+{% highlight r %}
+src <- sql("SELECT * FROM src")
+records <- sql("SELECT * FROM records")
+head(join(broadcast(src), records, src$key == records$key))
+{% endhighlight %}
+
+
+
+
+
+{% highlight sql %}
+-- We accept BROADCAST, BROADCASTJOIN and MAPJOIN for broadcast hint
+SELECT /*+ BROADCAST(r) */ * FROM records r JOIN src s ON r.key = s.key
+{% endhighlight %}
+
+
+
` to represe
[java-datasets]: api/java/index.html?org/apache/spark/sql/Dataset.html
Throughout this document, we will often refer to Scala/Java Datasets of `Row`s as DataFrames.
-
-# Getting Started
-
-## Starting Point: SparkSession
-
-
-
-
-The entry point into all functionality in Spark is the [`SparkSession`](api/scala/index.html#org.apache.spark.sql.SparkSession) class. To create a basic `SparkSession`, just use `SparkSession.builder()`:
-
-{% include_example init_session scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
-
-
-
-
-The entry point into all functionality in Spark is the [`SparkSession`](api/java/index.html#org.apache.spark.sql.SparkSession) class. To create a basic `SparkSession`, just use `SparkSession.builder()`:
-
-{% include_example init_session java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
-
-
-
-
-The entry point into all functionality in Spark is the [`SparkSession`](api/python/pyspark.sql.html#pyspark.sql.SparkSession) class. To create a basic `SparkSession`, just use `SparkSession.builder`:
-
-{% include_example init_session python/sql/basic.py %}
-
-
-
-
-The entry point into all functionality in Spark is the [`SparkSession`](api/R/sparkR.session.html) class. To initialize a basic `SparkSession`, just call `sparkR.session()`:
-
-{% include_example init_session r/RSparkSQLExample.R %}
-
-Note that when invoked for the first time, `sparkR.session()` initializes a global `SparkSession` singleton instance, and always returns a reference to this instance for successive invocations. In this way, users only need to initialize the `SparkSession` once, then SparkR functions like `read.df` will be able to access this global instance implicitly, and users don't need to pass the `SparkSession` instance around.
-
-
-
-With a `SparkSession`, applications can create DataFrames from an [existing `RDD`](#interoperating-with-rdds),
-from a Hive table, or from [Spark data sources](#data-sources).
-
-As an example, the following creates a DataFrame based on the content of a JSON file:
-
-{% include_example create_df scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
-
-
-
-With a `SparkSession`, applications can create DataFrames from an [existing `RDD`](#interoperating-with-rdds),
-from a Hive table, or from [Spark data sources](#data-sources).
-
-As an example, the following creates a DataFrame based on the content of a JSON file:
-
-{% include_example create_df java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
-
-
-
-With a `SparkSession`, applications can create DataFrames from an [existing `RDD`](#interoperating-with-rdds),
-from a Hive table, or from [Spark data sources](#data-sources).
-
-As an example, the following creates a DataFrame based on the content of a JSON file:
-
-{% include_example create_df python/sql/basic.py %}
-
-
-
-With a `SparkSession`, applications can create DataFrames from a local R data.frame,
-from a Hive table, or from [Spark data sources](#data-sources).
-
-As an example, the following creates a DataFrame based on the content of a JSON file:
-
-{% include_example create_df r/RSparkSQLExample.R %}
-
-
-
-
-{% include_example untyped_ops scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
-
-For a complete list of the types of operations that can be performed on a Dataset refer to the [API Documentation](api/scala/index.html#org.apache.spark.sql.Dataset).
-
-In addition to simple column references and expressions, Datasets also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/scala/index.html#org.apache.spark.sql.functions$).
-
-
-
-
-{% include_example untyped_ops java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
-
-For a complete list of the types of operations that can be performed on a Dataset refer to the [API Documentation](api/java/org/apache/spark/sql/Dataset.html).
-
-In addition to simple column references and expressions, Datasets also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/java/org/apache/spark/sql/functions.html).
-
-
-
-In Python, it's possible to access a DataFrame's columns either by attribute
-(`df.age`) or by indexing (`df['age']`). While the former is convenient for
-interactive data exploration, users are highly encouraged to use the
-latter form, which is future proof and won't break with column names that
-are also attributes on the DataFrame class.
-
-{% include_example untyped_ops python/sql/basic.py %}
-For a complete list of the types of operations that can be performed on a DataFrame refer to the [API Documentation](api/python/pyspark.sql.html#pyspark.sql.DataFrame).
-
-In addition to simple column references and expressions, DataFrames also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/python/pyspark.sql.html#module-pyspark.sql.functions).
-
-
-
-
-
-{% include_example untyped_ops r/RSparkSQLExample.R %}
-
-For a complete list of the types of operations that can be performed on a DataFrame refer to the [API Documentation](api/R/index.html).
-
-In addition to simple column references and expressions, DataFrames also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/R/SparkDataFrame.html).
-
-
-
-
-
-The `sql` function on a `SparkSession` enables applications to run SQL queries programmatically and returns the result as a `DataFrame`.
-
-{% include_example run_sql scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
-
-
-
-The `sql` function on a `SparkSession` enables applications to run SQL queries programmatically and returns the result as a `Dataset`.
-
-{% include_example run_sql java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
-
-
-
-The `sql` function on a `SparkSession` enables applications to run SQL queries programmatically and returns the result as a `DataFrame`.
-
-{% include_example run_sql python/sql/basic.py %}
-
-
-
-The `sql` function enables applications to run SQL queries programmatically and returns the result as a `SparkDataFrame`.
-
-{% include_example run_sql r/RSparkSQLExample.R %}
-
-
-
-
-{% include_example global_temp_view scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
-
-
-
-{% include_example global_temp_view java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
-
-
-
-{% include_example global_temp_view python/sql/basic.py %}
-
-
-
-
-{% highlight sql %}
-
-CREATE GLOBAL TEMPORARY VIEW temp_view AS SELECT a + 1, b * 2 FROM tbl
-
-SELECT * FROM global_temp.temp_view
-
-{% endhighlight %}
-
-
-
-
-{% include_example create_ds scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
-
-
-
-{% include_example create_ds java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
-
-
-
-
-
-The Scala interface for Spark SQL supports automatically converting an RDD containing case classes
-to a DataFrame. The case class
-defines the schema of the table. The names of the arguments to the case class are read using
-reflection and become the names of the columns. Case classes can also be nested or contain complex
-types such as `Seq`s or `Array`s. This RDD can be implicitly converted to a DataFrame and then be
-registered as a table. Tables can be used in subsequent SQL statements.
-
-{% include_example schema_inferring scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
-
-
-
-
-Spark SQL supports automatically converting an RDD of
-[JavaBeans](http://stackoverflow.com/questions/3295496/what-is-a-javabean-exactly) into a DataFrame.
-The `BeanInfo`, obtained using reflection, defines the schema of the table. Currently, Spark SQL
-does not support JavaBeans that contain `Map` field(s). Nested JavaBeans and `List` or `Array`
-fields are supported though. You can create a JavaBean by creating a class that implements
-Serializable and has getters and setters for all of its fields.
-
-{% include_example schema_inferring java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
-
-
-
-
-Spark SQL can convert an RDD of Row objects to a DataFrame, inferring the datatypes. Rows are constructed by passing a list of
-key/value pairs as kwargs to the Row class. The keys of this list define the column names of the table,
-and the types are inferred by sampling the whole dataset, similar to the inference that is performed on JSON files.
-
-{% include_example schema_inferring python/sql/basic.py %}
-
-
-
-
-
-
-When case classes cannot be defined ahead of time (for example,
-the structure of records is encoded in a string, or a text dataset will be parsed
-and fields will be projected differently for different users),
-a `DataFrame` can be created programmatically with three steps.
-
-1. Create an RDD of `Row`s from the original RDD;
-2. Create the schema represented by a `StructType` matching the structure of
-`Row`s in the RDD created in Step 1.
-3. Apply the schema to the RDD of `Row`s via `createDataFrame` method provided
-by `SparkSession`.
-
-For example:
-
-{% include_example programmatic_schema scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
-
-
-
-
-When JavaBean classes cannot be defined ahead of time (for example,
-the structure of records is encoded in a string, or a text dataset will be parsed and
-fields will be projected differently for different users),
-a `Dataset` can be created programmatically with three steps.
-
-1. Create an RDD of `Row`s from the original RDD;
-2. Create the schema represented by a `StructType` matching the structure of
-`Row`s in the RDD created in Step 1.
-3. Apply the schema to the RDD of `Row`s via `createDataFrame` method provided
-by `SparkSession`.
-
-For example:
-
-{% include_example programmatic_schema java/org/apache/spark/examples/sql/JavaSparkSQLExample.java %}
-
-
-
-
-When a dictionary of kwargs cannot be defined ahead of time (for example,
-the structure of records is encoded in a string, or a text dataset will be parsed and
-fields will be projected differently for different users),
-a `DataFrame` can be created programmatically with three steps.
-
-1. Create an RDD of tuples or lists from the original RDD;
-2. Create the schema represented by a `StructType` matching the structure of
-tuples or lists in the RDD created in the step 1.
-3. Apply the schema to the RDD via `createDataFrame` method provided by `SparkSession`.
-
-For example:
-
-{% include_example programmatic_schema python/sql/basic.py %}
-
-
-
-
-{% include_example untyped_custom_aggregation scala/org/apache/spark/examples/sql/UserDefinedUntypedAggregation.scala%}
-
-
-{% include_example untyped_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedUntypedAggregation.java%}
-
-
-
-{% include_example typed_custom_aggregation scala/org/apache/spark/examples/sql/UserDefinedTypedAggregation.scala%}
-
-
-{% include_example typed_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedTypedAggregation.java%}
-
-
-
-{% include_example generic_load_save_functions scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-
-
-
-{% include_example generic_load_save_functions java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-
-
-
-
-{% include_example generic_load_save_functions python/sql/datasource.py %}
-
-
-
-
-{% include_example generic_load_save_functions r/RSparkSQLExample.R %}
-
-
-
-
-{% include_example manual_load_options scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-
-
-
-{% include_example manual_load_options java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-
-
-
-{% include_example manual_load_options python/sql/datasource.py %}
-
-
-
-{% include_example manual_load_options r/RSparkSQLExample.R %}
-
-
-
-{% include_example manual_load_options_csv scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-
-
-
-{% include_example manual_load_options_csv java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-
-
-
-{% include_example manual_load_options_csv python/sql/datasource.py %}
-
-
-
-{% include_example manual_load_options_csv r/RSparkSQLExample.R %}
-
-
-
-
-{% include_example direct_sql scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-
-
-
-{% include_example direct_sql java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-
-
-
-{% include_example direct_sql python/sql/datasource.py %}
-
-
-
-{% include_example direct_sql r/RSparkSQLExample.R %}
-
-
-
-Scala/Java Any Language Meaning
- SaveMode.ErrorIfExists (default)"error" or "errorifexists" (default)
- When saving a DataFrame to a data source, if data already exists,
- an exception is expected to be thrown.
-
-
-
- SaveMode.Append"append"
- When saving a DataFrame to a data source, if data/table already exists,
- contents of the DataFrame are expected to be appended to existing data.
-
-
-
- SaveMode.Overwrite"overwrite"
- Overwrite mode means that when saving a DataFrame to a data source,
- if data/table already exists, existing data is expected to be overwritten by the contents of
- the DataFrame.
-
-
-
- SaveMode.Ignore"ignore"
- Ignore mode means that when saving a DataFrame to a data source, if data already exists,
- the save operation is expected to not save the contents of the DataFrame and to not
- change the existing data. This is similar to a CREATE TABLE IF NOT EXISTS in SQL.
-
-
-
-
-### Saving to Persistent Tables
-
-`DataFrames` can also be saved as persistent tables into Hive metastore using the `saveAsTable`
-command. Notice that an existing Hive deployment is not necessary to use this feature. Spark will create a
-default local Hive metastore (using Derby) for you. Unlike the `createOrReplaceTempView` command,
-`saveAsTable` will materialize the contents of the DataFrame and create a pointer to the data in the
-Hive metastore. Persistent tables will still exist even after your Spark program has restarted, as
-long as you maintain your connection to the same metastore. A DataFrame for a persistent table can
-be created by calling the `table` method on a `SparkSession` with the name of the table.
-
-For file-based data source, e.g. text, parquet, json, etc. you can specify a custom table path via the
-`path` option, e.g. `df.write.option("path", "/some/path").saveAsTable("t")`. When the table is dropped,
-the custom table path will not be removed and the table data is still there. If no custom table path is
-specified, Spark will write data to a default table path under the warehouse directory. When the table is
-dropped, the default table path will be removed too.
-
-Starting from Spark 2.1, persistent datasource tables have per-partition metadata stored in the Hive metastore. This brings several benefits:
-
-- Since the metastore can return only necessary partitions for a query, discovering all the partitions on the first query to the table is no longer needed.
-- Hive DDLs such as `ALTER TABLE PARTITION ... SET LOCATION` are now available for tables created with the Datasource API.
-
-Note that partition information is not gathered by default when creating external datasource tables (those with a `path` option). To sync the partition information in the metastore, you can invoke `MSCK REPAIR TABLE`.
-
-### Bucketing, Sorting and Partitioning
-
-For file-based data source, it is also possible to bucket and sort or partition the output.
-Bucketing and sorting are applicable only to persistent tables:
-
-
-
-
-{% include_example write_sorting_and_bucketing scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-
-
-
-{% include_example write_sorting_and_bucketing java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-
-
-
-{% include_example write_sorting_and_bucketing python/sql/datasource.py %}
-
-
-
-
-{% highlight sql %}
-
-CREATE TABLE users_bucketed_by_name(
- name STRING,
- favorite_color STRING,
- favorite_numbers array
-) USING parquet
-CLUSTERED BY(name) INTO 42 BUCKETS;
-
-{% endhighlight %}
-
-
-
-
-
-
-{% include_example write_partitioning scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-
-
-
-{% include_example write_partitioning java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-
-
-
-{% include_example write_partitioning python/sql/datasource.py %}
-
-
-
-
-{% highlight sql %}
-
-CREATE TABLE users_by_favorite_color(
- name STRING,
- favorite_color STRING,
- favorite_numbers array
-) USING csv PARTITIONED BY(favorite_color);
-
-{% endhighlight %}
-
-
-
-
-
-
-{% include_example write_partition_and_bucket scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-
-
-
-{% include_example write_partition_and_bucket java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-
-
-
-{% include_example write_partition_and_bucket python/sql/datasource.py %}
-
-
-
-
-{% highlight sql %}
-
-CREATE TABLE users_bucketed_and_partitioned(
- name STRING,
- favorite_color STRING,
- favorite_numbers array
-) USING parquet
-PARTITIONED BY (favorite_color)
-CLUSTERED BY(name) SORTED BY (favorite_numbers) INTO 42 BUCKETS;
-
-{% endhighlight %}
-
-
-
-
-
-
-{% include_example basic_parquet_example scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-
-
-
-{% include_example basic_parquet_example java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-
-
-
-
-{% include_example basic_parquet_example python/sql/datasource.py %}
-
-
-
-
-{% include_example basic_parquet_example r/RSparkSQLExample.R %}
-
-
-
-
-
-{% highlight sql %}
-
-CREATE TEMPORARY VIEW parquetTable
-USING org.apache.spark.sql.parquet
-OPTIONS (
- path "examples/src/main/resources/people.parquet"
-)
-
-SELECT * FROM parquetTable
-
-{% endhighlight %}
-
-
-
-
-
-
-{% include_example schema_merging scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-
-
-
-{% include_example schema_merging java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-
-
-
-
-{% include_example schema_merging python/sql/datasource.py %}
-
-
-
-
-{% include_example schema_merging r/RSparkSQLExample.R %}
-
-
-
-
-
-
-
-{% highlight scala %}
-// spark is an existing SparkSession
-spark.catalog.refreshTable("my_table")
-{% endhighlight %}
-
-
-
-
-
-{% highlight java %}
-// spark is an existing SparkSession
-spark.catalog().refreshTable("my_table");
-{% endhighlight %}
-
-
-
-
-
-{% highlight python %}
-# spark is an existing SparkSession
-spark.catalog.refreshTable("my_table")
-{% endhighlight %}
-
-
-
-
-
-{% highlight r %}
-refreshTable("my_table")
-{% endhighlight %}
-
-
-
-
-
-{% highlight sql %}
-REFRESH TABLE my_table;
-{% endhighlight %}
-
-
-
-
-Property Name Default Meaning
- spark.sql.parquet.binaryAsStringfalse
-
- Some other Parquet-producing systems, in particular Impala, Hive, and older versions of Spark SQL, do
- not differentiate between binary data and strings when writing out the Parquet schema. This
- flag tells Spark SQL to interpret binary data as a string to provide compatibility with these systems.
-
-
-
- spark.sql.parquet.int96AsTimestamptrue
-
- Some Parquet-producing systems, in particular Impala and Hive, store Timestamp into INT96. This
- flag tells Spark SQL to interpret INT96 data as a timestamp to provide compatibility with these systems.
-
-
-
- spark.sql.parquet.compression.codecsnappy
-
- Sets the compression codec used when writing Parquet files. If either `compression` or
- `parquet.compression` is specified in the table-specific options/properties, the precedence would be
- `compression`, `parquet.compression`, `spark.sql.parquet.compression.codec`. Acceptable values include:
- none, uncompressed, snappy, gzip, lzo, brotli, lz4, zstd.
-
-
-
- spark.sql.parquet.filterPushdowntrue
- Enables Parquet filter push-down optimization when set to true.
-
-
- spark.sql.hive.convertMetastoreParquettrue
-
- When set to false, Spark SQL will use the Hive SerDe for parquet tables instead of the built in
- support.
-
-
-
- spark.sql.parquet.mergeSchemafalse
-
-
- When true, the Parquet data source merges schemas collected from all data files, otherwise the
- schema is picked from the summary file or a random data file if no summary file is available.
-
-
-
-
- spark.sql.optimizer.metadataOnlytrue
-
-
- When true, enable the metadata-only query optimization that use the table's metadata to
- produce the partition columns instead of table scans. It applies when all the columns scanned
- are partition columns and the query has an aggregate operator that satisfies distinct
- semantics.
-
-
-
-
-
-## ORC Files
-
-Since Spark 2.3, Spark supports a vectorized ORC reader with a new ORC file format for ORC files.
-To do that, the following configurations are newly added. The vectorized reader is used for the
-native ORC tables (e.g., the ones created using the clause `USING ORC`) when `spark.sql.orc.impl`
-is set to `native` and `spark.sql.orc.enableVectorizedReader` is set to `true`. For the Hive ORC
-serde tables (e.g., the ones created using the clause `USING HIVE OPTIONS (fileFormat 'ORC')`),
-the vectorized reader is used when `spark.sql.hive.convertMetastoreOrc` is also set to `true`.
-
-
- Property Name Default Meaning
- spark.sql.orc.implnativeThe name of ORC implementation. It can be one of native and hive. native means the native ORC support that is built on Apache ORC 1.4. `hive` means the ORC library in Hive 1.2.1.
-
-
- spark.sql.orc.enableVectorizedReadertrueEnables vectorized orc decoding in native implementation. If false, a new non-vectorized ORC reader is used in native implementation. For hive implementation, this is ignored.
-
-
-
-## JSON Datasets
-
-
-
-Spark SQL can automatically infer the schema of a JSON dataset and load it as a `Dataset[Row]`.
-This conversion can be done using `SparkSession.read.json()` on either a `Dataset[String]`,
-or a JSON file.
-
-Note that the file that is offered as _a json file_ is not a typical JSON file. Each
-line must contain a separate, self-contained valid JSON object. For more information, please see
-[JSON Lines text format, also called newline-delimited JSON](http://jsonlines.org/).
-
-For a regular multi-line JSON file, set the `multiLine` option to `true`.
-
-{% include_example json_dataset scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-
-
-
-Spark SQL can automatically infer the schema of a JSON dataset and load it as a `Dataset`.
-This conversion can be done using `SparkSession.read().json()` on either a `Dataset`,
-or a JSON file.
-
-Note that the file that is offered as _a json file_ is not a typical JSON file. Each
-line must contain a separate, self-contained valid JSON object. For more information, please see
-[JSON Lines text format, also called newline-delimited JSON](http://jsonlines.org/).
-
-For a regular multi-line JSON file, set the `multiLine` option to `true`.
-
-{% include_example json_dataset java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-
-
-
-Spark SQL can automatically infer the schema of a JSON dataset and load it as a DataFrame.
-This conversion can be done using `SparkSession.read.json` on a JSON file.
-
-Note that the file that is offered as _a json file_ is not a typical JSON file. Each
-line must contain a separate, self-contained valid JSON object. For more information, please see
-[JSON Lines text format, also called newline-delimited JSON](http://jsonlines.org/).
-
-For a regular multi-line JSON file, set the `multiLine` parameter to `True`.
-
-{% include_example json_dataset python/sql/datasource.py %}
-
-
-
-Spark SQL can automatically infer the schema of a JSON dataset and load it as a DataFrame. using
-the `read.json()` function, which loads data from a directory of JSON files where each line of the
-files is a JSON object.
-
-Note that the file that is offered as _a json file_ is not a typical JSON file. Each
-line must contain a separate, self-contained valid JSON object. For more information, please see
-[JSON Lines text format, also called newline-delimited JSON](http://jsonlines.org/).
-
-For a regular multi-line JSON file, set a named parameter `multiLine` to `TRUE`.
-
-{% include_example json_dataset r/RSparkSQLExample.R %}
-
-
-
-
-
-{% highlight sql %}
-
-CREATE TEMPORARY VIEW jsonTable
-USING org.apache.spark.sql.json
-OPTIONS (
- path "examples/src/main/resources/people.json"
-)
-
-SELECT * FROM jsonTable
-
-{% endhighlight %}
-
-
-
-
-
-
-{% include_example spark_hive scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala %}
-
-
-
-{% include_example spark_hive java/org/apache/spark/examples/sql/hive/JavaSparkHiveExample.java %}
-
-
-
-{% include_example spark_hive python/sql/hive.py %}
-
-
-
-
-When working with Hive one must instantiate `SparkSession` with Hive support. This
-adds support for finding tables in the MetaStore and writing queries using HiveQL.
-
-{% include_example spark_hive r/RSparkSQLExample.R %}
-
-
-
- Property Name Meaning
- fileFormat
- A fileFormat is kind of a package of storage format specifications, including "serde", "input format" and
- "output format". Currently we support 6 fileFormats: 'sequencefile', 'rcfile', 'orc', 'parquet', 'textfile' and 'avro'.
-
-
-
-
- inputFormat, outputFormat
- These 2 options specify the name of a corresponding `InputFormat` and `OutputFormat` class as a string literal,
- e.g. `org.apache.hadoop.hive.ql.io.orc.OrcInputFormat`. These 2 options must be appeared in pair, and you can not
- specify them if you already specified the `fileFormat` option.
-
-
-
-
- serde
- This option specifies the name of a serde class. When the `fileFormat` option is specified, do not specify this option
- if the given `fileFormat` already include the information of serde. Currently "sequencefile", "textfile" and "rcfile"
- don't include the serde information and you can use this option with these 3 fileFormats.
-
-
-
-
- fieldDelim, escapeDelim, collectionDelim, mapkeyDelim, lineDelim
- These options can only be used with "textfile" fileFormat. They define how to read delimited files into rows.
-
-
-
-
-All other properties defined with `OPTIONS` will be regarded as Hive serde properties.
-
-### Interacting with Different Versions of Hive Metastore
-
-One of the most important pieces of Spark SQL's Hive support is interaction with Hive metastore,
-which enables Spark SQL to access metadata of Hive tables. Starting from Spark 1.4.0, a single binary
-build of Spark SQL can be used to query different versions of Hive metastores, using the configuration described below.
-Note that independent of the version of Hive that is being used to talk to the metastore, internally Spark SQL
-will compile against Hive 1.2.1 and use those classes for internal execution (serdes, UDFs, UDAFs, etc).
-
-The following options can be used to configure the version of Hive that is used to retrieve metadata:
-
-
- Property Name Default Meaning
- spark.sql.hive.metastore.version1.2.1
- Version of the Hive metastore. Available
- options are 0.12.0 through 2.3.3.
-
-
-
- spark.sql.hive.metastore.jarsbuiltin
- Location of the jars that should be used to instantiate the HiveMetastoreClient. This
- property can be one of three options:
-
- builtin-Phive is
- enabled. When this option is chosen, spark.sql.hive.metastore.version must be
- either 1.2.1 or not defined.
- mavenA classpath in the standard format for the JVM. This classpath must include all of Hive
- and its dependencies, including the correct version of Hadoop. These jars only need to be
- present on the driver, but if you are running in yarn cluster mode then you must ensure
- they are packaged with your application.
-
-
-
-
- spark.sql.hive.metastore.sharedPrefixescom.mysql.jdbc,
-
- A comma-separated list of class prefixes that should be loaded using the classloader that is
- shared between Spark SQL and a specific version of Hive. An example of classes that should
- be shared is JDBC drivers that are needed to talk to the metastore. Other classes that need
- to be shared are those that interact with classes that are already shared. For example,
- custom appenders that are used by log4j.
-
-
-
-
- spark.sql.hive.metastore.barrierPrefixes(empty)
-
- A comma separated list of class prefixes that should explicitly be reloaded for each version
- of Hive that Spark SQL is communicating with. For example, Hive UDFs that are declared in a
- prefix that typically would be shared (i.e. org.apache.spark.*).
-
-
-
-
-
-
-## JDBC To Other Databases
-
-Spark SQL also includes a data source that can read data from other databases using JDBC. This
-functionality should be preferred over using [JdbcRDD](api/scala/index.html#org.apache.spark.rdd.JdbcRDD).
-This is because the results are returned
-as a DataFrame and they can easily be processed in Spark SQL or joined with other data sources.
-The JDBC data source is also easier to use from Java or Python as it does not require the user to
-provide a ClassTag.
-(Note that this is different than the Spark SQL JDBC server, which allows other applications to
-run queries using Spark SQL).
-
-To get started you will need to include the JDBC driver for your particular database on the
-spark classpath. For example, to connect to postgres from the Spark Shell you would run the
-following command:
-
-{% highlight bash %}
-bin/spark-shell --driver-class-path postgresql-9.4.1207.jar --jars postgresql-9.4.1207.jar
-{% endhighlight %}
-
-Tables from the remote database can be loaded as a DataFrame or Spark SQL temporary view using
-the Data Sources API. Users can specify the JDBC connection properties in the data source options.
-user and password are normally provided as connection properties for
-logging into the data sources. In addition to the connection properties, Spark also supports
-the following case-insensitive options:
-
-
- Property Name Meaning
- url
- The JDBC URL to connect to. The source-specific connection properties may be specified in the URL. e.g., jdbc:postgresql://localhost/test?user=fred&password=secret
-
-
-
-
- dbtable
- The JDBC table that should be read from or written into. Note that when using it in the read
- path anything that is valid in a FROM clause of a SQL query can be used.
- For example, instead of a full table you could also use a subquery in parentheses. It is not
- allowed to specify `dbtable` and `query` options at the same time.
-
-
-
- query
- A query that will be used to read data into Spark. The specified query will be parenthesized and used
- as a subquery in the FROM clause. Spark will also assign an alias to the subquery clause.
- As an example, spark will issue a query of the following form to the JDBC Source. SELECT <columns> FROM (<user_specified_query>) spark_gen_alias
- It is not allowed to specify `dbtable` and `query` options at the same time.
- It is not allowed to spcify `query` and `partitionColumn` options at the same time. When specifying
- `partitionColumn` option is required, the subquery can be specified using `dbtable` option instead and
- partition columns can be qualified using the subquery alias provided as part of `dbtable`.
- spark.read.format("jdbc")
-
-
-
-
-
- driver
- The class name of the JDBC driver to use to connect to this URL.
-
-
-
-
- partitionColumn, lowerBound, upperBound
- These options must all be specified if any of them is specified. In addition,
- numPartitions must be specified. They describe how to partition the table when
- reading in parallel from multiple workers.
- partitionColumn must be a numeric, date, or timestamp column from the table in question.
- Notice that lowerBound and upperBound are just used to decide the
- partition stride, not for filtering the rows in table. So all rows in the table will be
- partitioned and returned. This option applies only to reading.
-
-
-
-
- numPartitions
- The maximum number of partitions that can be used for parallelism in table reading and
- writing. This also determines the maximum number of concurrent JDBC connections.
- If the number of partitions to write exceeds this limit, we decrease it to this limit by
- calling coalesce(numPartitions) before writing.
-
-
-
-
- queryTimeout
- The number of seconds the driver will wait for a Statement object to execute to the given
- number of seconds. Zero means there is no limit. In the write path, this option depends on
- how JDBC drivers implement the API setQueryTimeout, e.g., the h2 JDBC driver
- checks the timeout of each query instead of an entire JDBC batch.
- It defaults to 0.
-
-
-
-
- fetchsize
- The JDBC fetch size, which determines how many rows to fetch per round trip. This can help performance on JDBC drivers which default to low fetch size (eg. Oracle with 10 rows). This option applies only to reading.
-
-
-
-
- batchsize
- The JDBC batch size, which determines how many rows to insert per round trip. This can help performance on JDBC drivers. This option applies only to writing. It defaults to 1000.
-
-
-
-
- isolationLevel
- The transaction isolation level, which applies to current connection. It can be one of NONE, READ_COMMITTED, READ_UNCOMMITTED, REPEATABLE_READ, or SERIALIZABLE, corresponding to standard transaction isolation levels defined by JDBC's Connection object, with default of READ_UNCOMMITTED. This option applies only to writing. Please refer the documentation in java.sql.Connection.
-
-
-
-
- sessionInitStatement
- After each database session is opened to the remote DB and before starting to read data, this option executes a custom SQL statement (or a PL/SQL block). Use this to implement session initialization code. Example: option("sessionInitStatement", """BEGIN execute immediate 'alter session set "_serial_direct_read"=true'; END;""")
-
-
-
-
- truncate
- This is a JDBC writer related option. When SaveMode.Overwrite is enabled, this option causes Spark to truncate an existing table instead of dropping and recreating it. This can be more efficient, and prevents the table metadata (e.g., indices) from being removed. However, it will not work in some cases, such as when the new data has a different schema. It defaults to false. This option applies only to writing.
-
-
-
-
- cascadeTruncate
- This is a JDBC writer related option. If enabled and supported by the JDBC database (PostgreSQL and Oracle at the moment), this options allows execution of a TRUNCATE TABLE t CASCADE (in the case of PostgreSQL a TRUNCATE TABLE ONLY t CASCADE is executed to prevent inadvertently truncating descendant tables). This will affect other tables, and thus should be used with care. This option applies only to writing. It defaults to the default cascading truncate behaviour of the JDBC database in question, specified in the isCascadeTruncate in each JDBCDialect.
-
-
-
-
- createTableOptions
- This is a JDBC writer related option. If specified, this option allows setting of database-specific table and partition options when creating a table (e.g., CREATE TABLE t (name string) ENGINE=InnoDB.). This option applies only to writing.
-
-
-
-
- createTableColumnTypes
- The database column data types to use instead of the defaults, when creating the table. Data type information should be specified in the same format as CREATE TABLE columns syntax (e.g: "name CHAR(64), comments VARCHAR(1024)"). The specified types should be valid spark sql data types. This option applies only to writing.
-
-
-
-
- customSchema
- The custom schema to use for reading data from JDBC connectors. For example, "id DECIMAL(38, 0), name STRING". You can also specify partial fields, and the others use the default type mapping. For example, "id DECIMAL(38, 0)". The column names should be identical to the corresponding column names of JDBC table. Users can specify the corresponding data types of Spark SQL instead of using the defaults. This option applies only to reading.
-
-
-
-
- pushDownPredicate
- The option to enable or disable predicate push-down into the JDBC data source. The default value is true, in which case Spark will push down filters to the JDBC data source as much as possible. Otherwise, if set to false, no filter will be pushed down to the JDBC data source and thus all filters will be handled by Spark. Predicate push-down is usually turned off when the predicate filtering is performed faster by Spark than by the JDBC data source.
-
-
-
-
-
-
-
-{% include_example jdbc_dataset scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-
-
-
-{% include_example jdbc_dataset java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-
-
-
-{% include_example jdbc_dataset python/sql/datasource.py %}
-
-
-
-{% include_example jdbc_dataset r/RSparkSQLExample.R %}
-
-
-
-
-{% highlight sql %}
-
-CREATE TEMPORARY VIEW jdbcTable
-USING org.apache.spark.sql.jdbc
-OPTIONS (
- url "jdbc:postgresql:dbserver",
- dbtable "schema.tablename",
- user 'username',
- password 'password'
-)
-
-INSERT INTO TABLE jdbcTable
-SELECT * FROM resultTable
-{% endhighlight %}
-
-
-
-Property Name Default Meaning
- spark.sql.inMemoryColumnarStorage.compressedtrue
-
- When set to true Spark SQL will automatically select a compression codec for each column based
- on statistics of the data.
-
-
-
- spark.sql.inMemoryColumnarStorage.batchSize10000
-
- Controls the size of batches for columnar caching. Larger batch sizes can improve memory utilization
- and compression, but risk OOMs when caching data.
-
-
-
-
-
-## Other Configuration Options
-
-The following options can also be used to tune the performance of query execution. It is possible
-that these options will be deprecated in future release as more optimizations are performed automatically.
-
-
- Property Name Default Meaning
- spark.sql.files.maxPartitionBytes134217728 (128 MB)
-
- The maximum number of bytes to pack into a single partition when reading files.
-
-
-
- spark.sql.files.openCostInBytes4194304 (4 MB)
-
- The estimated cost to open a file, measured by the number of bytes could be scanned in the same
- time. This is used when putting multiple files into a partition. It is better to over estimated,
- then the partitions with small files will be faster than partitions with bigger files (which is
- scheduled first).
-
-
-
- spark.sql.broadcastTimeout300
-
-
- Timeout in seconds for the broadcast wait time in broadcast joins
-
-
-
-
- spark.sql.autoBroadcastJoinThreshold10485760 (10 MB)
-
- Configures the maximum size in bytes for a table that will be broadcast to all worker nodes when
- performing a join. By setting this value to -1 broadcasting can be disabled. Note that currently
- statistics are only supported for Hive Metastore tables where the command
- ANALYZE TABLE <tableName> COMPUTE STATISTICS noscan has been run.
-
-
-
- spark.sql.shuffle.partitions200
-
- Configures the number of partitions to use when shuffling data for joins or aggregations.
-
-
-
-
-## Broadcast Hint for SQL Queries
-
-The `BROADCAST` hint guides Spark to broadcast each specified table when joining them with another table or view.
-When Spark deciding the join methods, the broadcast hash join (i.e., BHJ) is preferred,
-even if the statistics is above the configuration `spark.sql.autoBroadcastJoinThreshold`.
-When both sides of a join are specified, Spark broadcasts the one having the lower statistics.
-Note Spark does not guarantee BHJ is always chosen, since not all cases (e.g. full outer join)
-support BHJ. When the broadcast nested loop join is selected, we still respect the hint.
-
-
-
-
-
-{% highlight scala %}
-import org.apache.spark.sql.functions.broadcast
-broadcast(spark.table("src")).join(spark.table("records"), "key").show()
-{% endhighlight %}
-
-
-
-
-
-{% highlight java %}
-import static org.apache.spark.sql.functions.broadcast;
-broadcast(spark.table("src")).join(spark.table("records"), "key").show();
-{% endhighlight %}
-
-
-
-
-
-{% highlight python %}
-from pyspark.sql.functions import broadcast
-broadcast(spark.table("src")).join(spark.table("records"), "key").show()
-{% endhighlight %}
-
-
-
-
-
-{% highlight r %}
-src <- sql("SELECT * FROM src")
-records <- sql("SELECT * FROM records")
-head(join(broadcast(src), records, src$key == records$key))
-{% endhighlight %}
-
-
-
-
-
-{% highlight sql %}
--- We accept BROADCAST, BROADCASTJOIN and MAPJOIN for broadcast hint
-SELECT /*+ BROADCAST(r) */ * FROM records r JOIN src s ON r.key = s.key
-{% endhighlight %}
-
-
-
-export HIVE_SERVER2_THRIFT_BIND_HOST=
-./sbin/start-thriftserver.sh \
- --master \
- ...
-{% endhighlight %}
-
-or system properties:
-
-{% highlight bash %}
-./sbin/start-thriftserver.sh \
- --hiveconf hive.server2.thrift.port= \
- --hiveconf hive.server2.thrift.bind.host= \
- --master
- ...
-{% endhighlight %}
-
-Now you can use beeline to test the Thrift JDBC/ODBC server:
-
- ./bin/beeline
-
-Connect to the JDBC/ODBC server in beeline with:
-
- beeline> !connect jdbc:hive2://localhost:10000
-
-Beeline will ask you for a username and password. In non-secure mode, simply enter the username on
-your machine and a blank password. For secure mode, please follow the instructions given in the
-[beeline documentation](https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Clients).
-
-Configuration of Hive is done by placing your `hive-site.xml`, `core-site.xml` and `hdfs-site.xml` files in `conf/`.
-
-You may also use the beeline script that comes with Hive.
-
-Thrift JDBC server also supports sending thrift RPC messages over HTTP transport.
-Use the following setting to enable HTTP mode as system property or in `hive-site.xml` file in `conf/`:
-
- hive.server2.transport.mode - Set this to value: http
- hive.server2.thrift.http.port - HTTP port number to listen on; default is 10001
- hive.server2.http.endpoint - HTTP endpoint; default is cliservice
-
-To test, use beeline to connect to the JDBC/ODBC server in http mode with:
-
- beeline> !connect jdbc:hive2://:/?hive.server2.transport.mode=http;hive.server2.thrift.http.path=
-
-
-## Running the Spark SQL CLI
-
-The Spark SQL CLI is a convenient tool to run the Hive metastore service in local mode and execute
-queries input from the command line. Note that the Spark SQL CLI cannot talk to the Thrift JDBC server.
-
-To start the Spark SQL CLI, run the following in the Spark directory:
-
- ./bin/spark-sql
-
-Configuration of Hive is done by placing your `hive-site.xml`, `core-site.xml` and `hdfs-site.xml` files in `conf/`.
-You may run `./bin/spark-sql --help` for a complete list of all available
-options.
-
-# PySpark Usage Guide for Pandas with Apache Arrow
-
-## Apache Arrow in Spark
-
-Apache Arrow is an in-memory columnar data format that is used in Spark to efficiently transfer
-data between JVM and Python processes. This currently is most beneficial to Python users that
-work with Pandas/NumPy data. Its usage is not automatic and might require some minor
-changes to configuration or code to take full advantage and ensure compatibility. This guide will
-give a high-level description of how to use Arrow in Spark and highlight any differences when
-working with Arrow-enabled data.
-
-### Ensure PyArrow Installed
-
-If you install PySpark using pip, then PyArrow can be brought in as an extra dependency of the
-SQL module with the command `pip install pyspark[sql]`. Otherwise, you must ensure that PyArrow
-is installed and available on all cluster nodes. The current supported version is 0.8.0.
-You can install using pip or conda from the conda-forge channel. See PyArrow
-[installation](https://arrow.apache.org/docs/python/install.html) for details.
-
-## Enabling for Conversion to/from Pandas
-
-Arrow is available as an optimization when converting a Spark DataFrame to a Pandas DataFrame
-using the call `toPandas()` and when creating a Spark DataFrame from a Pandas DataFrame with
-`createDataFrame(pandas_df)`. To use Arrow when executing these calls, users need to first set
-the Spark configuration 'spark.sql.execution.arrow.enabled' to 'true'. This is disabled by default.
-
-In addition, optimizations enabled by 'spark.sql.execution.arrow.enabled' could fallback automatically
-to non-Arrow optimization implementation if an error occurs before the actual computation within Spark.
-This can be controlled by 'spark.sql.execution.arrow.fallback.enabled'.
-
-
-
-{% include_example dataframe_with_arrow python/sql/arrow.py %}
-
-
-
-{% include_example scalar_pandas_udf python/sql/arrow.py %}
-
-
-
-{% include_example grouped_map_pandas_udf python/sql/arrow.py %}
-
-
-
-{% include_example grouped_agg_pandas_udf python/sql/arrow.py %}
-
-
-
-
- InputA \ InputB
-
-
- NullType
-
-
- IntegerType
-
-
- LongType
-
-
- DecimalType(38,0)*
-
-
- DoubleType
-
-
- DateType
-
-
- TimestampType
-
-
- StringType
-
-
-
-
- NullType
-
- NullType
- IntegerType
- LongType
- DecimalType(38,0)
- DoubleType
- DateType
- TimestampType
- StringType
-
-
-
- IntegerType
-
- IntegerType
- IntegerType
- LongType
- DecimalType(38,0)
- DoubleType
- StringType
- StringType
- StringType
-
-
-
- LongType
-
- LongType
- LongType
- LongType
- DecimalType(38,0)
- StringType
- StringType
- StringType
- StringType
-
-
-
- DecimalType(38,0)*
-
- DecimalType(38,0)
- DecimalType(38,0)
- DecimalType(38,0)
- DecimalType(38,0)
- StringType
- StringType
- StringType
- StringType
-
-
-
- DoubleType
-
- DoubleType
- DoubleType
- StringType
- StringType
- DoubleType
- StringType
- StringType
- StringType
-
-
-
- DateType
-
- DateType
- StringType
- StringType
- StringType
- StringType
- DateType
- TimestampType
- StringType
-
-
-
- TimestampType
-
- TimestampType
- StringType
- StringType
- StringType
- StringType
- TimestampType
- TimestampType
- StringType
-
-
-
- StringType
-
- StringType
- StringType
- StringType
- StringType
- StringType
- StringType
- StringType
- StringType
-
-
-
- Note that, for DecimalType(38,0)* , the table above intentionally does not cover all other combinations of scales and precisions because currently we only infer decimal type like `BigInteger`/`BigInt`. For example, 1.1 is inferred as double type.
- - In PySpark, now we need Pandas 0.19.2 or upper if you want to use Pandas related functionalities, such as `toPandas`, `createDataFrame` from Pandas DataFrame, etc.
- - In PySpark, the behavior of timestamp values for Pandas related functionalities was changed to respect session timezone. If you want to use the old behavior, you need to set a configuration `spark.sql.execution.pandas.respectSessionTimeZone` to `False`. See [SPARK-22395](https://issues.apache.org/jira/browse/SPARK-22395) for details.
- - In PySpark, `na.fill()` or `fillna` also accepts boolean and replaces nulls with booleans. In prior Spark versions, PySpark just ignores it and returns the original Dataset/DataFrame.
- - Since Spark 2.3, when either broadcast hash join or broadcast nested loop join is applicable, we prefer to broadcasting the table that is explicitly specified in a broadcast hint. For details, see the section [Broadcast Hint](#broadcast-hint-for-sql-queries) and [SPARK-22489](https://issues.apache.org/jira/browse/SPARK-22489).
- - Since Spark 2.3, when all inputs are binary, `functions.concat()` returns an output as binary. Otherwise, it returns as a string. Until Spark 2.3, it always returns as a string despite of input types. To keep the old behavior, set `spark.sql.function.concatBinaryAsString` to `true`.
- - Since Spark 2.3, when all inputs are binary, SQL `elt()` returns an output as binary. Otherwise, it returns as a string. Until Spark 2.3, it always returns as a string despite of input types. To keep the old behavior, set `spark.sql.function.eltOutputAsString` to `true`.
-
- - Since Spark 2.3, by default arithmetic operations between decimals return a rounded value if an exact representation is not possible (instead of returning NULL). This is compliant with SQL ANSI 2011 specification and Hive's new behavior introduced in Hive 2.2 (HIVE-15331). This involves the following changes
- - The rules to determine the result type of an arithmetic operation have been updated. In particular, if the precision / scale needed are out of the range of available values, the scale is reduced up to 6, in order to prevent the truncation of the integer part of the decimals. All the arithmetic operations are affected by the change, ie. addition (`+`), subtraction (`-`), multiplication (`*`), division (`/`), remainder (`%`) and positive module (`pmod`).
- - Literal values used in SQL operations are converted to DECIMAL with the exact precision and scale needed by them.
- - The configuration `spark.sql.decimalOperations.allowPrecisionLoss` has been introduced. It defaults to `true`, which means the new behavior described here; if set to `false`, Spark uses previous rules, ie. it doesn't adjust the needed scale to represent the values and it returns NULL if an exact representation of the value is not possible.
- - In PySpark, `df.replace` does not allow to omit `value` when `to_replace` is not a dictionary. Previously, `value` could be omitted in the other cases and had `None` by default, which is counterintuitive and error-prone.
- - Un-aliased subquery's semantic has not been well defined with confusing behaviors. Since Spark 2.3, we invalidate such confusing cases, for example: `SELECT v.i from (SELECT i FROM v)`, Spark will throw an analysis exception in this case because users should not be able to use the qualifier inside a subquery. See [SPARK-20690](https://issues.apache.org/jira/browse/SPARK-20690) and [SPARK-21335](https://issues.apache.org/jira/browse/SPARK-21335) for more details.
-
-## Upgrading From Spark SQL 2.1 to 2.2
-
- - Spark 2.1.1 introduced a new configuration key: `spark.sql.hive.caseSensitiveInferenceMode`. It had a default setting of `NEVER_INFER`, which kept behavior identical to 2.1.0. However, Spark 2.2.0 changes this setting's default value to `INFER_AND_SAVE` to restore compatibility with reading Hive metastore tables whose underlying file schema have mixed-case column names. With the `INFER_AND_SAVE` configuration value, on first access Spark will perform schema inference on any Hive metastore table for which it has not already saved an inferred schema. Note that schema inference can be a very time-consuming operation for tables with thousands of partitions. If compatibility with mixed-case column names is not a concern, you can safely set `spark.sql.hive.caseSensitiveInferenceMode` to `NEVER_INFER` to avoid the initial overhead of schema inference. Note that with the new default `INFER_AND_SAVE` setting, the results of the schema inference are saved as a metastore key for future use. Therefore, the initial schema inference occurs only at a table's first access.
-
- - Since Spark 2.2.1 and 2.3.0, the schema is always inferred at runtime when the data source tables have the columns that exist in both partition schema and data schema. The inferred schema does not have the partitioned columns. When reading the table, Spark respects the partition values of these overlapping columns instead of the values stored in the data source files. In 2.2.0 and 2.1.x release, the inferred schema is partitioned but the data of the table is invisible to users (i.e., the result set is empty).
-
-## Upgrading From Spark SQL 2.0 to 2.1
-
- - Datasource tables now store partition metadata in the Hive metastore. This means that Hive DDLs such as `ALTER TABLE PARTITION ... SET LOCATION` are now available for tables created with the Datasource API.
- - Legacy datasource tables can be migrated to this format via the `MSCK REPAIR TABLE` command. Migrating legacy tables is recommended to take advantage of Hive DDL support and improved planning performance.
- - To determine if a table has been migrated, look for the `PartitionProvider: Catalog` attribute when issuing `DESCRIBE FORMATTED` on the table.
- - Changes to `INSERT OVERWRITE TABLE ... PARTITION ...` behavior for Datasource tables.
- - In prior Spark versions `INSERT OVERWRITE` overwrote the entire Datasource table, even when given a partition specification. Now only partitions matching the specification are overwritten.
- - Note that this still differs from the behavior of Hive tables, which is to overwrite only partitions overlapping with newly inserted data.
-
-## Upgrading From Spark SQL 1.6 to 2.0
-
- - `SparkSession` is now the new entry point of Spark that replaces the old `SQLContext` and
- `HiveContext`. Note that the old SQLContext and HiveContext are kept for backward compatibility. A new `catalog` interface is accessible from `SparkSession` - existing API on databases and tables access such as `listTables`, `createExternalTable`, `dropTempView`, `cacheTable` are moved here.
-
- - Dataset API and DataFrame API are unified. In Scala, `DataFrame` becomes a type alias for
- `Dataset[Row]`, while Java API users must replace `DataFrame` with `Dataset`. Both the typed
- transformations (e.g., `map`, `filter`, and `groupByKey`) and untyped transformations (e.g.,
- `select` and `groupBy`) are available on the Dataset class. Since compile-time type-safety in
- Python and R is not a language feature, the concept of Dataset does not apply to these languages’
- APIs. Instead, `DataFrame` remains the primary programming abstraction, which is analogous to the
- single-node data frame notion in these languages.
-
- - Dataset and DataFrame API `unionAll` has been deprecated and replaced by `union`
- - Dataset and DataFrame API `explode` has been deprecated, alternatively, use `functions.explode()` with `select` or `flatMap`
- - Dataset and DataFrame API `registerTempTable` has been deprecated and replaced by `createOrReplaceTempView`
-
- - Changes to `CREATE TABLE ... LOCATION` behavior for Hive tables.
- - From Spark 2.0, `CREATE TABLE ... LOCATION` is equivalent to `CREATE EXTERNAL TABLE ... LOCATION`
- in order to prevent accidental dropping the existing data in the user-provided locations.
- That means, a Hive table created in Spark SQL with the user-specified location is always a Hive external table.
- Dropping external tables will not remove the data. Users are not allowed to specify the location for Hive managed tables.
- Note that this is different from the Hive behavior.
- - As a result, `DROP TABLE` statements on those tables will not remove the data.
-
- - `spark.sql.parquet.cacheMetadata` is no longer used.
- See [SPARK-13664](https://issues.apache.org/jira/browse/SPARK-13664) for details.
-
-## Upgrading From Spark SQL 1.5 to 1.6
-
- - From Spark 1.6, by default, the Thrift server runs in multi-session mode. Which means each JDBC/ODBC
- connection owns a copy of their own SQL configuration and temporary function registry. Cached
- tables are still shared though. If you prefer to run the Thrift server in the old single-session
- mode, please set option `spark.sql.hive.thriftServer.singleSession` to `true`. You may either add
- this option to `spark-defaults.conf`, or pass it to `start-thriftserver.sh` via `--conf`:
-
- {% highlight bash %}
- ./sbin/start-thriftserver.sh \
- --conf spark.sql.hive.thriftServer.singleSession=true \
- ...
- {% endhighlight %}
- - Since 1.6.1, withColumn method in sparkR supports adding a new column to or replacing existing columns
- of the same name of a DataFrame.
-
- - From Spark 1.6, LongType casts to TimestampType expect seconds instead of microseconds. This
- change was made to match the behavior of Hive 1.2 for more consistent type casting to TimestampType
- from numeric types. See [SPARK-11724](https://issues.apache.org/jira/browse/SPARK-11724) for
- details.
-
-## Upgrading From Spark SQL 1.4 to 1.5
-
- - Optimized execution using manually managed memory (Tungsten) is now enabled by default, along with
- code generation for expression evaluation. These features can both be disabled by setting
- `spark.sql.tungsten.enabled` to `false`.
- - Parquet schema merging is no longer enabled by default. It can be re-enabled by setting
- `spark.sql.parquet.mergeSchema` to `true`.
- - Resolution of strings to columns in python now supports using dots (`.`) to qualify the column or
- access nested values. For example `df['table.column.nestedField']`. However, this means that if
- your column name contains any dots you must now escape them using backticks (e.g., ``table.`column.with.dots`.nested``).
- - In-memory columnar storage partition pruning is on by default. It can be disabled by setting
- `spark.sql.inMemoryColumnarStorage.partitionPruning` to `false`.
- - Unlimited precision decimal columns are no longer supported, instead Spark SQL enforces a maximum
- precision of 38. When inferring schema from `BigDecimal` objects, a precision of (38, 18) is now
- used. When no precision is specified in DDL then the default remains `Decimal(10, 0)`.
- - Timestamps are now stored at a precision of 1us, rather than 1ns
- - In the `sql` dialect, floating point numbers are now parsed as decimal. HiveQL parsing remains
- unchanged.
- - The canonical name of SQL/DataFrame functions are now lower case (e.g., sum vs SUM).
- - JSON data source will not automatically load new files that are created by other applications
- (i.e. files that are not inserted to the dataset through Spark SQL).
- For a JSON persistent table (i.e. the metadata of the table is stored in Hive Metastore),
- users can use `REFRESH TABLE` SQL command or `HiveContext`'s `refreshTable` method
- to include those new files to the table. For a DataFrame representing a JSON dataset, users need to recreate
- the DataFrame and the new DataFrame will include new files.
- - DataFrame.withColumn method in pySpark supports adding a new column or replacing existing columns of the same name.
-
-## Upgrading from Spark SQL 1.3 to 1.4
-
-#### DataFrame data reader/writer interface
-
-Based on user feedback, we created a new, more fluid API for reading data in (`SQLContext.read`)
-and writing data out (`DataFrame.write`),
-and deprecated the old APIs (e.g., `SQLContext.parquetFile`, `SQLContext.jsonFile`).
-
-See the API docs for `SQLContext.read` (
- Scala ,
- Java ,
- Python
-) and `DataFrame.write` (
- Scala ,
- Java ,
- Python
-) more information.
-
-
-#### DataFrame.groupBy retains grouping columns
-
-Based on user feedback, we changed the default behavior of `DataFrame.groupBy().agg()` to retain the
-grouping columns in the resulting `DataFrame`. To keep the behavior in 1.3, set `spark.sql.retainGroupColumns` to `false`.
-
-
-
-{% highlight scala %}
-
-// In 1.3.x, in order for the grouping column "department" to show up,
-// it must be included explicitly as part of the agg function call.
-df.groupBy("department").agg($"department", max("age"), sum("expense"))
-
-// In 1.4+, grouping column "department" is included automatically.
-df.groupBy("department").agg(max("age"), sum("expense"))
-
-// Revert to 1.3 behavior (not retaining grouping column) by:
-sqlContext.setConf("spark.sql.retainGroupColumns", "false")
-
-{% endhighlight %}
-
-
-
-{% highlight java %}
-
-// In 1.3.x, in order for the grouping column "department" to show up,
-// it must be included explicitly as part of the agg function call.
-df.groupBy("department").agg(col("department"), max("age"), sum("expense"));
-
-// In 1.4+, grouping column "department" is included automatically.
-df.groupBy("department").agg(max("age"), sum("expense"));
-
-// Revert to 1.3 behavior (not retaining grouping column) by:
-sqlContext.setConf("spark.sql.retainGroupColumns", "false");
-
-{% endhighlight %}
-
-
-
-{% highlight python %}
-
-import pyspark.sql.functions as func
-
-# In 1.3.x, in order for the grouping column "department" to show up,
-# it must be included explicitly as part of the agg function call.
-df.groupBy("department").agg(df["department"], func.max("age"), func.sum("expense"))
-
-# In 1.4+, grouping column "department" is included automatically.
-df.groupBy("department").agg(func.max("age"), func.sum("expense"))
-
-# Revert to 1.3.x behavior (not retaining grouping column) by:
-sqlContext.setConf("spark.sql.retainGroupColumns", "false")
-
-{% endhighlight %}
-
-
-
-
-{% highlight scala %}
-
-sqlContext.udf.register("strLen", (s: String) => s.length())
-
-{% endhighlight %}
-
-
-
-{% highlight java %}
-
-sqlContext.udf().register("strLen", (String s) -> s.length(), DataTypes.IntegerType);
-
-{% endhighlight %}
-
-
-
-
-
-All data types of Spark SQL are located in the package `org.apache.spark.sql.types`.
-You can access them by doing
-
-{% include_example data_types scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
-
-
-
- Data type
- Value type in Scala
- API to access or create a data type
-
- ByteType
- Byte
-
- ByteType
-
-
-
- ShortType
- Short
-
- ShortType
-
-
-
- IntegerType
- Int
-
- IntegerType
-
-
-
- LongType
- Long
-
- LongType
-
-
-
- FloatType
- Float
-
- FloatType
-
-
-
- DoubleType
- Double
-
- DoubleType
-
-
-
- DecimalType
- java.math.BigDecimal
-
- DecimalType
-
-
-
- StringType
- String
-
- StringType
-
-
-
- BinaryType
- Array[Byte]
-
- BinaryType
-
-
-
- BooleanType
- Boolean
-
- BooleanType
-
-
-
- TimestampType
- java.sql.Timestamp
-
- TimestampType
-
-
-
- DateType
- java.sql.Date
-
- DateType
-
-
-
- ArrayType
- scala.collection.Seq
-
- ArrayType(elementType , [containsNull ])Note: The default value of containsNull is true .
-
-
-
- MapType
- scala.collection.Map
-
- MapType(keyType , valueType , [valueContainsNull ])Note: The default value of valueContainsNull is true .
-
-
-
- StructType
- org.apache.spark.sql.Row
-
- StructType(fields )Note: fields is a Seq of StructFields. Also, two fields with the same
- name are not allowed.
-
-
-
- StructField
- The value type in Scala of the data type of this field
- (For example, Int for a StructField with the data type IntegerType)
-
- StructField(name , dataType , [nullable ])Note: The default value of nullable is true .
-
-
-
-
-
-
-
-
-All data types of Spark SQL are located in the package of
-`org.apache.spark.sql.types`. To access or create a data type,
-please use factory methods provided in
-`org.apache.spark.sql.types.DataTypes`.
-
-
-
- Data type
- Value type in Java
- API to access or create a data type
-
- ByteType
- byte or Byte
-
- DataTypes.ByteType
-
-
-
- ShortType
- short or Short
-
- DataTypes.ShortType
-
-
-
- IntegerType
- int or Integer
-
- DataTypes.IntegerType
-
-
-
- LongType
- long or Long
-
- DataTypes.LongType
-
-
-
- FloatType
- float or Float
-
- DataTypes.FloatType
-
-
-
- DoubleType
- double or Double
-
- DataTypes.DoubleType
-
-
-
- DecimalType
- java.math.BigDecimal
-
- DataTypes.createDecimalType()precision , scale ).
-
-
-
- StringType
- String
-
- DataTypes.StringType
-
-
-
- BinaryType
- byte[]
-
- DataTypes.BinaryType
-
-
-
- BooleanType
- boolean or Boolean
-
- DataTypes.BooleanType
-
-
-
- TimestampType
- java.sql.Timestamp
-
- DataTypes.TimestampType
-
-
-
- DateType
- java.sql.Date
-
- DataTypes.DateType
-
-
-
- ArrayType
- java.util.List
-
- DataTypes.createArrayType(elementType )Note: The value of containsNull will be true elementType , containsNull ).
-
-
-
- MapType
- java.util.Map
-
- DataTypes.createMapType(keyType , valueType )Note: The value of valueContainsNull will be true .keyType , valueType , valueContainsNull )
-
-
- StructType
- org.apache.spark.sql.Row
-
- DataTypes.createStructType(fields )Note: fields is a List or an array of StructFields.
- Also, two fields with the same name are not allowed.
-
-
-
- StructField
- The value type in Java of the data type of this field
- (For example, int for a StructField with the data type IntegerType)
-
- DataTypes.createStructField(name , dataType , nullable )
-
-
-
-
-
-
-
-
-All data types of Spark SQL are located in the package of `pyspark.sql.types`.
-You can access them by doing
-{% highlight python %}
-from pyspark.sql.types import *
-{% endhighlight %}
-
-
-
- Data type
- Value type in Python
- API to access or create a data type
-
- ByteType
-
- int or long Note: Numbers will be converted to 1-byte signed integer numbers at runtime.
- Please make sure that numbers are within the range of -128 to 127.
-
-
- ByteType()
-
-
-
- ShortType
-
- int or long Note: Numbers will be converted to 2-byte signed integer numbers at runtime.
- Please make sure that numbers are within the range of -32768 to 32767.
-
-
- ShortType()
-
-
-
- IntegerType
- int or long
-
- IntegerType()
-
-
-
- LongType
-
- long Note: Numbers will be converted to 8-byte signed integer numbers at runtime.
- Please make sure that numbers are within the range of
- -9223372036854775808 to 9223372036854775807.
- Otherwise, please convert data to decimal.Decimal and use DecimalType.
-
-
- LongType()
-
-
-
- FloatType
-
- float Note: Numbers will be converted to 4-byte single-precision floating
- point numbers at runtime.
-
-
- FloatType()
-
-
-
- DoubleType
- float
-
- DoubleType()
-
-
-
- DecimalType
- decimal.Decimal
-
- DecimalType()
-
-
-
- StringType
- string
-
- StringType()
-
-
-
- BinaryType
- bytearray
-
- BinaryType()
-
-
-
- BooleanType
- bool
-
- BooleanType()
-
-
-
- TimestampType
- datetime.datetime
-
- TimestampType()
-
-
-
- DateType
- datetime.date
-
- DateType()
-
-
-
- ArrayType
- list, tuple, or array
-
- ArrayType(elementType , [containsNull ])Note: The default value of containsNull is True .
-
-
-
- MapType
- dict
-
- MapType(keyType , valueType , [valueContainsNull ])Note: The default value of valueContainsNull is True .
-
-
-
- StructType
- list or tuple
-
- StructType(fields )Note: fields is a Seq of StructFields. Also, two fields with the same
- name are not allowed.
-
-
-
- StructField
- The value type in Python of the data type of this field
- (For example, Int for a StructField with the data type IntegerType)
-
- StructField(name , dataType , [nullable ])Note: The default value of nullable is True .
-
-
-
-
-
-
-
-
-
-
- Data type
- Value type in R
- API to access or create a data type
-
- ByteType
-
- integer Note: Numbers will be converted to 1-byte signed integer numbers at runtime.
- Please make sure that numbers are within the range of -128 to 127.
-
-
- "byte"
-
-
-
- ShortType
-
- integer Note: Numbers will be converted to 2-byte signed integer numbers at runtime.
- Please make sure that numbers are within the range of -32768 to 32767.
-
-
- "short"
-
-
-
- IntegerType
- integer
-
- "integer"
-
-
-
- LongType
-
- integer Note: Numbers will be converted to 8-byte signed integer numbers at runtime.
- Please make sure that numbers are within the range of
- -9223372036854775808 to 9223372036854775807.
- Otherwise, please convert data to decimal.Decimal and use DecimalType.
-
-
- "long"
-
-
-
- FloatType
-
- numeric Note: Numbers will be converted to 4-byte single-precision floating
- point numbers at runtime.
-
-
- "float"
-
-
-
- DoubleType
- numeric
-
- "double"
-
-
-
- DecimalType
- Not supported
-
- Not supported
-
-
-
- StringType
- character
-
- "string"
-
-
-
- BinaryType
- raw
-
- "binary"
-
-
-
- BooleanType
- logical
-
- "bool"
-
-
-
- TimestampType
- POSIXct
-
- "timestamp"
-
-
-
- DateType
- Date
-
- "date"
-
-
-
- ArrayType
- vector or list
-
- list(type="array", elementType=elementType , containsNull=[containsNull ])Note: The default value of containsNull is TRUE .
-
-
-
- MapType
- environment
-
- list(type="map", keyType=keyType , valueType=valueType , valueContainsNull=[valueContainsNull ])Note: The default value of valueContainsNull is TRUE .
-
-
-
- StructType
- named list
-
- list(type="struct", fields=fields )Note: fields is a Seq of StructFields. Also, two fields with the same
- name are not allowed.
-
-
-
- StructField
- The value type in R of the data type of this field
- (For example, integer for a StructField with the data type IntegerType)
-
- list(name=name , type=dataType , nullable=[nullable ])Note: The default value of nullable is TRUE .
-
-
-
-
-
-
-
+
+{% include_example dataframe_with_arrow python/sql/arrow.py %}
+
+
+
+{% include_example scalar_pandas_udf python/sql/arrow.py %}
+
+
+
+{% include_example grouped_map_pandas_udf python/sql/arrow.py %}
+
+
+
+{% include_example grouped_agg_pandas_udf python/sql/arrow.py %}
+
+
+
+
+All data types of Spark SQL are located in the package `org.apache.spark.sql.types`.
+You can access them by doing
+
+{% include_example data_types scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
+
+
+
+ Data type
+ Value type in Scala
+ API to access or create a data type
+
+ ByteType
+ Byte
+
+ ByteType
+
+
+
+ ShortType
+ Short
+
+ ShortType
+
+
+
+ IntegerType
+ Int
+
+ IntegerType
+
+
+
+ LongType
+ Long
+
+ LongType
+
+
+
+ FloatType
+ Float
+
+ FloatType
+
+
+
+ DoubleType
+ Double
+
+ DoubleType
+
+
+
+ DecimalType
+ java.math.BigDecimal
+
+ DecimalType
+
+
+
+ StringType
+ String
+
+ StringType
+
+
+
+ BinaryType
+ Array[Byte]
+
+ BinaryType
+
+
+
+ BooleanType
+ Boolean
+
+ BooleanType
+
+
+
+ TimestampType
+ java.sql.Timestamp
+
+ TimestampType
+
+
+
+ DateType
+ java.sql.Date
+
+ DateType
+
+
+
+ ArrayType
+ scala.collection.Seq
+
+ ArrayType(elementType , [containsNull ])Note: The default value of containsNull is true .
+
+
+
+ MapType
+ scala.collection.Map
+
+ MapType(keyType , valueType , [valueContainsNull ])Note: The default value of valueContainsNull is true .
+
+
+
+ StructType
+ org.apache.spark.sql.Row
+
+ StructType(fields )Note: fields is a Seq of StructFields. Also, two fields with the same
+ name are not allowed.
+
+
+
+ StructField
+ The value type in Scala of the data type of this field
+ (For example, Int for a StructField with the data type IntegerType)
+
+ StructField(name , dataType , [nullable ])Note: The default value of nullable is true .
+
+
+
+
+
+
+
+
+All data types of Spark SQL are located in the package of
+`org.apache.spark.sql.types`. To access or create a data type,
+please use factory methods provided in
+`org.apache.spark.sql.types.DataTypes`.
+
+
+
+ Data type
+ Value type in Java
+ API to access or create a data type
+
+ ByteType
+ byte or Byte
+
+ DataTypes.ByteType
+
+
+
+ ShortType
+ short or Short
+
+ DataTypes.ShortType
+
+
+
+ IntegerType
+ int or Integer
+
+ DataTypes.IntegerType
+
+
+
+ LongType
+ long or Long
+
+ DataTypes.LongType
+
+
+
+ FloatType
+ float or Float
+
+ DataTypes.FloatType
+
+
+
+ DoubleType
+ double or Double
+
+ DataTypes.DoubleType
+
+
+
+ DecimalType
+ java.math.BigDecimal
+
+ DataTypes.createDecimalType()precision , scale ).
+
+
+
+ StringType
+ String
+
+ DataTypes.StringType
+
+
+
+ BinaryType
+ byte[]
+
+ DataTypes.BinaryType
+
+
+
+ BooleanType
+ boolean or Boolean
+
+ DataTypes.BooleanType
+
+
+
+ TimestampType
+ java.sql.Timestamp
+
+ DataTypes.TimestampType
+
+
+
+ DateType
+ java.sql.Date
+
+ DataTypes.DateType
+
+
+
+ ArrayType
+ java.util.List
+
+ DataTypes.createArrayType(elementType )Note: The value of containsNull will be true elementType , containsNull ).
+
+
+
+ MapType
+ java.util.Map
+
+ DataTypes.createMapType(keyType , valueType )Note: The value of valueContainsNull will be true .keyType , valueType , valueContainsNull )
+
+
+ StructType
+ org.apache.spark.sql.Row
+
+ DataTypes.createStructType(fields )Note: fields is a List or an array of StructFields.
+ Also, two fields with the same name are not allowed.
+
+
+
+ StructField
+ The value type in Java of the data type of this field
+ (For example, int for a StructField with the data type IntegerType)
+
+ DataTypes.createStructField(name , dataType , nullable )
+
+
+
+
+
+
+
+
+All data types of Spark SQL are located in the package of `pyspark.sql.types`.
+You can access them by doing
+{% highlight python %}
+from pyspark.sql.types import *
+{% endhighlight %}
+
+
+
+ Data type
+ Value type in Python
+ API to access or create a data type
+
+ ByteType
+
+ int or long Note: Numbers will be converted to 1-byte signed integer numbers at runtime.
+ Please make sure that numbers are within the range of -128 to 127.
+
+
+ ByteType()
+
+
+
+ ShortType
+
+ int or long Note: Numbers will be converted to 2-byte signed integer numbers at runtime.
+ Please make sure that numbers are within the range of -32768 to 32767.
+
+
+ ShortType()
+
+
+
+ IntegerType
+ int or long
+
+ IntegerType()
+
+
+
+ LongType
+
+ long Note: Numbers will be converted to 8-byte signed integer numbers at runtime.
+ Please make sure that numbers are within the range of
+ -9223372036854775808 to 9223372036854775807.
+ Otherwise, please convert data to decimal.Decimal and use DecimalType.
+
+
+ LongType()
+
+
+
+ FloatType
+
+ float Note: Numbers will be converted to 4-byte single-precision floating
+ point numbers at runtime.
+
+
+ FloatType()
+
+
+
+ DoubleType
+ float
+
+ DoubleType()
+
+
+
+ DecimalType
+ decimal.Decimal
+
+ DecimalType()
+
+
+
+ StringType
+ string
+
+ StringType()
+
+
+
+ BinaryType
+ bytearray
+
+ BinaryType()
+
+
+
+ BooleanType
+ bool
+
+ BooleanType()
+
+
+
+ TimestampType
+ datetime.datetime
+
+ TimestampType()
+
+
+
+ DateType
+ datetime.date
+
+ DateType()
+
+
+
+ ArrayType
+ list, tuple, or array
+
+ ArrayType(elementType , [containsNull ])Note: The default value of containsNull is True .
+
+
+
+ MapType
+ dict
+
+ MapType(keyType , valueType , [valueContainsNull ])Note: The default value of valueContainsNull is True .
+
+
+
+ StructType
+ list or tuple
+
+ StructType(fields )Note: fields is a Seq of StructFields. Also, two fields with the same
+ name are not allowed.
+
+
+
+ StructField
+ The value type in Python of the data type of this field
+ (For example, Int for a StructField with the data type IntegerType)
+
+ StructField(name , dataType , [nullable ])Note: The default value of nullable is True .
+
+
+
+
+
+
+
+
+
+
+ Data type
+ Value type in R
+ API to access or create a data type
+
+ ByteType
+
+ integer Note: Numbers will be converted to 1-byte signed integer numbers at runtime.
+ Please make sure that numbers are within the range of -128 to 127.
+
+
+ "byte"
+
+
+
+ ShortType
+
+ integer Note: Numbers will be converted to 2-byte signed integer numbers at runtime.
+ Please make sure that numbers are within the range of -32768 to 32767.
+
+
+ "short"
+
+
+
+ IntegerType
+ integer
+
+ "integer"
+
+
+
+ LongType
+
+ integer Note: Numbers will be converted to 8-byte signed integer numbers at runtime.
+ Please make sure that numbers are within the range of
+ -9223372036854775808 to 9223372036854775807.
+ Otherwise, please convert data to decimal.Decimal and use DecimalType.
+
+
+ "long"
+
+
+
+ FloatType
+
+ numeric Note: Numbers will be converted to 4-byte single-precision floating
+ point numbers at runtime.
+
+
+ "float"
+
+
+
+ DoubleType
+ numeric
+
+ "double"
+
+
+
+ DecimalType
+ Not supported
+
+ Not supported
+
+
+
+ StringType
+ character
+
+ "string"
+
+
+
+ BinaryType
+ raw
+
+ "binary"
+
+
+
+ BooleanType
+ logical
+
+ "bool"
+
+
+
+ TimestampType
+ POSIXct
+
+ "timestamp"
+
+
+
+ DateType
+ Date
+
+ "date"
+
+
+
+ ArrayType
+ vector or list
+
+ list(type="array", elementType=elementType , containsNull=[containsNull ])Note: The default value of containsNull is TRUE .
+
+
+
+ MapType
+ environment
+
+ list(type="map", keyType=keyType , valueType=valueType , valueContainsNull=[valueContainsNull ])Note: The default value of valueContainsNull is TRUE .
+
+
+
+ StructType
+ named list
+
+ list(type="struct", fields=fields )Note: fields is a Seq of StructFields. Also, two fields with the same
+ name are not allowed.
+
+
+
+ StructField
+ The value type in R of the data type of this field
+ (For example, integer for a StructField with the data type IntegerType)
+
+ list(name=name , type=dataType , nullable=[nullable ])Note: The default value of nullable is TRUE .
+
+
+
+
+
+
+
org.apache.spark
spark-streaming_{{site.SCALA_BINARY_VERSION}}
{{site.SPARK_VERSION}}
+ provided
- libraryDependencies += "org.apache.spark" % "spark-streaming_{{site.SCALA_BINARY_VERSION}}" % "{{site.SPARK_VERSION}}"
+ libraryDependencies += "org.apache.spark" % "spark-streaming_{{site.SCALA_BINARY_VERSION}}" % "{{site.SPARK_VERSION}}" % "provided"
@@ -431,7 +432,7 @@ val ssc = new StreamingContext(conf, Seconds(1))
{% endhighlight %}
The `appName` parameter is a name for your application to show on the cluster UI.
-`master` is a [Spark, Mesos or YARN cluster URL](submitting-applications.html#master-urls),
+`master` is a [Spark, Mesos, Kubernetes or YARN cluster URL](submitting-applications.html#master-urls),
or a special __"local[\*]"__ string to run in local mode. In practice, when running on a cluster,
you will not want to hardcode `master` in the program,
but rather [launch the application with `spark-submit`](submitting-applications.html) and
diff --git a/docs/structured-streaming-kafka-integration.md b/docs/structured-streaming-kafka-integration.md
index 71fd5b10cc407..2d1510dc4d8d1 100644
--- a/docs/structured-streaming-kafka-integration.md
+++ b/docs/structured-streaming-kafka-integration.md
@@ -374,6 +374,18 @@ The following configurations are optional:
streaming and batch
Rate limit on maximum number of offsets processed per trigger interval. The specified total number of offsets will be proportionally split across topicPartitions of different volume.
+
+ minPartitions
+ int
+ none
+ streaming
+ Desired minimum number of partitions to read from Kafka.
+ By default, Spark has a 1-1 mapping of topicPartitions to Spark partitions consuming from Kafka.
+ If you set this option to a value greater than your topicPartitions, Spark will divvy up large
+ Kafka partitions to smaller pieces. Please note that this configuration is like a `hint`: the
+ number of Spark tasks will be **approximately** `minPartitions`. It can be less or more depending on
+ rounding errors or Kafka partitions that didn't receive any new data.
+
## Writing Data to Kafka
diff --git a/docs/structured-streaming-programming-guide.md b/docs/structured-streaming-programming-guide.md
index 73de1892977ac..abff126ce4145 100644
--- a/docs/structured-streaming-programming-guide.md
+++ b/docs/structured-streaming-programming-guide.md
@@ -12,7 +12,7 @@ Structured Streaming is a scalable and fault-tolerant stream processing engine b
Internally, by default, Structured Streaming queries are processed using a *micro-batch processing* engine, which processes data streams as a series of small batch jobs thereby achieving end-to-end latencies as low as 100 milliseconds and exactly-once fault-tolerance guarantees. However, since Spark 2.3, we have introduced a new low-latency processing mode called **Continuous Processing**, which can achieve end-to-end latencies as low as 1 millisecond with at-least-once guarantees. Without changing the Dataset/DataFrame operations in your queries, you will be able to choose the mode based on your application requirements.
-In this guide, we are going to walk you through the programming model and the APIs. We are going to explain the concepts mostly using the default micro-batch processing model, and then [later](#continuous-processing-experimental) discuss Continuous Processing model. First, let's start with a simple example of a Structured Streaming query - a streaming word count.
+In this guide, we are going to walk you through the programming model and the APIs. We are going to explain the concepts mostly using the default micro-batch processing model, and then [later](#continuous-processing) discuss Continuous Processing model. First, let's start with a simple example of a Structured Streaming query - a streaming word count.
# Quick Example
Let’s say you want to maintain a running word count of text data received from a data server listening on a TCP socket. Let’s see how you can express this using Structured Streaming. You can see the full code in
@@ -536,7 +536,7 @@ Here are the details of all the sources in Spark.
href="api/R/read.stream.html">R).
E.g. for "parquet" format options see DataStreamReader.parquet().
SQL Programming Guide for more details. E.g., for "parquet", see Parquet configuration section.
+ In addition, there are session configurations that affect certain file-formats. See the SQL Programming Guide for more details. E.g., for "parquet", see Parquet configuration section.
Yes
Supports glob paths, but does not support multiple comma-separated paths/globs.
@@ -922,7 +922,7 @@ late data for that aggregate any more. To enable this, in Spark 2.1, we have int
**watermarking**, which lets the engine automatically track the current event time in the data
and attempt to clean up old state accordingly. You can define the watermark of a query by
specifying the event time column and the threshold on how late the data is expected to be in terms of
-event time. For a specific window starting at time `T`, the engine will maintain state and allow late
+event time. For a specific window ending at time `T`, the engine will maintain state and allow late
data to update the state until `(max event time seen by the engine - late threshold > T)`.
In other words, late data within the threshold will be aggregated,
but data later than the threshold will start getting dropped
@@ -1480,9 +1480,9 @@ Additional details on supported joins:
- Joins can be cascaded, that is, you can do `df1.join(df2, ...).join(df3, ...).join(df4, ....)`.
-- As of Spark 2.3, you can use joins only when the query is in Append output mode. Other output modes are not yet supported.
+- As of Spark 2.4, you can use joins only when the query is in Append output mode. Other output modes are not yet supported.
-- As of Spark 2.3, you cannot use other non-map-like operations before joins. Here are a few examples of
+- As of Spark 2.4, you cannot use other non-map-like operations before joins. Here are a few examples of
what cannot be used.
- Cannot use streaming aggregations before joins.
@@ -1560,6 +1560,43 @@ streamingDf <- dropDuplicates(streamingDf, "guid", "eventTime")
+
+
+{% highlight scala %}
+inputStream1.withWatermark("eventTime1", "1 hour")
+ .join(
+ inputStream2.withWatermark("eventTime2", "2 hours"),
+ joinCondition)
+{% endhighlight %}
+
+
+
Foreach Sink Append, Update, Complete
None
- Depends on ForeachWriter implementation
- More details in the next section
+ Yes (at-least-once)
+ More details in the next section
+
+ ForeachBatch Sink Append, Update, Complete
+ None
+ Depends on the implementation
+ More details in the next section
+
+
Console Sink Append, Update, Complete
@@ -1989,22 +2034,220 @@ head(sql("select * from aggregates"))
-##### Using Foreach
-The `foreach` operation allows arbitrary operations to be computed on the output data. As of Spark 2.1, this is available only for Scala and Java. To use this, you will have to implement the interface `ForeachWriter`
-([Scala](api/scala/index.html#org.apache.spark.sql.ForeachWriter)/[Java](api/java/org/apache/spark/sql/ForeachWriter.html) docs),
-which has methods that get called whenever there is a sequence of rows generated as output after a trigger. Note the following important points.
+##### Using Foreach and ForeachBatch
+The `foreach` and `foreachBatch` operations allow you to apply arbitrary operations and writing
+logic on the output of a streaming query. They have slightly different use cases - while `foreach`
+allows custom write logic on every row, `foreachBatch` allows arbitrary operations
+and custom logic on the output of each micro-batch. Let's understand their usages in more detail.
+
+###### ForeachBatch
+`foreachBatch(...)` allows you to specify a function that is executed on
+the output data of every micro-batch of a streaming query. Since Spark 2.4, this is supported in Scala, Java and Python.
+It takes two parameters: a DataFrame or Dataset that has the output data of a micro-batch and the unique ID of the micro-batch.
+
+
+
+
+{% highlight scala %}
+streamingDF.writeStream.foreachBatch { (batchDF: DataFrame, batchId: Long) =>
+ // Transform and write batchDF
+}.start()
+{% endhighlight %}
+
+
+
+
+{% highlight java %}
+streamingDatasetOfString.writeStream().foreachBatch(
+ new VoidFunction2, Long> {
+ public void call(Dataset dataset, Long batchId) {
+ // Transform and write batchDF
+ }
+ }
+).start();
+{% endhighlight %}
+
+
+
+
+{% highlight python %}
+def foreach_batch_function(df, epoch_id):
+ # Transform and write batchDF
+ pass
+
+streamingDF.writeStream.foreachBatch(foreach_batch_function).start()
+{% endhighlight %}
+
+
+
+R is not yet supported.
+
+
+
+
+{% highlight scala %}
+streamingDF.writeStream.foreachBatch { (batchDF: DataFrame, batchId: Long) =>
+ batchDF.persist()
+ batchDF.write.format(...).save(...) // location 1
+ batchDF.write.format(...).save(...) // location 2
+ batchDF.unpersist()
+}
+{% endhighlight %}
-- The writer must be serializable, as it will be serialized and sent to the executors for execution.
+
+
+
+
+In Scala, you have to extend the class `ForeachWriter` ([docs](api/scala/index.html#org.apache.spark.sql.ForeachWriter)).
+
+{% highlight scala %}
+streamingDatasetOfString.writeStream.foreach(
+ new ForeachWriter[String] {
+
+ def open(partitionId: Long, version: Long): Boolean = {
+ // Open connection
+ }
+
+ def process(record: String): Unit = {
+ // Write string to connection
+ }
+
+ def close(errorOrNull: Throwable): Unit = {
+ // Close the connection
+ }
+ }
+).start()
+{% endhighlight %}
+
+
+
+
+In Java, you have to extend the class `ForeachWriter` ([docs](api/java/org/apache/spark/sql/ForeachWriter.html)).
+{% highlight java %}
+streamingDatasetOfString.writeStream().foreach(
+ new ForeachWriter[String] {
+
+ @Override public boolean open(long partitionId, long version) {
+ // Open connection
+ }
+
+ @Override public void process(String record) {
+ // Write string to connection
+ }
+
+ @Override public void close(Throwable errorOrNull) {
+ // Close the connection
+ }
+ }
+).start();
+
+{% endhighlight %}
+
+
+
+
+In Python, you can invoke foreach in two ways: in a function or in an object.
+The function offers a simple way to express your processing logic but does not allow you to
+deduplicate generated data when failures cause reprocessing of some input data.
+For that situation you must specify the processing logic in an object.
+
+- First, the function takes a row as input.
+
+{% highlight python %}
+def process_row(row):
+ # Write row to storage
+ pass
+
+query = streamingDF.writeStream.foreach(process_row).start()
+{% endhighlight %}
+
+- Second, the object has a process method and optional open and close methods:
+
+{% highlight python %}
+class ForeachWriter:
+ def open(self, partition_id, epoch_id):
+ # Open connection. This method is optional in Python.
+ pass
+
+ def process(self, row):
+ # Write row to connection. This method is NOT optional in Python.
+ pass
+
+ def close(self, error):
+ # Close the connection. This method in optional in Python.
+ pass
+
+query = streamingDF.writeStream.foreach(ForeachWriter()).start()
+{% endhighlight %}
+
+
+
+R is not yet supported.
+
+
Continuous with fixed checkpoint interval (experimental)
The query will be executed in the new low-latency, continuous processing mode. Read more
- about this in the Continuous Processing section below.
+ about this in the Continuous Processing section below.
@@ -2310,11 +2553,11 @@ spark.streams().awaitAnyTermination(); // block until any one of them terminat
{% highlight python %}
spark = ... # spark session
-spark.streams().active # get the list of currently active streaming queries
+spark.streams.active # get the list of currently active streaming queries
-spark.streams().get(id) # get a query object by its unique id
+spark.streams.get(id) # get a query object by its unique id
-spark.streams().awaitAnyTermination() # block until any one of them terminates
+spark.streams.awaitAnyTermination() # block until any one of them terminates
{% endhighlight %}
@@ -2709,6 +2952,78 @@ write.stream(aggDF, "memory", outputMode = "complete", checkpointLocation = "pat
+
+## Recovery Semantics after Changes in a Streaming Query
+There are limitations on what changes in a streaming query are allowed between restarts from the
+same checkpoint location. Here are a few kinds of changes that are either not allowed, or
+the effect of the change is not well-defined. For all of them:
+
+- The term *allowed* means you can do the specified change but whether the semantics of its effect
+ is well-defined depends on the query and the change.
+
+- The term *not allowed* means you should not do the specified change as the restarted query is likely
+ to fail with unpredictable errors. `sdf` represents a streaming DataFrame/Dataset
+ generated with sparkSession.readStream.
+
+**Types of changes**
+
+- *Changes in the number or type (i.e. different source) of input sources*: This is not allowed.
+
+- *Changes in the parameters of input sources*: Whether this is allowed and whether the semantics
+ of the change are well-defined depends on the source and the query. Here are a few examples.
+
+ - Addition/deletion/modification of rate limits is allowed: `spark.readStream.format("kafka").option("subscribe", "topic")` to `spark.readStream.format("kafka").option("subscribe", "topic").option("maxOffsetsPerTrigger", ...)`
+
+ - Changes to subscribed topics/files is generally not allowed as the results are unpredictable: `spark.readStream.format("kafka").option("subscribe", "topic")` to `spark.readStream.format("kafka").option("subscribe", "newTopic")`
+
+- *Changes in the type of output sink*: Changes between a few specific combinations of sinks
+ are allowed. This needs to be verified on a case-by-case basis. Here are a few examples.
+
+ - File sink to Kafka sink is allowed. Kafka will see only the new data.
+
+ - Kafka sink to file sink is not allowed.
+
+ - Kafka sink changed to foreach, or vice versa is allowed.
+
+- *Changes in the parameters of output sink*: Whether this is allowed and whether the semantics of
+ the change are well-defined depends on the sink and the query. Here are a few examples.
+
+ - Changes to output directory of a file sink is not allowed: `sdf.writeStream.format("parquet").option("path", "/somePath")` to `sdf.writeStream.format("parquet").option("path", "/anotherPath")`
+
+ - Changes to output topic is allowed: `sdf.writeStream.format("kafka").option("topic", "someTopic")` to `sdf.writeStream.format("kafka").option("topic", "anotherTopic")`
+
+ - Changes to the user-defined foreach sink (that is, the `ForeachWriter` code) is allowed, but the semantics of the change depends on the code.
+
+- *Changes in projection / filter / map-like operations**: Some cases are allowed. For example:
+
+ - Addition / deletion of filters is allowed: `sdf.selectExpr("a")` to `sdf.where(...).selectExpr("a").filter(...)`.
+
+ - Changes in projections with same output schema is allowed: `sdf.selectExpr("stringColumn AS json").writeStream` to `sdf.selectExpr("anotherStringColumn AS json").writeStream`
+
+ - Changes in projections with different output schema are conditionally allowed: `sdf.selectExpr("a").writeStream` to `sdf.selectExpr("b").writeStream` is allowed only if the output sink allows the schema change from `"a"` to `"b"`.
+
+- *Changes in stateful operations*: Some operations in streaming queries need to maintain
+ state data in order to continuously update the result. Structured Streaming automatically checkpoints
+ the state data to fault-tolerant storage (for example, HDFS, AWS S3, Azure Blob storage) and restores it after restart.
+ However, this assumes that the schema of the state data remains same across restarts. This means that
+ *any changes (that is, additions, deletions, or schema modifications) to the stateful operations of a streaming query are not allowed between restarts*.
+ Here is the list of stateful operations whose schema should not be changed between restarts in order to ensure state recovery:
+
+ - *Streaming aggregation*: For example, `sdf.groupBy("a").agg(...)`. Any change in number or type of grouping keys or aggregates is not allowed.
+
+ - *Streaming deduplication*: For example, `sdf.dropDuplicates("a")`. Any change in number or type of grouping keys or aggregates is not allowed.
+
+ - *Stream-stream join*: For example, `sdf1.join(sdf2, ...)` (i.e. both inputs are generated with `sparkSession.readStream`). Changes
+ in the schema or equi-joining columns are not allowed. Changes in join type (outer or inner) not allowed. Other changes in the join condition are ill-defined.
+
+ - *Arbitrary stateful operation*: For example, `sdf.groupByKey(...).mapGroupsWithState(...)` or `sdf.groupByKey(...).flatMapGroupsWithState(...)`.
+ Any change to the schema of the user-defined state and the type of timeout is not allowed.
+ Any change within the user-defined state-mapping function are allowed, but the semantic effect of the change depends on the user-defined logic.
+ If you really want to support state schema changes, then you can explicitly encode/decode your complex state data
+ structures into bytes using an encoding/decoding scheme that supports schema migration. For example,
+ if you save your state as Avro-encoded bytes, then you are free to change the Avro-state-schema between query
+ restarts as the binary state will always be restored successfully.
+
# Continuous Processing
## [Experimental]
{:.no_toc}
@@ -2722,12 +3037,6 @@ To run a supported query in continuous processing mode, all you need to do is sp
{% highlight scala %}
import org.apache.spark.sql.streaming.Trigger
-spark
- .readStream
- .format("rate")
- .option("rowsPerSecond", "10")
- .option("")
-
spark
.readStream
.format("kafka")
@@ -2787,7 +3096,7 @@ A checkpoint interval of 1 second means that the continuous processing engine wi
## Supported Queries
{:.no_toc}
-As of Spark 2.3, only the following type of queries are supported in the continuous processing mode.
+As of Spark 2.4, only the following type of queries are supported in the continuous processing mode.
- *Operations*: Only map-like Dataset/DataFrame operations are supported in continuous mode, that is, only projections (`select`, `map`, `flatMap`, `mapPartitions`, etc.) and selections (`where`, `filter`, etc.).
+ All SQL functions are supported except aggregation functions (since aggregations are not yet supported), `current_timestamp()` and `current_date()` (deterministic computations using time is challenging).
diff --git a/docs/submitting-applications.md b/docs/submitting-applications.md
index 77aa083c4a584..34cea82974547 100644
--- a/docs/submitting-applications.md
+++ b/docs/submitting-applications.md
@@ -167,7 +167,7 @@ The master URL passed to Spark can be in one of the following formats:
k8s://HOST:PORT Connect to a Kubernetes cluster in
cluster mode. Client mode is currently unsupported and will be supported in future releases.
- The HOST and PORT refer to the [Kubernetes API Server](https://kubernetes.io/docs/reference/generated/kube-apiserver/).
+ The HOST and PORT refer to the Kubernetes API Server .
It connects using TLS by default. In order to force it to use an unsecured connection, you can use
k8s://http://HOST:PORT.
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../pom.xml
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaPrefixSpanExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaPrefixSpanExample.java
new file mode 100644
index 0000000000000..98ffd4faf9575
--- /dev/null
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaPrefixSpanExample.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.ml;
+
+// $example on$
+
+import org.apache.spark.ml.fpm.PrefixSpan;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.types.*;
+
+import java.util.Arrays;
+import java.util.List;
+// $example off$
+
+/**
+ * An example demonstrating PrefixSpan.
+ * Run with
+ *
+ * bin/run-example ml.JavaPrefixSpanExample
+ *
+ */
+public class JavaPrefixSpanExample {
+ public static void main(String[] args) {
+ SparkSession spark = SparkSession
+ .builder()
+ .appName("JavaPrefixSpanExample")
+ .getOrCreate();
+
+ // $example on$
+ List data = Arrays.asList(
+ RowFactory.create(Arrays.asList(Arrays.asList(1, 2), Arrays.asList(3))),
+ RowFactory.create(Arrays.asList(Arrays.asList(1), Arrays.asList(3, 2), Arrays.asList(1,2))),
+ RowFactory.create(Arrays.asList(Arrays.asList(1, 2), Arrays.asList(5))),
+ RowFactory.create(Arrays.asList(Arrays.asList(6)))
+ );
+ StructType schema = new StructType(new StructField[]{ new StructField(
+ "sequence", new ArrayType(new ArrayType(DataTypes.IntegerType, true), true),
+ false, Metadata.empty())
+ });
+ Dataset sequenceDF = spark.createDataFrame(data, schema);
+
+ PrefixSpan prefixSpan = new PrefixSpan().setMinSupport(0.5).setMaxPatternLength(5);
+
+ // Finding frequent sequential patterns
+ prefixSpan.findFrequentSequentialPatterns(sequenceDF).show();
+ // $example off$
+
+ spark.stop();
+ }
+}
diff --git a/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java b/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
index ef3c904775697..97e9ca311267c 100644
--- a/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
@@ -123,6 +123,12 @@ private static void runBasicDataSourceExample(SparkSession spark) {
.option("header", "true")
.load("examples/src/main/resources/people.csv");
// $example off:manual_load_options_csv$
+ // $example on:manual_save_options_orc$
+ usersDF.write().format("orc")
+ .option("orc.bloom.filter.columns", "favorite_color")
+ .option("orc.dictionary.key.threshold", "1.0")
+ .save("users_with_options.orc");
+ // $example off:manual_save_options_orc$
// $example on:direct_sql$
Dataset sqlDF =
spark.sql("SELECT * FROM parquet.`examples/src/main/resources/users.parquet`");
diff --git a/examples/src/main/python/ml/prefixspan_example.py b/examples/src/main/python/ml/prefixspan_example.py
new file mode 100644
index 0000000000000..88d1d4197341b
--- /dev/null
+++ b/examples/src/main/python/ml/prefixspan_example.py
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+An example demonstrating PrefixSpan.
+Run with:
+ bin/spark-submit examples/src/main/python/ml/prefixspan_example.py
+"""
+# $example on$
+from pyspark.ml.fpm import PrefixSpan
+# $example off$
+from pyspark.sql import Row, SparkSession
+
+if __name__ == "__main__":
+ spark = SparkSession\
+ .builder\
+ .appName("PrefixSpanExample")\
+ .getOrCreate()
+ sc = spark.sparkContext
+
+ # $example on$
+ df = sc.parallelize([Row(sequence=[[1, 2], [3]]),
+ Row(sequence=[[1], [3, 2], [1, 2]]),
+ Row(sequence=[[1, 2], [5]]),
+ Row(sequence=[[6]])]).toDF()
+
+ prefixSpan = PrefixSpan(minSupport=0.5, maxPatternLength=5,
+ maxLocalProjDBSize=32000000)
+
+ # Find frequent sequential patterns.
+ prefixSpan.findFrequentSequentialPatterns(df).show()
+ # $example off$
+
+ spark.stop()
diff --git a/examples/src/main/python/sql/datasource.py b/examples/src/main/python/sql/datasource.py
index d8c879dfe02ed..888e92a620070 100644
--- a/examples/src/main/python/sql/datasource.py
+++ b/examples/src/main/python/sql/datasource.py
@@ -57,6 +57,14 @@ def basic_datasource_example(spark):
format="csv", sep=":", inferSchema="true", header="true")
# $example off:manual_load_options_csv$
+ # $example on:manual_save_options_orc$
+ df = spark.read.orc("examples/src/main/resources/users.orc")
+ (df.write.format("orc")
+ .option("orc.bloom.filter.columns", "favorite_color")
+ .option("orc.dictionary.key.threshold", "1.0")
+ .save("users_with_options.orc"))
+ # $example off:manual_save_options_orc$
+
# $example on:write_sorting_and_bucketing$
df.write.bucketBy(42, "name").sortBy("age").saveAsTable("people_bucketed")
# $example off:write_sorting_and_bucketing$
diff --git a/examples/src/main/r/RSparkSQLExample.R b/examples/src/main/r/RSparkSQLExample.R
index a5ed723da47ca..f3d2ab8cbc844 100644
--- a/examples/src/main/r/RSparkSQLExample.R
+++ b/examples/src/main/r/RSparkSQLExample.R
@@ -114,10 +114,14 @@ write.df(namesAndAges, "namesAndAges.parquet", "parquet")
# $example on:manual_load_options_csv$
-df <- read.df("examples/src/main/resources/people.csv", "csv")
+df <- read.df("examples/src/main/resources/people.csv", "csv", sep = ";", inferSchema = TRUE, header = TRUE)
namesAndAges <- select(df, "name", "age")
# $example off:manual_load_options_csv$
+# $example on:manual_save_options_orc$
+df <- read.df("examples/src/main/resources/users.orc", "orc")
+write.orc(df, "users_with_options.orc", orc.bloom.filter.columns = "favorite_color", orc.dictionary.key.threshold = 1.0)
+# $example off:manual_save_options_orc$
# $example on:direct_sql$
df <- sql("SELECT * FROM parquet.`examples/src/main/resources/users.parquet`")
diff --git a/examples/src/main/resources/users.orc b/examples/src/main/resources/users.orc
new file mode 100644
index 0000000000000..12478a5d03c26
Binary files /dev/null and b/examples/src/main/resources/users.orc differ
diff --git a/examples/src/main/scala/org/apache/spark/examples/graphx/Analytics.scala b/examples/src/main/scala/org/apache/spark/examples/graphx/Analytics.scala
index 815404d1218b7..0726fe970fca7 100644
--- a/examples/src/main/scala/org/apache/spark/examples/graphx/Analytics.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/graphx/Analytics.scala
@@ -24,14 +24,13 @@ import org.apache.spark._
import org.apache.spark.graphx._
import org.apache.spark.graphx.PartitionStrategy._
import org.apache.spark.graphx.lib._
-import org.apache.spark.internal.Logging
import org.apache.spark.storage.StorageLevel
/**
* Driver program for running graph algorithms.
*/
-object Analytics extends Logging {
+object Analytics {
def main(args: Array[String]): Unit = {
if (args.length < 2) {
@@ -101,7 +100,7 @@ object Analytics extends Logging {
println(s"GRAPHX: Total rank: ${pr.map(_._2).reduce(_ + _)}")
if (!outFname.isEmpty) {
- logWarning(s"Saving pageranks of pages to $outFname")
+ println(s"Saving pageranks of pages to $outFname")
pr.map { case (id, r) => id + "\t" + r }.saveAsTextFile(outFname)
}
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/PrefixSpanExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/PrefixSpanExample.scala
new file mode 100644
index 0000000000000..0a2d31097a024
--- /dev/null
+++ b/examples/src/main/scala/org/apache/spark/examples/ml/PrefixSpanExample.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.ml
+
+// scalastyle:off println
+
+// $example on$
+import org.apache.spark.ml.fpm.PrefixSpan
+// $example off$
+import org.apache.spark.sql.SparkSession
+
+/**
+ * An example demonstrating PrefixSpan.
+ * Run with
+ * {{{
+ * bin/run-example ml.PrefixSpanExample
+ * }}}
+ */
+object PrefixSpanExample {
+
+ def main(args: Array[String]): Unit = {
+ val spark = SparkSession
+ .builder
+ .appName(s"${this.getClass.getSimpleName}")
+ .getOrCreate()
+ import spark.implicits._
+
+ // $example on$
+ val smallTestData = Seq(
+ Seq(Seq(1, 2), Seq(3)),
+ Seq(Seq(1), Seq(3, 2), Seq(1, 2)),
+ Seq(Seq(1, 2), Seq(5)),
+ Seq(Seq(6)))
+
+ val df = smallTestData.toDF("sequence")
+ val result = new PrefixSpan()
+ .setMinSupport(0.5)
+ .setMaxPatternLength(5)
+ .setMaxLocalProjDBSize(32000000)
+ .findFrequentSequentialPatterns(df)
+ .show()
+ // $example off$
+
+ spark.stop()
+ }
+}
+// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala b/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
index 7d83aacb11548..49177334ebae4 100644
--- a/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
@@ -56,6 +56,12 @@ object SQLDataSourceExample {
.option("header", "true")
.load("examples/src/main/resources/people.csv")
// $example off:manual_load_options_csv$
+ // $example on:manual_save_options_orc$
+ usersDF.write.format("orc")
+ .option("orc.bloom.filter.columns", "favorite_color")
+ .option("orc.dictionary.key.threshold", "1.0")
+ .save("users_with_options.orc")
+ // $example off:manual_save_options_orc$
// $example on:direct_sql$
val sqlDF = spark.sql("SELECT * FROM parquet.`examples/src/main/resources/users.parquet`")
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala b/examples/src/main/scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala
index 70fb5b27366ec..a832276602b88 100644
--- a/examples/src/main/scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala
@@ -122,16 +122,16 @@ object SparkHiveExample {
val dataDir = "/tmp/parquet_data"
spark.range(10).write.parquet(dataDir)
// Create a Hive external Parquet table
- sql(s"CREATE EXTERNAL TABLE hive_ints(key int) STORED AS PARQUET LOCATION '$dataDir'")
+ sql(s"CREATE EXTERNAL TABLE hive_bigints(id bigint) STORED AS PARQUET LOCATION '$dataDir'")
// The Hive external table should already have data
- sql("SELECT * FROM hive_ints").show()
+ sql("SELECT * FROM hive_bigints").show()
// +---+
- // |key|
+ // | id|
// +---+
// | 0|
// | 1|
// | 2|
- // ...
+ // ... Order may vary, as spark processes the partitions in parallel.
// Turn on flag for Hive Dynamic Partitioning
spark.sqlContext.setConf("hive.exec.dynamic.partition", "true")
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredSessionization.scala b/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredSessionization.scala
index ed63fb677b9ef..29dbb0d95cc5e 100644
--- a/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredSessionization.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredSessionization.scala
@@ -70,7 +70,7 @@ object StructuredSessionization {
line.split(" ").map(word => Event(sessionId = word, timestamp))
}
- // Sessionize the events. Track number of events, start and end timestamps of session, and
+ // Sessionize the events. Track number of events, start and end timestamps of session,
// and report session updates.
val sessionUpdates = events
.groupByKey(event => event.sessionId)
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
index 25c7bf2871972..fc3f8fa53c7ae 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
@@ -23,7 +23,6 @@ import java.net.Socket
import java.nio.charset.StandardCharsets
import org.apache.spark.SparkConf
-import org.apache.spark.internal.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.receiver.Receiver
@@ -63,7 +62,7 @@ object CustomReceiver {
class CustomReceiver(host: String, port: Int)
- extends Receiver[String](StorageLevel.MEMORY_AND_DISK_2) with Logging {
+ extends Receiver[String](StorageLevel.MEMORY_AND_DISK_2) {
def onStart() {
// Start the thread that receives data over a connection
@@ -82,9 +81,9 @@ class CustomReceiver(host: String, port: Int)
var socket: Socket = null
var userInput: String = null
try {
- logInfo(s"Connecting to $host : $port")
+ println(s"Connecting to $host : $port")
socket = new Socket(host, port)
- logInfo(s"Connected to $host : $port")
+ println(s"Connected to $host : $port")
val reader = new BufferedReader(
new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))
userInput = reader.readLine()
@@ -94,7 +93,7 @@ class CustomReceiver(host: String, port: Int)
}
reader.close()
socket.close()
- logInfo("Stopped receiving")
+ println("Stopped receiving")
restart("Trying to connect again")
} catch {
case e: java.net.ConnectException =>
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/DirectKafkaWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/DirectKafkaWordCount.scala
index 2082fb71afdf1..3024b59480099 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/DirectKafkaWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/DirectKafkaWordCount.scala
@@ -40,7 +40,7 @@ object DirectKafkaWordCount {
def main(args: Array[String]) {
if (args.length < 3) {
System.err.println(s"""
- |Usage: DirectKafkaWordCount
+ |Usage: DirectKafkaWordCount
| is a list of one or more Kafka brokers
| is a consumer group name to consume from topics
| is a list of one or more kafka topics to consume from
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index 8f118ba48201b..f3182e76f3340 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
index 915769fa708b0..8641b9ffad8b0 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
@@ -51,13 +51,7 @@ case class AvroDataToCatalyst(child: Expression, jsonFormatSchema: String)
deserializer.deserialize(result)
}
- override def simpleString: String = {
- s"from_avro(${child.sql}, ${dataType.simpleString})"
- }
-
- override def sql: String = {
- s"from_avro(${child.sql}, ${dataType.catalogString})"
- }
+ override def prettyName: String = "from_avro"
override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val expr = ctx.addReferenceObj("this", this)
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 272e7d5b388d9..b10405cf542c1 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -223,6 +223,7 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
case (UNION, _) =>
val allTypes = avroType.getTypes.asScala
val nonNullTypes = allTypes.filter(_.getType != NULL)
+ val nonNullAvroType = Schema.createUnion(nonNullTypes.asJava)
if (nonNullTypes.nonEmpty) {
if (nonNullTypes.length == 1) {
newWriter(nonNullTypes.head, catalystType, path)
@@ -251,7 +252,7 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
(updater, ordinal, value) => {
val row = new SpecificInternalRow(st)
val fieldUpdater = new RowUpdater(row)
- val i = GenericData.get().resolveUnion(avroType, value)
+ val i = GenericData.get().resolveUnion(nonNullAvroType, value)
fieldWriters(i)(fieldUpdater, i, value)
updater.set(ordinal, row)
}
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
index 141ff3782adfb..6ed330d92f5e6 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
@@ -52,13 +52,7 @@ case class CatalystDataToAvro(child: Expression) extends UnaryExpression {
out.toByteArray
}
- override def simpleString: String = {
- s"to_avro(${child.sql}, ${child.dataType.simpleString})"
- }
-
- override def sql: String = {
- s"to_avro(${child.sql}, ${child.dataType.catalogString})"
- }
+ override def prettyName: String = "to_avro"
override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val expr = ctx.addReferenceObj("this", this)
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala
index bd1576587d7fa..64127af73881b 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala
@@ -43,6 +43,10 @@ object SchemaConverters {
* This function takes an avro schema and returns a sql schema.
*/
def toSqlType(avroSchema: Schema): SchemaType = {
+ toSqlTypeHelper(avroSchema, Set.empty)
+ }
+
+ def toSqlTypeHelper(avroSchema: Schema, existingRecordNames: Set[String]): SchemaType = {
avroSchema.getType match {
case INT => avroSchema.getLogicalType match {
case _: Date => SchemaType(DateType, nullable = false)
@@ -67,21 +71,28 @@ object SchemaConverters {
case ENUM => SchemaType(StringType, nullable = false)
case RECORD =>
+ if (existingRecordNames.contains(avroSchema.getFullName)) {
+ throw new IncompatibleSchemaException(s"""
+ |Found recursive reference in Avro schema, which can not be processed by Spark:
+ |${avroSchema.toString(true)}
+ """.stripMargin)
+ }
+ val newRecordNames = existingRecordNames + avroSchema.getFullName
val fields = avroSchema.getFields.asScala.map { f =>
- val schemaType = toSqlType(f.schema())
+ val schemaType = toSqlTypeHelper(f.schema(), newRecordNames)
StructField(f.name, schemaType.dataType, schemaType.nullable)
}
SchemaType(StructType(fields), nullable = false)
case ARRAY =>
- val schemaType = toSqlType(avroSchema.getElementType)
+ val schemaType = toSqlTypeHelper(avroSchema.getElementType, existingRecordNames)
SchemaType(
ArrayType(schemaType.dataType, containsNull = schemaType.nullable),
nullable = false)
case MAP =>
- val schemaType = toSqlType(avroSchema.getValueType)
+ val schemaType = toSqlTypeHelper(avroSchema.getValueType, existingRecordNames)
SchemaType(
MapType(StringType, schemaType.dataType, valueContainsNull = schemaType.nullable),
nullable = false)
@@ -91,13 +102,14 @@ object SchemaConverters {
// In case of a union with null, eliminate it and make a recursive call
val remainingUnionTypes = avroSchema.getTypes.asScala.filterNot(_.getType == NULL)
if (remainingUnionTypes.size == 1) {
- toSqlType(remainingUnionTypes.head).copy(nullable = true)
+ toSqlTypeHelper(remainingUnionTypes.head, existingRecordNames).copy(nullable = true)
} else {
- toSqlType(Schema.createUnion(remainingUnionTypes.asJava)).copy(nullable = true)
+ toSqlTypeHelper(Schema.createUnion(remainingUnionTypes.asJava), existingRecordNames)
+ .copy(nullable = true)
}
} else avroSchema.getTypes.asScala.map(_.getType) match {
case Seq(t1) =>
- toSqlType(avroSchema.getTypes.get(0))
+ toSqlTypeHelper(avroSchema.getTypes.get(0), existingRecordNames)
case Seq(t1, t2) if Set(t1, t2) == Set(INT, LONG) =>
SchemaType(LongType, nullable = false)
case Seq(t1, t2) if Set(t1, t2) == Set(FLOAT, DOUBLE) =>
@@ -107,7 +119,7 @@ object SchemaConverters {
// This is consistent with the behavior when converting between Avro and Parquet.
val fields = avroSchema.getTypes.asScala.zipWithIndex.map {
case (s, i) =>
- val schemaType = toSqlType(s)
+ val schemaType = toSqlTypeHelper(s, existingRecordNames)
// All fields are nullable because only one of them is set at a time
StructField(s"member$i", schemaType.dataType, nullable = true)
}
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroFunctionsSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroFunctionsSuite.scala
index 90a4cd6ccf9dd..399b6743318a7 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroFunctionsSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroFunctionsSuite.scala
@@ -17,10 +17,16 @@
package org.apache.spark.sql.avro
+import java.io.ByteArrayOutputStream
+
import org.apache.avro.Schema
+import org.apache.avro.generic.{GenericDatumWriter, GenericRecord, GenericRecordBuilder}
+import org.apache.avro.io.EncoderFactory
-import org.apache.spark.sql.QueryTest
-import org.apache.spark.sql.functions.struct
+import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.execution.LocalTableScanExec
+import org.apache.spark.sql.functions.{col, struct}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSQLContext
class AvroFunctionsSuite extends QueryTest with SharedSQLContext {
@@ -80,4 +86,37 @@ class AvroFunctionsSuite extends QueryTest with SharedSQLContext {
.select(from_avro($"avro", avroTypeArrStruct).as("array"))
checkAnswer(dfOne, readBackOne)
}
+
+ test("SPARK-27798: from_avro produces same value when converted to local relation") {
+ val simpleSchema =
+ """
+ |{
+ | "type": "record",
+ | "name" : "Payload",
+ | "fields" : [ {"name" : "message", "type" : "string" } ]
+ |}
+ """.stripMargin
+
+ def generateBinary(message: String, avroSchema: String): Array[Byte] = {
+ val schema = new Schema.Parser().parse(avroSchema)
+ val out = new ByteArrayOutputStream()
+ val writer = new GenericDatumWriter[GenericRecord](schema)
+ val encoder = EncoderFactory.get().binaryEncoder(out, null)
+ val rootRecord = new GenericRecordBuilder(schema).set("message", message).build()
+ writer.write(rootRecord, encoder)
+ encoder.flush()
+ out.toByteArray
+ }
+
+ // This bug is hit when the rule `ConvertToLocalRelation` is run. But the rule was excluded
+ // in `SharedSparkSession`.
+ withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> "") {
+ val df = Seq("one", "two", "three", "four").map(generateBinary(_, simpleSchema))
+ .toDF()
+ .withColumn("value", from_avro(col("value"), simpleSchema))
+
+ assert(df.queryExecution.executedPlan.isInstanceOf[LocalTableScanExec])
+ assert(df.collect().map(_.get(0)) === Seq(Row("one"), Row("two"), Row("three"), Row("four")))
+ }
+ }
}
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index 9ad4388414eaa..d8e5297665eb6 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -266,6 +266,32 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils {
}
}
+ test("SPARK-27858 Union type: More than one non-null type") {
+ withTempDir { dir =>
+ val complexNullUnionType = Schema.createUnion(
+ List(Schema.create(Type.INT), Schema.create(Type.NULL), Schema.create(Type.STRING)).asJava)
+ val fields = Seq(
+ new Field("field1", complexNullUnionType, "doc", null.asInstanceOf[AnyVal])).asJava
+ val schema = Schema.createRecord("name", "docs", "namespace", false)
+ schema.setFields(fields)
+ val datumWriter = new GenericDatumWriter[GenericRecord](schema)
+ val dataFileWriter = new DataFileWriter[GenericRecord](datumWriter)
+ dataFileWriter.create(schema, new File(s"$dir.avro"))
+ val avroRec = new GenericData.Record(schema)
+ avroRec.put("field1", 42)
+ dataFileWriter.append(avroRec)
+ val avroRec2 = new GenericData.Record(schema)
+ avroRec2.put("field1", "Alice")
+ dataFileWriter.append(avroRec2)
+ dataFileWriter.flush()
+ dataFileWriter.close()
+
+ val df = spark.read.format("avro").load(s"$dir.avro")
+ assert(df.schema === StructType.fromDDL("field1 struct"))
+ assert(df.collect().toSet == Set(Row(Row(42, null)), Row(Row(null, "Alice"))))
+ }
+ }
+
test("Complex Union Type") {
withTempPath { dir =>
val fixedSchema = Schema.createFixed("fixed_name", "doc", "namespace", 4)
@@ -1266,4 +1292,69 @@ class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils {
checkCodec(df, path, "xz")
}
}
+
+ private def checkSchemaWithRecursiveLoop(avroSchema: String): Unit = {
+ val message = intercept[IncompatibleSchemaException] {
+ SchemaConverters.toSqlType(new Schema.Parser().parse(avroSchema))
+ }.getMessage
+
+ assert(message.contains("Found recursive reference in Avro schema"))
+ }
+
+ test("Detect recursive loop") {
+ checkSchemaWithRecursiveLoop("""
+ |{
+ | "type": "record",
+ | "name": "LongList",
+ | "fields" : [
+ | {"name": "value", "type": "long"}, // each element has a long
+ | {"name": "next", "type": ["null", "LongList"]} // optional next element
+ | ]
+ |}
+ """.stripMargin)
+
+ checkSchemaWithRecursiveLoop("""
+ |{
+ | "type": "record",
+ | "name": "LongList",
+ | "fields": [
+ | {
+ | "name": "value",
+ | "type": {
+ | "type": "record",
+ | "name": "foo",
+ | "fields": [
+ | {
+ | "name": "parent",
+ | "type": "LongList"
+ | }
+ | ]
+ | }
+ | }
+ | ]
+ |}
+ """.stripMargin)
+
+ checkSchemaWithRecursiveLoop("""
+ |{
+ | "type": "record",
+ | "name": "LongList",
+ | "fields" : [
+ | {"name": "value", "type": "long"},
+ | {"name": "array", "type": {"type": "array", "items": "LongList"}}
+ | ]
+ |}
+ """.stripMargin)
+
+ checkSchemaWithRecursiveLoop("""
+ |{
+ | "type": "record",
+ | "name": "LongList",
+ | "fields" : [
+ | {"name": "value", "type": "long"},
+ | {"name": "map", "type": {"type": "map", "values": "LongList"}}
+ | ]
+ |}
+ """.stripMargin)
+ }
}
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/execution/benchmark/AvroWriteBenchmark.scala b/external/avro/src/test/scala/org/apache/spark/sql/execution/benchmark/AvroWriteBenchmark.scala
new file mode 100644
index 0000000000000..df13b4a1c2d3a
--- /dev/null
+++ b/external/avro/src/test/scala/org/apache/spark/sql/execution/benchmark/AvroWriteBenchmark.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.benchmark
+
+/**
+ * Benchmark to measure Avro data sources write performance.
+ * Usage:
+ * 1. with spark-submit: bin/spark-submit --class
+ * 2. with sbt: build/sbt "avro/test:runMain "
+ */
+object AvroWriteBenchmark extends DataSourceWriteBenchmark {
+ def main(args: Array[String]): Unit = {
+ /*
+ Intel(R) Core(TM) i7-6920HQ CPU @ 2.90GHz
+ Avro writer benchmark: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
+ ------------------------------------------------------------------------------------------------
+ Output Single Int Column 2481 / 2499 6.3 157.8 1.0X
+ Output Single Double Column 2705 / 2710 5.8 172.0 0.9X
+ Output Int and String Column 5539 / 5639 2.8 352.2 0.4X
+ Output Partitions 4613 / 5004 3.4 293.3 0.5X
+ Output Buckets 5554 / 5561 2.8 353.1 0.4X
+ */
+ runBenchmark("Avro")
+ }
+}
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index 431339d412194..7e39950866805 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
@@ -150,5 +150,11 @@
10.5.0.5
jar
+
+ com.microsoft.sqlserver
+ mssql-jdbc
+ 7.2.1.jre8
+ test
+
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
index f5930bc281e8c..32e56f03ee521 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
@@ -158,4 +158,30 @@ class DB2IntegrationSuite extends DockerJDBCIntegrationSuite {
assert(rows(0).getInt(1) == 20)
assert(rows(0).getString(2) == "1")
}
+
+ test("query JDBC option") {
+ val expectedResult = Set(
+ (42, "fred"),
+ (17, "dave")
+ ).map { case (x, y) =>
+ Row(Integer.valueOf(x), String.valueOf(y))
+ }
+
+ val query = "SELECT x, y FROM tbl WHERE x > 10"
+ // query option to pass on the query string.
+ val df = spark.read.format("jdbc")
+ .option("url", jdbcUrl)
+ .option("query", query)
+ .load()
+ assert(df.collect.toSet === expectedResult)
+
+ // query option in the create table path.
+ sql(
+ s"""
+ |CREATE OR REPLACE TEMPORARY VIEW queryOption
+ |USING org.apache.spark.sql.jdbc
+ |OPTIONS (url '$jdbcUrl', query '$query')
+ """.stripMargin.replaceAll("\n", " "))
+ assert(sql("select x, y from queryOption").collect.toSet == expectedResult)
+ }
}
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
new file mode 100644
index 0000000000000..efd7ca74c796b
--- /dev/null
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc
+
+import java.math.BigDecimal
+import java.sql.{Connection, Date, Timestamp}
+import java.util.Properties
+
+import org.apache.spark.tags.DockerTest
+
+@DockerTest
+class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite {
+ override val db = new DatabaseOnDocker {
+ override val imageName = "mcr.microsoft.com/mssql/server:2017-GA-ubuntu"
+ override val env = Map(
+ "SA_PASSWORD" -> "Sapass123",
+ "ACCEPT_EULA" -> "Y"
+ )
+ override val usesIpc = false
+ override val jdbcPort: Int = 1433
+
+ override def getJdbcUrl(ip: String, port: Int): String =
+ s"jdbc:sqlserver://$ip:$port;user=sa;password=Sapass123;"
+
+ override def getStartupProcessName: Option[String] = None
+ }
+
+ override def dataPreparation(conn: Connection): Unit = {
+ conn.prepareStatement("CREATE TABLE tbl (x INT, y VARCHAR (50))").executeUpdate()
+ conn.prepareStatement("INSERT INTO tbl VALUES (42,'fred')").executeUpdate()
+ conn.prepareStatement("INSERT INTO tbl VALUES (17,'dave')").executeUpdate()
+
+ conn.prepareStatement(
+ """
+ |CREATE TABLE numbers (
+ |a BIT,
+ |b TINYINT, c SMALLINT, d INT, e BIGINT,
+ |f FLOAT, f1 FLOAT(24),
+ |g REAL,
+ |h DECIMAL(5,2), i NUMERIC(10,5),
+ |j MONEY, k SMALLMONEY)
+ """.stripMargin).executeUpdate()
+ conn.prepareStatement(
+ """
+ |INSERT INTO numbers VALUES (
+ |0,
+ |255, 32767, 2147483647, 9223372036854775807,
+ |123456789012345.123456789012345, 123456789012345.123456789012345,
+ |123456789012345.123456789012345,
+ |123, 12345.12,
+ |922337203685477.58, 214748.3647)
+ """.stripMargin).executeUpdate()
+
+ conn.prepareStatement(
+ """
+ |CREATE TABLE dates (
+ |a DATE, b DATETIME, c DATETIME2,
+ |d DATETIMEOFFSET, e SMALLDATETIME,
+ |f TIME)
+ """.stripMargin).executeUpdate()
+ conn.prepareStatement(
+ """
+ |INSERT INTO dates VALUES (
+ |'1991-11-09', '1999-01-01 13:23:35', '9999-12-31 23:59:59',
+ |'1901-05-09 23:59:59 +14:00', '1996-01-01 23:23:45',
+ |'13:31:24')
+ """.stripMargin).executeUpdate()
+
+ conn.prepareStatement(
+ """
+ |CREATE TABLE strings (
+ |a CHAR(10), b VARCHAR(10),
+ |c NCHAR(10), d NVARCHAR(10),
+ |e BINARY(4), f VARBINARY(4),
+ |g TEXT, h NTEXT,
+ |i IMAGE)
+ """.stripMargin).executeUpdate()
+ conn.prepareStatement(
+ """
+ |INSERT INTO strings VALUES (
+ |'the', 'quick',
+ |'brown', 'fox',
+ |123456, 123456,
+ |'the', 'lazy',
+ |'dog')
+ """.stripMargin).executeUpdate()
+ }
+
+ test("Basic test") {
+ val df = spark.read.jdbc(jdbcUrl, "tbl", new Properties)
+ val rows = df.collect()
+ assert(rows.length == 2)
+ val types = rows(0).toSeq.map(x => x.getClass.toString)
+ assert(types.length == 2)
+ assert(types(0).equals("class java.lang.Integer"))
+ assert(types(1).equals("class java.lang.String"))
+ }
+
+ test("Numeric types") {
+ val df = spark.read.jdbc(jdbcUrl, "numbers", new Properties)
+ val rows = df.collect()
+ assert(rows.length == 1)
+ val row = rows(0)
+ val types = row.toSeq.map(x => x.getClass.toString)
+ assert(types.length == 12)
+ assert(types(0).equals("class java.lang.Boolean"))
+ assert(types(1).equals("class java.lang.Integer"))
+ assert(types(2).equals("class java.lang.Short"))
+ assert(types(3).equals("class java.lang.Integer"))
+ assert(types(4).equals("class java.lang.Long"))
+ assert(types(5).equals("class java.lang.Double"))
+ assert(types(6).equals("class java.lang.Float"))
+ assert(types(7).equals("class java.lang.Float"))
+ assert(types(8).equals("class java.math.BigDecimal"))
+ assert(types(9).equals("class java.math.BigDecimal"))
+ assert(types(10).equals("class java.math.BigDecimal"))
+ assert(types(11).equals("class java.math.BigDecimal"))
+ assert(row.getBoolean(0) == false)
+ assert(row.getInt(1) == 255)
+ assert(row.getShort(2) == 32767)
+ assert(row.getInt(3) == 2147483647)
+ assert(row.getLong(4) == 9223372036854775807L)
+ assert(row.getDouble(5) == 1.2345678901234512E14) // float = float(53) has 15-digits precision
+ assert(row.getFloat(6) == 1.23456788103168E14) // float(24) has 7-digits precision
+ assert(row.getFloat(7) == 1.23456788103168E14) // real = float(24)
+ assert(row.getAs[BigDecimal](8).equals(new BigDecimal("123.00")))
+ assert(row.getAs[BigDecimal](9).equals(new BigDecimal("12345.12000")))
+ assert(row.getAs[BigDecimal](10).equals(new BigDecimal("922337203685477.5800")))
+ assert(row.getAs[BigDecimal](11).equals(new BigDecimal("214748.3647")))
+ }
+
+ test("Date types") {
+ val df = spark.read.jdbc(jdbcUrl, "dates", new Properties)
+ val rows = df.collect()
+ assert(rows.length == 1)
+ val row = rows(0)
+ val types = row.toSeq.map(x => x.getClass.toString)
+ assert(types.length == 6)
+ assert(types(0).equals("class java.sql.Date"))
+ assert(types(1).equals("class java.sql.Timestamp"))
+ assert(types(2).equals("class java.sql.Timestamp"))
+ assert(types(3).equals("class java.lang.String"))
+ assert(types(4).equals("class java.sql.Timestamp"))
+ assert(types(5).equals("class java.sql.Timestamp"))
+ assert(row.getAs[Date](0).equals(Date.valueOf("1991-11-09")))
+ assert(row.getAs[Timestamp](1).equals(Timestamp.valueOf("1999-01-01 13:23:35.0")))
+ assert(row.getAs[Timestamp](2).equals(Timestamp.valueOf("9999-12-31 23:59:59.0")))
+ assert(row.getString(3).equals("1901-05-09 23:59:59.0000000 +14:00"))
+ assert(row.getAs[Timestamp](4).equals(Timestamp.valueOf("1996-01-01 23:24:00.0")))
+ assert(row.getAs[Timestamp](5).equals(Timestamp.valueOf("1900-01-01 13:31:24.0")))
+ }
+
+ test("String types") {
+ val df = spark.read.jdbc(jdbcUrl, "strings", new Properties)
+ val rows = df.collect()
+ assert(rows.length == 1)
+ val row = rows(0)
+ val types = row.toSeq.map(x => x.getClass.toString)
+ assert(types.length == 9)
+ assert(types(0).equals("class java.lang.String"))
+ assert(types(1).equals("class java.lang.String"))
+ assert(types(2).equals("class java.lang.String"))
+ assert(types(3).equals("class java.lang.String"))
+ assert(types(4).equals("class [B"))
+ assert(types(5).equals("class [B"))
+ assert(types(6).equals("class java.lang.String"))
+ assert(types(7).equals("class java.lang.String"))
+ assert(types(8).equals("class [B"))
+ assert(row.getString(0).length == 10)
+ assert(row.getString(0).trim.equals("the"))
+ assert(row.getString(1).equals("quick"))
+ assert(row.getString(2).length == 10)
+ assert(row.getString(2).trim.equals("brown"))
+ assert(row.getString(3).equals("fox"))
+ assert(java.util.Arrays.equals(row.getAs[Array[Byte]](4), Array[Byte](0, 1, -30, 64)))
+ assert(java.util.Arrays.equals(row.getAs[Array[Byte]](5), Array[Byte](0, 1, -30, 64)))
+ assert(row.getString(6).equals("the"))
+ assert(row.getString(7).equals("lazy"))
+ assert(java.util.Arrays.equals(row.getAs[Array[Byte]](8), Array[Byte](100, 111, 103)))
+ }
+
+ test("Basic write test") {
+ val df1 = spark.read.jdbc(jdbcUrl, "numbers", new Properties)
+ val df2 = spark.read.jdbc(jdbcUrl, "dates", new Properties)
+ val df3 = spark.read.jdbc(jdbcUrl, "strings", new Properties)
+ df1.write.jdbc(jdbcUrl, "numberscopy", new Properties)
+ df2.write.jdbc(jdbcUrl, "datescopy", new Properties)
+ df3.write.jdbc(jdbcUrl, "stringscopy", new Properties)
+ }
+}
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
index a70ed98b52d5d..9cd5c4ec41a52 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
@@ -21,6 +21,7 @@ import java.math.BigDecimal
import java.sql.{Connection, Date, Timestamp}
import java.util.Properties
+import org.apache.spark.sql.{Row, SaveMode}
import org.apache.spark.tags.DockerTest
@DockerTest
@@ -152,4 +153,30 @@ class MySQLIntegrationSuite extends DockerJDBCIntegrationSuite {
df2.write.jdbc(jdbcUrl, "datescopy", new Properties)
df3.write.jdbc(jdbcUrl, "stringscopy", new Properties)
}
+
+ test("query JDBC option") {
+ val expectedResult = Set(
+ (42, "fred"),
+ (17, "dave")
+ ).map { case (x, y) =>
+ Row(Integer.valueOf(x), String.valueOf(y))
+ }
+
+ val query = "SELECT x, y FROM tbl WHERE x > 10"
+ // query option to pass on the query string.
+ val df = spark.read.format("jdbc")
+ .option("url", jdbcUrl)
+ .option("query", query)
+ .load()
+ assert(df.collect.toSet === expectedResult)
+
+ // query option in the create table path.
+ sql(
+ s"""
+ |CREATE OR REPLACE TEMPORARY VIEW queryOption
+ |USING org.apache.spark.sql.jdbc
+ |OPTIONS (url '$jdbcUrl', query '$query')
+ """.stripMargin.replaceAll("\n", " "))
+ assert(sql("select x, y from queryOption").collect.toSet == expectedResult)
+ }
}
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
index 09a2cd83aed6b..540de0fcad36f 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
@@ -442,6 +442,12 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSQLCo
.option("lowerBound", "2018-07-06")
.option("upperBound", "2018-07-20")
.option("numPartitions", 3)
+ // oracle.jdbc.mapDateToTimestamp defaults to true. If this flag is not disabled, column d
+ // (Oracle DATE) will be resolved as Catalyst Timestamp, which will fail bound evaluation of
+ // the partition column. E.g. 2018-07-06 cannot be evaluated as Timestamp, and the error
+ // message says: Timestamp format must be yyyy-mm-dd hh:mm:ss[.fffffffff].
+ .option("oracle.jdbc.mapDateToTimestamp", "false")
+ .option("sessionInitStatement", "ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD'")
.load()
df1.logicalPlan match {
@@ -462,6 +468,9 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSQLCo
.option("lowerBound", "2018-07-04 03:30:00.0")
.option("upperBound", "2018-07-27 14:11:05.0")
.option("numPartitions", 2)
+ .option("oracle.jdbc.mapDateToTimestamp", "false")
+ .option("sessionInitStatement",
+ "ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF'")
.load()
df2.logicalPlan match {
@@ -473,4 +482,32 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSQLCo
}
assert(df2.collect.toSet === expectedResult)
}
+
+ test("query JDBC option") {
+ val expectedResult = Set(
+ (1, "1991-11-09", "1996-01-01 01:23:45")
+ ).map { case (id, date, timestamp) =>
+ Row(BigDecimal.valueOf(id), Date.valueOf(date), Timestamp.valueOf(timestamp))
+ }
+
+ val query = "SELECT id, d, t FROM datetime WHERE id = 1"
+ // query option to pass on the query string.
+ val df = spark.read.format("jdbc")
+ .option("url", jdbcUrl)
+ .option("query", query)
+ .option("oracle.jdbc.mapDateToTimestamp", "false")
+ .load()
+ assert(df.collect.toSet === expectedResult)
+
+ // query option in the create table path.
+ sql(
+ s"""
+ |CREATE OR REPLACE TEMPORARY VIEW queryOption
+ |USING org.apache.spark.sql.jdbc
+ |OPTIONS (url '$jdbcUrl',
+ | query '$query',
+ | oracle.jdbc.mapDateToTimestamp false)
+ """.stripMargin.replaceAll("\n", " "))
+ assert(sql("select id, d, t from queryOption").collect.toSet == expectedResult)
+ }
}
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
index be32cb89f4886..7caf3d6ba59f5 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
@@ -21,6 +21,7 @@ import java.sql.Connection
import java.util.Properties
import org.apache.spark.sql.Column
+import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.expressions.Literal
import org.apache.spark.sql.types.{ArrayType, DecimalType, FloatType, ShortType}
import org.apache.spark.tags.DockerTest
@@ -46,14 +47,15 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
conn.prepareStatement("CREATE TABLE bar (c0 text, c1 integer, c2 double precision, c3 bigint, "
+ "c4 bit(1), c5 bit(10), c6 bytea, c7 boolean, c8 inet, c9 cidr, "
+ "c10 integer[], c11 text[], c12 real[], c13 numeric(2,2)[], c14 enum_type, "
- + "c15 float4, c16 smallint)").executeUpdate()
+ + "c15 float4, c16 smallint, c17 numeric[])").executeUpdate()
conn.prepareStatement("INSERT INTO bar VALUES ('hello', 42, 1.25, 123456789012345, B'0', "
+ "B'1000100101', E'\\\\xDEADBEEF', true, '172.16.0.42', '192.168.0.0/16', "
- + """'{1, 2}', '{"a", null, "b"}', '{0.11, 0.22}', '{0.11, 0.22}', 'd1', 1.01, 1)"""
+ + """'{1, 2}', '{"a", null, "b"}', '{0.11, 0.22}', '{0.11, 0.22}', 'd1', 1.01, 1, """
+ + "'{111.2222, 333.4444}')"
).executeUpdate()
conn.prepareStatement("INSERT INTO bar VALUES (null, null, null, null, null, "
+ "null, null, null, null, null, "
- + "null, null, null, null, null, null, null)"
+ + "null, null, null, null, null, null, null, null)"
).executeUpdate()
conn.prepareStatement("CREATE TABLE ts_with_timezone " +
@@ -85,7 +87,7 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
assert(rows.length == 2)
// Test the types, and values using the first row.
val types = rows(0).toSeq.map(x => x.getClass)
- assert(types.length == 17)
+ assert(types.length == 18)
assert(classOf[String].isAssignableFrom(types(0)))
assert(classOf[java.lang.Integer].isAssignableFrom(types(1)))
assert(classOf[java.lang.Double].isAssignableFrom(types(2)))
@@ -103,6 +105,7 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
assert(classOf[String].isAssignableFrom(types(14)))
assert(classOf[java.lang.Float].isAssignableFrom(types(15)))
assert(classOf[java.lang.Short].isAssignableFrom(types(16)))
+ assert(classOf[Seq[BigDecimal]].isAssignableFrom(types(17)))
assert(rows(0).getString(0).equals("hello"))
assert(rows(0).getInt(1) == 42)
assert(rows(0).getDouble(2) == 1.25)
@@ -123,6 +126,8 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
assert(rows(0).getString(14) == "d1")
assert(rows(0).getFloat(15) == 1.01f)
assert(rows(0).getShort(16) == 1)
+ assert(rows(0).getSeq(17) ==
+ Seq("111.222200000000000000", "333.444400000000000000").map(BigDecimal(_).bigDecimal))
// Test reading null values using the second row.
assert(0.until(16).forall(rows(1).isNullAt(_)))
@@ -176,4 +181,29 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
assert(rows(0).getSeq(8) == Seq("""{"a": "foo", "b": "bar"}""", """{"a": 1, "b": 2}"""))
assert(rows(0).getSeq(9) == Seq("""{"a": 1, "b": 2, "c": 3}"""))
}
+
+ test("query JDBC option") {
+ val expectedResult = Set(
+ (42, 123456789012345L)
+ ).map { case (c1, c3) =>
+ Row(Integer.valueOf(c1), java.lang.Long.valueOf(c3))
+ }
+
+ val query = "SELECT c1, c3 FROM bar WHERE c1 IS NOT NULL"
+ // query option to pass on the query string.
+ val df = spark.read.format("jdbc")
+ .option("url", jdbcUrl)
+ .option("query", query)
+ .load()
+ assert(df.collect.toSet === expectedResult)
+
+ // query option in the create table path.
+ sql(
+ s"""
+ |CREATE OR REPLACE TEMPORARY VIEW queryOption
+ |USING org.apache.spark.sql.jdbc
+ |OPTIONS (url '$jdbcUrl', query '$query')
+ """.stripMargin.replaceAll("\n", " "))
+ assert(sql("select c1, c3 from queryOption").collect.toSet == expectedResult)
+ }
}
diff --git a/external/docker/spark-test/base/Dockerfile b/external/docker/spark-test/base/Dockerfile
index c70cd71367679..03caeb64b5343 100644
--- a/external/docker/spark-test/base/Dockerfile
+++ b/external/docker/spark-test/base/Dockerfile
@@ -32,6 +32,6 @@ ENV SPARK_HOME /opt/spark
ENV PATH $SPARK_HOME:$SCALA_HOME/bin:$PATH
# Install Scala
-ADD http://www.scala-lang.org/files/archive/scala-$SCALA_VERSION.tgz /
+ADD https://www.scala-lang.org/files/archive/scala-$SCALA_VERSION.tgz /
RUN (cd / && gunzip < scala-$SCALA_VERSION.tgz)|(cd /opt && tar -xvf -)
RUN rm /scala-$SCALA_VERSION.tgz
diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml
index 7cd1ec4c9c09a..bf7e4985ae8d2 100644
--- a/external/flume-assembly/pom.xml
+++ b/external/flume-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml
index f810aa80e8780..94afe25bbda13 100644
--- a/external/flume-sink/pom.xml
+++ b/external/flume-sink/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/external/flume/pom.xml b/external/flume/pom.xml
index 498e88f665eb5..1187a466dce75 100644
--- a/external/flume/pom.xml
+++ b/external/flume/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index f80f8e3a0183d..497a94f48251e 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 8588e8be052eb..1c8419221f5fb 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReadSupport.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala
similarity index 74%
rename from external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReadSupport.scala
rename to external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala
index 1753a28fba2fb..561d501359321 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReadSupport.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala
@@ -25,15 +25,16 @@ import org.apache.kafka.common.TopicPartition
import org.apache.spark.TaskContext
import org.apache.spark.internal.Logging
+import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.UnsafeRow
import org.apache.spark.sql.kafka010.KafkaSourceProvider.{INSTRUCTION_FOR_FAIL_ON_DATA_LOSS_FALSE, INSTRUCTION_FOR_FAIL_ON_DATA_LOSS_TRUE}
import org.apache.spark.sql.sources.v2.reader._
-import org.apache.spark.sql.sources.v2.reader.streaming._
+import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousInputPartitionReader, ContinuousReader, Offset, PartitionOffset}
import org.apache.spark.sql.types.StructType
/**
- * A [[ContinuousReadSupport]] for data from kafka.
+ * A [[ContinuousReader]] for data from kafka.
*
* @param offsetReader a reader used to get kafka offsets. Note that the actual data will be
* read by per-task consumers generated later.
@@ -46,49 +47,70 @@ import org.apache.spark.sql.types.StructType
* scenarios, where some offsets after the specified initial ones can't be
* properly read.
*/
-class KafkaContinuousReadSupport(
+class KafkaContinuousReader(
offsetReader: KafkaOffsetReader,
kafkaParams: ju.Map[String, Object],
sourceOptions: Map[String, String],
metadataPath: String,
initialOffsets: KafkaOffsetRangeLimit,
failOnDataLoss: Boolean)
- extends ContinuousReadSupport with Logging {
+ extends ContinuousReader with Logging {
+
+ private lazy val session = SparkSession.getActiveSession.get
+ private lazy val sc = session.sparkContext
private val pollTimeoutMs = sourceOptions.getOrElse("kafkaConsumer.pollTimeoutMs", "512").toLong
- override def initialOffset(): Offset = {
- val offsets = initialOffsets match {
- case EarliestOffsetRangeLimit => KafkaSourceOffset(offsetReader.fetchEarliestOffsets())
- case LatestOffsetRangeLimit => KafkaSourceOffset(offsetReader.fetchLatestOffsets())
- case SpecificOffsetRangeLimit(p) => offsetReader.fetchSpecificOffsets(p, reportDataLoss)
- }
- logInfo(s"Initial offsets: $offsets")
- offsets
- }
+ // Initialized when creating reader factories. If this diverges from the partitions at the latest
+ // offsets, we need to reconfigure.
+ // Exposed outside this object only for unit tests.
+ @volatile private[sql] var knownPartitions: Set[TopicPartition] = _
- override def fullSchema(): StructType = KafkaOffsetReader.kafkaSchema
+ override def readSchema: StructType = KafkaOffsetReader.kafkaSchema
- override def newScanConfigBuilder(start: Offset): ScanConfigBuilder = {
- new KafkaContinuousScanConfigBuilder(fullSchema(), start, offsetReader, reportDataLoss)
+ private var offset: Offset = _
+ override def setStartOffset(start: ju.Optional[Offset]): Unit = {
+ offset = start.orElse {
+ val offsets = initialOffsets match {
+ case EarliestOffsetRangeLimit => KafkaSourceOffset(offsetReader.fetchEarliestOffsets())
+ case LatestOffsetRangeLimit => KafkaSourceOffset(offsetReader.fetchLatestOffsets(None))
+ case SpecificOffsetRangeLimit(p) => offsetReader.fetchSpecificOffsets(p, reportDataLoss)
+ }
+ logInfo(s"Initial offsets: $offsets")
+ offsets
+ }
}
+ override def getStartOffset(): Offset = offset
+
override def deserializeOffset(json: String): Offset = {
KafkaSourceOffset(JsonUtils.partitionOffsets(json))
}
- override def planInputPartitions(config: ScanConfig): Array[InputPartition] = {
- val startOffsets = config.asInstanceOf[KafkaContinuousScanConfig].startOffsets
+ override def planInputPartitions(): ju.List[InputPartition[InternalRow]] = {
+ import scala.collection.JavaConverters._
+
+ val oldStartPartitionOffsets = KafkaSourceOffset.getPartitionOffsets(offset)
+
+ val currentPartitionSet = offsetReader.fetchEarliestOffsets().keySet
+ val newPartitions = currentPartitionSet.diff(oldStartPartitionOffsets.keySet)
+ val newPartitionOffsets = offsetReader.fetchEarliestOffsets(newPartitions.toSeq)
+
+ val deletedPartitions = oldStartPartitionOffsets.keySet.diff(currentPartitionSet)
+ if (deletedPartitions.nonEmpty) {
+ reportDataLoss(s"Some partitions were deleted: $deletedPartitions")
+ }
+
+ val startOffsets = newPartitionOffsets ++
+ oldStartPartitionOffsets.filterKeys(!deletedPartitions.contains(_))
+ knownPartitions = startOffsets.keySet
+
startOffsets.toSeq.map {
case (topicPartition, start) =>
KafkaContinuousInputPartition(
- topicPartition, start, kafkaParams, pollTimeoutMs, failOnDataLoss)
- }.toArray
- }
-
- override def createContinuousReaderFactory(
- config: ScanConfig): ContinuousPartitionReaderFactory = {
- KafkaContinuousReaderFactory
+ topicPartition, start, kafkaParams, pollTimeoutMs, failOnDataLoss
+ ): InputPartition[InternalRow]
+ }.asJava
}
/** Stop this source and free any resources it has allocated. */
@@ -105,9 +127,8 @@ class KafkaContinuousReadSupport(
KafkaSourceOffset(mergedMap)
}
- override def needsReconfiguration(config: ScanConfig): Boolean = {
- val knownPartitions = config.asInstanceOf[KafkaContinuousScanConfig].knownPartitions
- offsetReader.fetchLatestOffsets().keySet != knownPartitions
+ override def needsReconfiguration(): Boolean = {
+ knownPartitions != null && offsetReader.fetchLatestOffsets(None).keySet != knownPartitions
}
override def toString(): String = s"KafkaSource[$offsetReader]"
@@ -141,51 +162,23 @@ case class KafkaContinuousInputPartition(
startOffset: Long,
kafkaParams: ju.Map[String, Object],
pollTimeoutMs: Long,
- failOnDataLoss: Boolean) extends InputPartition
-
-object KafkaContinuousReaderFactory extends ContinuousPartitionReaderFactory {
- override def createReader(partition: InputPartition): ContinuousPartitionReader[InternalRow] = {
- val p = partition.asInstanceOf[KafkaContinuousInputPartition]
- new KafkaContinuousPartitionReader(
- p.topicPartition, p.startOffset, p.kafkaParams, p.pollTimeoutMs, p.failOnDataLoss)
+ failOnDataLoss: Boolean) extends ContinuousInputPartition[InternalRow] {
+
+ override def createContinuousReader(
+ offset: PartitionOffset): InputPartitionReader[InternalRow] = {
+ val kafkaOffset = offset.asInstanceOf[KafkaSourcePartitionOffset]
+ require(kafkaOffset.topicPartition == topicPartition,
+ s"Expected topicPartition: $topicPartition, but got: ${kafkaOffset.topicPartition}")
+ new KafkaContinuousInputPartitionReader(
+ topicPartition, kafkaOffset.partitionOffset, kafkaParams, pollTimeoutMs, failOnDataLoss)
}
-}
-
-class KafkaContinuousScanConfigBuilder(
- schema: StructType,
- startOffset: Offset,
- offsetReader: KafkaOffsetReader,
- reportDataLoss: String => Unit)
- extends ScanConfigBuilder {
-
- override def build(): ScanConfig = {
- val oldStartPartitionOffsets = KafkaSourceOffset.getPartitionOffsets(startOffset)
-
- val currentPartitionSet = offsetReader.fetchEarliestOffsets().keySet
- val newPartitions = currentPartitionSet.diff(oldStartPartitionOffsets.keySet)
- val newPartitionOffsets = offsetReader.fetchEarliestOffsets(newPartitions.toSeq)
- val deletedPartitions = oldStartPartitionOffsets.keySet.diff(currentPartitionSet)
- if (deletedPartitions.nonEmpty) {
- reportDataLoss(s"Some partitions were deleted: $deletedPartitions")
- }
-
- val startOffsets = newPartitionOffsets ++
- oldStartPartitionOffsets.filterKeys(!deletedPartitions.contains(_))
- KafkaContinuousScanConfig(schema, startOffsets)
+ override def createPartitionReader(): KafkaContinuousInputPartitionReader = {
+ new KafkaContinuousInputPartitionReader(
+ topicPartition, startOffset, kafkaParams, pollTimeoutMs, failOnDataLoss)
}
}
-case class KafkaContinuousScanConfig(
- readSchema: StructType,
- startOffsets: Map[TopicPartition, Long])
- extends ScanConfig {
-
- // Created when building the scan config builder. If this diverges from the partitions at the
- // latest offsets, we need to reconfigure the kafka read support.
- def knownPartitions: Set[TopicPartition] = startOffsets.keySet
-}
-
/**
* A per-task data reader for continuous Kafka processing.
*
@@ -196,12 +189,12 @@ case class KafkaContinuousScanConfig(
* @param failOnDataLoss Flag indicating whether data reader should fail if some offsets
* are skipped.
*/
-class KafkaContinuousPartitionReader(
+class KafkaContinuousInputPartitionReader(
topicPartition: TopicPartition,
startOffset: Long,
kafkaParams: ju.Map[String, Object],
pollTimeoutMs: Long,
- failOnDataLoss: Boolean) extends ContinuousPartitionReader[InternalRow] {
+ failOnDataLoss: Boolean) extends ContinuousInputPartitionReader[InternalRow] {
private val consumer = KafkaDataConsumer.acquire(topicPartition, kafkaParams, useCache = false)
private val converter = new KafkaRecordToUnsafeRowConverter
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaDataConsumer.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaDataConsumer.scala
index ceb9e318b283b..7b1314bc8c3c0 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaDataConsumer.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaDataConsumer.scala
@@ -134,6 +134,8 @@ private[kafka010] case class InternalKafkaConsumer(
/** Reset the internal pre-fetched data. */
def reset(): Unit = {
_records = ju.Collections.emptyListIterator()
+ _nextOffsetInFetchedData = UNKNOWN_OFFSET
+ _offsetAfterPoll = UNKNOWN_OFFSET
}
/**
@@ -361,8 +363,9 @@ private[kafka010] case class InternalKafkaConsumer(
if (offset < fetchedData.offsetAfterPoll) {
// Offsets in [offset, fetchedData.offsetAfterPoll) are invisible. Return a record to ask
// the next call to start from `fetchedData.offsetAfterPoll`.
+ val nextOffsetToFetch = fetchedData.offsetAfterPoll
fetchedData.reset()
- return fetchedRecord.withRecord(null, fetchedData.offsetAfterPoll)
+ return fetchedRecord.withRecord(null, nextOffsetToFetch)
} else {
// Fetch records from Kafka and update `fetchedData`.
fetchData(offset, pollTimeoutMs)
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchReadSupport.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchReader.scala
similarity index 79%
rename from external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchReadSupport.scala
rename to external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchReader.scala
index bb4de674c3c72..1333bc261020f 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchReadSupport.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchReader.scala
@@ -21,6 +21,8 @@ import java.{util => ju}
import java.io._
import java.nio.charset.StandardCharsets
+import scala.collection.JavaConverters._
+
import org.apache.commons.io.IOUtils
import org.apache.spark.SparkEnv
@@ -29,17 +31,16 @@ import org.apache.spark.scheduler.ExecutorCacheTaskLocation
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.UnsafeRow
-import org.apache.spark.sql.execution.streaming.{HDFSMetadataLog, SerializedOffset, SimpleStreamingScanConfig, SimpleStreamingScanConfigBuilder}
-import org.apache.spark.sql.execution.streaming.sources.RateControlMicroBatchReadSupport
+import org.apache.spark.sql.execution.streaming.{HDFSMetadataLog, SerializedOffset}
import org.apache.spark.sql.kafka010.KafkaSourceProvider.{INSTRUCTION_FOR_FAIL_ON_DATA_LOSS_FALSE, INSTRUCTION_FOR_FAIL_ON_DATA_LOSS_TRUE}
import org.apache.spark.sql.sources.v2.DataSourceOptions
-import org.apache.spark.sql.sources.v2.reader._
-import org.apache.spark.sql.sources.v2.reader.streaming.{MicroBatchReadSupport, Offset}
+import org.apache.spark.sql.sources.v2.reader.{InputPartition, InputPartitionReader}
+import org.apache.spark.sql.sources.v2.reader.streaming.{MicroBatchReader, Offset}
import org.apache.spark.sql.types.StructType
import org.apache.spark.util.UninterruptibleThread
/**
- * A [[MicroBatchReadSupport]] that reads data from Kafka.
+ * A [[MicroBatchReader]] that reads data from Kafka.
*
* The [[KafkaSourceOffset]] is the custom [[Offset]] defined for this source that contains
* a map of TopicPartition -> offset. Note that this offset is 1 + (available offset). For
@@ -54,13 +55,17 @@ import org.apache.spark.util.UninterruptibleThread
* To avoid this issue, you should make sure stopping the query before stopping the Kafka brokers
* and not use wrong broker addresses.
*/
-private[kafka010] class KafkaMicroBatchReadSupport(
+private[kafka010] class KafkaMicroBatchReader(
kafkaOffsetReader: KafkaOffsetReader,
executorKafkaParams: ju.Map[String, Object],
options: DataSourceOptions,
metadataPath: String,
startingOffsets: KafkaOffsetRangeLimit,
- failOnDataLoss: Boolean) extends RateControlMicroBatchReadSupport with Logging {
+ failOnDataLoss: Boolean)
+ extends MicroBatchReader with Logging {
+
+ private var startPartitionOffsets: PartitionOffsetMap = _
+ private var endPartitionOffsets: PartitionOffsetMap = _
private val pollTimeoutMs = options.getLong(
"kafkaConsumer.pollTimeoutMs",
@@ -70,40 +75,35 @@ private[kafka010] class KafkaMicroBatchReadSupport(
Option(options.get("maxOffsetsPerTrigger").orElse(null)).map(_.toLong)
private val rangeCalculator = KafkaOffsetRangeCalculator(options)
-
- private var endPartitionOffsets: KafkaSourceOffset = _
-
/**
* Lazily initialize `initialPartitionOffsets` to make sure that `KafkaConsumer.poll` is only
* called in StreamExecutionThread. Otherwise, interrupting a thread while running
* `KafkaConsumer.poll` may hang forever (KAFKA-1894).
*/
- override def initialOffset(): Offset = {
- KafkaSourceOffset(getOrCreateInitialPartitionOffsets())
- }
-
- override def latestOffset(start: Offset): Offset = {
- val startPartitionOffsets = start.asInstanceOf[KafkaSourceOffset].partitionToOffsets
- val latestPartitionOffsets = kafkaOffsetReader.fetchLatestOffsets()
- endPartitionOffsets = KafkaSourceOffset(maxOffsetsPerTrigger.map { maxOffsets =>
- rateLimit(maxOffsets, startPartitionOffsets, latestPartitionOffsets)
- }.getOrElse {
- latestPartitionOffsets
- })
- endPartitionOffsets
- }
-
- override def fullSchema(): StructType = KafkaOffsetReader.kafkaSchema
-
- override def newScanConfigBuilder(start: Offset, end: Offset): ScanConfigBuilder = {
- new SimpleStreamingScanConfigBuilder(fullSchema(), start, Some(end))
+ private lazy val initialPartitionOffsets = getOrCreateInitialPartitionOffsets()
+
+ override def setOffsetRange(start: ju.Optional[Offset], end: ju.Optional[Offset]): Unit = {
+ // Make sure initialPartitionOffsets is initialized
+ initialPartitionOffsets
+
+ startPartitionOffsets = Option(start.orElse(null))
+ .map(_.asInstanceOf[KafkaSourceOffset].partitionToOffsets)
+ .getOrElse(initialPartitionOffsets)
+
+ endPartitionOffsets = Option(end.orElse(null))
+ .map(_.asInstanceOf[KafkaSourceOffset].partitionToOffsets)
+ .getOrElse {
+ val latestPartitionOffsets =
+ kafkaOffsetReader.fetchLatestOffsets(Some(startPartitionOffsets))
+ maxOffsetsPerTrigger.map { maxOffsets =>
+ rateLimit(maxOffsets, startPartitionOffsets, latestPartitionOffsets)
+ }.getOrElse {
+ latestPartitionOffsets
+ }
+ }
}
- override def planInputPartitions(config: ScanConfig): Array[InputPartition] = {
- val sc = config.asInstanceOf[SimpleStreamingScanConfig]
- val startPartitionOffsets = sc.start.asInstanceOf[KafkaSourceOffset].partitionToOffsets
- val endPartitionOffsets = sc.end.get.asInstanceOf[KafkaSourceOffset].partitionToOffsets
-
+ override def planInputPartitions(): ju.List[InputPartition[InternalRow]] = {
// Find the new partitions, and get their earliest offsets
val newPartitions = endPartitionOffsets.keySet.diff(startPartitionOffsets.keySet)
val newPartitionInitialOffsets = kafkaOffsetReader.fetchEarliestOffsets(newPartitions.toSeq)
@@ -133,10 +133,21 @@ private[kafka010] class KafkaMicroBatchReadSupport(
}.toSeq
logDebug("TopicPartitions: " + topicPartitions.mkString(", "))
+ val fromOffsets = startPartitionOffsets ++ newPartitionInitialOffsets
+ val untilOffsets = endPartitionOffsets
+ untilOffsets.foreach { case (tp, untilOffset) =>
+ fromOffsets.get(tp).foreach { fromOffset =>
+ if (untilOffset < fromOffset) {
+ reportDataLoss(s"Partition $tp's offset was changed from " +
+ s"$fromOffset to $untilOffset, some data may have been missed")
+ }
+ }
+ }
+
// Calculate offset ranges
val offsetRanges = rangeCalculator.getRanges(
- fromOffsets = startPartitionOffsets ++ newPartitionInitialOffsets,
- untilOffsets = endPartitionOffsets,
+ fromOffsets = fromOffsets,
+ untilOffsets = untilOffsets,
executorLocations = getSortedExecutorList())
// Reuse Kafka consumers only when all the offset ranges have distinct TopicPartitions,
@@ -145,19 +156,26 @@ private[kafka010] class KafkaMicroBatchReadSupport(
// Generate factories based on the offset ranges
offsetRanges.map { range =>
- KafkaMicroBatchInputPartition(
- range, executorKafkaParams, pollTimeoutMs, failOnDataLoss, reuseKafkaConsumer)
- }.toArray
+ new KafkaMicroBatchInputPartition(
+ range, executorKafkaParams, pollTimeoutMs, failOnDataLoss, reuseKafkaConsumer
+ ): InputPartition[InternalRow]
+ }.asJava
}
- override def createReaderFactory(config: ScanConfig): PartitionReaderFactory = {
- KafkaMicroBatchReaderFactory
+ override def getStartOffset: Offset = {
+ KafkaSourceOffset(startPartitionOffsets)
+ }
+
+ override def getEndOffset: Offset = {
+ KafkaSourceOffset(endPartitionOffsets)
}
override def deserializeOffset(json: String): Offset = {
KafkaSourceOffset(JsonUtils.partitionOffsets(json))
}
+ override def readSchema(): StructType = KafkaOffsetReader.kafkaSchema
+
override def commit(end: Offset): Unit = {}
override def stop(): Unit = {
@@ -186,7 +204,7 @@ private[kafka010] class KafkaMicroBatchReadSupport(
case EarliestOffsetRangeLimit =>
KafkaSourceOffset(kafkaOffsetReader.fetchEarliestOffsets())
case LatestOffsetRangeLimit =>
- KafkaSourceOffset(kafkaOffsetReader.fetchLatestOffsets())
+ KafkaSourceOffset(kafkaOffsetReader.fetchLatestOffsets(None))
case SpecificOffsetRangeLimit(p) =>
kafkaOffsetReader.fetchSpecificOffsets(p, reportDataLoss)
}
@@ -221,7 +239,15 @@ private[kafka010] class KafkaMicroBatchReadSupport(
val begin = from.get(tp).getOrElse(fromNew(tp))
val prorate = limit * (size / total)
// Don't completely starve small topicpartitions
- val off = begin + (if (prorate < 1) Math.ceil(prorate) else Math.floor(prorate)).toLong
+ val prorateLong = (if (prorate < 1) Math.ceil(prorate) else Math.floor(prorate)).toLong
+ // need to be careful of integer overflow
+ // therefore added canary checks where to see if off variable could be overflowed
+ // refer to [https://issues.apache.org/jira/browse/SPARK-26718]
+ val off = if (prorateLong > Long.MaxValue - begin) {
+ Long.MaxValue
+ } else {
+ begin + prorateLong
+ }
// Paranoia, make sure not to return an offset that's past end
Math.min(end, off)
}.getOrElse(end)
@@ -300,23 +326,22 @@ private[kafka010] case class KafkaMicroBatchInputPartition(
executorKafkaParams: ju.Map[String, Object],
pollTimeoutMs: Long,
failOnDataLoss: Boolean,
- reuseKafkaConsumer: Boolean) extends InputPartition
+ reuseKafkaConsumer: Boolean) extends InputPartition[InternalRow] {
-private[kafka010] object KafkaMicroBatchReaderFactory extends PartitionReaderFactory {
- override def createReader(partition: InputPartition): PartitionReader[InternalRow] = {
- val p = partition.asInstanceOf[KafkaMicroBatchInputPartition]
- KafkaMicroBatchPartitionReader(p.offsetRange, p.executorKafkaParams, p.pollTimeoutMs,
- p.failOnDataLoss, p.reuseKafkaConsumer)
- }
+ override def preferredLocations(): Array[String] = offsetRange.preferredLoc.toArray
+
+ override def createPartitionReader(): InputPartitionReader[InternalRow] =
+ new KafkaMicroBatchInputPartitionReader(offsetRange, executorKafkaParams, pollTimeoutMs,
+ failOnDataLoss, reuseKafkaConsumer)
}
-/** A [[PartitionReader]] for reading Kafka data in a micro-batch streaming query. */
-private[kafka010] case class KafkaMicroBatchPartitionReader(
+/** A [[InputPartitionReader]] for reading Kafka data in a micro-batch streaming query. */
+private[kafka010] case class KafkaMicroBatchInputPartitionReader(
offsetRange: KafkaOffsetRange,
executorKafkaParams: ju.Map[String, Object],
pollTimeoutMs: Long,
failOnDataLoss: Boolean,
- reuseKafkaConsumer: Boolean) extends PartitionReader[InternalRow] with Logging {
+ reuseKafkaConsumer: Boolean) extends InputPartitionReader[InternalRow] with Logging {
private val consumer = KafkaDataConsumer.acquire(
offsetRange.topicPartition, executorKafkaParams, reuseKafkaConsumer)
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculator.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculator.scala
index 6631ae84167c8..554a3c9bd02c2 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculator.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculator.scala
@@ -29,7 +29,6 @@ import org.apache.spark.sql.sources.v2.DataSourceOptions
private[kafka010] class KafkaOffsetRangeCalculator(val minPartitions: Option[Int]) {
require(minPartitions.isEmpty || minPartitions.get > 0)
- import KafkaOffsetRangeCalculator._
/**
* Calculate the offset ranges that we are going to process this batch. If `minPartitions`
* is not set or is set less than or equal the number of `topicPartitions` that we're going to
@@ -38,6 +37,8 @@ private[kafka010] class KafkaOffsetRangeCalculator(val minPartitions: Option[Int
* the read tasks of the skewed partitions to multiple Spark tasks.
* The number of Spark tasks will be *approximately* `numPartitions`. It can be less or more
* depending on rounding errors or Kafka partitions that didn't receive any new data.
+ *
+ * Empty ranges (`KafkaOffsetRange.size <= 0`) will be dropped.
*/
def getRanges(
fromOffsets: PartitionOffsetMap,
@@ -60,19 +61,23 @@ private[kafka010] class KafkaOffsetRangeCalculator(val minPartitions: Option[Int
// Splits offset ranges with relatively large amount of data to smaller ones.
val totalSize = offsetRanges.map(_.size).sum
- val idealRangeSize = totalSize.toDouble / minPartitions.get
-
offsetRanges.flatMap { range =>
- // Split the current range into subranges as close to the ideal range size
- val numSplitsInRange = math.round(range.size.toDouble / idealRangeSize).toInt
-
- (0 until numSplitsInRange).map { i =>
- val splitStart = range.fromOffset + range.size * (i.toDouble / numSplitsInRange)
- val splitEnd = range.fromOffset + range.size * ((i.toDouble + 1) / numSplitsInRange)
- KafkaOffsetRange(
- range.topicPartition, splitStart.toLong, splitEnd.toLong, preferredLoc = None)
+ val tp = range.topicPartition
+ val size = range.size
+ // number of partitions to divvy up this topic partition to
+ val parts = math.max(math.round(size.toDouble / totalSize * minPartitions.get), 1).toInt
+ var remaining = size
+ var startOffset = range.fromOffset
+ (0 until parts).map { part =>
+ // Fine to do integer division. Last partition will consume all the round off errors
+ val thisPartition = remaining / (parts - part)
+ remaining -= thisPartition
+ val endOffset = math.min(startOffset + thisPartition, range.untilOffset)
+ val offsetRange = KafkaOffsetRange(tp, startOffset, endOffset, None)
+ startOffset = endOffset
+ offsetRange
}
- }
+ }.filter(_.size > 0)
}
}
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReader.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReader.scala
index 82066697cb95a..fc443d22bf5a2 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReader.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReader.scala
@@ -21,6 +21,7 @@ import java.{util => ju}
import java.util.concurrent.{Executors, ThreadFactory}
import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
import scala.concurrent.{ExecutionContext, Future}
import scala.concurrent.duration.Duration
import scala.util.control.NonFatal
@@ -137,6 +138,12 @@ private[kafka010] class KafkaOffsetReader(
// Poll to get the latest assigned partitions
consumer.poll(0)
val partitions = consumer.assignment()
+
+ // Call `position` to wait until the potential offset request triggered by `poll(0)` is
+ // done. This is a workaround for KAFKA-7703, which an async `seekToBeginning` triggered by
+ // `poll(0)` may reset offsets that should have been set by another request.
+ partitions.asScala.map(p => p -> consumer.position(p)).foreach(_ => {})
+
consumer.pause(partitions)
assert(partitions.asScala == partitionOffsets.keySet,
"If startingOffsets contains specific offsets, you must specify all TopicPartitions.\n" +
@@ -192,19 +199,82 @@ private[kafka010] class KafkaOffsetReader(
/**
* Fetch the latest offsets for the topic partitions that are indicated
* in the [[ConsumerStrategy]].
+ *
+ * Kafka may return earliest offsets when we are requesting latest offsets if `poll` is called
+ * right before `seekToEnd` (KAFKA-7703). As a workaround, we will call `position` right after
+ * `poll` to wait until the potential offset request triggered by `poll(0)` is done.
+ *
+ * In addition, to avoid other unknown issues, we also use the given `knownOffsets` to audit the
+ * latest offsets returned by Kafka. If we find some incorrect offsets (a latest offset is less
+ * than an offset in `knownOffsets`), we will retry at most `maxOffsetFetchAttempts` times. When
+ * a topic is recreated, the latest offsets may be less than offsets in `knownOffsets`. We cannot
+ * distinguish this with KAFKA-7703, so we just return whatever we get from Kafka after retrying.
*/
- def fetchLatestOffsets(): Map[TopicPartition, Long] = runUninterruptibly {
+ def fetchLatestOffsets(
+ knownOffsets: Option[PartitionOffsetMap]): PartitionOffsetMap = runUninterruptibly {
withRetriesWithoutInterrupt {
// Poll to get the latest assigned partitions
consumer.poll(0)
val partitions = consumer.assignment()
+
+ // Call `position` to wait until the potential offset request triggered by `poll(0)` is
+ // done. This is a workaround for KAFKA-7703, which an async `seekToBeginning` triggered by
+ // `poll(0)` may reset offsets that should have been set by another request.
+ partitions.asScala.map(p => p -> consumer.position(p)).foreach(_ => {})
+
consumer.pause(partitions)
logDebug(s"Partitions assigned to consumer: $partitions. Seeking to the end.")
- consumer.seekToEnd(partitions)
- val partitionOffsets = partitions.asScala.map(p => p -> consumer.position(p)).toMap
- logDebug(s"Got latest offsets for partition : $partitionOffsets")
- partitionOffsets
+ if (knownOffsets.isEmpty) {
+ consumer.seekToEnd(partitions)
+ partitions.asScala.map(p => p -> consumer.position(p)).toMap
+ } else {
+ var partitionOffsets: PartitionOffsetMap = Map.empty
+
+ /**
+ * Compare `knownOffsets` and `partitionOffsets`. Returns all partitions that have incorrect
+ * latest offset (offset in `knownOffsets` is great than the one in `partitionOffsets`).
+ */
+ def findIncorrectOffsets(): Seq[(TopicPartition, Long, Long)] = {
+ var incorrectOffsets = ArrayBuffer[(TopicPartition, Long, Long)]()
+ partitionOffsets.foreach { case (tp, offset) =>
+ knownOffsets.foreach(_.get(tp).foreach { knownOffset =>
+ if (knownOffset > offset) {
+ val incorrectOffset = (tp, knownOffset, offset)
+ incorrectOffsets += incorrectOffset
+ }
+ })
+ }
+ incorrectOffsets
+ }
+
+ // Retry to fetch latest offsets when detecting incorrect offsets. We don't use
+ // `withRetriesWithoutInterrupt` to retry because:
+ //
+ // - `withRetriesWithoutInterrupt` will reset the consumer for each attempt but a fresh
+ // consumer has a much bigger chance to hit KAFKA-7703.
+ // - Avoid calling `consumer.poll(0)` which may cause KAFKA-7703.
+ var incorrectOffsets: Seq[(TopicPartition, Long, Long)] = Nil
+ var attempt = 0
+ do {
+ consumer.seekToEnd(partitions)
+ partitionOffsets = partitions.asScala.map(p => p -> consumer.position(p)).toMap
+ attempt += 1
+
+ incorrectOffsets = findIncorrectOffsets()
+ if (incorrectOffsets.nonEmpty) {
+ logWarning("Found incorrect offsets in some partitions " +
+ s"(partition, previous offset, fetched offset): $incorrectOffsets")
+ if (attempt < maxOffsetFetchAttempts) {
+ logWarning("Retrying to fetch latest offsets because of incorrect offsets")
+ Thread.sleep(offsetFetchAttemptIntervalMs)
+ }
+ }
+ } while (incorrectOffsets.nonEmpty && attempt < maxOffsetFetchAttempts)
+
+ logDebug(s"Got latest offsets for partition : $partitionOffsets")
+ partitionOffsets
+ }
}
}
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRecordToUnsafeRowConverter.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRecordToUnsafeRowConverter.scala
index f35a143e00374..306ef10b775a9 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRecordToUnsafeRowConverter.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRecordToUnsafeRowConverter.scala
@@ -30,13 +30,18 @@ private[kafka010] class KafkaRecordToUnsafeRowConverter {
def toUnsafeRow(record: ConsumerRecord[Array[Byte], Array[Byte]]): UnsafeRow = {
rowWriter.reset()
+ rowWriter.zeroOutNullBytes()
if (record.key == null) {
rowWriter.setNullAt(0)
} else {
rowWriter.write(0, record.key)
}
- rowWriter.write(1, record.value)
+ if (record.value == null) {
+ rowWriter.setNullAt(1)
+ } else {
+ rowWriter.write(1, record.value)
+ }
rowWriter.write(2, UTF8String.fromString(record.topic))
rowWriter.write(3, record.partition)
rowWriter.write(4, record.offset)
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala
index 9d856c9494e10..e6f9d1259e43e 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala
@@ -17,7 +17,6 @@
package org.apache.spark.sql.kafka010
-import java.{util => ju}
import java.util.UUID
import org.apache.kafka.common.TopicPartition
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
index 66ec7e0cd084a..464ad648d4bfc 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
@@ -130,7 +130,7 @@ private[kafka010] class KafkaSource(
metadataLog.get(0).getOrElse {
val offsets = startingOffsets match {
case EarliestOffsetRangeLimit => KafkaSourceOffset(kafkaReader.fetchEarliestOffsets())
- case LatestOffsetRangeLimit => KafkaSourceOffset(kafkaReader.fetchLatestOffsets())
+ case LatestOffsetRangeLimit => KafkaSourceOffset(kafkaReader.fetchLatestOffsets(None))
case SpecificOffsetRangeLimit(p) => kafkaReader.fetchSpecificOffsets(p, reportDataLoss)
}
metadataLog.add(0, offsets)
@@ -148,7 +148,8 @@ private[kafka010] class KafkaSource(
// Make sure initialPartitionOffsets is initialized
initialPartitionOffsets
- val latest = kafkaReader.fetchLatestOffsets()
+ val latest = kafkaReader.fetchLatestOffsets(
+ currentPartitionOffsets.orElse(Some(initialPartitionOffsets)))
val offsets = maxOffsetsPerTrigger match {
case None =>
latest
@@ -189,7 +190,15 @@ private[kafka010] class KafkaSource(
val prorate = limit * (size / total)
logDebug(s"rateLimit $tp prorated amount is $prorate")
// Don't completely starve small topicpartitions
- val off = begin + (if (prorate < 1) Math.ceil(prorate) else Math.floor(prorate)).toLong
+ val prorateLong = (if (prorate < 1) Math.ceil(prorate) else Math.floor(prorate)).toLong
+ // need to be careful of integer overflow
+ // therefore added canary checks where to see if off variable could be overflowed
+ // refer to [https://issues.apache.org/jira/browse/SPARK-26718]
+ val off = if (prorateLong > Long.MaxValue - begin) {
+ Long.MaxValue
+ } else {
+ begin + prorateLong
+ }
logDebug(s"rateLimit $tp new offset is $off")
// Paranoia, make sure not to return an offset that's past end
Math.min(end, off)
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
index 28c9853bfea9c..d225c1ea6b7f1 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
@@ -30,8 +30,9 @@ import org.apache.spark.internal.Logging
import org.apache.spark.sql.{AnalysisException, DataFrame, SaveMode, SparkSession, SQLContext}
import org.apache.spark.sql.execution.streaming.{Sink, Source}
import org.apache.spark.sql.sources._
-import org.apache.spark.sql.sources.v2._
-import org.apache.spark.sql.sources.v2.writer.streaming.StreamingWriteSupport
+import org.apache.spark.sql.sources.v2.{ContinuousReadSupport, DataSourceOptions, MicroBatchReadSupport, StreamWriteSupport}
+import org.apache.spark.sql.sources.v2.reader.streaming.ContinuousInputPartitionReader
+import org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter
import org.apache.spark.sql.streaming.OutputMode
import org.apache.spark.sql.types.StructType
@@ -45,9 +46,9 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
with StreamSinkProvider
with RelationProvider
with CreatableRelationProvider
- with StreamingWriteSupportProvider
- with ContinuousReadSupportProvider
- with MicroBatchReadSupportProvider
+ with StreamWriteSupport
+ with ContinuousReadSupport
+ with MicroBatchReadSupport
with Logging {
import KafkaSourceProvider._
@@ -107,12 +108,13 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
}
/**
- * Creates a [[org.apache.spark.sql.sources.v2.reader.streaming.MicroBatchReadSupport]] to read
- * batches of Kafka data in a micro-batch streaming query.
+ * Creates a [[org.apache.spark.sql.sources.v2.reader.streaming.MicroBatchReader]] to read batches
+ * of Kafka data in a micro-batch streaming query.
*/
- override def createMicroBatchReadSupport(
+ override def createMicroBatchReader(
+ schema: Optional[StructType],
metadataPath: String,
- options: DataSourceOptions): KafkaMicroBatchReadSupport = {
+ options: DataSourceOptions): KafkaMicroBatchReader = {
val parameters = options.asMap().asScala.toMap
validateStreamOptions(parameters)
@@ -138,7 +140,7 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
parameters,
driverGroupIdPrefix = s"$uniqueGroupId-driver")
- new KafkaMicroBatchReadSupport(
+ new KafkaMicroBatchReader(
kafkaOffsetReader,
kafkaParamsForExecutors(specifiedKafkaParams, uniqueGroupId),
options,
@@ -148,12 +150,13 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
}
/**
- * Creates a [[org.apache.spark.sql.sources.v2.reader.streaming.ContinuousReadSupport]] to read
+ * Creates a [[ContinuousInputPartitionReader]] to read
* Kafka data in a continuous streaming query.
*/
- override def createContinuousReadSupport(
+ override def createContinuousReader(
+ schema: Optional[StructType],
metadataPath: String,
- options: DataSourceOptions): KafkaContinuousReadSupport = {
+ options: DataSourceOptions): KafkaContinuousReader = {
val parameters = options.asMap().asScala.toMap
validateStreamOptions(parameters)
// Each running query should use its own group id. Otherwise, the query may be only assigned
@@ -178,7 +181,7 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
parameters,
driverGroupIdPrefix = s"$uniqueGroupId-driver")
- new KafkaContinuousReadSupport(
+ new KafkaContinuousReader(
kafkaOffsetReader,
kafkaParamsForExecutors(specifiedKafkaParams, uniqueGroupId),
parameters,
@@ -267,11 +270,11 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
}
}
- override def createStreamingWriteSupport(
+ override def createStreamWriter(
queryId: String,
schema: StructType,
mode: OutputMode,
- options: DataSourceOptions): StreamingWriteSupport = {
+ options: DataSourceOptions): StreamWriter = {
import scala.collection.JavaConverters._
val spark = SparkSession.getActiveSession.get
@@ -282,7 +285,7 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
KafkaWriter.validateQuery(
schema.toAttributes, new java.util.HashMap[String, Object](producerParams.asJava), topic)
- new KafkaStreamingWriteSupport(topic, producerParams, schema)
+ new KafkaStreamWriter(topic, producerParams, schema)
}
private def strategy(caseInsensitiveParams: Map[String, String]) =
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaStreamingWriteSupport.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaStreamWriter.scala
similarity index 88%
rename from external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaStreamingWriteSupport.scala
rename to external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaStreamWriter.scala
index dc19312f79a22..97c577d5a8b91 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaStreamingWriteSupport.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaStreamWriter.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.kafka010.KafkaWriter.validateQuery
import org.apache.spark.sql.sources.v2.writer._
-import org.apache.spark.sql.sources.v2.writer.streaming.{StreamingDataWriterFactory, StreamingWriteSupport}
+import org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter
import org.apache.spark.sql.types.StructType
/**
@@ -33,20 +33,20 @@ import org.apache.spark.sql.types.StructType
case object KafkaWriterCommitMessage extends WriterCommitMessage
/**
- * A [[StreamingWriteSupport]] for Kafka writing. Responsible for generating the writer factory.
+ * A [[StreamWriter]] for Kafka writing. Responsible for generating the writer factory.
*
* @param topic The topic this writer is responsible for. If None, topic will be inferred from
* a `topic` field in the incoming data.
* @param producerParams Parameters for Kafka producers in each task.
* @param schema The schema of the input data.
*/
-class KafkaStreamingWriteSupport(
+class KafkaStreamWriter(
topic: Option[String], producerParams: Map[String, String], schema: StructType)
- extends StreamingWriteSupport {
+ extends StreamWriter {
validateQuery(schema.toAttributes, producerParams.toMap[String, Object].asJava, topic)
- override def createStreamingWriterFactory(): KafkaStreamWriterFactory =
+ override def createWriterFactory(): KafkaStreamWriterFactory =
KafkaStreamWriterFactory(topic, producerParams, schema)
override def commit(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}
@@ -54,8 +54,8 @@ class KafkaStreamingWriteSupport(
}
/**
- * A [[DataWriterFactory]] for Kafka writing. Will be serialized and sent to executors to generate
- * the per-task data writers.
+ * A [[StreamingDataWriterFactory]] for Kafka writing. Will be serialized and sent to executors to
+ * generate the per-task data writers.
* @param topic The topic that should be written to. If None, topic will be inferred from
* a `topic` field in the incoming data.
* @param producerParams Parameters for Kafka producers in each task.
@@ -63,9 +63,9 @@ class KafkaStreamingWriteSupport(
*/
case class KafkaStreamWriterFactory(
topic: Option[String], producerParams: Map[String, String], schema: StructType)
- extends StreamingDataWriterFactory {
+ extends DataWriterFactory[InternalRow] {
- override def createWriter(
+ override def createDataWriter(
partitionId: Int,
taskId: Long,
epochId: Long): DataWriter[InternalRow] = {
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala
index af510219a6f6f..649cb725d7d17 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.kafka010
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.spark.sql.Dataset
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec
+import org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation
import org.apache.spark.sql.execution.streaming.continuous.ContinuousTrigger
import org.apache.spark.sql.streaming.Trigger
@@ -169,6 +169,10 @@ class KafkaContinuousSourceSuite extends KafkaSourceSuiteBase with KafkaContinuo
}
}
}
+
+ test("SPARK-27494: read kafka record containing null key/values.") {
+ testNullableKeyValue(ContinuousTrigger(100))
+ }
}
class KafkaContinuousSourceTopicDeletionSuite extends KafkaContinuousTest {
@@ -207,13 +211,11 @@ class KafkaContinuousSourceTopicDeletionSuite extends KafkaContinuousTest {
testUtils.createTopic(topic2, partitions = 5)
eventually(timeout(streamingTimeout)) {
assert(
- query.lastExecution.executedPlan.collectFirst {
- case scan: DataSourceV2ScanExec
- if scan.readSupport.isInstanceOf[KafkaContinuousReadSupport] =>
- scan.scanConfig.asInstanceOf[KafkaContinuousScanConfig]
- }.exists { config =>
+ query.lastExecution.logical.collectFirst {
+ case StreamingDataSourceV2Relation(_, _, _, r: KafkaContinuousReader) => r
+ }.exists { r =>
// Ensure the new topic is present and the old topic is gone.
- config.knownPartitions.exists(_.topic == topic2)
+ r.knownPartitions.exists(_.topic == topic2)
},
s"query never reconfigured to new topic $topic2")
}
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
index fa6bdc20bd4f9..fa1468a3943c8 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
@@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicInteger
import org.apache.spark.SparkContext
import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd, SparkListenerTaskStart}
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec
+import org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation
import org.apache.spark.sql.execution.streaming.StreamExecution
import org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution
import org.apache.spark.sql.streaming.Trigger
@@ -46,10 +46,8 @@ trait KafkaContinuousTest extends KafkaSourceTest {
testUtils.addPartitions(topic, newCount)
eventually(timeout(streamingTimeout)) {
assert(
- query.lastExecution.executedPlan.collectFirst {
- case scan: DataSourceV2ScanExec
- if scan.readSupport.isInstanceOf[KafkaContinuousReadSupport] =>
- scan.scanConfig.asInstanceOf[KafkaContinuousScanConfig]
+ query.lastExecution.logical.collectFirst {
+ case StreamingDataSourceV2Relation(_, _, _, r: KafkaContinuousReader) => r
}.exists(_.knownPartitions.size == newCount),
s"query never reconfigured to $newCount partitions")
}
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
index 8e246dbbf5d70..36da2f13a6159 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.kafka010
import java.io._
import java.nio.charset.StandardCharsets.UTF_8
import java.nio.file.{Files, Paths}
-import java.util.Locale
+import java.util.{Locale, Optional}
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.atomic.AtomicInteger
@@ -28,21 +28,24 @@ import scala.collection.JavaConverters._
import scala.io.Source
import scala.util.Random
-import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord, RecordMetadata}
+import org.apache.kafka.clients.producer.{ProducerRecord, RecordMetadata}
import org.apache.kafka.common.TopicPartition
import org.scalatest.concurrent.PatienceConfiguration.Timeout
import org.scalatest.time.SpanSugar._
-import org.apache.spark.sql.{ForeachWriter, SparkSession}
+import org.apache.spark.sql.{Dataset, ForeachWriter, SparkSession}
import org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation
import org.apache.spark.sql.execution.streaming._
import org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution
import org.apache.spark.sql.functions.{count, window}
import org.apache.spark.sql.kafka010.KafkaSourceProvider._
import org.apache.spark.sql.sources.v2.DataSourceOptions
+import org.apache.spark.sql.sources.v2.reader.streaming.{Offset => OffsetV2}
import org.apache.spark.sql.streaming.{ProcessingTime, StreamTest}
+import org.apache.spark.sql.streaming.Trigger
import org.apache.spark.sql.streaming.util.StreamManualClock
import org.apache.spark.sql.test.SharedSQLContext
+import org.apache.spark.sql.types.StructType
abstract class KafkaSourceTest extends StreamTest with SharedSQLContext with KafkaTest {
@@ -112,16 +115,14 @@ abstract class KafkaSourceTest extends StreamTest with SharedSQLContext with Kaf
query.nonEmpty,
"Cannot add data when there is no query for finding the active kafka source")
- val sources: Seq[BaseStreamingSource] = {
+ val sources = {
query.get.logicalPlan.collect {
case StreamingExecutionRelation(source: KafkaSource, _) => source
- case StreamingExecutionRelation(source: KafkaMicroBatchReadSupport, _) => source
+ case StreamingExecutionRelation(source: KafkaMicroBatchReader, _) => source
} ++ (query.get.lastExecution match {
case null => Seq()
case e => e.logical.collect {
- case r: StreamingDataSourceV2Relation
- if r.readSupport.isInstanceOf[KafkaContinuousReadSupport] =>
- r.readSupport.asInstanceOf[KafkaContinuousReadSupport]
+ case StreamingDataSourceV2Relation(_, _, _, reader: KafkaContinuousReader) => reader
}
})
}.distinct
@@ -199,6 +200,41 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
StopStream)
}
+ test("SPARK-26718 Rate limit set to Long.Max should not overflow integer " +
+ "during end offset calculation") {
+ val topic = newTopic()
+ testUtils.createTopic(topic, partitions = 1)
+ // fill in 5 messages to trigger potential integer overflow
+ testUtils.sendMessages(topic, (0 until 5).map(_.toString).toArray, Some(0))
+
+ val partitionOffsets = Map(
+ new TopicPartition(topic, 0) -> 5L
+ )
+ val startingOffsets = JsonUtils.partitionOffsets(partitionOffsets)
+
+ val kafka = spark
+ .readStream
+ .format("kafka")
+ .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+ // use latest to force begin to be 5
+ .option("startingOffsets", startingOffsets)
+ // use Long.Max to try to trigger overflow
+ .option("maxOffsetsPerTrigger", Long.MaxValue)
+ .option("subscribe", topic)
+ .option("kafka.metadata.max.age.ms", "1")
+ .load()
+ .selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
+ .as[(String, String)]
+ val mapped: org.apache.spark.sql.Dataset[_] = kafka.map(kv => kv._2.toInt)
+
+ testStream(mapped)(
+ makeSureGetOffsetCalled,
+ AddKafkaData(Set(topic), 30, 31, 32, 33, 34),
+ CheckAnswer(30, 31, 32, 33, 34),
+ StopStream
+ )
+ }
+
test("maxOffsetsPerTrigger") {
val topic = newTopic()
testUtils.createTopic(topic, partitions = 3)
@@ -327,6 +363,54 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
)
}
+ test("subscribe topic by pattern with topic recreation between batches") {
+ val topicPrefix = newTopic()
+ val topic = topicPrefix + "-good"
+ val topic2 = topicPrefix + "-bad"
+ testUtils.createTopic(topic, partitions = 1)
+ testUtils.sendMessages(topic, Array("1", "3"))
+ testUtils.createTopic(topic2, partitions = 1)
+ testUtils.sendMessages(topic2, Array("2", "4"))
+
+ val reader = spark
+ .readStream
+ .format("kafka")
+ .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+ .option("kafka.metadata.max.age.ms", "1")
+ .option("kafka.default.api.timeout.ms", "3000")
+ .option("startingOffsets", "earliest")
+ .option("subscribePattern", s"$topicPrefix-.*")
+
+ val ds = reader.load()
+ .selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
+ .as[(String, String)]
+ .map(kv => kv._2.toInt)
+
+ testStream(ds)(
+ StartStream(),
+ AssertOnQuery { q =>
+ q.processAllAvailable()
+ true
+ },
+ CheckAnswer(1, 2, 3, 4),
+ // Restart the stream in this test to make the test stable. When recreating a topic when a
+ // consumer is alive, it may not be able to see the recreated topic even if a fresh consumer
+ // has seen it.
+ StopStream,
+ // Recreate `topic2` and wait until it's available
+ WithOffsetSync(new TopicPartition(topic2, 0), expectedOffset = 1) { () =>
+ testUtils.deleteTopic(topic2)
+ testUtils.createTopic(topic2)
+ testUtils.sendMessages(topic2, Array("6"))
+ },
+ StartStream(),
+ ExpectFailure[IllegalStateException](e => {
+ // The offset of `topic2` should be changed from 2 to 1
+ assert(e.getMessage.contains("was changed from 2 to 1"))
+ })
+ )
+ }
+
test("ensure that initial offset are written with an extra byte in the beginning (SPARK-19517)") {
withTempDir { metadataPath =>
val topic = "kafka-initial-offset-current"
@@ -853,6 +937,62 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
)
}
}
+
+ test("SPARK-25495: FetchedData.reset should reset all fields") {
+ val topic = newTopic()
+ val topicPartition = new TopicPartition(topic, 0)
+ testUtils.createTopic(topic, partitions = 1)
+
+ val ds = spark
+ .readStream
+ .format("kafka")
+ .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+ .option("kafka.metadata.max.age.ms", "1")
+ .option("kafka.isolation.level", "read_committed")
+ .option("subscribe", topic)
+ .option("startingOffsets", "earliest")
+ .load()
+ .select($"value".as[String])
+
+ testUtils.withTranscationalProducer { producer =>
+ producer.beginTransaction()
+ (0 to 3).foreach { i =>
+ producer.send(new ProducerRecord[String, String](topic, i.toString)).get()
+ }
+ producer.commitTransaction()
+ }
+ testUtils.waitUntilOffsetAppears(topicPartition, 5)
+
+ val q = ds.writeStream.foreachBatch { (ds: Dataset[String], epochId: Long) =>
+ if (epochId == 0) {
+ // Send more message before the tasks of the current batch start reading the current batch
+ // data, so that the executors will prefetch messages in the next batch and drop them. In
+ // this case, if we forget to reset `FetchedData._nextOffsetInFetchedData` or
+ // `FetchedData._offsetAfterPoll` (See SPARK-25495), the next batch will see incorrect
+ // values and return wrong results hence fail the test.
+ testUtils.withTranscationalProducer { producer =>
+ producer.beginTransaction()
+ (4 to 7).foreach { i =>
+ producer.send(new ProducerRecord[String, String](topic, i.toString)).get()
+ }
+ producer.commitTransaction()
+ }
+ testUtils.waitUntilOffsetAppears(topicPartition, 10)
+ checkDatasetUnorderly(ds, (0 to 3).map(_.toString): _*)
+ } else {
+ checkDatasetUnorderly(ds, (4 to 7).map(_.toString): _*)
+ }
+ }.start()
+ try {
+ q.processAllAvailable()
+ } finally {
+ q.stop()
+ }
+ }
+
+ test("SPARK-27494: read kafka record containing null key/values.") {
+ testNullableKeyValue(Trigger.ProcessingTime(100))
+ }
}
@@ -905,7 +1045,7 @@ class KafkaMicroBatchV2SourceSuite extends KafkaMicroBatchSourceSuiteBase {
makeSureGetOffsetCalled,
AssertOnQuery { query =>
query.logicalPlan.collect {
- case StreamingExecutionRelation(_: KafkaMicroBatchReadSupport, _) => true
+ case StreamingExecutionRelation(_: KafkaMicroBatchReader, _) => true
}.nonEmpty
}
)
@@ -930,16 +1070,17 @@ class KafkaMicroBatchV2SourceSuite extends KafkaMicroBatchSourceSuiteBase {
"kafka.bootstrap.servers" -> testUtils.brokerAddress,
"subscribe" -> topic
) ++ Option(minPartitions).map { p => "minPartitions" -> p}
- val readSupport = provider.createMicroBatchReadSupport(
- dir.getAbsolutePath, new DataSourceOptions(options.asJava))
- val config = readSupport.newScanConfigBuilder(
- KafkaSourceOffset(Map(tp -> 0L)),
- KafkaSourceOffset(Map(tp -> 100L))).build()
- val inputPartitions = readSupport.planInputPartitions(config)
+ val reader = provider.createMicroBatchReader(
+ Optional.empty[StructType], dir.getAbsolutePath, new DataSourceOptions(options.asJava))
+ reader.setOffsetRange(
+ Optional.of[OffsetV2](KafkaSourceOffset(Map(tp -> 0L))),
+ Optional.of[OffsetV2](KafkaSourceOffset(Map(tp -> 100L)))
+ )
+ val factories = reader.planInputPartitions().asScala
.map(_.asInstanceOf[KafkaMicroBatchInputPartition])
- withClue(s"minPartitions = $minPartitions generated factories $inputPartitions\n\t") {
- assert(inputPartitions.size == numPartitionsGenerated)
- inputPartitions.foreach { f => assert(f.reuseKafkaConsumer == reusesConsumers) }
+ withClue(s"minPartitions = $minPartitions generated factories $factories\n\t") {
+ assert(factories.size == numPartitionsGenerated)
+ factories.foreach { f => assert(f.reuseKafkaConsumer == reusesConsumers) }
}
}
}
@@ -1325,6 +1466,60 @@ abstract class KafkaSourceSuiteBase extends KafkaSourceTest {
CheckAnswer(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)
)
}
+
+ protected def testNullableKeyValue(trigger: Trigger): Unit = {
+ val table = "kafka_null_key_value_source_test"
+ withTable(table) {
+ val topic = newTopic()
+ testUtils.createTopic(topic)
+ testUtils.withTranscationalProducer { producer =>
+ val df = spark
+ .readStream
+ .format("kafka")
+ .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+ .option("kafka.isolation.level", "read_committed")
+ .option("startingOffsets", "earliest")
+ .option("subscribe", topic)
+ .load()
+ .selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)")
+ .as[(String, String)]
+
+ val q = df
+ .writeStream
+ .format("memory")
+ .queryName(table)
+ .trigger(trigger)
+ .start()
+ try {
+ var idx = 0
+ producer.beginTransaction()
+ val expected1 = Seq.tabulate(5) { _ =>
+ producer.send(new ProducerRecord[String, String](topic, null, null)).get()
+ (null, null)
+ }.asInstanceOf[Seq[(String, String)]]
+
+ val expected2 = Seq.tabulate(5) { _ =>
+ idx += 1
+ producer.send(new ProducerRecord[String, String](topic, idx.toString, null)).get()
+ (idx.toString, null)
+ }.asInstanceOf[Seq[(String, String)]]
+
+ val expected3 = Seq.tabulate(5) { _ =>
+ idx += 1
+ producer.send(new ProducerRecord[String, String](topic, null, idx.toString)).get()
+ (null, idx.toString)
+ }.asInstanceOf[Seq[(String, String)]]
+
+ producer.commitTransaction()
+ eventually(timeout(streamingTimeout)) {
+ checkAnswer(spark.table(table), (expected1 ++ expected2 ++ expected3).toDF())
+ }
+ } finally {
+ q.stop()
+ }
+ }
+ }
+ }
}
object KafkaSourceSuite {
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculatorSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculatorSuite.scala
index 2ccf3e291bea7..5088583520b87 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculatorSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaOffsetRangeCalculatorSuite.scala
@@ -141,6 +141,21 @@ class KafkaOffsetRangeCalculatorSuite extends SparkFunSuite {
KafkaOffsetRange(tp2, 14, 21, None)))
}
+ testWithMinPartitions("SPARK-28489: never drop offsets", 6) { calc =>
+ assert(
+ calc.getRanges(
+ fromOffsets = Map(tp1 -> 0, tp2 -> 0, tp3 -> 0),
+ untilOffsets = Map(tp1 -> 10, tp2 -> 10, tp3 -> 1)) ==
+ Seq(
+ KafkaOffsetRange(tp1, 0, 3, None),
+ KafkaOffsetRange(tp1, 3, 6, None),
+ KafkaOffsetRange(tp1, 6, 10, None),
+ KafkaOffsetRange(tp2, 0, 3, None),
+ KafkaOffsetRange(tp2, 3, 6, None),
+ KafkaOffsetRange(tp2, 6, 10, None),
+ KafkaOffsetRange(tp3, 0, 1, None)))
+ }
+
private val tp1 = new TopicPartition("t1", 1)
private val tp2 = new TopicPartition("t2", 1)
private val tp3 = new TopicPartition("t3", 1)
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index a97fd35bfbb73..42f2fc645fff5 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
index 661b67a8ab68a..c20e0dbc8c824 100644
--- a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
+++ b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.streaming.kafka010
import java.io.File
import java.lang.{ Long => JLong }
import java.util.{ Arrays, HashMap => JHashMap, Map => JMap, UUID }
+import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.atomic.AtomicLong
@@ -423,7 +424,7 @@ class DirectKafkaStreamSuite
)
val collectedData = new ConcurrentLinkedQueue[String]()
- val committed = new JHashMap[TopicPartition, OffsetAndMetadata]()
+ val committed = new ConcurrentHashMap[TopicPartition, OffsetAndMetadata]()
// Send data to Kafka and wait for it to be received
def sendDataAndWaitForReceive(data: Seq[Int]) {
@@ -452,6 +453,7 @@ class DirectKafkaStreamSuite
logError("commit failed", e)
} else {
committed.putAll(m)
+ logDebug(s"commit succeeded: $m")
}
}
})
@@ -462,8 +464,10 @@ class DirectKafkaStreamSuite
for (i <- (1 to 10).grouped(4)) {
sendDataAndWaitForReceive(i)
}
+ eventually(timeout(10.seconds), interval(50.milliseconds)) {
+ assert(!committed.isEmpty)
+ }
ssc.stop()
- assert(! committed.isEmpty)
val consumer = new KafkaConsumer[String, String](kafkaParams)
consumer.subscribe(Arrays.asList(topic))
consumer.poll(0)
diff --git a/external/kafka-0-8-assembly/pom.xml b/external/kafka-0-8-assembly/pom.xml
index 6be17a81f3fed..13e4bc29b6d5b 100644
--- a/external/kafka-0-8-assembly/pom.xml
+++ b/external/kafka-0-8-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml
index 6d1c4789f382d..f9eac52fe5064 100644
--- a/external/kafka-0-8/pom.xml
+++ b/external/kafka-0-8/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index 68fded515626b..3971bb0c80720 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 4915893965595..3888bbb4846fc 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
index 5fb83b26f8382..11e949536f2b6 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
@@ -68,7 +68,7 @@ private[kinesis] class KinesisCheckpointer(
if (checkpointer != null) {
try {
// We must call `checkpoint()` with no parameter to finish reading shards.
- // See an URL below for details:
+ // See a URL below for details:
// https://forums.aws.amazon.com/thread.jspa?threadID=244218
KinesisRecordProcessor.retryRandom(checkpointer.checkpoint(), 4, 100)
} catch {
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/SparkAWSCredentials.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/SparkAWSCredentials.scala
index 9facfe8ff2b0f..97400c195476c 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/SparkAWSCredentials.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/SparkAWSCredentials.scala
@@ -102,8 +102,8 @@ object SparkAWSCredentials {
*
* @note The given AWS keypair will be saved in DStream checkpoints if checkpointing is
* enabled. Make sure that your checkpoint directory is secure. Prefer using the
- * [[http://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default default provider chain]]
- * instead if possible.
+ * default provider chain instead if possible
+ * (http://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default).
*
* @param accessKeyId AWS access key ID
* @param secretKey AWS secret key
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index 027157e53d511..53ef67e1315e3 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/graphx/pom.xml b/graphx/pom.xml
index 0f5dc548600b2..6eabd19d3d2e5 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../pom.xml
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgeRDDImpl.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgeRDDImpl.scala
index 376c7b06f9d2b..eb8abd1846d0e 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgeRDDImpl.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/EdgeRDDImpl.scala
@@ -87,7 +87,7 @@ class EdgeRDDImpl[ED: ClassTag, VD: ClassTag] private[graphx] (
/** The number of edges in the RDD. */
override def count(): Long = {
- partitionsRDD.map(_._2.size.toLong).reduce(_ + _)
+ partitionsRDD.map(_._2.size.toLong).fold(0)(_ + _)
}
override def mapValues[ED2: ClassTag](f: Edge[ED] => ED2): EdgeRDDImpl[ED2, VD] =
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
index 34e1253ff42ab..fb1410950a259 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
@@ -103,15 +103,16 @@ class GraphImpl[VD: ClassTag, ED: ClassTag] protected (
(part, (e.srcId, e.dstId, e.attr))
}
.partitionBy(new HashPartitioner(numPartitions))
- .mapPartitionsWithIndex( { (pid, iter) =>
- val builder = new EdgePartitionBuilder[ED, VD]()(edTag, vdTag)
- iter.foreach { message =>
- val data = message._2
- builder.add(data._1, data._2, data._3)
- }
- val edgePartition = builder.toEdgePartition
- Iterator((pid, edgePartition))
- }, preservesPartitioning = true)).cache()
+ .mapPartitionsWithIndex(
+ { (pid: Int, iter: Iterator[(PartitionID, (VertexId, VertexId, ED))]) =>
+ val builder = new EdgePartitionBuilder[ED, VD]()(edTag, vdTag)
+ iter.foreach { message =>
+ val data = message._2
+ builder.add(data._1, data._2, data._3)
+ }
+ val edgePartition = builder.toEdgePartition
+ Iterator((pid, edgePartition))
+ }, preservesPartitioning = true)).cache()
GraphImpl.fromExistingRDDs(vertices.withEdges(newEdges), newEdges)
}
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexRDDImpl.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexRDDImpl.scala
index 3c6f22d97360d..2da9762fb0452 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexRDDImpl.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/VertexRDDImpl.scala
@@ -87,7 +87,7 @@ class VertexRDDImpl[VD] private[graphx] (
/** The number of vertices in the RDD. */
override def count(): Long = {
- partitionsRDD.map(_.size.toLong).reduce(_ + _)
+ partitionsRDD.map(_.size.toLong).fold(0)(_ + _)
}
override private[graphx] def mapVertexPartitions[VD2: ClassTag](
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
index 96b635f9a144e..1305c059b89ce 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
@@ -198,9 +198,11 @@ object PageRank extends Logging {
val zero = Vectors.sparse(sources.size, List()).asBreeze
// map of vid -> vector where for each vid, the _position of vid in source_ is set to 1.0
- val sourcesInitMap = sources.zipWithIndex.toMap.mapValues { i =>
- Vectors.sparse(sources.size, Array(i), Array(1.0)).asBreeze
- }
+ val sourcesInitMap = sources.zipWithIndex.map { case (vid, i) =>
+ val v = Vectors.sparse(sources.size, Array(i), Array(1.0)).asBreeze
+ (vid, v)
+ }.toMap
+
val sc = graph.vertices.sparkContext
val sourcesInitMapBC = sc.broadcast(sourcesInitMap)
// Initialize the PageRank graph with each edge attribute having
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala
index 59fdd855e6f37..2847a4e172d40 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/SVDPlusPlus.scala
@@ -72,7 +72,7 @@ object SVDPlusPlus {
// calculate global rating mean
edges.cache()
- val (rs, rc) = edges.map(e => (e.attr, 1L)).reduce((a, b) => (a._1 + b._1, a._2 + b._2))
+ val (rs, rc) = edges.map(e => (e.attr, 1L)).fold((0, 0))((a, b) => (a._1 + b._1, a._2 + b._2))
val u = rs / rc
// construct graph
diff --git a/graphx/src/test/scala/org/apache/spark/graphx/EdgeRDDSuite.scala b/graphx/src/test/scala/org/apache/spark/graphx/EdgeRDDSuite.scala
index 7a24e320c3e04..8fd3e6f5229cc 100644
--- a/graphx/src/test/scala/org/apache/spark/graphx/EdgeRDDSuite.scala
+++ b/graphx/src/test/scala/org/apache/spark/graphx/EdgeRDDSuite.scala
@@ -60,4 +60,14 @@ class EdgeRDDSuite extends SparkFunSuite with LocalSparkContext {
}
}
+ test("count") {
+ withSpark { sc =>
+ val empty = EdgeRDD.fromEdges(sc.emptyRDD[Edge[Int]])
+ assert(empty.count === 0)
+
+ val edges = List(Edge(0, 1, ()), Edge(1, 2, ()), Edge(2, 0, ()))
+ val nonempty = EdgeRDD.fromEdges(sc.parallelize(edges))
+ assert(nonempty.count === edges.size)
+ }
+ }
}
diff --git a/graphx/src/test/scala/org/apache/spark/graphx/VertexRDDSuite.scala b/graphx/src/test/scala/org/apache/spark/graphx/VertexRDDSuite.scala
index 8e630435279de..434e6a84edf6d 100644
--- a/graphx/src/test/scala/org/apache/spark/graphx/VertexRDDSuite.scala
+++ b/graphx/src/test/scala/org/apache/spark/graphx/VertexRDDSuite.scala
@@ -223,4 +223,15 @@ class VertexRDDSuite extends SparkFunSuite with LocalSparkContext {
assert(verts.collect().toSeq === data) // test checkpointed RDD
}
}
+
+ test("count") {
+ withSpark { sc =>
+ val empty = VertexRDD(sc.emptyRDD[(Long, Unit)])
+ assert(empty.count === 0)
+
+ val n = 100
+ val nonempty = vertices(sc, n)
+ assert(nonempty.count === n + 1)
+ }
+ }
}
diff --git a/graphx/src/test/scala/org/apache/spark/graphx/lib/SVDPlusPlusSuite.scala b/graphx/src/test/scala/org/apache/spark/graphx/lib/SVDPlusPlusSuite.scala
index 2991438f5e57e..da0457c354b51 100644
--- a/graphx/src/test/scala/org/apache/spark/graphx/lib/SVDPlusPlusSuite.scala
+++ b/graphx/src/test/scala/org/apache/spark/graphx/lib/SVDPlusPlusSuite.scala
@@ -40,4 +40,13 @@ class SVDPlusPlusSuite extends SparkFunSuite with LocalSparkContext {
}
}
+ test("Test SVD++ with no edges") {
+ withSpark { sc =>
+ val edges = sc.emptyRDD[Edge[Double]]
+ val conf = new SVDPlusPlus.Conf(10, 2, 0.0, 5.0, 0.007, 0.007, 0.005, 0.015) // 2 iterations
+ val (graph, _) = SVDPlusPlus.run(edges, conf)
+ assert(graph.vertices.count == 0)
+ assert(graph.edges.count == 0)
+ }
+ }
}
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index 2c39a7df0146e..17c49175e469a 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../pom.xml
@@ -102,6 +102,11 @@
com.fasterxml.jackson.core
jackson-annotations
+
+
+ com.amazonaws
+ aws-java-sdk
+
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 912eb6b6d2a08..6178fdb07ea38 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../pom.xml
diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
index b343094b2e7b8..e694e9066f12e 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
@@ -158,7 +158,7 @@ public void testPySparkLauncher() throws Exception {
Map env = new HashMap<>();
List cmd = buildCommand(sparkSubmitArgs, env);
- assertEquals("python", cmd.get(cmd.size() - 1));
+ assertTrue(Arrays.asList("python", "python2", "python3").contains(cmd.get(cmd.size() - 1)));
assertEquals(
String.format("\"%s\" \"foo\" \"%s\" \"bar\" \"%s\"",
parser.MASTER, parser.DEPLOY_MODE, SparkSubmitCommandBuilder.PYSPARK_SHELL_RESOURCE),
diff --git a/licenses-binary/LICENSE-mustache.txt b/licenses-binary/LICENSE-mustache.txt
new file mode 100644
index 0000000000000..038cbb95d1cd0
--- /dev/null
+++ b/licenses-binary/LICENSE-mustache.txt
@@ -0,0 +1,11 @@
+The MIT License
+
+Copyright (c) 2009 Chris Wanstrath (Ruby)
+Copyright (c) 2010-2014 Jan Lehnardt (JavaScript)
+Copyright (c) 2010-2015 The mustache.js community
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
diff --git a/licenses/LICENSE-mustache.txt b/licenses/LICENSE-mustache.txt
new file mode 100644
index 0000000000000..038cbb95d1cd0
--- /dev/null
+++ b/licenses/LICENSE-mustache.txt
@@ -0,0 +1,11 @@
+The MIT License
+
+Copyright (c) 2009 Chris Wanstrath (Ruby)
+Copyright (c) 2010-2014 Jan Lehnardt (JavaScript)
+Copyright (c) 2010-2015 The mustache.js community
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index 53286fe93478d..70cc43474def4 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../pom.xml
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
index 5824e463ca1aa..827b6202a3d47 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
@@ -603,6 +603,15 @@ class SparseVector @Since("2.0.0") (
private[spark] override def asBreeze: BV[Double] = new BSV[Double](indices, values, size)
+ override def apply(i: Int): Double = {
+ if (i < 0 || i >= size) {
+ throw new IndexOutOfBoundsException(s"Index $i out of bounds [0, $size)")
+ }
+
+ val j = util.Arrays.binarySearch(indices, i)
+ if (j < 0) 0.0 else values(j)
+ }
+
override def foreachActive(f: (Int, Double) => Unit): Unit = {
var i = 0
val localValuesSize = values.length
diff --git a/mllib/pom.xml b/mllib/pom.xml
index f07d7f24fd312..789854df887c2 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../pom.xml
diff --git a/mllib/src/main/scala/org/apache/spark/ml/ann/Layer.scala b/mllib/src/main/scala/org/apache/spark/ml/ann/Layer.scala
index 014ff07c21158..2b4b0fc55b955 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/ann/Layer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/ann/Layer.scala
@@ -371,7 +371,7 @@ private[ann] trait TopologyModel extends Serializable {
def forward(data: BDM[Double], includeLastLayer: Boolean): Array[BDM[Double]]
/**
- * Prediction of the model. See {@link ProbabilisticClassificationModel}
+ * Prediction of the model. See `ProbabilisticClassificationModel``
*
* @param features input features
* @return prediction
@@ -379,7 +379,7 @@ private[ann] trait TopologyModel extends Serializable {
def predict(features: Vector): Vector
/**
- * Raw prediction of the model. See {@link ProbabilisticClassificationModel}
+ * Raw prediction of the model. See `ProbabilisticClassificationModel`
*
* @param features input features
* @return raw prediction
@@ -389,7 +389,7 @@ private[ann] trait TopologyModel extends Serializable {
def predictRaw(features: Vector): Vector
/**
- * Probability of the model. See {@link ProbabilisticClassificationModel}
+ * Probability of the model. See `ProbabilisticClassificationModel`
*
* @param rawPrediction raw prediction vector
* @return probability
diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
index 1cd2b1ad84092..756dd677cb579 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
@@ -121,7 +121,7 @@ sealed abstract class Attribute extends Serializable {
private[attribute] trait AttributeFactory {
/**
- * Creates an [[Attribute]] from a [[Metadata]] instance.
+ * Creates an [[Attribute]] from a `Metadata` instance.
*/
private[attribute] def fromMetadata(metadata: Metadata): Attribute
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.scala
index 8a57bfc029d14..6648e78d8eafa 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.scala
@@ -168,7 +168,7 @@ object DecisionTreeClassifier extends DefaultParamsReadable[DecisionTreeClassifi
@Since("1.4.0")
class DecisionTreeClassificationModel private[ml] (
@Since("1.4.0")override val uid: String,
- @Since("1.4.0")override val rootNode: ClassificationNode,
+ @Since("1.4.0")override val rootNode: Node,
@Since("1.6.0")override val numFeatures: Int,
@Since("1.5.0")override val numClasses: Int)
extends ProbabilisticClassificationModel[Vector, DecisionTreeClassificationModel]
@@ -181,7 +181,7 @@ class DecisionTreeClassificationModel private[ml] (
* Construct a decision tree classification model.
* @param rootNode Root node of tree, with other nodes attached.
*/
- private[ml] def this(rootNode: ClassificationNode, numFeatures: Int, numClasses: Int) =
+ private[ml] def this(rootNode: Node, numFeatures: Int, numClasses: Int) =
this(Identifiable.randomUID("dtc"), rootNode, numFeatures, numClasses)
override def predict(features: Vector): Double = {
@@ -279,9 +279,8 @@ object DecisionTreeClassificationModel extends MLReadable[DecisionTreeClassifica
val metadata = DefaultParamsReader.loadMetadata(path, sc, className)
val numFeatures = (metadata.metadata \ "numFeatures").extract[Int]
val numClasses = (metadata.metadata \ "numClasses").extract[Int]
- val root = loadTreeNodes(path, metadata, sparkSession, isClassification = true)
- val model = new DecisionTreeClassificationModel(metadata.uid,
- root.asInstanceOf[ClassificationNode], numFeatures, numClasses)
+ val root = loadTreeNodes(path, metadata, sparkSession)
+ val model = new DecisionTreeClassificationModel(metadata.uid, root, numFeatures, numClasses)
metadata.getAndSetParams(model)
model
}
@@ -296,10 +295,9 @@ object DecisionTreeClassificationModel extends MLReadable[DecisionTreeClassifica
require(oldModel.algo == OldAlgo.Classification,
s"Cannot convert non-classification DecisionTreeModel (old API) to" +
s" DecisionTreeClassificationModel (new API). Algo is: ${oldModel.algo}")
- val rootNode = Node.fromOld(oldModel.topNode, categoricalFeatures, isClassification = true)
+ val rootNode = Node.fromOld(oldModel.topNode, categoricalFeatures)
val uid = if (parent != null) parent.uid else Identifiable.randomUID("dtc")
// Can't infer number of features from old model, so default to -1
- new DecisionTreeClassificationModel(uid,
- rootNode.asInstanceOf[ClassificationNode], numFeatures, -1)
+ new DecisionTreeClassificationModel(uid, rootNode, numFeatures, -1)
}
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/GBTClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/GBTClassifier.scala
index 33acd9914073f..62cfa39746ff0 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/GBTClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/GBTClassifier.scala
@@ -412,14 +412,14 @@ object GBTClassificationModel extends MLReadable[GBTClassificationModel] {
override def load(path: String): GBTClassificationModel = {
implicit val format = DefaultFormats
val (metadata: Metadata, treesData: Array[(Metadata, Node)], treeWeights: Array[Double]) =
- EnsembleModelReadWrite.loadImpl(path, sparkSession, className, treeClassName, false)
+ EnsembleModelReadWrite.loadImpl(path, sparkSession, className, treeClassName)
val numFeatures = (metadata.metadata \ numFeaturesKey).extract[Int]
val numTrees = (metadata.metadata \ numTreesKey).extract[Int]
val trees: Array[DecisionTreeRegressionModel] = treesData.map {
case (treeMetadata, root) =>
- val tree = new DecisionTreeRegressionModel(treeMetadata.uid,
- root.asInstanceOf[RegressionNode], numFeatures)
+ val tree =
+ new DecisionTreeRegressionModel(treeMetadata.uid, root, numFeatures)
treeMetadata.getAndSetParams(tree)
tree
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
index 1b5c02fc9a576..9f9d4eeaf2afa 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
@@ -313,7 +313,8 @@ class LinearSVCModel private[classification] (
setDefault(threshold, 0.0)
@Since("2.2.0")
- def setWeightCol(value: Double): this.type = set(threshold, value)
+ @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.4.4")
+ def setWeightCol(value: Double): this.type = this
private val margin: Vector => Double = (features) => {
BLAS.dot(features, coefficients) + intercept
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
index 94887ac346fec..57132381b6474 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
@@ -313,15 +313,15 @@ object RandomForestClassificationModel extends MLReadable[RandomForestClassifica
override def load(path: String): RandomForestClassificationModel = {
implicit val format = DefaultFormats
val (metadata: Metadata, treesData: Array[(Metadata, Node)], _) =
- EnsembleModelReadWrite.loadImpl(path, sparkSession, className, treeClassName, true)
+ EnsembleModelReadWrite.loadImpl(path, sparkSession, className, treeClassName)
val numFeatures = (metadata.metadata \ "numFeatures").extract[Int]
val numClasses = (metadata.metadata \ "numClasses").extract[Int]
val numTrees = (metadata.metadata \ "numTrees").extract[Int]
val trees: Array[DecisionTreeClassificationModel] = treesData.map {
case (treeMetadata, root) =>
- val tree = new DecisionTreeClassificationModel(treeMetadata.uid,
- root.asInstanceOf[ClassificationNode], numFeatures, numClasses)
+ val tree =
+ new DecisionTreeClassificationModel(treeMetadata.uid, root, numFeatures, numClasses)
treeMetadata.getAndSetParams(tree)
tree
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
index 8904193cae94c..5cb16cc765887 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
@@ -104,10 +104,6 @@ class BisectingKMeansModel private[ml] (
@Since("2.1.0")
def setPredictionCol(value: String): this.type = set(predictionCol, value)
- /** @group expertSetParam */
- @Since("2.4.0")
- def setDistanceMeasure(value: String): this.type = set(distanceMeasure, value)
-
@Since("2.0.0")
override def transform(dataset: Dataset[_]): DataFrame = {
transformSchema(dataset.schema, logging = true)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index 88abc1605d69f..6c0b49c040726 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -36,6 +36,7 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
import org.apache.spark.sql.functions.udf
import org.apache.spark.sql.types.{IntegerType, StructType}
+import org.apache.spark.storage.StorageLevel
/**
@@ -342,10 +343,15 @@ class GaussianMixture @Since("2.0.0") (
val sc = dataset.sparkSession.sparkContext
val numClusters = $(k)
+ val handlePersistence = dataset.storageLevel == StorageLevel.NONE
val instances = dataset
.select(DatasetUtils.columnToVector(dataset, getFeaturesCol)).rdd.map {
case Row(features: Vector) => features
- }.cache()
+ }
+
+ if (handlePersistence) {
+ instances.persist(StorageLevel.MEMORY_AND_DISK)
+ }
// Extract the number of features.
val numFeatures = instances.first().size
@@ -422,8 +428,10 @@ class GaussianMixture @Since("2.0.0") (
logLikelihood = sums.logLikelihood // this is the freshly computed log-likelihood
iter += 1
}
+ if (handlePersistence) {
+ instances.unpersist()
+ }
- instances.unpersist(false)
val gaussianDists = gaussians.map { case (mean, covVec) =>
val cov = GaussianMixture.unpackUpperTriangularMatrix(numFeatures, covVec.values)
new MultivariateGaussian(mean, cov)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
index 50867f776c522..84e73dc19a392 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
@@ -570,13 +570,11 @@ abstract class LDAModel private[ml] (
class LocalLDAModel private[ml] (
uid: String,
vocabSize: Int,
- private[clustering] val oldLocalModel_ : OldLocalLDAModel,
+ private[clustering] val oldLocalModel : OldLocalLDAModel,
sparkSession: SparkSession)
extends LDAModel(uid, vocabSize, sparkSession) {
- override private[clustering] def oldLocalModel: OldLocalLDAModel = {
- oldLocalModel_.setSeed(getSeed)
- }
+ oldLocalModel.setSeed(getSeed)
@Since("1.6.0")
override def copy(extra: ParamMap): LocalLDAModel = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala
index dc38ee326e5e9..dc18e1d34880a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/FeatureHasher.scala
@@ -29,7 +29,7 @@ import org.apache.spark.mllib.feature.{HashingTF => OldHashingTF}
import org.apache.spark.sql.{DataFrame, Dataset, Row}
import org.apache.spark.sql.functions._
import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.hash.Murmur3_x86_32.{hashInt, hashLong, hashUnsafeBytes2Block}
+import org.apache.spark.unsafe.hash.Murmur3_x86_32.{hashInt, hashLong, hashUnsafeBytes2}
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.Utils
import org.apache.spark.util.collection.OpenHashMap
@@ -244,7 +244,8 @@ object FeatureHasher extends DefaultParamsReadable[FeatureHasher] {
case f: Float => hashInt(java.lang.Float.floatToIntBits(f), seed)
case d: Double => hashLong(java.lang.Double.doubleToLongBits(d), seed)
case s: String =>
- hashUnsafeBytes2Block(UTF8String.fromString(s).getMemoryBlock, seed)
+ val utf8 = UTF8String.fromString(s)
+ hashUnsafeBytes2(utf8.getBaseObject, utf8.getBaseOffset, utf8.numBytes(), seed)
case _ => throw new SparkException("FeatureHasher with murmur3 algorithm does not " +
s"support type ${term.getClass.getCanonicalName} of input data.")
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/image/ImageSchema.scala b/mllib/src/main/scala/org/apache/spark/ml/image/ImageSchema.scala
index dcc40b6668c7a..0b13eefdf3f5f 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/image/ImageSchema.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/image/ImageSchema.scala
@@ -198,6 +198,8 @@ object ImageSchema {
* @return DataFrame with a single column "image" of images;
* see ImageSchema for the details
*/
+ @deprecated("use `spark.read.format(\"image\").load(path)` and this `readImages` will be " +
+ "removed in 3.0.0.", "2.4.0")
def readImages(path: String): DataFrame = readImages(path, null, false, -1, false, 1.0, 0)
/**
@@ -218,6 +220,8 @@ object ImageSchema {
* @return DataFrame with a single column "image" of images;
* see ImageSchema for the details
*/
+ @deprecated("use `spark.read.format(\"image\").load(path)` and this `readImages` will be " +
+ "removed in 3.0.0.", "2.4.0")
def readImages(
path: String,
sparkSession: SparkSession,
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/AFTSurvivalRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/AFTSurvivalRegressionWrapper.scala
index 48485e02edda8..1b5f77a9ae897 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/r/AFTSurvivalRegressionWrapper.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/r/AFTSurvivalRegressionWrapper.scala
@@ -62,7 +62,7 @@ private[r] object AFTSurvivalRegressionWrapper extends MLReadable[AFTSurvivalReg
private val FORMULA_REGEXP = """Surv\(([^,]+), ([^,]+)\) ~ (.+)""".r
private def formulaRewrite(formula: String): (String, String) = {
- var rewritedFormula: String = null
+ var rewrittenFormula: String = null
var censorCol: String = null
try {
val FORMULA_REGEXP(label, censor, features) = formula
@@ -71,14 +71,14 @@ private[r] object AFTSurvivalRegressionWrapper extends MLReadable[AFTSurvivalReg
throw new UnsupportedOperationException(
"Terms of survreg formula can not support dot operator.")
}
- rewritedFormula = label.trim + "~" + features.trim
+ rewrittenFormula = label.trim + "~" + features.trim
censorCol = censor.trim
} catch {
case e: MatchError =>
throw new SparkException(s"Could not parse formula: $formula")
}
- (rewritedFormula, censorCol)
+ (rewrittenFormula, censorCol)
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala
index 018290f81842f..6fa656275c1fd 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala
@@ -160,7 +160,7 @@ object DecisionTreeRegressor extends DefaultParamsReadable[DecisionTreeRegressor
@Since("1.4.0")
class DecisionTreeRegressionModel private[ml] (
override val uid: String,
- override val rootNode: RegressionNode,
+ override val rootNode: Node,
override val numFeatures: Int)
extends PredictionModel[Vector, DecisionTreeRegressionModel]
with DecisionTreeModel with DecisionTreeRegressorParams with MLWritable with Serializable {
@@ -175,7 +175,7 @@ class DecisionTreeRegressionModel private[ml] (
* Construct a decision tree regression model.
* @param rootNode Root node of tree, with other nodes attached.
*/
- private[ml] def this(rootNode: RegressionNode, numFeatures: Int) =
+ private[ml] def this(rootNode: Node, numFeatures: Int) =
this(Identifiable.randomUID("dtr"), rootNode, numFeatures)
override def predict(features: Vector): Double = {
@@ -279,9 +279,8 @@ object DecisionTreeRegressionModel extends MLReadable[DecisionTreeRegressionMode
implicit val format = DefaultFormats
val metadata = DefaultParamsReader.loadMetadata(path, sc, className)
val numFeatures = (metadata.metadata \ "numFeatures").extract[Int]
- val root = loadTreeNodes(path, metadata, sparkSession, isClassification = false)
- val model = new DecisionTreeRegressionModel(metadata.uid,
- root.asInstanceOf[RegressionNode], numFeatures)
+ val root = loadTreeNodes(path, metadata, sparkSession)
+ val model = new DecisionTreeRegressionModel(metadata.uid, root, numFeatures)
metadata.getAndSetParams(model)
model
}
@@ -296,8 +295,8 @@ object DecisionTreeRegressionModel extends MLReadable[DecisionTreeRegressionMode
require(oldModel.algo == OldAlgo.Regression,
s"Cannot convert non-regression DecisionTreeModel (old API) to" +
s" DecisionTreeRegressionModel (new API). Algo is: ${oldModel.algo}")
- val rootNode = Node.fromOld(oldModel.topNode, categoricalFeatures, isClassification = false)
+ val rootNode = Node.fromOld(oldModel.topNode, categoricalFeatures)
val uid = if (parent != null) parent.uid else Identifiable.randomUID("dtr")
- new DecisionTreeRegressionModel(uid, rootNode.asInstanceOf[RegressionNode], numFeatures)
+ new DecisionTreeRegressionModel(uid, rootNode, numFeatures)
}
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala
index 3305881b0ccc6..07f88d8d5f84d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala
@@ -338,15 +338,15 @@ object GBTRegressionModel extends MLReadable[GBTRegressionModel] {
override def load(path: String): GBTRegressionModel = {
implicit val format = DefaultFormats
val (metadata: Metadata, treesData: Array[(Metadata, Node)], treeWeights: Array[Double]) =
- EnsembleModelReadWrite.loadImpl(path, sparkSession, className, treeClassName, false)
+ EnsembleModelReadWrite.loadImpl(path, sparkSession, className, treeClassName)
val numFeatures = (metadata.metadata \ "numFeatures").extract[Int]
val numTrees = (metadata.metadata \ "numTrees").extract[Int]
val trees: Array[DecisionTreeRegressionModel] = treesData.map {
case (treeMetadata, root) =>
- val tree = new DecisionTreeRegressionModel(treeMetadata.uid,
- root.asInstanceOf[RegressionNode], numFeatures)
+ val tree =
+ new DecisionTreeRegressionModel(treeMetadata.uid, root, numFeatures)
treeMetadata.getAndSetParams(tree)
tree
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
index ce6c12cc368dd..92853b7db3b8c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
@@ -109,12 +109,13 @@ private[regression] trait LinearRegressionParams extends PredictorParams
schema: StructType,
fitting: Boolean,
featuresDataType: DataType): StructType = {
- if ($(loss) == Huber) {
- require($(solver)!= Normal, "LinearRegression with huber loss doesn't support " +
- "normal solver, please change solver to auto or l-bfgs.")
- require($(elasticNetParam) == 0.0, "LinearRegression with huber loss only supports " +
- s"L2 regularization, but got elasticNetParam = $getElasticNetParam.")
-
+ if (fitting) {
+ if ($(loss) == Huber) {
+ require($(solver)!= Normal, "LinearRegression with huber loss doesn't support " +
+ "normal solver, please change solver to auto or l-bfgs.")
+ require($(elasticNetParam) == 0.0, "LinearRegression with huber loss only supports " +
+ s"L2 regularization, but got elasticNetParam = $getElasticNetParam.")
+ }
}
super.validateAndTransformSchema(schema, fitting, featuresDataType)
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
index 35875724b3cfa..82bf66ff66d8a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
@@ -271,13 +271,13 @@ object RandomForestRegressionModel extends MLReadable[RandomForestRegressionMode
override def load(path: String): RandomForestRegressionModel = {
implicit val format = DefaultFormats
val (metadata: Metadata, treesData: Array[(Metadata, Node)], treeWeights: Array[Double]) =
- EnsembleModelReadWrite.loadImpl(path, sparkSession, className, treeClassName, false)
+ EnsembleModelReadWrite.loadImpl(path, sparkSession, className, treeClassName)
val numFeatures = (metadata.metadata \ "numFeatures").extract[Int]
val numTrees = (metadata.metadata \ "numTrees").extract[Int]
val trees: Array[DecisionTreeRegressionModel] = treesData.map { case (treeMetadata, root) =>
- val tree = new DecisionTreeRegressionModel(treeMetadata.uid,
- root.asInstanceOf[RegressionNode], numFeatures)
+ val tree =
+ new DecisionTreeRegressionModel(treeMetadata.uid, root, numFeatures)
treeMetadata.getAndSetParams(tree)
tree
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageDataSource.scala b/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageDataSource.scala
index a111c95248cf5..d4d74082dc8c5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageDataSource.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageDataSource.scala
@@ -19,14 +19,17 @@ package org.apache.spark.ml.source.image
/**
* `image` package implements Spark SQL data source API for loading image data as `DataFrame`.
- * The loaded `DataFrame` has one `StructType` column: `image`.
+ * It can load compressed image (jpeg, png, etc.) into raw image representation via `ImageIO`
+ * in Java library.
+ * The loaded `DataFrame` has one `StructType` column: `image`, containing image data stored
+ * as image schema.
* The schema of the `image` column is:
- * - origin: String (represents the file path of the image)
- * - height: Int (height of the image)
- * - width: Int (width of the image)
- * - nChannels: Int (number of the image channels)
- * - mode: Int (OpenCV-compatible type)
- * - data: BinaryType (Image bytes in OpenCV-compatible order: row-wise BGR in most cases)
+ * - origin: `StringType` (represents the file path of the image)
+ * - height: `IntegerType` (height of the image)
+ * - width: `IntegerType` (width of the image)
+ * - nChannels: `IntegerType` (number of image channels)
+ * - mode: `IntegerType` (OpenCV-compatible type)
+ * - data: `BinaryType` (Image bytes in OpenCV-compatible order: row-wise BGR in most cases)
*
* To use image data source, you need to set "image" as the format in `DataFrameReader` and
* optionally specify the data source options, for example:
diff --git a/mllib/src/main/scala/org/apache/spark/ml/stat/Correlation.scala b/mllib/src/main/scala/org/apache/spark/ml/stat/Correlation.scala
index 6e885d7c8aec5..8167ea68a7150 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/stat/Correlation.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/stat/Correlation.scala
@@ -49,7 +49,7 @@ object Correlation {
* Supported: `pearson` (default), `spearman`
* @return A dataframe that contains the correlation matrix of the column of vectors. This
* dataframe contains a single row and a single column of name
- * '$METHODNAME($COLUMN)'.
+ * `$METHODNAME($COLUMN)`.
* @throws IllegalArgumentException if the column is not a valid column in the dataset, or if
* the content of this column is not of type Vector.
*
diff --git a/mllib/src/main/scala/org/apache/spark/ml/stat/Summarizer.scala b/mllib/src/main/scala/org/apache/spark/ml/stat/Summarizer.scala
index d40827edb6d64..ed7d7e0852647 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/stat/Summarizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/stat/Summarizer.scala
@@ -96,7 +96,7 @@ object Summarizer extends Logging {
* - numNonzeros: a vector with the number of non-zeros for each coefficients
* - max: the maximum for each coefficient.
* - min: the minimum for each coefficient.
- * - normL2: the Euclidian norm for each coefficient.
+ * - normL2: the Euclidean norm for each coefficient.
* - normL1: the L1 norm of each coefficient (sum of the absolute values).
* @param metrics metrics that can be provided.
* @return a builder.
@@ -536,7 +536,7 @@ private[ml] object SummaryBuilderImpl extends Logging {
}
/**
- * L2 (Euclidian) norm of each dimension.
+ * L2 (Euclidean) norm of each dimension.
*/
def normL2: Vector = {
require(requestedMetrics.contains(NormL2))
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala
index 0242bc76698d0..d30be452a436e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala
@@ -17,16 +17,14 @@
package org.apache.spark.ml.tree
-import org.apache.spark.annotation.Since
import org.apache.spark.ml.linalg.Vector
import org.apache.spark.mllib.tree.impurity.ImpurityCalculator
-import org.apache.spark.mllib.tree.model.{ImpurityStats, InformationGainStats => OldInformationGainStats,
- Node => OldNode, Predict => OldPredict}
+import org.apache.spark.mllib.tree.model.{ImpurityStats, InformationGainStats => OldInformationGainStats, Node => OldNode, Predict => OldPredict}
/**
* Decision tree node interface.
*/
-sealed trait Node extends Serializable {
+sealed abstract class Node extends Serializable {
// TODO: Add aggregate stats (once available). This will happen after we move the DecisionTree
// code into the new API and deprecate the old API. SPARK-3727
@@ -86,86 +84,35 @@ private[ml] object Node {
/**
* Create a new Node from the old Node format, recursively creating child nodes as needed.
*/
- def fromOld(
- oldNode: OldNode,
- categoricalFeatures: Map[Int, Int],
- isClassification: Boolean): Node = {
+ def fromOld(oldNode: OldNode, categoricalFeatures: Map[Int, Int]): Node = {
if (oldNode.isLeaf) {
// TODO: Once the implementation has been moved to this API, then include sufficient
// statistics here.
- if (isClassification) {
- new ClassificationLeafNode(prediction = oldNode.predict.predict,
- impurity = oldNode.impurity, impurityStats = null)
- } else {
- new RegressionLeafNode(prediction = oldNode.predict.predict,
- impurity = oldNode.impurity, impurityStats = null)
- }
+ new LeafNode(prediction = oldNode.predict.predict,
+ impurity = oldNode.impurity, impurityStats = null)
} else {
val gain = if (oldNode.stats.nonEmpty) {
oldNode.stats.get.gain
} else {
0.0
}
- if (isClassification) {
- new ClassificationInternalNode(prediction = oldNode.predict.predict,
- impurity = oldNode.impurity, gain = gain,
- leftChild = fromOld(oldNode.leftNode.get, categoricalFeatures, true)
- .asInstanceOf[ClassificationNode],
- rightChild = fromOld(oldNode.rightNode.get, categoricalFeatures, true)
- .asInstanceOf[ClassificationNode],
- split = Split.fromOld(oldNode.split.get, categoricalFeatures), impurityStats = null)
- } else {
- new RegressionInternalNode(prediction = oldNode.predict.predict,
- impurity = oldNode.impurity, gain = gain,
- leftChild = fromOld(oldNode.leftNode.get, categoricalFeatures, false)
- .asInstanceOf[RegressionNode],
- rightChild = fromOld(oldNode.rightNode.get, categoricalFeatures, false)
- .asInstanceOf[RegressionNode],
- split = Split.fromOld(oldNode.split.get, categoricalFeatures), impurityStats = null)
- }
+ new InternalNode(prediction = oldNode.predict.predict, impurity = oldNode.impurity,
+ gain = gain, leftChild = fromOld(oldNode.leftNode.get, categoricalFeatures),
+ rightChild = fromOld(oldNode.rightNode.get, categoricalFeatures),
+ split = Split.fromOld(oldNode.split.get, categoricalFeatures), impurityStats = null)
}
}
}
-@Since("2.4.0")
-sealed trait ClassificationNode extends Node {
-
- /**
- * Get count of training examples for specified label in this node
- * @param label label number in the range [0, numClasses)
- */
- @Since("2.4.0")
- def getLabelCount(label: Int): Double = {
- require(label >= 0 && label < impurityStats.stats.length,
- "label should be in the range between 0 (inclusive) " +
- s"and ${impurityStats.stats.length} (exclusive).")
- impurityStats.stats(label)
- }
-}
-
-@Since("2.4.0")
-sealed trait RegressionNode extends Node {
-
- /** Number of training data points in this node */
- @Since("2.4.0")
- def getCount: Double = impurityStats.stats(0)
-
- /** Sum over training data points of the labels in this node */
- @Since("2.4.0")
- def getSum: Double = impurityStats.stats(1)
-
- /** Sum over training data points of the square of the labels in this node */
- @Since("2.4.0")
- def getSumOfSquares: Double = impurityStats.stats(2)
-}
-
-@Since("2.4.0")
-sealed trait LeafNode extends Node {
-
- /** Prediction this node makes. */
- def prediction: Double
-
- def impurity: Double
+/**
+ * Decision tree leaf node.
+ * @param prediction Prediction this node makes
+ * @param impurity Impurity measure at this node (for training data)
+ */
+class LeafNode private[ml] (
+ override val prediction: Double,
+ override val impurity: Double,
+ override private[ml] val impurityStats: ImpurityCalculator) extends Node {
override def toString: String =
s"LeafNode(prediction = $prediction, impurity = $impurity)"
@@ -188,58 +135,32 @@ sealed trait LeafNode extends Node {
override private[ml] def maxSplitFeatureIndex(): Int = -1
-}
-
-/**
- * Decision tree leaf node for classification.
- */
-@Since("2.4.0")
-class ClassificationLeafNode private[ml] (
- override val prediction: Double,
- override val impurity: Double,
- override private[ml] val impurityStats: ImpurityCalculator)
- extends ClassificationNode with LeafNode {
-
override private[tree] def deepCopy(): Node = {
- new ClassificationLeafNode(prediction, impurity, impurityStats)
+ new LeafNode(prediction, impurity, impurityStats)
}
}
/**
- * Decision tree leaf node for regression.
+ * Internal Decision Tree node.
+ * @param prediction Prediction this node would make if it were a leaf node
+ * @param impurity Impurity measure at this node (for training data)
+ * @param gain Information gain value. Values less than 0 indicate missing values;
+ * this quirk will be removed with future updates.
+ * @param leftChild Left-hand child node
+ * @param rightChild Right-hand child node
+ * @param split Information about the test used to split to the left or right child.
*/
-@Since("2.4.0")
-class RegressionLeafNode private[ml] (
+class InternalNode private[ml] (
override val prediction: Double,
override val impurity: Double,
- override private[ml] val impurityStats: ImpurityCalculator)
- extends RegressionNode with LeafNode {
-
- override private[tree] def deepCopy(): Node = {
- new RegressionLeafNode(prediction, impurity, impurityStats)
- }
-}
-
-/**
- * Internal Decision Tree node.
- */
-@Since("2.4.0")
-sealed trait InternalNode extends Node {
-
- /**
- * Information gain value. Values less than 0 indicate missing values;
- * this quirk will be removed with future updates.
- */
- def gain: Double
-
- /** Left-hand child node */
- def leftChild: Node
-
- /** Right-hand child node */
- def rightChild: Node
+ val gain: Double,
+ val leftChild: Node,
+ val rightChild: Node,
+ val split: Split,
+ override private[ml] val impurityStats: ImpurityCalculator) extends Node {
- /** Information about the test used to split to the left or right child. */
- def split: Split
+ // Note to developers: The constructor argument impurityStats should be reconsidered before we
+ // make the constructor public. We may be able to improve the representation.
override def toString: String = {
s"InternalNode(prediction = $prediction, impurity = $impurity, split = $split)"
@@ -284,6 +205,11 @@ sealed trait InternalNode extends Node {
math.max(split.featureIndex,
math.max(leftChild.maxSplitFeatureIndex(), rightChild.maxSplitFeatureIndex()))
}
+
+ override private[tree] def deepCopy(): Node = {
+ new InternalNode(prediction, impurity, gain, leftChild.deepCopy(), rightChild.deepCopy(),
+ split, impurityStats)
+ }
}
private object InternalNode {
@@ -314,57 +240,6 @@ private object InternalNode {
}
}
-/**
- * Internal Decision Tree node for regression.
- */
-@Since("2.4.0")
-class ClassificationInternalNode private[ml] (
- override val prediction: Double,
- override val impurity: Double,
- override val gain: Double,
- override val leftChild: ClassificationNode,
- override val rightChild: ClassificationNode,
- override val split: Split,
- override private[ml] val impurityStats: ImpurityCalculator)
- extends ClassificationNode with InternalNode {
-
- // Note to developers: The constructor argument impurityStats should be reconsidered before we
- // make the constructor public. We may be able to improve the representation.
-
- override private[tree] def deepCopy(): Node = {
- new ClassificationInternalNode(prediction, impurity, gain,
- leftChild.deepCopy().asInstanceOf[ClassificationNode],
- rightChild.deepCopy().asInstanceOf[ClassificationNode],
- split, impurityStats)
- }
-}
-
-/**
- * Internal Decision Tree node for regression.
- */
-@Since("2.4.0")
-class RegressionInternalNode private[ml] (
- override val prediction: Double,
- override val impurity: Double,
- override val gain: Double,
- override val leftChild: RegressionNode,
- override val rightChild: RegressionNode,
- override val split: Split,
- override private[ml] val impurityStats: ImpurityCalculator)
- extends RegressionNode with InternalNode {
-
- // Note to developers: The constructor argument impurityStats should be reconsidered before we
- // make the constructor public. We may be able to improve the representation.
-
- override private[tree] def deepCopy(): Node = {
- new RegressionInternalNode(prediction, impurity, gain,
- leftChild.deepCopy().asInstanceOf[RegressionNode],
- rightChild.deepCopy().asInstanceOf[RegressionNode],
- split, impurityStats)
- }
-}
-
-
/**
* Version of a node used in learning. This uses vars so that we can modify nodes as we split the
* tree by adding children, etc.
@@ -390,52 +265,30 @@ private[tree] class LearningNode(
var isLeaf: Boolean,
var stats: ImpurityStats) extends Serializable {
- def toNode(isClassification: Boolean): Node = toNode(isClassification, prune = true)
-
- def toClassificationNode(prune: Boolean = true): ClassificationNode = {
- toNode(true, prune).asInstanceOf[ClassificationNode]
- }
-
- def toRegressionNode(prune: Boolean = true): RegressionNode = {
- toNode(false, prune).asInstanceOf[RegressionNode]
- }
+ def toNode: Node = toNode(prune = true)
/**
* Convert this [[LearningNode]] to a regular [[Node]], and recurse on any children.
*/
- def toNode(isClassification: Boolean, prune: Boolean): Node = {
+ def toNode(prune: Boolean = true): Node = {
if (!leftChild.isEmpty || !rightChild.isEmpty) {
assert(leftChild.nonEmpty && rightChild.nonEmpty && split.nonEmpty && stats != null,
"Unknown error during Decision Tree learning. Could not convert LearningNode to Node.")
- (leftChild.get.toNode(isClassification, prune),
- rightChild.get.toNode(isClassification, prune)) match {
+ (leftChild.get.toNode(prune), rightChild.get.toNode(prune)) match {
case (l: LeafNode, r: LeafNode) if prune && l.prediction == r.prediction =>
- if (isClassification) {
- new ClassificationLeafNode(l.prediction, stats.impurity, stats.impurityCalculator)
- } else {
- new RegressionLeafNode(l.prediction, stats.impurity, stats.impurityCalculator)
- }
+ new LeafNode(l.prediction, stats.impurity, stats.impurityCalculator)
case (l, r) =>
- if (isClassification) {
- new ClassificationInternalNode(stats.impurityCalculator.predict, stats.impurity,
- stats.gain, l.asInstanceOf[ClassificationNode], r.asInstanceOf[ClassificationNode],
- split.get, stats.impurityCalculator)
- } else {
- new RegressionInternalNode(stats.impurityCalculator.predict, stats.impurity, stats.gain,
- l.asInstanceOf[RegressionNode], r.asInstanceOf[RegressionNode],
- split.get, stats.impurityCalculator)
- }
+ new InternalNode(stats.impurityCalculator.predict, stats.impurity, stats.gain,
+ l, r, split.get, stats.impurityCalculator)
}
} else {
- // Here we want to keep same behavior with the old mllib.DecisionTreeModel
- val impurity = if (stats.valid) stats.impurity else -1.0
- if (isClassification) {
- new ClassificationLeafNode(stats.impurityCalculator.predict, impurity,
+ if (stats.valid) {
+ new LeafNode(stats.impurityCalculator.predict, stats.impurity,
stats.impurityCalculator)
} else {
- new RegressionLeafNode(stats.impurityCalculator.predict, impurity,
- stats.impurityCalculator)
+ // Here we want to keep same behavior with the old mllib.DecisionTreeModel
+ new LeafNode(stats.impurityCalculator.predict, -1.0, stats.impurityCalculator)
}
}
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala
index 4cdd17266b771..822abd2d3522d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala
@@ -226,23 +226,23 @@ private[spark] object RandomForest extends Logging with Serializable {
case Some(uid) =>
if (strategy.algo == OldAlgo.Classification) {
topNodes.map { rootNode =>
- new DecisionTreeClassificationModel(uid, rootNode.toClassificationNode(prune),
- numFeatures, strategy.getNumClasses)
+ new DecisionTreeClassificationModel(uid, rootNode.toNode(prune), numFeatures,
+ strategy.getNumClasses)
}
} else {
topNodes.map { rootNode =>
- new DecisionTreeRegressionModel(uid, rootNode.toRegressionNode(prune), numFeatures)
+ new DecisionTreeRegressionModel(uid, rootNode.toNode(prune), numFeatures)
}
}
case None =>
if (strategy.algo == OldAlgo.Classification) {
topNodes.map { rootNode =>
- new DecisionTreeClassificationModel(rootNode.toClassificationNode(prune), numFeatures,
+ new DecisionTreeClassificationModel(rootNode.toNode(prune), numFeatures,
strategy.getNumClasses)
}
} else {
topNodes.map(rootNode =>
- new DecisionTreeRegressionModel(rootNode.toRegressionNode(prune), numFeatures))
+ new DecisionTreeRegressionModel(rootNode.toNode(prune), numFeatures))
}
}
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
index f027b14f1d476..4aa4c3617e7fd 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
@@ -219,10 +219,8 @@ private[ml] object TreeEnsembleModel {
importances.changeValue(feature, scaledGain, _ + scaledGain)
computeFeatureImportance(n.leftChild, importances)
computeFeatureImportance(n.rightChild, importances)
- case _: LeafNode =>
+ case n: LeafNode =>
// do nothing
- case _ =>
- throw new IllegalArgumentException(s"Unknown node type: ${node.getClass.toString}")
}
}
@@ -319,8 +317,6 @@ private[ml] object DecisionTreeModelReadWrite {
(Seq(NodeData(id, node.prediction, node.impurity, node.impurityStats.stats,
-1.0, -1, -1, SplitData(-1, Array.empty[Double], -1))),
id)
- case _ =>
- throw new IllegalArgumentException(s"Unknown node type: ${node.getClass.toString}")
}
}
@@ -331,7 +327,7 @@ private[ml] object DecisionTreeModelReadWrite {
def loadTreeNodes(
path: String,
metadata: DefaultParamsReader.Metadata,
- sparkSession: SparkSession, isClassification: Boolean): Node = {
+ sparkSession: SparkSession): Node = {
import sparkSession.implicits._
implicit val format = DefaultFormats
@@ -343,7 +339,7 @@ private[ml] object DecisionTreeModelReadWrite {
val dataPath = new Path(path, "data").toString
val data = sparkSession.read.parquet(dataPath).as[NodeData]
- buildTreeFromNodes(data.collect(), impurityType, isClassification)
+ buildTreeFromNodes(data.collect(), impurityType)
}
/**
@@ -352,8 +348,7 @@ private[ml] object DecisionTreeModelReadWrite {
* @param impurityType Impurity type for this tree
* @return Root node of reconstructed tree
*/
- def buildTreeFromNodes(data: Array[NodeData], impurityType: String,
- isClassification: Boolean): Node = {
+ def buildTreeFromNodes(data: Array[NodeData], impurityType: String): Node = {
// Load all nodes, sorted by ID.
val nodes = data.sortBy(_.id)
// Sanity checks; could remove
@@ -369,21 +364,10 @@ private[ml] object DecisionTreeModelReadWrite {
val node = if (n.leftChild != -1) {
val leftChild = finalNodes(n.leftChild)
val rightChild = finalNodes(n.rightChild)
- if (isClassification) {
- new ClassificationInternalNode(n.prediction, n.impurity, n.gain,
- leftChild.asInstanceOf[ClassificationNode], rightChild.asInstanceOf[ClassificationNode],
- n.split.getSplit, impurityStats)
- } else {
- new RegressionInternalNode(n.prediction, n.impurity, n.gain,
- leftChild.asInstanceOf[RegressionNode], rightChild.asInstanceOf[RegressionNode],
- n.split.getSplit, impurityStats)
- }
+ new InternalNode(n.prediction, n.impurity, n.gain, leftChild, rightChild,
+ n.split.getSplit, impurityStats)
} else {
- if (isClassification) {
- new ClassificationLeafNode(n.prediction, n.impurity, impurityStats)
- } else {
- new RegressionLeafNode(n.prediction, n.impurity, impurityStats)
- }
+ new LeafNode(n.prediction, n.impurity, impurityStats)
}
finalNodes(n.id) = node
}
@@ -437,8 +421,7 @@ private[ml] object EnsembleModelReadWrite {
path: String,
sql: SparkSession,
className: String,
- treeClassName: String,
- isClassification: Boolean): (Metadata, Array[(Metadata, Node)], Array[Double]) = {
+ treeClassName: String): (Metadata, Array[(Metadata, Node)], Array[Double]) = {
import sql.implicits._
implicit val format = DefaultFormats
val metadata = DefaultParamsReader.loadMetadata(path, sql.sparkContext, className)
@@ -466,8 +449,7 @@ private[ml] object EnsembleModelReadWrite {
val rootNodesRDD: RDD[(Int, Node)] =
nodeData.rdd.map(d => (d.treeID, d.nodeData)).groupByKey().map {
case (treeID: Int, nodeData: Iterable[NodeData]) =>
- treeID -> DecisionTreeModelReadWrite.buildTreeFromNodes(
- nodeData.toArray, impurityType, isClassification)
+ treeID -> DecisionTreeModelReadWrite.buildTreeFromNodes(nodeData.toArray, impurityType)
}
val rootNodes: Array[Node] = rootNodesRDD.sortByKey().values.collect()
(metadata, treesMetadata.zip(rootNodes), treesWeights)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
index 00157fe63af41..767c151c8293c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
@@ -40,43 +40,43 @@ private[ml] trait DecisionTreeParams extends PredictorParams
with HasCheckpointInterval with HasSeed {
/**
- * Maximum depth of the tree (>= 0).
+ * Maximum depth of the tree (nonnegative).
* E.g., depth 0 means 1 leaf node; depth 1 means 1 internal node + 2 leaf nodes.
* (default = 5)
* @group param
*/
final val maxDepth: IntParam =
- new IntParam(this, "maxDepth", "Maximum depth of the tree. (>= 0)" +
+ new IntParam(this, "maxDepth", "Maximum depth of the tree. (Nonnegative)" +
" E.g., depth 0 means 1 leaf node; depth 1 means 1 internal node + 2 leaf nodes.",
ParamValidators.gtEq(0))
/**
* Maximum number of bins used for discretizing continuous features and for choosing how to split
* on features at each node. More bins give higher granularity.
- * Must be >= 2 and >= number of categories in any categorical feature.
+ * Must be at least 2 and at least number of categories in any categorical feature.
* (default = 32)
* @group param
*/
final val maxBins: IntParam = new IntParam(this, "maxBins", "Max number of bins for" +
- " discretizing continuous features. Must be >=2 and >= number of categories for any" +
- " categorical feature.", ParamValidators.gtEq(2))
+ " discretizing continuous features. Must be at least 2 and at least number of categories" +
+ " for any categorical feature.", ParamValidators.gtEq(2))
/**
* Minimum number of instances each child must have after split.
* If a split causes the left or right child to have fewer than minInstancesPerNode,
* the split will be discarded as invalid.
- * Should be >= 1.
+ * Must be at least 1.
* (default = 1)
* @group param
*/
final val minInstancesPerNode: IntParam = new IntParam(this, "minInstancesPerNode", "Minimum" +
" number of instances each child must have after split. If a split causes the left or right" +
" child to have fewer than minInstancesPerNode, the split will be discarded as invalid." +
- " Should be >= 1.", ParamValidators.gtEq(1))
+ " Must be at least 1.", ParamValidators.gtEq(1))
/**
* Minimum information gain for a split to be considered at a tree node.
- * Should be >= 0.0.
+ * Should be at least 0.0.
* (default = 0.0)
* @group param
*/
@@ -372,7 +372,7 @@ private[ml] trait TreeEnsembleParams extends DecisionTreeParams {
* Supported options:
* - "auto": Choose automatically for task:
* If numTrees == 1, set to "all."
- * If numTrees > 1 (forest), set to "sqrt" for classification and
+ * If numTrees greater than 1 (forest), set to "sqrt" for classification and
* to "onethird" for regression.
* - "all": use all features
* - "onethird": use 1/3 of the features
@@ -424,8 +424,8 @@ private[ml] trait TreeEnsembleParams extends DecisionTreeParams {
private[ml] trait RandomForestParams extends TreeEnsembleParams {
/**
- * Number of trees to train (>= 1).
- * If 1, then no bootstrapping is used. If > 1, then bootstrapping is done.
+ * Number of trees to train (at least 1).
+ * If 1, then no bootstrapping is used. If greater than 1, then bootstrapping is done.
* TODO: Change to always do bootstrapping (simpler). SPARK-7130
* (default = 20)
*
@@ -434,7 +434,8 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams {
* are a bit different.
* @group param
*/
- final val numTrees: IntParam = new IntParam(this, "numTrees", "Number of trees to train (>= 1)",
+ final val numTrees: IntParam =
+ new IntParam(this, "numTrees", "Number of trees to train (at least 1)",
ParamValidators.gtEq(1))
setDefault(numTrees -> 20)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala b/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala
index 49654918bd8f8..e7c1a50f1a5e9 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala
@@ -17,6 +17,7 @@
package org.apache.spark.ml.util
+import java.io.{PrintWriter, StringWriter}
import java.util.UUID
import scala.util.{Failure, Success, Try}
@@ -160,8 +161,9 @@ private[spark] class Instrumentation private () extends Logging {
* Logs an exception raised during a training session.
*/
def logFailure(e: Throwable): Unit = {
- val msg = e.getStackTrace.mkString("\n")
- super.logError(msg)
+ val msg = new StringWriter()
+ e.printStackTrace(new PrintWriter(msg))
+ super.logError(msg.toString)
}
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
index 9d115afcea75d..4c5794fbffc8e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
@@ -109,10 +109,10 @@ class BisectingKMeansModel private[clustering] (
@Since("2.0.0")
override def save(sc: SparkContext, path: String): Unit = {
- BisectingKMeansModel.SaveLoadV1_0.save(sc, this, path)
+ BisectingKMeansModel.SaveLoadV2_0.save(sc, this, path)
}
- override protected def formatVersion: String = "1.0"
+ override protected def formatVersion: String = "2.0"
}
@Since("2.0.0")
@@ -126,7 +126,7 @@ object BisectingKMeansModel extends Loader[BisectingKMeansModel] {
val model = SaveLoadV1_0.load(sc, path)
model
case (SaveLoadV2_0.thisClassName, SaveLoadV2_0.thisFormatVersion) =>
- val model = SaveLoadV1_0.load(sc, path)
+ val model = SaveLoadV2_0.load(sc, path)
model
case _ => throw new Exception(
s"BisectingKMeansModel.load did not recognize model with (className, format version):" +
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala
index d5c8188144ce2..b0709547ab1be 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala
@@ -113,7 +113,7 @@ class KMeansModel (@Since("1.0.0") val clusterCenters: Array[Vector],
KMeansModel.SaveLoadV2_0.save(sc, this, path)
}
- override protected def formatVersion: String = "1.0"
+ override protected def formatVersion: String = "2.0"
}
@Since("1.4.0")
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/StreamingKMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/StreamingKMeans.scala
index ed8543da4d4ce..ff4ca0ac40fe2 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/StreamingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/StreamingKMeans.scala
@@ -42,8 +42,8 @@ import org.apache.spark.util.random.XORShiftRandom
*
* $$
* \begin{align}
- * c_t+1 &= [(c_t * n_t * a) + (x_t * m_t)] / [n_t + m_t] \\
- * n_t+1 &= n_t * a + m_t
+ * c_{t+1} &= [(c_t * n_t * a) + (x_t * m_t)] / [n_t + m_t] \\
+ * n_{t+1} &= n_t * a + m_t
* \end{align}
* $$
*
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala
index 2cfcf38eb4ca8..764806bc64a97 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala
@@ -175,12 +175,15 @@ class BinaryClassificationMetrics @Since("1.3.0") (
grouping = Int.MaxValue
}
counts.mapPartitions(_.grouped(grouping.toInt).map { pairs =>
- // The score of the combined point will be just the first one's score
- val firstScore = pairs.head._1
- // The point will contain all counts in this chunk
+ // The score of the combined point will be just the last one's score, which is also
+ // the minimal in each chunk since all scores are already sorted in descending.
+ val lastScore = pairs.last._1
+ // The combined point will contain all counts in this chunk. Thus, calculated
+ // metrics (like precision, recall, etc.) on its score (or so-called threshold) are
+ // the same as those without sampling.
val agg = new BinaryLabelCounter()
pairs.foreach(pair => agg += pair._2)
- (firstScore, agg)
+ (lastScore, agg)
})
}
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RankingMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RankingMetrics.scala
index b98aa0534152b..4935d11411134 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RankingMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/RankingMetrics.scala
@@ -138,6 +138,8 @@ class RankingMetrics[T: ClassTag](predictionAndLabels: RDD[(Array[T], Array[T])]
var dcg = 0.0
var i = 0
while (i < n) {
+ // Base of the log doesn't matter for calculating NDCG,
+ // if the relevance value is binary.
val gain = 1.0 / math.log(i + 2)
if (i < pred.length && labSet.contains(pred(i))) {
dcg += gain
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/HashingTF.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/HashingTF.scala
index 7b73b286fb91c..8935c8496cdbb 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/feature/HashingTF.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/HashingTF.scala
@@ -160,7 +160,7 @@ object HashingTF {
case d: Double => hashLong(java.lang.Double.doubleToLongBits(d), seed)
case s: String =>
val utf8 = UTF8String.fromString(s)
- hashUnsafeBytesBlock(utf8.getMemoryBlock(), seed)
+ hashUnsafeBytes(utf8.getBaseObject, utf8.getBaseOffset, utf8.numBytes(), seed)
case _ => throw new SparkException("HashingTF with murmur3 algorithm does not " +
s"support type ${term.getClass.getCanonicalName} of input data.")
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala
index b8c306d86bace..bb5d02e8ad37f 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala
@@ -45,7 +45,7 @@ import org.apache.spark.util.random.XORShiftRandom
*/
private case class VocabWord(
var word: String,
- var cn: Int,
+ var cn: Long,
var point: Array[Int],
var code: Array[Int],
var codeLen: Int
@@ -194,7 +194,7 @@ class Word2Vec extends Serializable with Logging {
new Array[Int](MAX_CODE_LENGTH),
0))
.collect()
- .sortWith((a, b) => a.cn > b.cn)
+ .sortBy(_.cn)(Ordering[Long].reverse)
vocabSize = vocab.length
require(vocabSize > 0, "The vocabulary size should be > 0. You may need to check " +
@@ -232,7 +232,7 @@ class Word2Vec extends Serializable with Logging {
a += 1
}
while (a < 2 * vocabSize) {
- count(a) = 1e9.toInt
+ count(a) = Long.MaxValue
a += 1
}
var pos1 = vocabSize - 1
@@ -267,6 +267,8 @@ class Word2Vec extends Serializable with Logging {
min2i = pos2
pos2 += 1
}
+ assert(count(min1i) < Long.MaxValue)
+ assert(count(min2i) < Long.MaxValue)
count(vocabSize + a) = count(min1i) + count(min2i)
parentNode(min1i) = vocabSize + a
parentNode(min2i) = vocabSize + a
@@ -436,9 +438,20 @@ class Word2Vec extends Serializable with Logging {
}
}.flatten
}
- val synAgg = partial.reduceByKey { case (v1, v2) =>
- blas.saxpy(vectorSize, 1.0f, v2, 1, v1, 1)
- v1
+ // SPARK-24666: do normalization for aggregating weights from partitions.
+ // Original Word2Vec either single-thread or multi-thread which do Hogwild-style aggregation.
+ // Our approach needs to do extra normalization, otherwise adding weights continuously may
+ // cause overflow on float and lead to infinity/-infinity weights.
+ val synAgg = partial.mapPartitions { iter =>
+ iter.map { case (id, vec) =>
+ (id, (vec, 1))
+ }
+ }.reduceByKey { case ((v1, count1), (v2, count2)) =>
+ blas.saxpy(vectorSize, 1.0f, v2, 1, v1, 1)
+ (v1, count1 + count2)
+ }.map { case (id, (vec, count)) =>
+ blas.sscal(vectorSize, 1.0f / count, vec, 1)
+ (id, vec)
}.collect()
var i = 0
while (i < synAgg.length) {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
index bf9b4cfe15b2c..e474cfa002fad 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
@@ -30,6 +30,7 @@ import org.apache.spark.ml.{linalg => newlinalg}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeArrayData}
import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.array.ByteArrayMethods
/**
* Trait for a local matrix.
@@ -456,7 +457,7 @@ object DenseMatrix {
*/
@Since("1.3.0")
def zeros(numRows: Int, numCols: Int): DenseMatrix = {
- require(numRows.toLong * numCols <= Int.MaxValue,
+ require(numRows.toLong * numCols <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
s"$numRows x $numCols dense matrix is too large to allocate")
new DenseMatrix(numRows, numCols, new Array[Double](numRows * numCols))
}
@@ -469,7 +470,7 @@ object DenseMatrix {
*/
@Since("1.3.0")
def ones(numRows: Int, numCols: Int): DenseMatrix = {
- require(numRows.toLong * numCols <= Int.MaxValue,
+ require(numRows.toLong * numCols <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
s"$numRows x $numCols dense matrix is too large to allocate")
new DenseMatrix(numRows, numCols, Array.fill(numRows * numCols)(1.0))
}
@@ -499,7 +500,7 @@ object DenseMatrix {
*/
@Since("1.3.0")
def rand(numRows: Int, numCols: Int, rng: Random): DenseMatrix = {
- require(numRows.toLong * numCols <= Int.MaxValue,
+ require(numRows.toLong * numCols <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
s"$numRows x $numCols dense matrix is too large to allocate")
new DenseMatrix(numRows, numCols, Array.fill(numRows * numCols)(rng.nextDouble()))
}
@@ -513,7 +514,7 @@ object DenseMatrix {
*/
@Since("1.3.0")
def randn(numRows: Int, numCols: Int, rng: Random): DenseMatrix = {
- require(numRows.toLong * numCols <= Int.MaxValue,
+ require(numRows.toLong * numCols <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
s"$numRows x $numCols dense matrix is too large to allocate")
new DenseMatrix(numRows, numCols, Array.fill(numRows * numCols)(rng.nextGaussian()))
}
@@ -846,8 +847,8 @@ object SparseMatrix {
s"density must be a double in the range 0.0 <= d <= 1.0. Currently, density: $density")
val size = numRows.toLong * numCols
val expected = size * density
- assert(expected < Int.MaxValue,
- "The expected number of nonzeros cannot be greater than Int.MaxValue.")
+ assert(expected < ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
+ "The expected number of nonzeros cannot be greater than Int.MaxValue - 15.")
val nnz = math.ceil(expected).toInt
if (density == 0.0) {
new SparseMatrix(numRows, numCols, new Array[Int](numCols + 1), Array.empty, Array.empty)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
index 6e68d9684a672..fbc935862a25c 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
@@ -785,6 +785,15 @@ class SparseVector @Since("1.0.0") (
private[spark] override def asBreeze: BV[Double] = new BSV[Double](indices, values, size)
+ override def apply(i: Int): Double = {
+ if (i < 0 || i >= size) {
+ throw new IndexOutOfBoundsException(s"Index $i out of bounds [0, $size)")
+ }
+
+ val j = util.Arrays.binarySearch(indices, i)
+ if (j < 0) 0.0 else values(j)
+ }
+
@Since("1.6.0")
override def foreachActive(f: (Int, Double) => Unit): Unit = {
var i = 0
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
index 78a8810052aef..5109efb1e39e5 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
@@ -119,11 +119,25 @@ class RowMatrix @Since("1.0.0") (
val nt = if (n % 2 == 0) ((n / 2) * (n + 1)) else (n * ((n + 1) / 2))
// Compute the upper triangular part of the gram matrix.
- val GU = rows.treeAggregate(new BDV[Double](nt))(
- seqOp = (U, v) => {
+ val GU = rows.treeAggregate(null.asInstanceOf[BDV[Double]])(
+ seqOp = (maybeU, v) => {
+ val U =
+ if (maybeU == null) {
+ new BDV[Double](nt)
+ } else {
+ maybeU
+ }
BLAS.spr(1.0, v, U.data)
U
- }, combOp = (U1, U2) => U1 += U2)
+ }, combOp = (U1, U2) =>
+ if (U1 == null) {
+ U2
+ } else if (U2 == null) {
+ U1
+ } else {
+ U1 += U2
+ }
+ )
RowMatrix.triuToFull(n, GU.data)
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala
index 8121880cfb233..0554b6d8ff5b5 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala
@@ -273,7 +273,7 @@ class MultivariateOnlineSummarizer extends MultivariateStatisticalSummary with S
}
/**
- * L2 (Euclidian) norm of each dimension.
+ * L2 (Euclidean) norm of each dimension.
*
*/
@Since("1.2.0")
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/StreamingTestMethod.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/StreamingTestMethod.scala
index 14ac14d6d61f4..8f3d0f8b3214c 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/StreamingTestMethod.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/StreamingTestMethod.scala
@@ -33,7 +33,7 @@ import org.apache.spark.util.StatCounter
/**
* Significance testing methods for [[StreamingTest]]. New 2-sample statistical significance tests
* should extend [[StreamingTestMethod]] and introduce a new entry in
- * [[StreamingTestMethod.TEST_NAME_TO_OBJECT]]
+ * `StreamingTestMethod.TEST_NAME_TO_OBJECT`
*/
private[stat] sealed trait StreamingTestMethod extends Serializable {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala
index d3dbb4e754d3d..2930f4900d50e 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala
@@ -21,7 +21,7 @@ import org.apache.spark.SparkFunSuite
import org.apache.spark.ml.feature.LabeledPoint
import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.ml.param.ParamsSuite
-import org.apache.spark.ml.tree.ClassificationLeafNode
+import org.apache.spark.ml.tree.LeafNode
import org.apache.spark.ml.tree.impl.TreeTests
import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
import org.apache.spark.mllib.regression.{LabeledPoint => OldLabeledPoint}
@@ -61,8 +61,7 @@ class DecisionTreeClassifierSuite extends MLTest with DefaultReadWriteTest {
test("params") {
ParamsSuite.checkParams(new DecisionTreeClassifier)
- val model = new DecisionTreeClassificationModel("dtc",
- new ClassificationLeafNode(0.0, 0.0, null), 1, 2)
+ val model = new DecisionTreeClassificationModel("dtc", new LeafNode(0.0, 0.0, null), 1, 2)
ParamsSuite.checkParams(model)
}
@@ -376,32 +375,6 @@ class DecisionTreeClassifierSuite extends MLTest with DefaultReadWriteTest {
testDefaultReadWrite(model)
}
-
- test("label/impurity stats") {
- val arr = Array(
- LabeledPoint(0.0, Vectors.sparse(2, Seq((0, 0.0)))),
- LabeledPoint(1.0, Vectors.sparse(2, Seq((1, 1.0)))),
- LabeledPoint(0.0, Vectors.sparse(2, Seq((0, 1.0)))))
- val rdd = sc.parallelize(arr)
- val df = TreeTests.setMetadata(rdd, Map.empty[Int, Int], 2)
- val dt1 = new DecisionTreeClassifier()
- .setImpurity("entropy")
- .setMaxDepth(2)
- .setMinInstancesPerNode(2)
- val model1 = dt1.fit(df)
-
- val rootNode1 = model1.rootNode
- assert(Array(rootNode1.getLabelCount(0), rootNode1.getLabelCount(1)) === Array(2.0, 1.0))
-
- val dt2 = new DecisionTreeClassifier()
- .setImpurity("gini")
- .setMaxDepth(2)
- .setMinInstancesPerNode(2)
- val model2 = dt2.fit(df)
-
- val rootNode2 = model2.rootNode
- assert(Array(rootNode2.getLabelCount(0), rootNode2.getLabelCount(1)) === Array(2.0, 1.0))
- }
}
private[ml] object DecisionTreeClassifierSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala
index e6d2a8e2b900e..304977634189c 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.ml.feature.LabeledPoint
import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.ml.param.ParamsSuite
import org.apache.spark.ml.regression.DecisionTreeRegressionModel
-import org.apache.spark.ml.tree.RegressionLeafNode
+import org.apache.spark.ml.tree.LeafNode
import org.apache.spark.ml.tree.impl.{GradientBoostedTrees, TreeTests}
import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
import org.apache.spark.ml.util.TestingUtils._
@@ -70,7 +70,7 @@ class GBTClassifierSuite extends MLTest with DefaultReadWriteTest {
test("params") {
ParamsSuite.checkParams(new GBTClassifier)
val model = new GBTClassificationModel("gbtc",
- Array(new DecisionTreeRegressionModel("dtr", new RegressionLeafNode(0.0, 0.0, null), 1)),
+ Array(new DecisionTreeRegressionModel("dtr", new LeafNode(0.0, 0.0, null), 1)),
Array(1.0), 1, 2)
ParamsSuite.checkParams(model)
}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala
index 3062aa9f3d274..ba4a9cf082785 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala
@@ -21,7 +21,7 @@ import org.apache.spark.SparkFunSuite
import org.apache.spark.ml.feature.LabeledPoint
import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.ml.param.ParamsSuite
-import org.apache.spark.ml.tree.ClassificationLeafNode
+import org.apache.spark.ml.tree.LeafNode
import org.apache.spark.ml.tree.impl.TreeTests
import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
import org.apache.spark.mllib.regression.{LabeledPoint => OldLabeledPoint}
@@ -71,8 +71,7 @@ class RandomForestClassifierSuite extends MLTest with DefaultReadWriteTest {
test("params") {
ParamsSuite.checkParams(new RandomForestClassifier)
val model = new RandomForestClassificationModel("rfc",
- Array(new DecisionTreeClassificationModel("dtc",
- new ClassificationLeafNode(0.0, 0.0, null), 1, 2)), 2, 2)
+ Array(new DecisionTreeClassificationModel("dtc", new LeafNode(0.0, 0.0, null), 1, 2)), 2, 2)
ParamsSuite.checkParams(model)
}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
index ed15a1d88a269..a4d388fd321db 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
@@ -256,4 +256,9 @@ class VectorAssemblerSuite
assert(runWithMetadata("keep", additional_filter = "id1 > 2").count() == 4)
}
+ test("SPARK-25371: VectorAssembler with empty inputCols") {
+ val vectorAssembler = new VectorAssembler().setInputCols(Array()).setOutputCol("a")
+ val output = vectorAssembler.transform(dfWithNullsAndNaNs)
+ assert(output.select("a").limit(1).collect().head == Row(Vectors.sparse(0, Seq.empty)))
+ }
}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/Word2VecSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/Word2VecSuite.scala
index b59c4e7967338..c816a6cd9c965 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/Word2VecSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/Word2VecSuite.scala
@@ -75,14 +75,6 @@ class Word2VecSuite extends MLTest with DefaultReadWriteTest {
test("getVectors") {
val sentence = "a b " * 100 + "a c " * 10
val doc = sc.parallelize(Seq(sentence, sentence)).map(line => line.split(" "))
-
- val codes = Map(
- "a" -> Array(-0.2811822295188904, -0.6356269121170044, -0.3020961284637451),
- "b" -> Array(1.0309048891067505, -1.29472815990448, 0.22276712954044342),
- "c" -> Array(-0.08456747233867645, 0.5137411952018738, 0.11731560528278351)
- )
- val expectedVectors = codes.toSeq.sortBy(_._1).map { case (w, v) => Vectors.dense(v) }
-
val docDF = doc.zip(doc).toDF("text", "alsotext")
val model = new Word2Vec()
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala
index 9ae27339b11d5..29a438396516b 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala
@@ -191,20 +191,6 @@ class DecisionTreeRegressorSuite extends MLTest with DefaultReadWriteTest {
TreeTests.allParamSettings ++ Map("maxDepth" -> 0),
TreeTests.allParamSettings ++ Map("maxDepth" -> 0), checkModelData)
}
-
- test("label/impurity stats") {
- val categoricalFeatures = Map(0 -> 2, 1 -> 2)
- val df = TreeTests.setMetadata(categoricalDataPointsRDD, categoricalFeatures, numClasses = 0)
- val dtr = new DecisionTreeRegressor()
- .setImpurity("variance")
- .setMaxDepth(2)
- .setMaxBins(8)
- val model = dtr.fit(df)
- val statInfo = model.rootNode
-
- assert(statInfo.getCount == 1000.0 && statInfo.getSum == 600.0
- && statInfo.getSumOfSquares == 600.0)
- }
}
private[ml] object DecisionTreeRegressorSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
index 90ceb7dee38f7..36bff8301e713 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
@@ -187,6 +187,18 @@ class LinearRegressionSuite extends MLTest with DefaultReadWriteTest with PMMLRe
assert(model.numFeatures === numFeatures)
}
+ test("linear regression: can transform data with LinearRegressionModel") {
+ withClue("training related params like loss are only validated during fitting phase") {
+ val original = new LinearRegression().fit(datasetWithDenseFeature)
+
+ val deserialized = new LinearRegressionModel(uid = original.uid,
+ coefficients = original.coefficients,
+ intercept = original.intercept)
+ val output = deserialized.transform(datasetWithDenseFeature)
+ assert(output.collect().size > 0) // simple assertion to ensure no exception thrown
+ }
+ }
+
test("linear regression: illegal params") {
withClue("LinearRegression with huber loss only supports L2 regularization") {
intercept[IllegalArgumentException] {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/source/image/ImageFileFormatSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/source/image/ImageFileFormatSuite.scala
index 1a6a8d67d8d66..38e25131df867 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/source/image/ImageFileFormatSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/source/image/ImageFileFormatSuite.scala
@@ -17,6 +17,7 @@
package org.apache.spark.ml.source.image
+import java.net.URI
import java.nio.file.Paths
import org.apache.spark.SparkFunSuite
@@ -58,8 +59,14 @@ class ImageFileFormatSuite extends SparkFunSuite with MLlibTestSparkContext {
.load(filePath)
assert(df2.count() === 1)
val result = df2.head()
- assert(result === invalidImageRow(
- Paths.get(filePath).toAbsolutePath().normalize().toUri().toString))
+
+ val resultOrigin = result.getStruct(0).getString(0)
+ // covert `origin` to `java.net.URI` object and then compare.
+ // because `file:/path` and `file:///path` are both valid URI-ifications
+ assert(new URI(resultOrigin) === Paths.get(filePath).toAbsolutePath().normalize().toUri())
+
+ // Compare other columns in the row to be the same with the `invalidImageRow`
+ assert(result === invalidImageRow(resultOrigin))
}
test("image datasource partition test") {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tree/impl/RandomForestSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tree/impl/RandomForestSuite.scala
index 4dbbd75d2466d..743dacf146fe7 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tree/impl/RandomForestSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tree/impl/RandomForestSuite.scala
@@ -340,8 +340,8 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
assert(topNode.stats.impurity > 0.0)
// set impurity and predict for child nodes
- assert(topNode.leftChild.get.toNode(isClassification = true).prediction === 0.0)
- assert(topNode.rightChild.get.toNode(isClassification = true).prediction === 1.0)
+ assert(topNode.leftChild.get.toNode.prediction === 0.0)
+ assert(topNode.rightChild.get.toNode.prediction === 1.0)
assert(topNode.leftChild.get.stats.impurity === 0.0)
assert(topNode.rightChild.get.stats.impurity === 0.0)
}
@@ -382,8 +382,8 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
assert(topNode.stats.impurity > 0.0)
// set impurity and predict for child nodes
- assert(topNode.leftChild.get.toNode(isClassification = true).prediction === 0.0)
- assert(topNode.rightChild.get.toNode(isClassification = true).prediction === 1.0)
+ assert(topNode.leftChild.get.toNode.prediction === 0.0)
+ assert(topNode.rightChild.get.toNode.prediction === 1.0)
assert(topNode.leftChild.get.stats.impurity === 0.0)
assert(topNode.rightChild.get.stats.impurity === 0.0)
}
@@ -582,18 +582,18 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
left right
*/
val leftImp = new GiniCalculator(Array(3.0, 2.0, 1.0))
- val left = new ClassificationLeafNode(0.0, leftImp.calculate(), leftImp)
+ val left = new LeafNode(0.0, leftImp.calculate(), leftImp)
val rightImp = new GiniCalculator(Array(1.0, 2.0, 5.0))
- val right = new ClassificationLeafNode(2.0, rightImp.calculate(), rightImp)
+ val right = new LeafNode(2.0, rightImp.calculate(), rightImp)
- val parent = TreeTests.buildParentNode(left, right, new ContinuousSplit(0, 0.5), true)
+ val parent = TreeTests.buildParentNode(left, right, new ContinuousSplit(0, 0.5))
val parentImp = parent.impurityStats
val left2Imp = new GiniCalculator(Array(1.0, 6.0, 1.0))
- val left2 = new ClassificationLeafNode(0.0, left2Imp.calculate(), left2Imp)
+ val left2 = new LeafNode(0.0, left2Imp.calculate(), left2Imp)
- val grandParent = TreeTests.buildParentNode(left2, parent, new ContinuousSplit(1, 1.0), true)
+ val grandParent = TreeTests.buildParentNode(left2, parent, new ContinuousSplit(1, 1.0))
val grandImp = grandParent.impurityStats
// Test feature importance computed at different subtrees.
@@ -618,8 +618,8 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
// Forest consisting of (full tree) + (internal node with 2 leafs)
val trees = Array(parent, grandParent).map { root =>
- new DecisionTreeClassificationModel(root.asInstanceOf[ClassificationNode],
- numFeatures = 2, numClasses = 3).asInstanceOf[DecisionTreeModel]
+ new DecisionTreeClassificationModel(root, numFeatures = 2, numClasses = 3)
+ .asInstanceOf[DecisionTreeModel]
}
val importances: Vector = TreeEnsembleModel.featureImportances(trees, 2)
val tree2norm = feature0importance + feature1importance
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala b/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala
index 3f03d909d4a4c..b6894b30b0c2b 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala
@@ -159,7 +159,7 @@ private[ml] object TreeTests extends SparkFunSuite {
* @param split Split for parent node
* @return Parent node with children attached
*/
- def buildParentNode(left: Node, right: Node, split: Split, isClassification: Boolean): Node = {
+ def buildParentNode(left: Node, right: Node, split: Split): Node = {
val leftImp = left.impurityStats
val rightImp = right.impurityStats
val parentImp = leftImp.copy.add(rightImp)
@@ -168,15 +168,7 @@ private[ml] object TreeTests extends SparkFunSuite {
val gain = parentImp.calculate() -
(leftWeight * leftImp.calculate() + rightWeight * rightImp.calculate())
val pred = parentImp.predict
- if (isClassification) {
- new ClassificationInternalNode(pred, parentImp.calculate(), gain,
- left.asInstanceOf[ClassificationNode], right.asInstanceOf[ClassificationNode],
- split, parentImp)
- } else {
- new RegressionInternalNode(pred, parentImp.calculate(), gain,
- left.asInstanceOf[RegressionNode], right.asInstanceOf[RegressionNode],
- split, parentImp)
- }
+ new InternalNode(pred, parentImp.calculate(), gain, left, right, split, parentImp)
}
/**
diff --git a/mllib/src/test/scala/org/apache/spark/ml/util/MLTest.scala b/mllib/src/test/scala/org/apache/spark/ml/util/MLTest.scala
index 76d41f9b23715..acac171346a85 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/util/MLTest.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/util/MLTest.scala
@@ -21,12 +21,13 @@ import java.io.File
import org.scalatest.Suite
-import org.apache.spark.SparkContext
+import org.apache.spark.{DebugFilesystem, SparkConf, SparkContext}
import org.apache.spark.ml.{PredictionModel, Transformer}
import org.apache.spark.ml.linalg.Vector
import org.apache.spark.sql.{DataFrame, Dataset, Encoder, Row}
import org.apache.spark.sql.execution.streaming.MemoryStream
import org.apache.spark.sql.functions.col
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.streaming.StreamTest
import org.apache.spark.sql.test.TestSparkSession
import org.apache.spark.util.Utils
@@ -36,6 +37,13 @@ trait MLTest extends StreamTest with TempDirectory { self: Suite =>
@transient var sc: SparkContext = _
@transient var checkpointDir: String = _
+ protected override def sparkConf = {
+ new SparkConf()
+ .set("spark.hadoop.fs.file.impl", classOf[DebugFilesystem].getName)
+ .set("spark.unsafe.exceptionOnMemoryLeak", "true")
+ .set(SQLConf.CODEGEN_FALLBACK.key, "false")
+ }
+
protected override def createSparkSession: TestSparkSession = {
new TestSparkSession(new SparkContext("local[2]", "MLlibUnitTest", sparkConf))
}
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/clustering/BisectingKMeansSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/clustering/BisectingKMeansSuite.scala
index 35f7932ae8224..4a4d8b5c89de8 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/clustering/BisectingKMeansSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/clustering/BisectingKMeansSuite.scala
@@ -187,11 +187,12 @@ class BisectingKMeansSuite extends SparkFunSuite with MLlibTestSparkContext {
val points = (1 until 8).map(i => Vectors.dense(i))
val data = sc.parallelize(points, 2)
- val model = new BisectingKMeans().run(data)
+ val model = new BisectingKMeans().setDistanceMeasure(DistanceMeasure.COSINE).run(data)
try {
model.save(sc, path)
val sameModel = BisectingKMeansModel.load(sc, path)
assert(model.k === sameModel.k)
+ assert(model.distanceMeasure === sameModel.distanceMeasure)
model.clusterCenters.zip(sameModel.clusterCenters).foreach(c => c._1 === c._2)
} finally {
Utils.deleteRecursively(tempDir)
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala
index a08917ac1ebed..4cc9ee5a5cc9b 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala
@@ -155,6 +155,17 @@ class BinaryClassificationMetricsSuite extends SparkFunSuite with MLlibTestSpark
(1.0, 1.0), (1.0, 1.0)
) ==
downsampledROC)
+
+ val downsampledRecall = downsampled.recallByThreshold().collect().sorted.toList
+ assert(
+ // May have to add 1 if the sample factor didn't divide evenly
+ numBins + (if (scoreAndLabels.size % numBins == 0) 0 else 1) ==
+ downsampledRecall.size)
+ assert(
+ List(
+ (0.1, 1.0), (0.2, 1.0), (0.4, 0.75), (0.6, 0.75), (0.8, 0.25)
+ ) ==
+ downsampledRecall)
}
}
diff --git a/pom.xml b/pom.xml
index da526a1709e65..010b58b5f19da 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
pom
Spark Project Parent POM
http://spark.apache.org/
@@ -39,7 +39,7 @@
scm:git:git@github.com:apache/spark.git
- scm:git:https://git-wip-us.apache.org/repos/asf/spark.git
+ scm:git:https://gitbox.apache.org/repos/asf/spark.git
scm:git:git@github.com:apache/spark.git
HEAD
@@ -130,11 +130,11 @@
1.2.1
10.12.1.1
- 1.10.0
- 1.5.2
+ 1.10.1
+ 1.5.5
nohive
1.6.0
- 9.3.24.v20180605
+ 9.3.27.v20190418
3.1.0
0.9.3
2.4.0
@@ -142,7 +142,7 @@
3.1.5
1.8.2
hadoop2
- 1.8.10
+ 1.12.0
1.11.271
@@ -159,8 +159,9 @@
2.11
1.9.13
2.6.7
- 2.6.7.1
- 1.1.7.1
+ 2.6.7.1
+ 2.6.7.3
+ 1.1.7.3
1.1.2
1.2.0-incubating
1.10
@@ -226,7 +227,7 @@
-->
${session.executionRootDirectory}
- 512m
+ 1g
@@ -435,6 +436,11 @@
commons-collections
${commons.collections.version}
+
+ commons-beanutils
+ commons-beanutils
+ 1.9.4
+
org.apache.ivy
ivy
@@ -574,7 +580,7 @@
org.roaringbitmap
RoaringBitmap
- 0.5.11
+ 0.7.45
commons-net
@@ -584,7 +590,7 @@
io.netty
netty-all
- 4.1.17.Final
+ 4.1.42.Final
io.netty
@@ -641,7 +647,7 @@
com.fasterxml.jackson.module
jackson-module-scala_${scala.binary.version}
- ${fasterxml.jackson.databind.version}
+ ${fasterxml.jackson-module-scala.version}
com.google.guava
@@ -878,6 +884,11 @@
org.jboss.netty
netty
+
+
+ commons-beanutils
+ commons-beanutils-core
+
commons-logging
commons-logging
@@ -920,13 +931,11 @@
org.apache.avro
avro
${avro.version}
- ${hadoop.deps.scope}
org.apache.avro
avro-ipc
${avro.version}
- ${hadoop.deps.scope}
io.netty
@@ -1733,6 +1742,10 @@
${orc.classifier}
${orc.deps.scope}
+
+ javax.xml.bind
+ jaxb-api
+
org.apache.hadoop
hadoop-common
@@ -1978,6 +1991,7 @@
com.thoughtworks.paranamer
paranamer
${paranamer.version}
+ runtime
org.apache.arrow
@@ -2048,6 +2062,17 @@
+
+ enforce-no-duplicate-dependencies
+
+ enforce
+
+
+
+
+
+
@@ -2537,12 +2562,17 @@
org.apache.maven.plugins
maven-checkstyle-plugin
- 2.17
+ 3.0.0
false
true
- ${basedir}/src/main/java,${basedir}/src/main/scala
- ${basedir}/src/test/java
+
+ ${basedir}/src/main/java
+ ${basedir}/src/main/scala
+
+
+ ${basedir}/src/test/java
+
dev/checkstyle.xml
${basedir}/target/checkstyle-output.xml
${project.build.sourceEncoding}
@@ -2552,7 +2582,7 @@
com.puppycrawl.tools
checkstyle
- 8.2
+ 8.23
@@ -2683,7 +2713,7 @@
hadoop-2.7
- 2.7.7
+ 2.7.3
2.7.1
@@ -2716,6 +2746,13 @@
kubernetes
resource-managers/kubernetes/core
+
+
+
+
+
+ kubernetes-integration-tests
+
resource-managers/kubernetes/integration-tests
@@ -2760,7 +2797,7 @@
scala-2.12
- 2.12.6
+ 2.12.10
2.12
diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala
index adde213e361f0..fbf9b8e326d5c 100644
--- a/project/MimaBuild.scala
+++ b/project/MimaBuild.scala
@@ -88,7 +88,7 @@ object MimaBuild {
def mimaSettings(sparkHome: File, projectRef: ProjectRef) = {
val organization = "org.apache.spark"
- val previousSparkVersion = "2.2.0"
+ val previousSparkVersion = "2.3.0"
val project = projectRef.project
val fullId = "spark-" + project + "_2.11"
mimaDefaultSettings ++
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 7ff783da130af..4246355f088eb 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -93,13 +93,6 @@ object MimaExcludes {
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.ml.param.Params.defaultParamMap"),
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.ml.param.Params.org$apache$spark$ml$param$Params$_setter_$defaultParamMap_="),
- // [SPARK-14681][ML] Provide label/impurity stats for spark.ml decision tree nodes
- ProblemFilters.exclude[IncompatibleTemplateDefProblem]("org.apache.spark.ml.tree.LeafNode"),
- ProblemFilters.exclude[IncompatibleTemplateDefProblem]("org.apache.spark.ml.tree.InternalNode"),
- ProblemFilters.exclude[IncompatibleTemplateDefProblem]("org.apache.spark.ml.tree.Node"),
- ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.classification.DecisionTreeClassificationModel.this"),
- ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.regression.DecisionTreeRegressionModel.this"),
-
// [SPARK-7132][ML] Add fit with validation set to spark.ml GBT
ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.param.shared.HasValidationIndicatorCol.getValidationIndicatorCol"),
ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.param.shared.HasValidationIndicatorCol.org$apache$spark$ml$param$shared$HasValidationIndicatorCol$_setter_$validationIndicatorCol_="),
@@ -112,7 +105,50 @@ object MimaExcludes {
ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.param.shared.HasValidationIndicatorCol.validationIndicatorCol"),
// [SPARK-23042] Use OneHotEncoderModel to encode labels in MultilayerPerceptronClassifier
- ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ml.classification.LabelConverter")
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ml.classification.LabelConverter"),
+
+ // [SPARK-21842][MESOS] Support Kerberos ticket renewal and creation in Mesos
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.deploy.SparkHadoopUtil.getDateOfNextUpdate"),
+
+ // [SPARK-23366] Improve hot reading path in ReadAheadInputStream
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.io.ReadAheadInputStream.this"),
+
+ // [SPARK-22941][CORE] Do not exit JVM when submit fails with in-process launcher.
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.deploy.SparkSubmit.addJarToClasspath"),
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.deploy.SparkSubmit.mergeFileLists"),
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment$default$2"),
+
+ // Data Source V2 API changes
+ // TODO: they are unstable APIs and should not be tracked by mima.
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.sources.v2.ReadSupportWithSchema"),
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsScanColumnarBatch.createDataReaderFactories"),
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsScanColumnarBatch.createBatchDataReaderFactories"),
+ ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsScanColumnarBatch.planBatchInputPartitions"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.sources.v2.reader.SupportsScanUnsafeRow"),
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.sources.v2.reader.DataSourceReader.createDataReaderFactories"),
+ ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.sources.v2.reader.DataSourceReader.planInputPartitions"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.sources.v2.reader.SupportsPushDownCatalystFilters"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.sources.v2.reader.DataReader"),
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsReportStatistics.getStatistics"),
+ ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsReportStatistics.estimateStatistics"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.sources.v2.reader.DataReaderFactory"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.sources.v2.reader.streaming.ContinuousDataReader"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.sources.v2.writer.SupportsWriteInternalRow"),
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.sources.v2.writer.DataWriterFactory.createDataWriter"),
+ ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.sources.v2.writer.DataWriterFactory.createDataWriter"),
+
+ // Changes to HasRawPredictionCol.
+ ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.param.shared.HasRawPredictionCol.rawPredictionCol"),
+ ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.param.shared.HasRawPredictionCol.org$apache$spark$ml$param$shared$HasRawPredictionCol$_setter_$rawPredictionCol_="),
+ ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.param.shared.HasRawPredictionCol.getRawPredictionCol"),
+
+ // [SPARK-15526][ML][FOLLOWUP] Make JPMML provided scope to avoid including unshaded JARs
+ (problem: Problem) => problem match {
+ case MissingClassProblem(cls) =>
+ !cls.fullName.startsWith("org.spark_project.jpmml") &&
+ !cls.fullName.startsWith("org.spark_project.dmg.pmml")
+ case _ => true
+ }
)
// Exclude rules for 2.3.x
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index a5ed9088eaa4d..18ef6c70b79d8 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -218,7 +218,7 @@ object SparkBuild extends PomBuild {
.map(file),
incOptions := incOptions.value.withNameHashing(true),
publishMavenStyle := true,
- unidocGenjavadocVersion := "0.11",
+ unidocGenjavadocVersion := "0.14",
// Override SBT's default resolvers:
resolvers := Seq(
@@ -430,6 +430,84 @@ object SparkBuild extends PomBuild {
else x.settings(Seq[Setting[_]](): _*)
} ++ Seq[Project](OldDeps.project)
}
+
+ if (!sys.env.contains("SERIAL_SBT_TESTS")) {
+ allProjects.foreach(enable(SparkParallelTestGrouping.settings))
+ }
+}
+
+object SparkParallelTestGrouping {
+ // Settings for parallelizing tests. The basic strategy here is to run the slowest suites (or
+ // collections of suites) in their own forked JVMs, allowing us to gain parallelism within a
+ // SBT project. Here, we take a whitelisting approach where the default behavior is to run all
+ // tests sequentially in a single JVM, requiring us to manually opt-in to the extra parallelism.
+ //
+ // There are a reasons why such a whitelist approach is good:
+ //
+ // 1. Launching one JVM per suite adds significant overhead for short-running suites. In
+ // addition to JVM startup time and JIT warmup, it appears that initialization of Derby
+ // metastores can be very slow so creating a fresh warehouse per suite is inefficient.
+ //
+ // 2. When parallelizing within a project we need to give each forked JVM a different tmpdir
+ // so that the metastore warehouses do not collide. Unfortunately, it seems that there are
+ // some tests which have an overly tight dependency on the default tmpdir, so those fragile
+ // tests need to continue re-running in the default configuration (or need to be rewritten).
+ // Fixing that problem would be a huge amount of work for limited payoff in most cases
+ // because most test suites are short-running.
+ //
+
+ private val testsWhichShouldRunInTheirOwnDedicatedJvm = Set(
+ "org.apache.spark.DistributedSuite",
+ "org.apache.spark.sql.catalyst.expressions.DateExpressionsSuite",
+ "org.apache.spark.sql.catalyst.expressions.HashExpressionsSuite",
+ "org.apache.spark.sql.catalyst.expressions.CastSuite",
+ "org.apache.spark.sql.catalyst.expressions.MathExpressionsSuite",
+ "org.apache.spark.sql.hive.HiveExternalCatalogSuite",
+ "org.apache.spark.sql.hive.StatisticsSuite",
+ "org.apache.spark.sql.hive.execution.HiveCompatibilitySuite",
+ "org.apache.spark.sql.hive.client.VersionsSuite",
+ "org.apache.spark.sql.hive.client.HiveClientVersions",
+ "org.apache.spark.sql.hive.HiveExternalCatalogVersionsSuite",
+ "org.apache.spark.ml.classification.LogisticRegressionSuite",
+ "org.apache.spark.ml.classification.LinearSVCSuite",
+ "org.apache.spark.sql.SQLQueryTestSuite"
+ )
+
+ private val DEFAULT_TEST_GROUP = "default_test_group"
+
+ private def testNameToTestGroup(name: String): String = name match {
+ case _ if testsWhichShouldRunInTheirOwnDedicatedJvm.contains(name) => name
+ case _ => DEFAULT_TEST_GROUP
+ }
+
+ lazy val settings = Seq(
+ testGrouping in Test := {
+ val tests: Seq[TestDefinition] = (definedTests in Test).value
+ val defaultForkOptions = ForkOptions(
+ bootJars = Nil,
+ javaHome = javaHome.value,
+ connectInput = connectInput.value,
+ outputStrategy = outputStrategy.value,
+ runJVMOptions = (javaOptions in Test).value,
+ workingDirectory = Some(baseDirectory.value),
+ envVars = (envVars in Test).value
+ )
+ tests.groupBy(test => testNameToTestGroup(test.name)).map { case (groupName, groupTests) =>
+ val forkOptions = {
+ if (groupName == DEFAULT_TEST_GROUP) {
+ defaultForkOptions
+ } else {
+ defaultForkOptions.copy(runJVMOptions = defaultForkOptions.runJVMOptions ++
+ Seq(s"-Djava.io.tmpdir=${baseDirectory.value}/target/tmp/$groupName"))
+ }
+ }
+ new Tests.Group(
+ name = groupName,
+ tests = groupTests,
+ runPolicy = Tests.SubProcess(forkOptions))
+ }
+ }.toSeq
+ )
}
object Core {
@@ -471,6 +549,7 @@ object DockerIntegrationTests {
object DependencyOverrides {
lazy val settings = Seq(
dependencyOverrides += "com.google.guava" % "guava" % "14.0.1",
+ dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-databind" % "2.6.7.1",
dependencyOverrides += "jline" % "jline" % "2.14.6")
}
@@ -676,6 +755,7 @@ object Unidoc {
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/unsafe")))
.map(_.filterNot(_.getCanonicalPath.contains("python")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/util/collection")))
+ .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/util/kvstore")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/catalyst")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/execution")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/internal")))
@@ -843,8 +923,14 @@ object TestSettings {
testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
// Enable Junit testing.
libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test",
- // Only allow one test at a time, even across projects, since they run in the same JVM
- parallelExecution in Test := false,
+ // `parallelExecutionInTest` controls whether test suites belonging to the same SBT project
+ // can run in parallel with one another. It does NOT control whether tests execute in parallel
+ // within the same JVM (which is controlled by `testForkedParallel`) or whether test cases
+ // within the same suite can run in parallel (which is a ScalaTest runner option which is passed
+ // to the underlying runner but is not a SBT-level configuration). This needs to be `true` in
+ // order for the extra parallelism enabled by `SparkParallelTestGrouping` to take effect.
+ // The `SERIAL_SBT_TESTS` check is here so the extra parallelism can be feature-flagged.
+ parallelExecution in Test := { if (sys.env.contains("SERIAL_SBT_TESTS")) false else true },
// Make sure the test temp directory exists.
resourceGenerators in Test += Def.macroValueI(resourceManaged in Test map { outDir: File =>
var dir = new File(testTempDir)
@@ -865,7 +951,12 @@ object TestSettings {
}
Seq.empty[File]
}).value,
- concurrentRestrictions in Global += Tags.limit(Tags.Test, 1)
+ concurrentRestrictions in Global := {
+ // The number of concurrent test groups is empirically chosen based on experience
+ // with Jenkins flakiness.
+ if (sys.env.contains("SERIAL_SBT_TESTS")) (concurrentRestrictions in Global).value
+ else Seq(Tags.limit(Tags.ForkedTestGroup, 4))
+ }
)
}
diff --git a/project/plugins.sbt b/project/plugins.sbt
index ffbd417b0f145..96640a0c2eb1e 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,7 +1,7 @@
addSbtPlugin("com.etsy" % "sbt-checkstyle-plugin" % "3.1.1")
// sbt-checkstyle-plugin uses an old version of checkstyle. Match it to Maven's.
-libraryDependencies += "com.puppycrawl.tools" % "checkstyle" % "8.2"
+libraryDependencies += "com.puppycrawl.tools" % "checkstyle" % "8.23"
// checkstyle uses guava 23.0.
libraryDependencies += "com.google.guava" % "guava" % "23.0"
@@ -15,6 +15,9 @@ addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.0")
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
+// SPARK-29560 Only sbt-mima-plugin needs this repo
+resolvers += Resolver.url("bintray",
+ new java.net.URL("https://dl.bintray.com/typesafe/sbt-plugins"))(Resolver.defaultIvyPatterns)
addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.1.17")
// sbt 1.0.0 support: https://github.com/AlpineNow/junit_xml_listener/issues/6
diff --git a/python/docs/index.rst b/python/docs/index.rst
index 421c8de86a3cc..0e7b62361802a 100644
--- a/python/docs/index.rst
+++ b/python/docs/index.rst
@@ -37,7 +37,7 @@ Core classes:
A Discretized Stream (DStream), the basic abstraction in Spark Streaming.
- :class:`pyspark.sql.SQLContext`
+ :class:`pyspark.sql.SparkSession`
Main entry point for DataFrame and SQL functionality.
diff --git a/python/pyspark/accumulators.py b/python/pyspark/accumulators.py
index 30ad04297c682..855d8fb4a859f 100644
--- a/python/pyspark/accumulators.py
+++ b/python/pyspark/accumulators.py
@@ -109,10 +109,14 @@
def _deserialize_accumulator(aid, zero_value, accum_param):
from pyspark.accumulators import _accumulatorRegistry
- accum = Accumulator(aid, zero_value, accum_param)
- accum._deserialized = True
- _accumulatorRegistry[aid] = accum
- return accum
+ # If this certain accumulator was deserialized, don't overwrite it.
+ if aid in _accumulatorRegistry:
+ return _accumulatorRegistry[aid]
+ else:
+ accum = Accumulator(aid, zero_value, accum_param)
+ accum._deserialized = True
+ _accumulatorRegistry[aid] = accum
+ return accum
class Accumulator(object):
@@ -258,9 +262,10 @@ def authenticate_and_accum_updates():
raise Exception(
"The value of the provided token to the AccumulatorServer is not correct.")
- # first we keep polling till we've received the authentication token
- poll(authenticate_and_accum_updates)
- # now we've authenticated, don't need to check for the token anymore
+ if auth_token is not None:
+ # first we keep polling till we've received the authentication token
+ poll(authenticate_and_accum_updates)
+ # now we've authenticated if needed, don't need to check for the token anymore
poll(accum_updates)
diff --git a/python/pyspark/broadcast.py b/python/pyspark/broadcast.py
index b3dfc99962a35..29358b5740e51 100644
--- a/python/pyspark/broadcast.py
+++ b/python/pyspark/broadcast.py
@@ -15,13 +15,16 @@
# limitations under the License.
#
+import gc
import os
+import socket
import sys
-import gc
from tempfile import NamedTemporaryFile
import threading
from pyspark.cloudpickle import print_exec
+from pyspark.java_gateway import local_connect_and_auth
+from pyspark.serializers import ChunkedStream
from pyspark.util import _exception_message
if sys.version < '3':
@@ -64,19 +67,46 @@ class Broadcast(object):
>>> large_broadcast = sc.broadcast(range(10000))
"""
- def __init__(self, sc=None, value=None, pickle_registry=None, path=None):
+ def __init__(self, sc=None, value=None, pickle_registry=None, path=None,
+ sock_file=None):
"""
Should not be called directly by users -- use L{SparkContext.broadcast()}
instead.
"""
if sc is not None:
+ # we're on the driver. We want the pickled data to end up in a file (maybe encrypted)
f = NamedTemporaryFile(delete=False, dir=sc._temp_dir)
- self._path = self.dump(value, f)
- self._jbroadcast = sc._jvm.PythonRDD.readBroadcastFromFile(sc._jsc, self._path)
+ self._path = f.name
+ self._sc = sc
+ self._python_broadcast = sc._jvm.PythonRDD.setupBroadcast(self._path)
+ if sc._encryption_enabled:
+ # with encryption, we ask the jvm to do the encryption for us, we send it data
+ # over a socket
+ port, auth_secret = self._python_broadcast.setupEncryptionServer()
+ (encryption_sock_file, _) = local_connect_and_auth(port, auth_secret)
+ broadcast_out = ChunkedStream(encryption_sock_file, 8192)
+ else:
+ # no encryption, we can just write pickled data directly to the file from python
+ broadcast_out = f
+ self.dump(value, broadcast_out)
+ if sc._encryption_enabled:
+ self._python_broadcast.waitTillDataReceived()
+ self._jbroadcast = sc._jsc.broadcast(self._python_broadcast)
self._pickle_registry = pickle_registry
else:
+ # we're on an executor
self._jbroadcast = None
- self._path = path
+ self._sc = None
+ self._python_broadcast = None
+ if sock_file is not None:
+ # the jvm is doing decryption for us. Read the value
+ # immediately from the sock_file
+ self._value = self.load(sock_file)
+ else:
+ # the jvm just dumps the pickled data in path -- we'll unpickle lazily when
+ # the value is requested
+ assert(path is not None)
+ self._path = path
def dump(self, value, f):
try:
@@ -89,24 +119,33 @@ def dump(self, value, f):
print_exec(sys.stderr)
raise pickle.PicklingError(msg)
f.close()
- return f.name
- def load(self, path):
+ def load_from_path(self, path):
with open(path, 'rb', 1 << 20) as f:
- # pickle.load() may create lots of objects, disable GC
- # temporary for better performance
- gc.disable()
- try:
- return pickle.load(f)
- finally:
- gc.enable()
+ return self.load(f)
+
+ def load(self, file):
+ # "file" could also be a socket
+ gc.disable()
+ try:
+ return pickle.load(file)
+ finally:
+ gc.enable()
@property
def value(self):
""" Return the broadcasted value
"""
if not hasattr(self, "_value") and self._path is not None:
- self._value = self.load(self._path)
+ # we only need to decrypt it here when encryption is enabled and
+ # if its on the driver, since executor decryption is handled already
+ if self._sc is not None and self._sc._encryption_enabled:
+ port, auth_secret = self._python_broadcast.setupDecryptionServer()
+ (decrypted_sock_file, _) = local_connect_and_auth(port, auth_secret)
+ self._python_broadcast.waitTillBroadcastDataSent()
+ return self.load(decrypted_sock_file)
+ else:
+ self._value = self.load_from_path(self._path)
return self._value
def unpersist(self, blocking=False):
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 4cabae4b2f50b..1e0dd10c1c428 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -33,9 +33,9 @@
from pyspark.broadcast import Broadcast, BroadcastPickleRegistry
from pyspark.conf import SparkConf
from pyspark.files import SparkFiles
-from pyspark.java_gateway import launch_gateway
+from pyspark.java_gateway import launch_gateway, local_connect_and_auth
from pyspark.serializers import PickleSerializer, BatchedSerializer, UTF8Deserializer, \
- PairDeserializer, AutoBatchedSerializer, NoOpSerializer
+ PairDeserializer, AutoBatchedSerializer, NoOpSerializer, ChunkedStream
from pyspark.storagelevel import StorageLevel
from pyspark.rdd import RDD, _load_from_socket, ignore_unicode_prefix
from pyspark.traceback_utils import CallSite, first_spark_call
@@ -63,6 +63,10 @@ class SparkContext(object):
Main entry point for Spark functionality. A SparkContext represents the
connection to a Spark cluster, and can be used to create L{RDD} and
broadcast variables on that cluster.
+
+ .. note:: :class:`SparkContext` instance is not supported to share across multiple
+ processes out of the box, and PySpark does not guarantee multi-processing execution.
+ Use threads instead for concurrent processing purpose.
"""
_gateway = None
@@ -112,6 +116,20 @@ def __init__(self, master=None, appName=None, sparkHome=None, pyFiles=None,
ValueError:...
"""
self._callsite = first_spark_call() or CallSite(None, None, None)
+ if gateway is not None and gateway.gateway_parameters.auth_token is None:
+ allow_insecure_env = os.environ.get("PYSPARK_ALLOW_INSECURE_GATEWAY", "0")
+ if allow_insecure_env == "1" or allow_insecure_env.lower() == "true":
+ warnings.warn(
+ "You are passing in an insecure Py4j gateway. This "
+ "presents a security risk, and will be completely forbidden in Spark 3.0")
+ else:
+ raise ValueError(
+ "You are trying to pass an insecure Py4j gateway to Spark. This"
+ " presents a security risk. If you are sure you understand and accept this"
+ " risk, you can set the environment variable"
+ " 'PYSPARK_ALLOW_INSECURE_GATEWAY=1', but"
+ " note this option will be removed in Spark 3.0")
+
SparkContext._ensure_initialized(self, gateway=gateway, conf=conf)
try:
self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,
@@ -189,6 +207,11 @@ def _do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize,
self._javaAccumulator = self._jvm.PythonAccumulatorV2(host, port, auth_token)
self._jsc.sc().register(self._javaAccumulator)
+ # If encryption is enabled, we need to setup a server in the jvm to read broadcast
+ # data via a socket.
+ # scala's mangled names w/ $ in them require special treatment.
+ self._encryption_enabled = self._jvm.PythonUtils.getEncryptionEnabled(self._jsc)
+
self.pythonExec = os.environ.get("PYSPARK_PYTHON", 'python')
self.pythonVer = "%d.%d" % sys.version_info[:2]
@@ -498,23 +521,48 @@ def f(split, iterator):
def reader_func(temp_filename):
return self._jvm.PythonRDD.readRDDFromFile(self._jsc, temp_filename, numSlices)
- jrdd = self._serialize_to_jvm(c, serializer, reader_func)
+ def createRDDServer():
+ return self._jvm.PythonParallelizeServer(self._jsc.sc(), numSlices)
+
+ jrdd = self._serialize_to_jvm(c, serializer, reader_func, createRDDServer)
return RDD(jrdd, self, serializer)
- def _serialize_to_jvm(self, data, serializer, reader_func):
- """
- Calling the Java parallelize() method with an ArrayList is too slow,
- because it sends O(n) Py4J commands. As an alternative, serialized
- objects are written to a file and loaded through textFile().
- """
- tempFile = NamedTemporaryFile(delete=False, dir=self._temp_dir)
- try:
- serializer.dump_stream(data, tempFile)
- tempFile.close()
- return reader_func(tempFile.name)
- finally:
- # readRDDFromFile eagerily reads the file so we can delete right after.
- os.unlink(tempFile.name)
+ def _serialize_to_jvm(self, data, serializer, reader_func, createRDDServer):
+ """
+ Using py4j to send a large dataset to the jvm is really slow, so we use either a file
+ or a socket if we have encryption enabled.
+ :param data:
+ :param serializer:
+ :param reader_func: A function which takes a filename and reads in the data in the jvm and
+ returns a JavaRDD. Only used when encryption is disabled.
+ :param createRDDServer: A function which creates a PythonRDDServer in the jvm to
+ accept the serialized data, for use when encryption is enabled.
+ :return:
+ """
+ if self._encryption_enabled:
+ # with encryption, we open a server in java and send the data directly
+ server = createRDDServer()
+ (sock_file, _) = local_connect_and_auth(server.port(), server.secret())
+ chunked_out = ChunkedStream(sock_file, 8192)
+ serializer.dump_stream(data, chunked_out)
+ chunked_out.close()
+ # this call will block until the server has read all the data and processed it (or
+ # throws an exception)
+ r = server.getResult()
+ return r
+ else:
+ # without encryption, we serialize to a file, and we read the file in java and
+ # parallelize from there.
+ tempFile = NamedTemporaryFile(delete=False, dir=self._temp_dir)
+ try:
+ try:
+ serializer.dump_stream(data, tempFile)
+ finally:
+ tempFile.close()
+ return reader_func(tempFile.name)
+ finally:
+ # we eagerily reads the file so we can delete right after.
+ os.unlink(tempFile.name)
def pickleFile(self, name, minPartitions=None):
"""
@@ -889,7 +937,7 @@ def addPyFile(self, path):
def setCheckpointDir(self, dirName):
"""
Set the directory under which RDDs are going to be checkpointed. The
- directory must be a HDFS path if running on a cluster.
+ directory must be an HDFS path if running on a cluster.
"""
self._jsc.sc().setCheckpointDir(dirName)
diff --git a/python/pyspark/java_gateway.py b/python/pyspark/java_gateway.py
index c8c5f801f89bb..feb6b7bd6aa3d 100644
--- a/python/pyspark/java_gateway.py
+++ b/python/pyspark/java_gateway.py
@@ -41,8 +41,20 @@ def launch_gateway(conf=None):
"""
launch jvm gateway
:param conf: spark configuration passed to spark-submit
- :return:
+ :return: a JVM gateway
"""
+ return _launch_gateway(conf)
+
+
+def _launch_gateway(conf=None, insecure=False):
+ """
+ launch jvm gateway
+ :param conf: spark configuration passed to spark-submit
+ :param insecure: True to create an insecure gateway; only for testing
+ :return: a JVM gateway
+ """
+ if insecure and os.environ.get("SPARK_TESTING", "0") != "1":
+ raise ValueError("creating insecure gateways is only for testing")
if "PYSPARK_GATEWAY_PORT" in os.environ:
gateway_port = int(os.environ["PYSPARK_GATEWAY_PORT"])
gateway_secret = os.environ["PYSPARK_GATEWAY_SECRET"]
@@ -74,6 +86,8 @@ def launch_gateway(conf=None):
env = dict(os.environ)
env["_PYSPARK_DRIVER_CONN_INFO_PATH"] = conn_info_file
+ if insecure:
+ env["_PYSPARK_CREATE_INSECURE_GATEWAY"] = "1"
# Launch the Java gateway.
# We open a pipe to stdin so that the Java gateway can die when the pipe is broken
@@ -116,9 +130,10 @@ def killChild():
atexit.register(killChild)
# Connect to the gateway
- gateway = JavaGateway(
- gateway_parameters=GatewayParameters(port=gateway_port, auth_token=gateway_secret,
- auto_convert=True))
+ gateway_params = GatewayParameters(port=gateway_port, auto_convert=True)
+ if not insecure:
+ gateway_params.auth_token = gateway_secret
+ gateway = JavaGateway(gateway_parameters=gateway_params)
# Import the classes used by PySpark
java_import(gateway.jvm, "org.apache.spark.SparkConf")
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index d5963f4f7042c..ce028512357f2 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -773,8 +773,8 @@ def roc(self):
which is a Dataframe having two fields (FPR, TPR) with
(0.0, 0.0) prepended and (1.0, 1.0) appended to it.
- .. seealso:: `Wikipedia reference \
- `_
+ .. seealso:: `Wikipedia reference
+ `_
.. note:: This ignores instance weights (setting all to 1.0) from
`LogisticRegression.weightCol`. This will change in later Spark
diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index ab449bc3f8f51..5ef4e765ea4e1 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -1202,21 +1202,21 @@ class PowerIterationClustering(HasMaxIter, HasWeightCol, JavaParams, JavaMLReada
.. note:: Experimental
Power Iteration Clustering (PIC), a scalable graph clustering algorithm developed by
- Lin and Cohen . From the abstract:
+ `Lin and Cohen `_. From the abstract:
PIC finds a very low-dimensional embedding of a dataset using truncated power
iteration on a normalized pair-wise similarity matrix of the data.
This class is not yet an Estimator/Transformer, use :py:func:`assignClusters` method
to run the PowerIterationClustering algorithm.
- .. seealso:: `Wikipedia on Spectral clustering \
- `_
+ .. seealso:: `Wikipedia on Spectral clustering
+ `_
- >>> data = [(1, 0, 0.5), \
- (2, 0, 0.5), (2, 1, 0.7), \
- (3, 0, 0.5), (3, 1, 0.7), (3, 2, 0.9), \
- (4, 0, 0.5), (4, 1, 0.7), (4, 2, 0.9), (4, 3, 1.1), \
- (5, 0, 0.5), (5, 1, 0.7), (5, 2, 0.9), (5, 3, 1.1), (5, 4, 1.3)]
+ >>> data = [(1, 0, 0.5),
+ ... (2, 0, 0.5), (2, 1, 0.7),
+ ... (3, 0, 0.5), (3, 1, 0.7), (3, 2, 0.9),
+ ... (4, 0, 0.5), (4, 1, 0.7), (4, 2, 0.9), (4, 3, 1.1),
+ ... (5, 0, 0.5), (5, 1, 0.7), (5, 2, 0.9), (5, 3, 1.1), (5, 4, 1.3)]
>>> df = spark.createDataFrame(data).toDF("src", "dst", "weight")
>>> pic = PowerIterationClustering(k=2, maxIter=40, weightCol="weight")
>>> assignments = pic.assignClusters(df)
diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py
index 760aa82168f5a..bc4f4c9ab5cd5 100755
--- a/python/pyspark/ml/feature.py
+++ b/python/pyspark/ml/feature.py
@@ -193,6 +193,7 @@ def approxSimilarityJoin(self, datasetA, datasetB, threshold, distCol="distCol")
"datasetA" and "datasetB", and a column "distCol" is added to show the distance
between each pair.
"""
+ threshold = TypeConverters.toFloat(threshold)
return self._call_java("approxSimilarityJoin", datasetA, datasetB, threshold, distCol)
@@ -207,8 +208,8 @@ class BucketedRandomProjectionLSH(JavaEstimator, LSHParams, HasInputCol, HasOutp
distance space. The output will be vectors of configurable dimension. Hash values in the same
dimension are calculated by the same hash function.
- .. seealso:: `Stable Distributions \
- `_
+ .. seealso:: `Stable Distributions
+ `_
.. seealso:: `Hashing for Similarity Search: A Survey `_
>>> from pyspark.ml.linalg import Vectors
@@ -240,6 +241,16 @@ class BucketedRandomProjectionLSH(JavaEstimator, LSHParams, HasInputCol, HasOutp
| 3| 6| 2.23606797749979|
+---+---+-----------------+
...
+ >>> model.approxSimilarityJoin(df, df2, 3, distCol="EuclideanDistance").select(
+ ... col("datasetA.id").alias("idA"),
+ ... col("datasetB.id").alias("idB"),
+ ... col("EuclideanDistance")).show()
+ +---+---+-----------------+
+ |idA|idB|EuclideanDistance|
+ +---+---+-----------------+
+ | 3| 6| 2.23606797749979|
+ +---+---+-----------------+
+ ...
>>> brpPath = temp_path + "/brp"
>>> brp.save(brpPath)
>>> brp2 = BucketedRandomProjectionLSH.load(brpPath)
@@ -303,7 +314,7 @@ def _create_model(self, java_model):
class BucketedRandomProjectionLSHModel(LSHModel, JavaMLReadable, JavaMLWritable):
- """
+ r"""
.. note:: Experimental
Model fitted by :py:class:`BucketedRandomProjectionLSH`, where multiple random vectors are
@@ -653,8 +664,8 @@ class DCT(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWrit
The return vector is scaled such that the transform matrix is
unitary (aka scaled DCT-II).
- .. seealso:: `More information on Wikipedia \
- `_.
+ .. seealso:: `More information on Wikipedia
+ `_.
>>> from pyspark.ml.linalg import Vectors
>>> df1 = spark.createDataFrame([(Vectors.dense([5.0, 8.0, 6.0]),)], ["vec"])
@@ -1353,7 +1364,7 @@ def _create_model(self, java_model):
class MinHashLSHModel(LSHModel, JavaMLReadable, JavaMLWritable):
- """
+ r"""
.. note:: Experimental
Model produced by :py:class:`MinHashLSH`, where where multiple hash functions are stored. Each
@@ -1362,8 +1373,8 @@ class MinHashLSHModel(LSHModel, JavaMLReadable, JavaMLWritable):
:math:`h_i(x) = ((x \cdot a_i + b_i) \mod prime)` This hash family is approximately min-wise
independent according to the reference.
- .. seealso:: Tom Bohman, Colin Cooper, and Alan Frieze. "Min-wise independent linear \
- permutations." Electronic Journal of Combinatorics 7 (2000): R26.
+ .. seealso:: Tom Bohman, Colin Cooper, and Alan Frieze. "Min-wise independent linear
+ permutations." Electronic Journal of Combinatorics 7 (2000): R26.
.. versionadded:: 2.2.0
"""
diff --git a/python/pyspark/ml/fpm.py b/python/pyspark/ml/fpm.py
index c2b29b73460ff..886ad8409ca66 100644
--- a/python/pyspark/ml/fpm.py
+++ b/python/pyspark/ml/fpm.py
@@ -158,7 +158,7 @@ class FPGrowth(JavaEstimator, HasItemsCol, HasPredictionCol,
HasMinSupport, HasNumPartitions, HasMinConfidence,
JavaMLWritable, JavaMLReadable):
- """
+ r"""
.. note:: Experimental
A parallel FP-growth algorithm to mine frequent itemsets. The algorithm is described in
diff --git a/python/pyspark/ml/image.py b/python/pyspark/ml/image.py
index ef6785b4a8ed4..a1aacea88e42e 100644
--- a/python/pyspark/ml/image.py
+++ b/python/pyspark/ml/image.py
@@ -25,8 +25,11 @@
"""
import sys
+import warnings
import numpy as np
+from distutils.version import LooseVersion
+
from pyspark import SparkContext
from pyspark.sql.types import Row, _create_row, _parse_datatype_json_string
from pyspark.sql import DataFrame, SparkSession
@@ -188,7 +191,11 @@ def toImage(self, array, origin=""):
# Running `bytearray(numpy.array([1]))` fails in specific Python versions
# with a specific Numpy version, for example in Python 3.6.0 and NumPy 1.13.3.
# Here, it avoids it by converting it to bytes.
- data = bytearray(array.astype(dtype=np.uint8).ravel().tobytes())
+ if LooseVersion(np.__version__) >= LooseVersion('1.9'):
+ data = bytearray(array.astype(dtype=np.uint8).ravel().tobytes())
+ else:
+ # Numpy prior to 1.9 don't have `tobytes` method.
+ data = bytearray(array.astype(dtype=np.uint8).ravel())
# Creating new Row with _create_row(), because Row(name = value, ... )
# orders fields by name, which conflicts with expected schema order
@@ -207,6 +214,9 @@ def readImages(self, path, recursive=False, numPartitions=-1,
.. note:: If sample ratio is less than 1, sampling uses a PathFilter that is efficient but
potentially non-deterministic.
+ .. note:: Deprecated in 2.4.0. Use `spark.read.format("image").load(path)` instead and
+ this `readImages` will be removed in 3.0.0.
+
:param str path: Path to the image directory.
:param bool recursive: Recursive search flag.
:param int numPartitions: Number of DataFrame partitions.
@@ -222,7 +232,8 @@ def readImages(self, path, recursive=False, numPartitions=-1,
.. versionadded:: 2.3.0
"""
-
+ warnings.warn("`ImageSchema.readImage` is deprecated. " +
+ "Use `spark.read.format(\"image\").load(path)` instead.", DeprecationWarning)
spark = SparkSession.builder.getOrCreate()
image_schema = spark._jvm.org.apache.spark.ml.image.ImageSchema
jsession = spark._jsparkSession
diff --git a/python/pyspark/ml/linalg/__init__.py b/python/pyspark/ml/linalg/__init__.py
index 2548fd0f50b33..9261d65916b3c 100644
--- a/python/pyspark/ml/linalg/__init__.py
+++ b/python/pyspark/ml/linalg/__init__.py
@@ -270,6 +270,8 @@ class DenseVector(Vector):
DenseVector([3.0, 2.0])
>>> u % 2
DenseVector([1.0, 0.0])
+ >>> -v
+ DenseVector([-1.0, -2.0])
"""
def __init__(self, ar):
if isinstance(ar, bytes):
@@ -384,14 +386,14 @@ def squared_distance(self, other):
def toArray(self):
"""
- Returns an numpy.ndarray
+ Returns the underlying numpy.ndarray
"""
return self.array
@property
def values(self):
"""
- Returns a list of values
+ Returns the underlying numpy.ndarray
"""
return self.array
@@ -436,6 +438,9 @@ def __hash__(self):
def __getattr__(self, item):
return getattr(self.array, item)
+ def __neg__(self):
+ return DenseVector(-self.array)
+
def _delegate(op):
def func(self, other):
if isinstance(other, DenseVector):
@@ -443,7 +448,6 @@ def func(self, other):
return DenseVector(getattr(self.array, op)(other))
return func
- __neg__ = _delegate("__neg__")
__add__ = _delegate("__add__")
__sub__ = _delegate("__sub__")
__mul__ = _delegate("__mul__")
@@ -677,7 +681,7 @@ def squared_distance(self, other):
def toArray(self):
"""
- Returns a copy of this SparseVector as a 1-dimensional NumPy array.
+ Returns a copy of this SparseVector as a 1-dimensional numpy.ndarray.
"""
arr = np.zeros((self.size,), dtype=np.float64)
arr[self.indices] = self.values
@@ -858,7 +862,7 @@ def __init__(self, numRows, numCols, isTransposed=False):
def toArray(self):
"""
- Returns its elements in a NumPy ndarray.
+ Returns its elements in a numpy.ndarray.
"""
raise NotImplementedError
@@ -933,7 +937,7 @@ def __repr__(self):
def toArray(self):
"""
- Return an numpy.ndarray
+ Return a numpy.ndarray
>>> m = DenseMatrix(2, 2, range(4))
>>> m.toArray()
@@ -1117,7 +1121,7 @@ def __getitem__(self, indices):
def toArray(self):
"""
- Return an numpy.ndarray
+ Return a numpy.ndarray
"""
A = np.zeros((self.numRows, self.numCols), dtype=np.float64, order='F')
for k in xrange(self.colPtrs.size - 1):
diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py
index 513ca5a9df85e..98f4361351847 100644
--- a/python/pyspark/ml/regression.py
+++ b/python/pyspark/ml/regression.py
@@ -188,8 +188,8 @@ def intercept(self):
@property
@since("2.3.0")
def scale(self):
- """
- The value by which \|y - X'w\| is scaled down when loss is "huber", otherwise 1.0.
+ r"""
+ The value by which :math:`\|y - X'w\|` is scaled down when loss is "huber", otherwise 1.0.
"""
return self._call_java("scale")
@@ -279,12 +279,12 @@ def featuresCol(self):
@property
@since("2.0.0")
def explainedVariance(self):
- """
+ r"""
Returns the explained variance regression score.
- explainedVariance = 1 - variance(y - \hat{y}) / variance(y)
+ explainedVariance = :math:`1 - \frac{variance(y - \hat{y})}{variance(y)}`
- .. seealso:: `Wikipedia explain variation \
- `_
+ .. seealso:: `Wikipedia explain variation
+ `_
.. note:: This ignores instance weights (setting all to 1.0) from
`LinearRegression.weightCol`. This will change in later Spark
@@ -339,8 +339,8 @@ def r2(self):
"""
Returns R^2, the coefficient of determination.
- .. seealso:: `Wikipedia coefficient of determination \
- `_
+ .. seealso:: `Wikipedia coefficient of determination
+ `_
.. note:: This ignores instance weights (setting all to 1.0) from
`LinearRegression.weightCol`. This will change in later Spark
@@ -354,8 +354,8 @@ def r2adj(self):
"""
Returns Adjusted R^2, the adjusted coefficient of determination.
- .. seealso:: `Wikipedia coefficient of determination, Adjusted R^2 \
- `_
+ .. seealso:: `Wikipedia coefficient of determination, Adjusted R^2
+ `_
.. note:: This ignores instance weights (setting all to 1.0) from
`LinearRegression.weightCol`. This will change in later Spark versions.
diff --git a/python/pyspark/ml/stat.py b/python/pyspark/ml/stat.py
index 370154fc6d62a..3f421024acdce 100644
--- a/python/pyspark/ml/stat.py
+++ b/python/pyspark/ml/stat.py
@@ -336,7 +336,7 @@ def metrics(*metrics):
- numNonzeros: a vector with the number of non-zeros for each coefficients
- max: the maximum for each coefficient.
- min: the minimum for each coefficient.
- - normL2: the Euclidian norm for each coefficient.
+ - normL2: the Euclidean norm for each coefficient.
- normL1: the L1 norm of each coefficient (sum of the absolute values).
:param metrics:
diff --git a/python/pyspark/mllib/clustering.py b/python/pyspark/mllib/clustering.py
index b09469b9f5c2d..b1a8af6bcc094 100644
--- a/python/pyspark/mllib/clustering.py
+++ b/python/pyspark/mllib/clustering.py
@@ -647,7 +647,7 @@ class PowerIterationClustering(object):
@classmethod
@since('1.5.0')
def train(cls, rdd, k, maxIterations=100, initMode="random"):
- """
+ r"""
:param rdd:
An RDD of (i, j, s\ :sub:`ij`\) tuples representing the
affinity matrix, which is the matrix A in the PIC paper. The
diff --git a/python/pyspark/mllib/evaluation.py b/python/pyspark/mllib/evaluation.py
index 6c65da58e4e2b..0bb0ca37c1ab6 100644
--- a/python/pyspark/mllib/evaluation.py
+++ b/python/pyspark/mllib/evaluation.py
@@ -117,9 +117,9 @@ def __init__(self, predictionAndObservations):
@property
@since('1.4.0')
def explainedVariance(self):
- """
+ r"""
Returns the explained variance regression score.
- explainedVariance = 1 - variance(y - \hat{y}) / variance(y)
+ explainedVariance = :math:`1 - \frac{variance(y - \hat{y})}{variance(y)}`
"""
return self.call("explainedVariance")
diff --git a/python/pyspark/mllib/feature.py b/python/pyspark/mllib/feature.py
index 40ecd2e0ff4be..6d7d4d61db043 100644
--- a/python/pyspark/mllib/feature.py
+++ b/python/pyspark/mllib/feature.py
@@ -59,7 +59,7 @@ def transform(self, vector):
class Normalizer(VectorTransformer):
- """
+ r"""
Normalizes samples individually to unit L\ :sup:`p`\ norm
For any 1 <= `p` < float('inf'), normalizes samples using
diff --git a/python/pyspark/mllib/linalg/__init__.py b/python/pyspark/mllib/linalg/__init__.py
index 4afd6666400b0..94a3e2af4d2d1 100644
--- a/python/pyspark/mllib/linalg/__init__.py
+++ b/python/pyspark/mllib/linalg/__init__.py
@@ -281,6 +281,8 @@ class DenseVector(Vector):
DenseVector([3.0, 2.0])
>>> u % 2
DenseVector([1.0, 0.0])
+ >>> -v
+ DenseVector([-1.0, -2.0])
"""
def __init__(self, ar):
if isinstance(ar, bytes):
@@ -480,6 +482,9 @@ def __hash__(self):
def __getattr__(self, item):
return getattr(self.array, item)
+ def __neg__(self):
+ return DenseVector(-self.array)
+
def _delegate(op):
def func(self, other):
if isinstance(other, DenseVector):
@@ -487,7 +492,6 @@ def func(self, other):
return DenseVector(getattr(self.array, op)(other))
return func
- __neg__ = _delegate("__neg__")
__add__ = _delegate("__add__")
__sub__ = _delegate("__sub__")
__mul__ = _delegate("__mul__")
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py
index b317156885e51..ccf39e1ffbe96 100644
--- a/python/pyspark/rdd.py
+++ b/python/pyspark/rdd.py
@@ -2399,7 +2399,7 @@ def barrier(self):
:return: an :class:`RDDBarrier` instance that provides actions within a barrier stage.
.. seealso:: :class:`BarrierTaskContext`
- .. seealso:: `SPIP: Barrier Execution Mode \
+ .. seealso:: `SPIP: Barrier Execution Mode
`_
.. seealso:: `Design Doc `_
diff --git a/python/pyspark/serializers.py b/python/pyspark/serializers.py
index 48006778e86f2..28c8d12c23ee7 100644
--- a/python/pyspark/serializers.py
+++ b/python/pyspark/serializers.py
@@ -51,6 +51,7 @@
"""
import sys
+from distutils.version import LooseVersion
from itertools import chain, product
import marshal
import struct
@@ -204,7 +205,10 @@ def dump_stream(self, iterator, stream):
def load_stream(self, stream):
import pyarrow as pa
- reader = pa.open_stream(stream)
+ if LooseVersion(pa.__version__) >= "0.12.0":
+ reader = pa.ipc.open_stream(stream)
+ else:
+ reader = pa.open_stream(stream)
for batch in reader:
yield batch
@@ -248,7 +252,10 @@ def create_array(s, t):
# TODO: see ARROW-2432. Remove when the minimum PyArrow version becomes 0.10.0.
return pa.Array.from_pandas(s.apply(
lambda v: decimal.Decimal('NaN') if v is None else v), mask=mask, type=t)
- return pa.Array.from_pandas(s, mask=mask, type=t)
+ elif LooseVersion(pa.__version__) < LooseVersion("0.11.0"):
+ # TODO: see ARROW-1949. Remove when the minimum PyArrow version becomes 0.11.0.
+ return pa.Array.from_pandas(s, mask=mask, type=t)
+ return pa.Array.from_pandas(s, mask=mask, type=t, safe=False)
arrs = [create_array(s, t) for s, t in series]
return pa.RecordBatch.from_arrays(arrs, ["_%d" % i for i in xrange(len(arrs))])
@@ -295,7 +302,10 @@ def load_stream(self, stream):
Deserialize ArrowRecordBatches to an Arrow table and return as a list of pandas.Series.
"""
import pyarrow as pa
- reader = pa.open_stream(stream)
+ if LooseVersion(pa.__version__) >= "0.12.0":
+ reader = pa.ipc.open_stream(stream)
+ else:
+ reader = pa.open_stream(stream)
for batch in reader:
yield [self.arrow_to_pandas(c) for c in pa.Table.from_batches([batch]).itercolumns()]
@@ -731,6 +741,57 @@ def write_with_length(obj, stream):
stream.write(obj)
+class ChunkedStream(object):
+
+ """
+ This is a file-like object takes a stream of data, of unknown length, and breaks it into fixed
+ length frames. The intended use case is serializing large data and sending it immediately over
+ a socket -- we do not want to buffer the entire data before sending it, but the receiving end
+ needs to know whether or not there is more data coming.
+
+ It works by buffering the incoming data in some fixed-size chunks. If the buffer is full, it
+ first sends the buffer size, then the data. This repeats as long as there is more data to send.
+ When this is closed, it sends the length of whatever data is in the buffer, then that data, and
+ finally a "length" of -1 to indicate the stream has completed.
+ """
+
+ def __init__(self, wrapped, buffer_size):
+ self.buffer_size = buffer_size
+ self.buffer = bytearray(buffer_size)
+ self.current_pos = 0
+ self.wrapped = wrapped
+
+ def write(self, bytes):
+ byte_pos = 0
+ byte_remaining = len(bytes)
+ while byte_remaining > 0:
+ new_pos = byte_remaining + self.current_pos
+ if new_pos < self.buffer_size:
+ # just put it in our buffer
+ self.buffer[self.current_pos:new_pos] = bytes[byte_pos:]
+ self.current_pos = new_pos
+ byte_remaining = 0
+ else:
+ # fill the buffer, send the length then the contents, and start filling again
+ space_left = self.buffer_size - self.current_pos
+ new_byte_pos = byte_pos + space_left
+ self.buffer[self.current_pos:self.buffer_size] = bytes[byte_pos:new_byte_pos]
+ write_int(self.buffer_size, self.wrapped)
+ self.wrapped.write(self.buffer)
+ byte_remaining -= space_left
+ byte_pos = new_byte_pos
+ self.current_pos = 0
+
+ def close(self):
+ # if there is anything left in the buffer, write it out first
+ if self.current_pos > 0:
+ write_int(self.current_pos, self.wrapped)
+ self.wrapped.write(self.buffer[:self.current_pos])
+ # -1 length indicates to the receiving end that we're done.
+ write_int(-1, self.wrapped)
+ self.wrapped.close()
+
+
if __name__ == '__main__':
import doctest
(failure_count, test_count) = doctest.testmod()
diff --git a/python/pyspark/shell.py b/python/pyspark/shell.py
index 472c3cd4452f0..65e3bdbc05ce8 100644
--- a/python/pyspark/shell.py
+++ b/python/pyspark/shell.py
@@ -54,7 +54,7 @@
sqlContext = spark._wrapped
sqlCtx = sqlContext
-print("""Welcome to
+print(r"""Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py
index e7dec11c69b57..7f12d2324e715 100644
--- a/python/pyspark/sql/column.py
+++ b/python/pyspark/sql/column.py
@@ -668,9 +668,17 @@ def over(self, window):
:return: a Column
>>> from pyspark.sql import Window
- >>> window = Window.partitionBy("name").orderBy("age").rowsBetween(-1, 1)
+ >>> window = Window.partitionBy("name").orderBy("age") \
+ .rowsBetween(Window.unboundedPreceding, Window.currentRow)
>>> from pyspark.sql.functions import rank, min
- >>> # df.select(rank().over(window), min('age').over(window))
+ >>> df.withColumn("rank", rank().over(window)) \
+ .withColumn("min", min('age').over(window)).show()
+ +---+-----+----+---+
+ |age| name|rank|min|
+ +---+-----+----+---+
+ | 5| Bob| 1| 5|
+ | 2|Alice| 1| 2|
+ +---+-----+----+---+
"""
from pyspark.sql.window import WindowSpec
if not isinstance(window, WindowSpec):
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 9c094dd9a9033..f58c9c34ed103 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -318,7 +318,7 @@ def registerDataFrameAsTable(self, df, tableName):
@since(1.6)
def dropTempTable(self, tableName):
- """ Remove the temp table from catalog.
+ """ Remove the temporary table from catalog.
>>> sqlContext.registerDataFrameAsTable(df, "table1")
>>> sqlContext.dropTempTable("table1")
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 1affc9b4fcf6c..a2651d2781c21 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -22,8 +22,10 @@
basestring = unicode = str
long = int
from functools import reduce
+ from html import escape as html_escape
else:
from itertools import imap as map
+ from cgi import escape as html_escape
import warnings
@@ -55,7 +57,7 @@ class DataFrame(object):
Once created, it can be manipulated using the various domain-specific-language
(DSL) functions defined in: :class:`DataFrame`, :class:`Column`.
- To select a column from the data frame, use the apply method::
+ To select a column from the :class:`DataFrame`, use the apply method::
ageCol = people.age
@@ -121,7 +123,7 @@ def toJSON(self, use_unicode=True):
@since(1.3)
def registerTempTable(self, name):
- """Registers this DataFrame as a temporary table using the given name.
+ """Registers this :class:`DataFrame` as a temporary table using the given name.
The lifetime of this temporary table is tied to the :class:`SparkSession`
that was used to create this :class:`DataFrame`.
@@ -140,7 +142,7 @@ def registerTempTable(self, name):
@since(2.0)
def createTempView(self, name):
- """Creates a local temporary view with this DataFrame.
+ """Creates a local temporary view with this :class:`DataFrame`.
The lifetime of this temporary table is tied to the :class:`SparkSession`
that was used to create this :class:`DataFrame`.
@@ -162,7 +164,7 @@ def createTempView(self, name):
@since(2.0)
def createOrReplaceTempView(self, name):
- """Creates or replaces a local temporary view with this DataFrame.
+ """Creates or replaces a local temporary view with this :class:`DataFrame`.
The lifetime of this temporary table is tied to the :class:`SparkSession`
that was used to create this :class:`DataFrame`.
@@ -344,10 +346,10 @@ def show(self, n=20, truncate=True, vertical=False):
"""Prints the first ``n`` rows to the console.
:param n: Number of rows to show.
- :param truncate: If set to True, truncate strings longer than 20 chars by default.
+ :param truncate: If set to ``True``, truncate strings longer than 20 chars by default.
If set to a number greater than one, truncates long strings to length ``truncate``
and align cells right.
- :param vertical: If set to True, print output rows vertically (one line
+ :param vertical: If set to ``True``, print output rows vertically (one line
per column value).
>>> df
@@ -389,11 +391,10 @@ def __repr__(self):
return "DataFrame[%s]" % (", ".join("%s: %s" % c for c in self.dtypes))
def _repr_html_(self):
- """Returns a dataframe with html code when you enabled eager evaluation
+ """Returns a :class:`DataFrame` with html code when you enabled eager evaluation
by 'spark.sql.repl.eagerEval.enabled', this only called by REPL you are
using support eager evaluation with HTML.
"""
- import cgi
if not self._support_repr_html:
self._support_repr_html = True
if self.sql_ctx._conf.isReplEagerEvalEnabled():
@@ -408,11 +409,11 @@ def _repr_html_(self):
html = "\n"
# generate table head
- html += "%s ".join(map(lambda x: cgi.escape(x), head))
+ html += " %s ".join(map(lambda x: html_escape(x), head))
# generate table rows
for row in row_data:
html += " %s ".join(
- map(lambda x: cgi.escape(x), row))
+ map(lambda x: html_escape(x), row))
html += "
\n"
if has_more_data:
html += "only showing top %d %s\n" % (
@@ -428,7 +429,7 @@ def checkpoint(self, eager=True):
plan may grow exponentially. It will be saved to files inside the checkpoint
directory set with L{SparkContext.setCheckpointDir()}.
- :param eager: Whether to checkpoint this DataFrame immediately
+ :param eager: Whether to checkpoint this :class:`DataFrame` immediately
.. note:: Experimental
"""
@@ -442,7 +443,7 @@ def localCheckpoint(self, eager=True):
algorithms where the plan may grow exponentially. Local checkpoints are stored in the
executors using the caching subsystem and therefore they are not reliable.
- :param eager: Whether to checkpoint this DataFrame immediately
+ :param eager: Whether to checkpoint this :class:`DataFrame` immediately
.. note:: Experimental
"""
@@ -485,7 +486,7 @@ def withWatermark(self, eventTime, delayThreshold):
@since(2.2)
def hint(self, name, *parameters):
- """Specifies some hint on the current DataFrame.
+ """Specifies some hint on the current :class:`DataFrame`.
:param name: A name of the hint.
:param parameters: Optional parameters.
@@ -538,7 +539,8 @@ def collect(self):
def toLocalIterator(self):
"""
Returns an iterator that contains all of the rows in this :class:`DataFrame`.
- The iterator will consume as much memory as the largest partition in this DataFrame.
+ The iterator will consume as much memory as the largest partition in this
+ :class:`DataFrame`.
>>> list(df.toLocalIterator())
[Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]
@@ -644,7 +646,7 @@ def unpersist(self, blocking=False):
"""Marks the :class:`DataFrame` as non-persistent, and remove all blocks for it from
memory and disk.
- .. note:: `blocking` default has changed to False to match Scala in 2.0.
+ .. note:: `blocking` default has changed to ``False`` to match Scala in 2.0.
"""
self.is_cached = False
self._jdf.unpersist(blocking)
@@ -679,7 +681,7 @@ def coalesce(self, numPartitions):
def repartition(self, numPartitions, *cols):
"""
Returns a new :class:`DataFrame` partitioned by the given partitioning expressions. The
- resulting DataFrame is hash partitioned.
+ resulting :class:`DataFrame` is hash partitioned.
:param numPartitions:
can be an int to specify the target number of partitions or a Column.
@@ -741,7 +743,7 @@ def repartition(self, numPartitions, *cols):
def repartitionByRange(self, numPartitions, *cols):
"""
Returns a new :class:`DataFrame` partitioned by the given partitioning expressions. The
- resulting DataFrame is range partitioned.
+ resulting :class:`DataFrame` is range partitioned.
:param numPartitions:
can be an int to specify the target number of partitions or a Column.
@@ -796,7 +798,7 @@ def distinct(self):
def sample(self, withReplacement=None, fraction=None, seed=None):
"""Returns a sampled subset of this :class:`DataFrame`.
- :param withReplacement: Sample with replacement or not (default False).
+ :param withReplacement: Sample with replacement or not (default ``False``).
:param fraction: Fraction of rows to generate, range [0.0, 1.0].
:param seed: Seed for sampling (default a random seed).
@@ -868,7 +870,7 @@ def sampleBy(self, col, fractions, seed=None):
sampling fraction for each stratum. If a stratum is not
specified, we treat its fraction as zero.
:param seed: random seed
- :return: a new DataFrame that represents the stratified sample
+ :return: a new :class:`DataFrame` that represents the stratified sample
>>> from pyspark.sql.functions import col
>>> dataset = sqlContext.range(0, 100).select((col("id") % 3).alias("key"))
@@ -897,8 +899,8 @@ def sampleBy(self, col, fractions, seed=None):
def randomSplit(self, weights, seed=None):
"""Randomly splits this :class:`DataFrame` with the provided weights.
- :param weights: list of doubles as weights with which to split the DataFrame. Weights will
- be normalized if they don't sum up to 1.0.
+ :param weights: list of doubles as weights with which to split the :class:`DataFrame`.
+ Weights will be normalized if they don't sum up to 1.0.
:param seed: The seed for sampling.
>>> splits = df4.randomSplit([1.0, 2.0], 24)
@@ -963,7 +965,7 @@ def colRegex(self, colName):
def alias(self, alias):
"""Returns a new :class:`DataFrame` with an alias set.
- :param alias: string, an alias name to be set for the DataFrame.
+ :param alias: string, an alias name to be set for the :class:`DataFrame`.
>>> from pyspark.sql.functions import *
>>> df_as1 = df.alias("df_as1")
@@ -1054,7 +1056,7 @@ def sortWithinPartitions(self, *cols, **kwargs):
"""Returns a new :class:`DataFrame` with each partition sorted by the specified column(s).
:param cols: list of :class:`Column` or column names to sort by.
- :param ascending: boolean or list of boolean (default True).
+ :param ascending: boolean or list of boolean (default ``True``).
Sort ascending vs. descending. Specify list for multiple sort orders.
If a list is specified, length of the list must equal length of the `cols`.
@@ -1075,7 +1077,7 @@ def sort(self, *cols, **kwargs):
"""Returns a new :class:`DataFrame` sorted by the specified column(s).
:param cols: list of :class:`Column` or column names to sort by.
- :param ascending: boolean or list of boolean (default True).
+ :param ascending: boolean or list of boolean (default ``True``).
Sort ascending vs. descending. Specify list for multiple sort orders.
If a list is specified, length of the list must equal length of the `cols`.
@@ -1142,7 +1144,8 @@ def describe(self, *cols):
given, this function computes statistics for all numerical or string columns.
.. note:: This function is meant for exploratory data analysis, as we make no
- guarantee about the backward compatibility of the schema of the resulting DataFrame.
+ guarantee about the backward compatibility of the schema of the resulting
+ :class:`DataFrame`.
>>> df.describe(['age']).show()
+-------+------------------+
@@ -1186,7 +1189,8 @@ def summary(self, *statistics):
approximate quartiles (percentiles at 25%, 50%, and 75%), and max.
.. note:: This function is meant for exploratory data analysis, as we make no
- guarantee about the backward compatibility of the schema of the resulting DataFrame.
+ guarantee about the backward compatibility of the schema of the resulting
+ :class:`DataFrame`.
>>> df.summary().show()
+-------+------------------+-----+
@@ -1308,7 +1312,7 @@ def select(self, *cols):
:param cols: list of column names (string) or expressions (:class:`Column`).
If one of the column names is '*', that column is expanded to include all columns
- in the current DataFrame.
+ in the current :class:`DataFrame`.
>>> df.select('*').collect()
[Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]
@@ -1412,7 +1416,7 @@ def rollup(self, *cols):
def cube(self, *cols):
"""
Create a multi-dimensional cube for the current :class:`DataFrame` using
- the specified columns, so we can run aggregation on them.
+ the specified columns, so we can run aggregations on them.
>>> df.cube("name", df.age).count().orderBy("name", "age").show()
+-----+----+-----+
@@ -1446,7 +1450,8 @@ def agg(self, *exprs):
@since(2.0)
def union(self, other):
- """ Return a new :class:`DataFrame` containing union of rows in this and another frame.
+ """ Return a new :class:`DataFrame` containing union of rows in this and another
+ :class:`DataFrame`.
This is equivalent to `UNION ALL` in SQL. To do a SQL-style set union
(that does deduplication of elements), use this function followed by :func:`distinct`.
@@ -1457,7 +1462,8 @@ def union(self, other):
@since(1.3)
def unionAll(self, other):
- """ Return a new :class:`DataFrame` containing union of rows in this and another frame.
+ """ Return a new :class:`DataFrame` containing union of rows in this and another
+ :class:`DataFrame`.
This is equivalent to `UNION ALL` in SQL. To do a SQL-style set union
(that does deduplication of elements), use this function followed by :func:`distinct`.
@@ -1471,7 +1477,8 @@ def unionAll(self, other):
@since(2.3)
def unionByName(self, other):
- """ Returns a new :class:`DataFrame` containing union of rows in this and another frame.
+ """ Returns a new :class:`DataFrame` containing union of rows in this and another
+ :class:`DataFrame`.
This is different from both `UNION ALL` and `UNION DISTINCT` in SQL. To do a SQL-style set
union (that does deduplication of elements), use this function followed by :func:`distinct`.
@@ -1494,7 +1501,7 @@ def unionByName(self, other):
@since(1.3)
def intersect(self, other):
""" Return a new :class:`DataFrame` containing rows only in
- both this frame and another frame.
+ both this :class:`DataFrame` and another :class:`DataFrame`.
This is equivalent to `INTERSECT` in SQL.
"""
@@ -1524,8 +1531,8 @@ def intersectAll(self, other):
@since(1.3)
def subtract(self, other):
- """ Return a new :class:`DataFrame` containing rows in this frame
- but not in another frame.
+ """ Return a new :class:`DataFrame` containing rows in this :class:`DataFrame`
+ but not in another :class:`DataFrame`.
This is equivalent to `EXCEPT DISTINCT` in SQL.
@@ -1815,12 +1822,12 @@ def all_of_(xs):
def approxQuantile(self, col, probabilities, relativeError):
"""
Calculates the approximate quantiles of numerical columns of a
- DataFrame.
+ :class:`DataFrame`.
The result of this algorithm has the following deterministic bound:
- If the DataFrame has N elements and if we request the quantile at
+ If the :class:`DataFrame` has N elements and if we request the quantile at
probability `p` up to error `err`, then the algorithm will return
- a sample `x` from the DataFrame so that the *exact* rank of `x` is
+ a sample `x` from the :class:`DataFrame` so that the *exact* rank of `x` is
close to (p * N). More precisely,
floor((p - err) * N) <= rank(x) <= ceil((p + err) * N).
@@ -1888,7 +1895,7 @@ def approxQuantile(self, col, probabilities, relativeError):
@since(1.4)
def corr(self, col1, col2, method=None):
"""
- Calculates the correlation of two columns of a DataFrame as a double value.
+ Calculates the correlation of two columns of a :class:`DataFrame` as a double value.
Currently only supports the Pearson Correlation Coefficient.
:func:`DataFrame.corr` and :func:`DataFrameStatFunctions.corr` are aliases of each other.
@@ -1936,7 +1943,7 @@ def crosstab(self, col1, col2):
:param col1: The name of the first column. Distinct items will make the first item of
each row.
:param col2: The name of the second column. Distinct items will make the column names
- of the DataFrame.
+ of the :class:`DataFrame`.
"""
if not isinstance(col1, basestring):
raise ValueError("col1 should be a string.")
@@ -1953,7 +1960,8 @@ def freqItems(self, cols, support=None):
:func:`DataFrame.freqItems` and :func:`DataFrameStatFunctions.freqItems` are aliases.
.. note:: This function is meant for exploratory data analysis, as we make no
- guarantee about the backward compatibility of the schema of the resulting DataFrame.
+ guarantee about the backward compatibility of the schema of the resulting
+ :class:`DataFrame`.
:param cols: Names of the columns to calculate frequent items for as a list or tuple of
strings.
@@ -1975,8 +1983,8 @@ def withColumn(self, colName, col):
Returns a new :class:`DataFrame` by adding a column or replacing the
existing column that has the same name.
- The column expression must be an expression over this DataFrame; attempting to add
- a column from some other dataframe will raise an error.
+ The column expression must be an expression over this :class:`DataFrame`; attempting to add
+ a column from some other :class:`DataFrame` will raise an error.
:param colName: string, name of the new column.
:param col: a :class:`Column` expression for the new column.
@@ -2061,8 +2069,8 @@ def toPandas(self):
This is only available if Pandas is installed and available.
- .. note:: This method should only be used if the resulting Pandas's DataFrame is expected
- to be small, as all the data is loaded into the driver's memory.
+ .. note:: This method should only be used if the resulting Pandas's :class:`DataFrame` is
+ expected to be small, as all the data is loaded into the driver's memory.
.. note:: Usage with spark.sql.execution.arrow.enabled=True is experimental.
@@ -2174,9 +2182,13 @@ def _collectAsArrow(self):
.. note:: Experimental.
"""
- with SCCallSiteSync(self._sc) as css:
- sock_info = self._jdf.collectAsArrowToPython()
- return list(_load_from_socket(sock_info, ArrowStreamSerializer()))
+ with SCCallSiteSync(self._sc):
+ from pyspark.rdd import _load_from_socket
+ port, auth_secret, jsocket_auth_server = self._jdf.collectAsArrowToPython()
+ try:
+ return list(_load_from_socket((port, auth_secret), ArrowStreamSerializer()))
+ finally:
+ jsocket_auth_server.getResult() # Join serving thread and raise any exceptions
##########################################################################################
# Pandas compatibility
@@ -2207,8 +2219,9 @@ def _to_scala_map(sc, jm):
def _to_corrected_pandas_type(dt):
"""
- When converting Spark SQL records to Pandas DataFrame, the inferred data type may be wrong.
- This method gets the corrected data type for Pandas if that type may be inferred uncorrectly.
+ When converting Spark SQL records to Pandas :class:`DataFrame`, the inferred data type may be
+ wrong. This method gets the corrected data type for Pandas if that type may be inferred
+ uncorrectly.
"""
import numpy as np
if type(dt) == ByteType:
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 864780e0be9bd..b9649807138ce 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -25,9 +25,12 @@
if sys.version < "3":
from itertools import imap as map
+if sys.version >= '3':
+ basestring = str
+
from pyspark import since, SparkContext
from pyspark.rdd import ignore_unicode_prefix, PythonEvalType
-from pyspark.sql.column import Column, _to_java_column, _to_seq
+from pyspark.sql.column import Column, _to_java_column, _to_seq, _create_column_from_literal
from pyspark.sql.dataframe import DataFrame
from pyspark.sql.types import StringType, DataType
# Keep UserDefinedFunction import for backwards compatible import; moved in SPARK-22409
@@ -171,14 +174,14 @@ def _():
"""
_functions_1_6 = {
# unary math functions
- 'stddev': 'Aggregate function: returns the unbiased sample standard deviation of' +
- ' the expression in a group.',
+ 'stddev': 'Aggregate function: alias for stddev_samp.',
'stddev_samp': 'Aggregate function: returns the unbiased sample standard deviation of' +
' the expression in a group.',
'stddev_pop': 'Aggregate function: returns population standard deviation of' +
' the expression in a group.',
- 'variance': 'Aggregate function: returns the population variance of the values in a group.',
- 'var_samp': 'Aggregate function: returns the unbiased variance of the values in a group.',
+ 'variance': 'Aggregate function: alias for var_samp.',
+ 'var_samp': 'Aggregate function: returns the unbiased sample variance of' +
+ ' the values in a group.',
'var_pop': 'Aggregate function: returns the population variance of the values in a group.',
'skewness': 'Aggregate function: returns the skewness of the values in a group.',
'kurtosis': 'Aggregate function: returns the kurtosis of the values in a group.',
@@ -283,7 +286,8 @@ def approxCountDistinct(col, rsd=None):
@since(2.1)
def approx_count_distinct(col, rsd=None):
- """Aggregate function: returns a new :class:`Column` for approximate distinct count of column `col`.
+ """Aggregate function: returns a new :class:`Column` for approximate distinct count of
+ column `col`.
:param rsd: maximum estimation error allowed (default = 0.05). For rsd < 0.01, it is more
efficient to use :func:`countDistinct`
@@ -346,7 +350,8 @@ def coalesce(*cols):
@since(1.6)
def corr(col1, col2):
- """Returns a new :class:`Column` for the Pearson Correlation Coefficient for ``col1`` and ``col2``.
+ """Returns a new :class:`Column` for the Pearson Correlation Coefficient for ``col1``
+ and ``col2``.
>>> a = range(20)
>>> b = [2 * x for x in range(20)]
@@ -853,36 +858,6 @@ def ntile(n):
return Column(sc._jvm.functions.ntile(int(n)))
-@since(2.4)
-def unboundedPreceding():
- """
- Window function: returns the special frame boundary that represents the first row
- in the window partition.
- """
- sc = SparkContext._active_spark_context
- return Column(sc._jvm.functions.unboundedPreceding())
-
-
-@since(2.4)
-def unboundedFollowing():
- """
- Window function: returns the special frame boundary that represents the last row
- in the window partition.
- """
- sc = SparkContext._active_spark_context
- return Column(sc._jvm.functions.unboundedFollowing())
-
-
-@since(2.4)
-def currentRow():
- """
- Window function: returns the special frame boundary that represents the current row
- in the window partition.
- """
- sc = SparkContext._active_spark_context
- return Column(sc._jvm.functions.currentRow())
-
-
# ---------------------- Date/Timestamp functions ------------------------------
@since(1.5)
@@ -1281,9 +1256,18 @@ def unix_timestamp(timestamp=None, format='yyyy-MM-dd HH:mm:ss'):
@since(1.5)
def from_utc_timestamp(timestamp, tz):
"""
- Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders
- that time as a timestamp in the given time zone. For example, 'GMT+1' would yield
- '2017-07-14 03:40:00.0'.
+ This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
+ takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in UTC, and
+ renders that timestamp as a timestamp in the given time zone.
+
+ However, timestamp in Spark represents number of microseconds from the Unix epoch, which is not
+ timezone-agnostic. So in Spark this function just shift the timestamp value from UTC timezone to
+ the given timezone.
+
+ This function may return confusing result if the input is a string with timezone, e.g.
+ '2018-03-13T06:18:23+00:00'. The reason is that, Spark firstly cast the string to timestamp
+ according to the timezone in the string, and finally display the result by converting the
+ timestamp to string according to the session local timezone.
:param timestamp: the column that contains timestamps
:param tz: a string that has the ID of timezone, e.g. "GMT", "America/Los_Angeles", etc
@@ -1306,9 +1290,18 @@ def from_utc_timestamp(timestamp, tz):
@since(1.5)
def to_utc_timestamp(timestamp, tz):
"""
- Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time
- zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield
- '2017-07-14 01:40:00.0'.
+ This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
+ takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in the given
+ timezone, and renders that timestamp as a timestamp in UTC.
+
+ However, timestamp in Spark represents number of microseconds from the Unix epoch, which is not
+ timezone-agnostic. So in Spark this function just shift the timestamp value from the given
+ timezone to UTC timezone.
+
+ This function may return confusing result if the input is a string with timezone, e.g.
+ '2018-03-13T06:18:23+00:00'. The reason is that, Spark firstly cast the string to timestamp
+ according to the timezone in the string, and finally display the result by converting the
+ timestamp to string according to the session local timezone.
:param timestamp: the column that contains timestamps
:param tz: a string that has the ID of timezone, e.g. "GMT", "America/Los_Angeles", etc
@@ -1528,8 +1521,9 @@ def format_string(format, *cols):
"""
Formats the arguments in printf-style and returns the result as a string column.
- :param col: the column name of the numeric value to be formatted
- :param d: the N decimal places
+ :param format: string that can contain embedded format tags and used as result column's value
+ :param cols: list of column names (string) or list of :class:`Column` expressions to
+ be used in formatting
>>> df = spark.createDataFrame([(5, "hello")], ['a', 'b'])
>>> df.select(format_string('%d %s', df.a, df.b).alias('v')).collect()
@@ -1688,14 +1682,14 @@ def split(str, pattern):
@ignore_unicode_prefix
@since(1.5)
def regexp_extract(str, pattern, idx):
- """Extract a specific group matched by a Java regex, from the specified string column.
+ r"""Extract a specific group matched by a Java regex, from the specified string column.
If the regex did not match, or the specified group did not match, an empty string is returned.
>>> df = spark.createDataFrame([('100-200',)], ['str'])
- >>> df.select(regexp_extract('str', '(\d+)-(\d+)', 1).alias('d')).collect()
+ >>> df.select(regexp_extract('str', r'(\d+)-(\d+)', 1).alias('d')).collect()
[Row(d=u'100')]
>>> df = spark.createDataFrame([('foo',)], ['str'])
- >>> df.select(regexp_extract('str', '(\d+)', 1).alias('d')).collect()
+ >>> df.select(regexp_extract('str', r'(\d+)', 1).alias('d')).collect()
[Row(d=u'')]
>>> df = spark.createDataFrame([('aaaac',)], ['str'])
>>> df.select(regexp_extract('str', '(a+)(b)?(c)', 2).alias('d')).collect()
@@ -1709,10 +1703,10 @@ def regexp_extract(str, pattern, idx):
@ignore_unicode_prefix
@since(1.5)
def regexp_replace(str, pattern, replacement):
- """Replace all substrings of the specified string value that match regexp with rep.
+ r"""Replace all substrings of the specified string value that match regexp with rep.
>>> df = spark.createDataFrame([('100-200',)], ['str'])
- >>> df.select(regexp_replace('str', '(\\d+)', '--').alias('d')).collect()
+ >>> df.select(regexp_replace('str', r'(\d+)', '--').alias('d')).collect()
[Row(d=u'-----')]
"""
sc = SparkContext._active_spark_context
@@ -1913,7 +1907,12 @@ def arrays_overlap(a1, a2):
def slice(x, start, length):
"""
Collection function: returns an array containing all the elements in `x` from index `start`
- (or starting from the end if `start` is negative) with the specified `length`.
+ (array indices start at 1, or from the end if `start` is negative) with the specified `length`.
+
+ :param x: the array to be sliced
+ :param start: the starting index
+ :param length: the length of the slice
+
>>> df = spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x'])
>>> df.select(slice(df.x, 2, 2).alias("sliced")).collect()
[Row(sliced=[2, 3]), Row(sliced=[5])]
@@ -1996,11 +1995,12 @@ def element_at(col, extraction):
[Row(element_at(data, 1)=u'a'), Row(element_at(data, 1)=None)]
>>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},), ({},)], ['data'])
- >>> df.select(element_at(df.data, "a")).collect()
+ >>> df.select(element_at(df.data, lit("a"))).collect()
[Row(element_at(data, a)=1.0), Row(element_at(data, a)=None)]
"""
sc = SparkContext._active_spark_context
- return Column(sc._jvm.functions.element_at(_to_java_column(col), extraction))
+ return Column(sc._jvm.functions.element_at(
+ _to_java_column(col), lit(extraction)._jc)) # noqa: F821 'lit' is dynamically defined.
@since(2.4)
@@ -2326,23 +2326,25 @@ def to_json(col, options={}):
@ignore_unicode_prefix
@since(2.4)
-def schema_of_json(col):
+def schema_of_json(json):
"""
- Parses a column containing a JSON string and infers its schema in DDL format.
+ Parses a JSON string and infers its schema in DDL format.
- :param col: string column in json format
+ :param json: a JSON string or a string literal containing a JSON string.
- >>> from pyspark.sql.types import *
- >>> data = [(1, '{"a": 1}')]
- >>> df = spark.createDataFrame(data, ("key", "value"))
- >>> df.select(schema_of_json(df.value).alias("json")).collect()
- [Row(json=u'struct')]
- >>> df.select(schema_of_json(lit('{"a": 0}')).alias("json")).collect()
+ >>> df = spark.range(1)
+ >>> df.select(schema_of_json('{"a": 0}').alias("json")).collect()
[Row(json=u'struct')]
"""
+ if isinstance(json, basestring):
+ col = _create_column_from_literal(json)
+ elif isinstance(json, Column):
+ col = _to_java_column(json)
+ else:
+ raise TypeError("schema argument should be a column or string")
sc = SparkContext._active_spark_context
- jc = sc._jvm.functions.schema_of_json(_to_java_column(col))
+ jc = sc._jvm.functions.schema_of_json(col)
return Column(jc)
@@ -2520,26 +2522,6 @@ def map_values(col):
return Column(sc._jvm.functions.map_values(_to_java_column(col)))
-@since(2.4)
-def map_entries(col):
- """
- Collection function: Returns an unordered array of all entries in the given map.
-
- :param col: name of column or expression
-
- >>> from pyspark.sql.functions import map_entries
- >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data")
- >>> df.select(map_entries("data").alias("entries")).show()
- +----------------+
- | entries|
- +----------------+
- |[[1, a], [2, b]]|
- +----------------+
- """
- sc = SparkContext._active_spark_context
- return Column(sc._jvm.functions.map_entries(_to_java_column(col)))
-
-
@since(2.4)
def map_from_entries(col):
"""
@@ -2720,9 +2702,10 @@ def pandas_udf(f=None, returnType=None, functionType=None):
1. SCALAR
A scalar UDF defines a transformation: One or more `pandas.Series` -> A `pandas.Series`.
- The returnType should be a primitive data type, e.g., :class:`DoubleType`.
The length of the returned `pandas.Series` must be of the same as the input `pandas.Series`.
+ :class:`MapType`, :class:`StructType` are currently not supported as output types.
+
Scalar UDFs are used with :meth:`pyspark.sql.DataFrame.withColumn` and
:meth:`pyspark.sql.DataFrame.select`.
@@ -2783,14 +2766,14 @@ def pandas_udf(f=None, returnType=None, functionType=None):
+---+-------------------+
Alternatively, the user can define a function that takes two arguments.
- In this case, the grouping key will be passed as the first argument and the data will
- be passed as the second argument. The grouping key will be passed as a tuple of numpy
+ In this case, the grouping key(s) will be passed as the first argument and the data will
+ be passed as the second argument. The grouping key(s) will be passed as a tuple of numpy
data types, e.g., `numpy.int32` and `numpy.float64`. The data will still be passed in
as a `pandas.DataFrame` containing all columns from the original Spark DataFrame.
- This is useful when the user does not want to hardcode grouping key in the function.
+ This is useful when the user does not want to hardcode grouping key(s) in the function.
- >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
>>> import pandas as pd # doctest: +SKIP
+ >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
>>> df = spark.createDataFrame(
... [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
... ("id", "v")) # doctest: +SKIP
@@ -2806,6 +2789,22 @@ def pandas_udf(f=None, returnType=None, functionType=None):
| 1|1.5|
| 2|6.0|
+---+---+
+ >>> @pandas_udf(
+ ... "id long, `ceil(v / 2)` long, v double",
+ ... PandasUDFType.GROUPED_MAP) # doctest: +SKIP
+ >>> def sum_udf(key, pdf):
+ ... # key is a tuple of two numpy.int64s, which is the values
+ ... # of 'id' and 'ceil(df.v / 2)' for the current group
+ ... return pd.DataFrame([key + (pdf.v.sum(),)])
+ >>> df.groupby(df.id, ceil(df.v / 2)).apply(sum_udf).show() # doctest: +SKIP
+ +---+-----------+----+
+ | id|ceil(v / 2)| v|
+ +---+-----------+----+
+ | 2| 5|10.0|
+ | 1| 1| 3.0|
+ | 2| 3| 5.0|
+ | 2| 2| 3.0|
+ +---+-----------+----+
.. note:: If returning a new `pandas.DataFrame` constructed with a dictionary, it is
recommended to explicitly index the columns by name to ensure the positions are correct,
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 49f4e6b2ede1b..1c5f8e34e374d 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -349,8 +349,8 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
negativeInf=None, dateFormat=None, timestampFormat=None, maxColumns=None,
maxCharsPerColumn=None, maxMalformedLogPerPartition=None, mode=None,
columnNameOfCorruptRecord=None, multiLine=None, charToEscapeQuoteEscaping=None,
- samplingRatio=None, enforceSchema=None):
- """Loads a CSV file and returns the result as a :class:`DataFrame`.
+ samplingRatio=None, enforceSchema=None, emptyValue=None):
+ r"""Loads a CSV file and returns the result as a :class:`DataFrame`.
This function will go through the input once to determine the input schema if
``inferSchema`` is enabled. To avoid going through the entire data once, disable
@@ -417,7 +417,11 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
:param maxMalformedLogPerPartition: this parameter is no longer used since Spark 2.2.0.
If specified, it is ignored.
:param mode: allows a mode for dealing with corrupt records during parsing. If None is
- set, it uses the default value, ``PERMISSIVE``.
+ set, it uses the default value, ``PERMISSIVE``. Note that Spark tries to
+ parse only required columns in CSV under column pruning. Therefore, corrupt
+ records can be different based on required set of fields. This behavior can
+ be controlled by ``spark.sql.csv.parser.columnPruning.enabled``
+ (enabled by default).
* ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
into a field configured by ``columnNameOfCorruptRecord``, and sets other \
@@ -444,6 +448,8 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
different, ``\0`` otherwise.
:param samplingRatio: defines fraction of rows used for schema inferring.
If None is set, it uses the default value, ``1.0``.
+ :param emptyValue: sets the string representation of an empty value. If None is set, it uses
+ the default value, empty string.
>>> df = spark.read.csv('python/test_support/sql/ages.csv')
>>> df.dtypes
@@ -463,7 +469,7 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
maxMalformedLogPerPartition=maxMalformedLogPerPartition, mode=mode,
columnNameOfCorruptRecord=columnNameOfCorruptRecord, multiLine=multiLine,
charToEscapeQuoteEscaping=charToEscapeQuoteEscaping, samplingRatio=samplingRatio,
- enforceSchema=enforceSchema)
+ enforceSchema=enforceSchema, emptyValue=emptyValue)
if isinstance(path, basestring):
path = [path]
if type(path) == list:
@@ -494,8 +500,6 @@ def func(iterator):
def orc(self, path):
"""Loads ORC files, returning the result as a :class:`DataFrame`.
- .. note:: Currently ORC support is only available together with Hive support.
-
>>> df = spark.read.orc('python/test_support/sql/orc_partitioned')
>>> df.dtypes
[('a', 'bigint'), ('b', 'int'), ('c', 'int')]
@@ -517,12 +521,13 @@ def jdbc(self, url, table, column=None, lowerBound=None, upperBound=None, numPar
If both ``column`` and ``predicates`` are specified, ``column`` will be used.
- .. note:: Don't create too many partitions in parallel on a large cluster; \
- otherwise Spark might crash your external database systems.
+ .. note:: Don't create too many partitions in parallel on a large cluster;
+ otherwise Spark might crash your external database systems.
:param url: a JDBC URL of the form ``jdbc:subprotocol:subname``
:param table: the name of the table
- :param column: the name of an integer column that will be used for partitioning;
+ :param column: the name of a column of numeric, date, or timestamp type
+ that will be used for partitioning;
if this parameter is specified, then ``numPartitions``, ``lowerBound``
(inclusive), and ``upperBound`` (exclusive) will form partition strides
for generated WHERE clause expressions used to split the column
@@ -721,7 +726,7 @@ def save(self, path=None, format=None, mode=None, partitionBy=None, **options):
:param partitionBy: names of partitioning columns
:param options: all other string options
- >>> df.write.mode('append').parquet(os.path.join(tempfile.mkdtemp(), 'data'))
+ >>> df.write.mode("append").save(os.path.join(tempfile.mkdtemp(), 'data'))
"""
self.mode(mode).options(**options)
if partitionBy is not None:
@@ -859,8 +864,8 @@ def text(self, path, compression=None, lineSep=None):
def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=None,
header=None, nullValue=None, escapeQuotes=None, quoteAll=None, dateFormat=None,
timestampFormat=None, ignoreLeadingWhiteSpace=None, ignoreTrailingWhiteSpace=None,
- charToEscapeQuoteEscaping=None, encoding=None):
- """Saves the content of the :class:`DataFrame` in CSV format at the specified path.
+ charToEscapeQuoteEscaping=None, encoding=None, emptyValue=None):
+ r"""Saves the content of the :class:`DataFrame` in CSV format at the specified path.
:param path: the path in any Hadoop supported file system
:param mode: specifies the behavior of the save operation when data already exists.
@@ -911,6 +916,8 @@ def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=No
different, ``\0`` otherwise..
:param encoding: sets the encoding (charset) of saved csv files. If None is set,
the default UTF-8 charset will be used.
+ :param emptyValue: sets the string representation of an empty value. If None is set, it uses
+ the default value, ``""``.
>>> df.write.csv(os.path.join(tempfile.mkdtemp(), 'data'))
"""
@@ -921,15 +928,13 @@ def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=No
ignoreLeadingWhiteSpace=ignoreLeadingWhiteSpace,
ignoreTrailingWhiteSpace=ignoreTrailingWhiteSpace,
charToEscapeQuoteEscaping=charToEscapeQuoteEscaping,
- encoding=encoding)
+ encoding=encoding, emptyValue=emptyValue)
self._jwrite.csv(path)
@since(1.5)
def orc(self, path, mode=None, partitionBy=None, compression=None):
"""Saves the content of the :class:`DataFrame` in ORC format at the specified path.
- .. note:: Currently ORC support is only available together with Hive support.
-
:param path: the path in any Hadoop supported file system
:param mode: specifies the behavior of the save operation when data already exists.
@@ -958,8 +963,8 @@ def orc(self, path, mode=None, partitionBy=None, compression=None):
def jdbc(self, url, table, mode=None, properties=None):
"""Saves the content of the :class:`DataFrame` to an external database table via JDBC.
- .. note:: Don't create too many partitions in parallel on a large cluster; \
- otherwise Spark might crash your external database systems.
+ .. note:: Don't create too many partitions in parallel on a large cluster;
+ otherwise Spark might crash your external database systems.
:param url: a JDBC URL of the form ``jdbc:subprotocol:subname``
:param table: Name of the table in the external database.
diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
index 87d8d6a59a6e9..fc18f57d56aad 100644
--- a/python/pyspark/sql/session.py
+++ b/python/pyspark/sql/session.py
@@ -135,7 +135,7 @@ def appName(self, name):
@since(2.0)
def enableHiveSupport(self):
"""Enables Hive support, including connectivity to a persistent Hive metastore, support
- for Hive serdes, and Hive user-defined functions.
+ for Hive SerDes, and Hive user-defined functions.
"""
return self.config("spark.sql.catalogImplementation", "hive")
@@ -186,7 +186,7 @@ def getOrCreate(self):
return session
builder = Builder()
- """A class attribute having a :class:`Builder` to construct :class:`SparkSession` instances"""
+ """A class attribute having a :class:`Builder` to construct :class:`SparkSession` instances."""
_instantiatedSession = None
@@ -281,7 +281,7 @@ def conf(self):
@since(2.0)
def catalog(self):
"""Interface through which the user may create, drop, alter or query underlying
- databases, tables, functions etc.
+ databases, tables, functions, etc.
:return: :class:`Catalog`
"""
@@ -539,12 +539,18 @@ def _create_from_pandas_with_arrow(self, pdf, schema, timezone):
struct.names[i] = name
schema = struct
+ jsqlContext = self._wrapped._jsqlContext
+
def reader_func(temp_filename):
- return self._jvm.PythonSQLUtils.arrowReadStreamFromFile(
- self._wrapped._jsqlContext, temp_filename, schema.json())
+ return self._jvm.PythonSQLUtils.readArrowStreamFromFile(jsqlContext, temp_filename)
+
+ def create_RDD_server():
+ return self._jvm.ArrowRDDServer(jsqlContext)
# Create Spark DataFrame from Arrow stream file, using one batch per partition
- jdf = self._sc._serialize_to_jvm(batches, ArrowStreamSerializer(), reader_func)
+ jrdd = self._sc._serialize_to_jvm(batches, ArrowStreamSerializer(), reader_func,
+ create_RDD_server)
+ jdf = self._jvm.PythonSQLUtils.toDataFrame(jrdd, schema.json(), jsqlContext)
df = DataFrame(jdf, self._wrapped)
df._schema = schema
return df
@@ -586,20 +592,20 @@ def createDataFrame(self, data, schema=None, samplingRatio=None, verifySchema=Tr
will be inferred from ``data``.
When ``schema`` is ``None``, it will try to infer the schema (column names and types)
- from ``data``, which should be an RDD of :class:`Row`,
- or :class:`namedtuple`, or :class:`dict`.
+ from ``data``, which should be an RDD of either :class:`Row`,
+ :class:`namedtuple`, or :class:`dict`.
When ``schema`` is :class:`pyspark.sql.types.DataType` or a datatype string, it must match
the real data, or an exception will be thrown at runtime. If the given schema is not
:class:`pyspark.sql.types.StructType`, it will be wrapped into a
- :class:`pyspark.sql.types.StructType` as its only field, and the field name will be "value",
- each record will also be wrapped into a tuple, which can be converted to row later.
+ :class:`pyspark.sql.types.StructType` as its only field, and the field name will be "value".
+ Each record will also be wrapped into a tuple, which can be converted to row later.
If schema inference is needed, ``samplingRatio`` is used to determined the ratio of
rows used for schema inference. The first row will be used if ``samplingRatio`` is ``None``.
- :param data: an RDD of any kind of SQL data representation(e.g. row, tuple, int, boolean,
- etc.), or :class:`list`, or :class:`pandas.DataFrame`.
+ :param data: an RDD of any kind of SQL data representation (e.g. row, tuple, int, boolean,
+ etc.), :class:`list`, or :class:`pandas.DataFrame`.
:param schema: a :class:`pyspark.sql.types.DataType` or a datatype string or a list of
column names, default is ``None``. The data type string format equals to
:class:`pyspark.sql.types.DataType.simpleString`, except that top level struct type can
@@ -801,7 +807,7 @@ def readStream(self):
@since(2.0)
def streams(self):
"""Returns a :class:`StreamingQueryManager` that allows managing all the
- :class:`StreamingQuery` StreamingQueries active on `this` context.
+ :class:`StreamingQuery` instances active on `this` context.
.. note:: Evolving.
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index ee13778a7dcd6..73f0a59a5c79a 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -564,8 +564,8 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
negativeInf=None, dateFormat=None, timestampFormat=None, maxColumns=None,
maxCharsPerColumn=None, maxMalformedLogPerPartition=None, mode=None,
columnNameOfCorruptRecord=None, multiLine=None, charToEscapeQuoteEscaping=None,
- enforceSchema=None):
- """Loads a CSV file stream and returns the result as a :class:`DataFrame`.
+ enforceSchema=None, emptyValue=None):
+ r"""Loads a CSV file stream and returns the result as a :class:`DataFrame`.
This function will go through the input once to determine the input schema if
``inferSchema`` is enabled. To avoid going through the entire data once, disable
@@ -658,6 +658,8 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
the quote character. If None is set, the default value is
escape character when escape and quote characters are
different, ``\0`` otherwise..
+ :param emptyValue: sets the string representation of an empty value. If None is set, it uses
+ the default value, empty string.
>>> csv_sdf = spark.readStream.csv(tempfile.mkdtemp(), schema = sdf_schema)
>>> csv_sdf.isStreaming
@@ -674,7 +676,8 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
maxCharsPerColumn=maxCharsPerColumn,
maxMalformedLogPerPartition=maxMalformedLogPerPartition, mode=mode,
columnNameOfCorruptRecord=columnNameOfCorruptRecord, multiLine=multiLine,
- charToEscapeQuoteEscaping=charToEscapeQuoteEscaping, enforceSchema=enforceSchema)
+ charToEscapeQuoteEscaping=charToEscapeQuoteEscaping, enforceSchema=enforceSchema,
+ emptyValue=emptyValue)
if isinstance(path, basestring):
return self._df(self._jreader.csv(path))
else:
@@ -809,10 +812,13 @@ def trigger(self, processingTime=None, once=None, continuous=None):
.. note:: Evolving.
:param processingTime: a processing time interval as a string, e.g. '5 seconds', '1 minute'.
- Set a trigger that runs a query periodically based on the processing
- time. Only one trigger can be set.
+ Set a trigger that runs a microbatch query periodically based on the
+ processing time. Only one trigger can be set.
:param once: if set to True, set a trigger that processes only one batch of data in a
streaming query then terminates the query. Only one trigger can be set.
+ :param continuous: a time interval as a string, e.g. '5 seconds', '1 minute'.
+ Set a trigger that runs a continuous query with a given checkpoint
+ interval. Only one trigger can be set.
>>> # trigger the query for execution every 5 seconds
>>> writer = sdf.writeStream.trigger(processingTime='5 seconds')
@@ -1032,7 +1038,7 @@ def foreachBatch(self, func):
>>> def func(batch_df, batch_id):
... batch_df.collect()
...
- >>> writer = sdf.writeStream.foreach(func)
+ >>> writer = sdf.writeStream.foreachBatch(func)
"""
from pyspark.java_gateway import ensure_callback_server_started
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index 81c0af0b3d81b..949ce3b824011 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -26,6 +26,7 @@
import pydoc
import shutil
import tempfile
+import threading
import pickle
import functools
import time
@@ -79,7 +80,7 @@
_have_pyarrow = _pyarrow_requirement_message is None
_test_compiled = _test_not_compiled_message is None
-from pyspark import SparkContext
+from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession, SQLContext, HiveContext, Column, Row
from pyspark.sql.types import *
from pyspark.sql.types import UserDefinedType, _infer_type, _make_type_verifier
@@ -87,7 +88,7 @@
from pyspark.sql.types import _array_unsigned_int_typecode_ctype_mappings
from pyspark.sql.types import _merge_type
from pyspark.tests import QuietTest, ReusedPySparkTestCase, PySparkTestCase, SparkSubmitTests
-from pyspark.sql.functions import UserDefinedFunction, sha2, lit
+from pyspark.sql.functions import UserDefinedFunction, sha2, lit, input_file_name
from pyspark.sql.window import Window
from pyspark.sql.utils import AnalysisException, ParseException, IllegalArgumentException
@@ -228,12 +229,12 @@ def sql_conf(self, pairs):
class ReusedSQLTestCase(ReusedPySparkTestCase, SQLTestUtils):
@classmethod
def setUpClass(cls):
- ReusedPySparkTestCase.setUpClass()
+ super(ReusedSQLTestCase, cls).setUpClass()
cls.spark = SparkSession(cls.sc)
@classmethod
def tearDownClass(cls):
- ReusedPySparkTestCase.tearDownClass()
+ super(ReusedSQLTestCase, cls).tearDownClass()
cls.spark.stop()
def assertPandasEqual(self, expected, result):
@@ -277,6 +278,10 @@ def test_struct_field_type_name(self):
struct_field = StructField("a", IntegerType())
self.assertRaises(TypeError, struct_field.typeName)
+ def test_invalid_create_row(self):
+ row_class = Row("c1", "c2")
+ self.assertRaises(ValueError, lambda: row_class(1, 2, 3))
+
class SQLTests(ReusedSQLTestCase):
@@ -547,6 +552,60 @@ def test_udf_in_filter_on_top_of_join(self):
df = left.crossJoin(right).filter(f("a", "b"))
self.assertEqual(df.collect(), [Row(a=1, b=1)])
+ def test_udf_in_join_condition(self):
+ # regression test for SPARK-25314
+ from pyspark.sql.functions import udf
+ left = self.spark.createDataFrame([Row(a=1)])
+ right = self.spark.createDataFrame([Row(b=1)])
+ f = udf(lambda a, b: a == b, BooleanType())
+ df = left.join(right, f("a", "b"))
+ with self.assertRaisesRegexp(AnalysisException, 'Detected implicit cartesian product'):
+ df.collect()
+ with self.sql_conf({"spark.sql.crossJoin.enabled": True}):
+ self.assertEqual(df.collect(), [Row(a=1, b=1)])
+
+ def test_udf_in_left_outer_join_condition(self):
+ # regression test for SPARK-26147
+ from pyspark.sql.functions import udf, col
+ left = self.spark.createDataFrame([Row(a=1)])
+ right = self.spark.createDataFrame([Row(b=1)])
+ f = udf(lambda a: str(a), StringType())
+ # The join condition can't be pushed down, as it refers to attributes from both sides.
+ # The Python UDF only refer to attributes from one side, so it's evaluable.
+ df = left.join(right, f("a") == col("b").cast("string"), how="left_outer")
+ with self.sql_conf({"spark.sql.crossJoin.enabled": True}):
+ self.assertEqual(df.collect(), [Row(a=1, b=1)])
+
+ def test_udf_and_common_filter_in_join_condition(self):
+ # regression test for SPARK-25314
+ # test the complex scenario with both udf and common filter
+ from pyspark.sql.functions import udf
+ left = self.spark.createDataFrame([Row(a=1, a1=1, a2=1), Row(a=2, a1=2, a2=2)])
+ right = self.spark.createDataFrame([Row(b=1, b1=1, b2=1), Row(b=1, b1=3, b2=1)])
+ f = udf(lambda a, b: a == b, BooleanType())
+ df = left.join(right, [f("a", "b"), left.a1 == right.b1])
+ # do not need spark.sql.crossJoin.enabled=true for udf is not the only join condition.
+ self.assertEqual(df.collect(), [Row(a=1, a1=1, a2=1, b=1, b1=1, b2=1)])
+
+ def test_udf_not_supported_in_join_condition(self):
+ # regression test for SPARK-25314
+ # test python udf is not supported in join type except inner join.
+ from pyspark.sql.functions import udf
+ left = self.spark.createDataFrame([Row(a=1, a1=1, a2=1), Row(a=2, a1=2, a2=2)])
+ right = self.spark.createDataFrame([Row(b=1, b1=1, b2=1), Row(b=1, b1=3, b2=1)])
+ f = udf(lambda a, b: a == b, BooleanType())
+
+ def runWithJoinType(join_type, type_string):
+ with self.assertRaisesRegexp(
+ AnalysisException,
+ 'Using PythonUDF.*%s is not supported.' % type_string):
+ left.join(right, [f("a", "b"), left.a1 == right.b1], join_type).collect()
+ runWithJoinType("full", "FullOuter")
+ runWithJoinType("left", "LeftOuter")
+ runWithJoinType("right", "RightOuter")
+ runWithJoinType("leftanti", "LeftAnti")
+ runWithJoinType("leftsemi", "LeftSemi")
+
def test_udf_without_arguments(self):
self.spark.catalog.registerFunction("foo", lambda: "bar")
[row] = self.spark.sql("SELECT foo()").collect()
@@ -773,6 +832,29 @@ def filename(path):
row2 = df2.select(sameText(df2['file'])).first()
self.assertTrue(row2[0].find("people.json") != -1)
+ def test_input_file_name_reset_for_rdd(self):
+ rdd = self.sc.textFile('python/test_support/hello/hello.txt').map(lambda x: {'data': x})
+ df = self.spark.createDataFrame(rdd, "data STRING")
+ df.select(input_file_name().alias('file')).collect()
+
+ non_file_df = self.spark.range(100).select(input_file_name())
+
+ results = non_file_df.collect()
+ self.assertTrue(len(results) == 100)
+
+ # [SPARK-24605]: if everything was properly reset after the last job, this should return
+ # empty string rather than the file read in the last job.
+ for result in results:
+ self.assertEqual(result[0], '')
+
+ def test_input_file_name_udf(self):
+ from pyspark.sql.functions import udf, input_file_name
+
+ df = self.spark.read.text('python/test_support/hello/hello.txt')
+ df = df.select(udf(lambda x: x)("value"), input_file_name().alias('file'))
+ file_name = df.collect()[0].file
+ self.assertTrue("python/test_support/hello/hello.txt" in file_name)
+
def test_udf_defers_judf_initialization(self):
# This is separate of UDFInitializationTests
# to avoid context initialization
@@ -1163,7 +1245,7 @@ def test_simple_udt_in_df(self):
df = self.spark.createDataFrame(
[(i % 3, PythonOnlyPoint(float(i), float(i))) for i in range(10)],
schema=schema)
- df.show()
+ df.collect()
def test_nested_udt_in_df(self):
schema = StructType().add("key", LongType()).add("val", ArrayType(PythonOnlyUDT()))
@@ -1493,8 +1575,7 @@ def test_array_contains_function(self):
from pyspark.sql.functions import array_contains
df = self.spark.createDataFrame([(["1", "2", "3"],), ([],)], ['data'])
- actual = df.select(array_contains(df.data, 1).alias('b')).collect()
- # The value argument can be implicitly castable to the element's type of the array.
+ actual = df.select(array_contains(df.data, "1").alias('b')).collect()
self.assertEqual([Row(b=True), Row(b=False)], actual)
def test_between_function(self):
@@ -3261,7 +3342,7 @@ def test_create_dataframe_from_pandas_with_timestamp(self):
import pandas as pd
from datetime import datetime
pdf = pd.DataFrame({"ts": [datetime(2017, 10, 31, 1, 1, 1)],
- "d": [pd.Timestamp.now().date()]})
+ "d": [pd.Timestamp.now().date()]}, columns=["d", "ts"])
# test types are inferred correctly without specifying schema
df = self.spark.createDataFrame(pdf)
self.assertTrue(isinstance(df.schema['ts'].dataType, TimestampType))
@@ -3488,6 +3569,31 @@ def test_repr_behaviors(self):
self.assertEquals(None, df._repr_html_())
self.assertEquals(expected, df.__repr__())
+ # SPARK-25591
+ def test_same_accumulator_in_udfs(self):
+ from pyspark.sql.functions import udf
+
+ data_schema = StructType([StructField("a", IntegerType(), True),
+ StructField("b", IntegerType(), True)])
+ data = self.spark.createDataFrame([[1, 2]], schema=data_schema)
+
+ test_accum = self.sc.accumulator(0)
+
+ def first_udf(x):
+ test_accum.add(1)
+ return x
+
+ def second_udf(x):
+ test_accum.add(100)
+ return x
+
+ func_udf = udf(first_udf, IntegerType())
+ func_udf2 = udf(second_udf, IntegerType())
+ data = data.withColumn("out1", func_udf(data["a"]))
+ data = data.withColumn("out2", func_udf2(data["b"]))
+ data.collect()
+ self.assertEqual(test_accum.value, 101)
+
class HiveSparkSubmitTests(SparkSubmitTests):
@@ -3609,6 +3715,7 @@ def test_query_execution_listener_on_collect(self):
self.spark._jvm.OnSuccessCall.isCalled(),
"The callback from the query execution listener should not be called before 'collect'")
self.spark.sql("SELECT * FROM range(1)").collect()
+ self.spark.sparkContext._jsc.sc().listenerBus().waitUntilEmpty(10000)
self.assertTrue(
self.spark._jvm.OnSuccessCall.isCalled(),
"The callback from the query execution listener should be called after 'collect'")
@@ -3623,6 +3730,7 @@ def test_query_execution_listener_on_collect_with_arrow(self):
"The callback from the query execution listener should not be "
"called before 'toPandas'")
self.spark.sql("SELECT * FROM range(1)").toPandas()
+ self.spark.sparkContext._jsc.sc().listenerBus().waitUntilEmpty(10000)
self.assertTrue(
self.spark._jvm.OnSuccessCall.isCalled(),
"The callback from the query execution listener should be called after 'toPandas'")
@@ -4101,7 +4209,8 @@ def setUpClass(cls):
from decimal import Decimal
from distutils.version import LooseVersion
import pyarrow as pa
- ReusedSQLTestCase.setUpClass()
+ super(ArrowTests, cls).setUpClass()
+ cls.warnings_lock = threading.Lock()
# Synchronize default timezone between Python and Java
cls.tz_prev = os.environ.get("TZ", None) # save current tz if set
@@ -4142,7 +4251,7 @@ def tearDownClass(cls):
if cls.tz_prev is not None:
os.environ["TZ"] = cls.tz_prev
time.tzset()
- ReusedSQLTestCase.tearDownClass()
+ super(ArrowTests, cls).tearDownClass()
def create_pandas_data_frame(self):
import pandas as pd
@@ -4162,15 +4271,18 @@ def test_toPandas_fallback_enabled(self):
schema = StructType([StructField("map", MapType(StringType(), IntegerType()), True)])
df = self.spark.createDataFrame([({u'a': 1},)], schema=schema)
with QuietTest(self.sc):
- with warnings.catch_warnings(record=True) as warns:
- pdf = df.toPandas()
- # Catch and check the last UserWarning.
- user_warns = [
- warn.message for warn in warns if isinstance(warn.message, UserWarning)]
- self.assertTrue(len(user_warns) > 0)
- self.assertTrue(
- "Attempting non-optimization" in _exception_message(user_warns[-1]))
- self.assertPandasEqual(pdf, pd.DataFrame({u'map': [{u'a': 1}]}))
+ with self.warnings_lock:
+ with warnings.catch_warnings(record=True) as warns:
+ # we want the warnings to appear even if this test is run from a subclass
+ warnings.simplefilter("always")
+ pdf = df.toPandas()
+ # Catch and check the last UserWarning.
+ user_warns = [
+ warn.message for warn in warns if isinstance(warn.message, UserWarning)]
+ self.assertTrue(len(user_warns) > 0)
+ self.assertTrue(
+ "Attempting non-optimization" in _exception_message(user_warns[-1]))
+ self.assertPandasEqual(pdf, pd.DataFrame({u'map': [{u'a': 1}]}))
def test_toPandas_fallback_disabled(self):
from distutils.version import LooseVersion
@@ -4179,8 +4291,9 @@ def test_toPandas_fallback_disabled(self):
schema = StructType([StructField("map", MapType(StringType(), IntegerType()), True)])
df = self.spark.createDataFrame([(None,)], schema=schema)
with QuietTest(self.sc):
- with self.assertRaisesRegexp(Exception, 'Unsupported type'):
- df.toPandas()
+ with self.warnings_lock:
+ with self.assertRaisesRegexp(Exception, 'Unsupported type'):
+ df.toPandas()
# TODO: remove BinaryType check once minimum pyarrow version is 0.10.0
if LooseVersion(pa.__version__) < LooseVersion("0.10.0"):
@@ -4392,6 +4505,8 @@ def test_createDataFrame_fallback_enabled(self):
with QuietTest(self.sc):
with self.sql_conf({"spark.sql.execution.arrow.fallback.enabled": True}):
with warnings.catch_warnings(record=True) as warns:
+ # we want the warnings to appear even if this test is run from a subclass
+ warnings.simplefilter("always")
df = self.spark.createDataFrame(
pd.DataFrame([[{u'a': 1}]]), "a: map")
# Catch and check the last UserWarning.
@@ -4435,10 +4550,44 @@ def test_timestamp_dst(self):
self.assertPandasEqual(pdf, df_from_pandas.toPandas())
+@unittest.skipIf(
+ not _have_pandas or not _have_pyarrow,
+ _pandas_requirement_message or _pyarrow_requirement_message)
+class MaxResultArrowTests(unittest.TestCase):
+ # These tests are separate as 'spark.driver.maxResultSize' configuration
+ # is a static configuration to Spark context.
+
+ @classmethod
+ def setUpClass(cls):
+ cls.spark = SparkSession(SparkContext(
+ 'local[4]', cls.__name__, conf=SparkConf().set("spark.driver.maxResultSize", "10k")))
+
+ # Explicitly enable Arrow and disable fallback.
+ cls.spark.conf.set("spark.sql.execution.arrow.enabled", "true")
+ cls.spark.conf.set("spark.sql.execution.arrow.fallback.enabled", "false")
+
+ @classmethod
+ def tearDownClass(cls):
+ if hasattr(cls, "spark"):
+ cls.spark.stop()
+
+ def test_exception_by_max_results(self):
+ with self.assertRaisesRegexp(Exception, "is bigger than"):
+ self.spark.range(0, 10000, 1, 100).toPandas()
+
+
+class EncryptionArrowTests(ArrowTests):
+
+ @classmethod
+ def conf(cls):
+ return super(EncryptionArrowTests, cls).conf().set("spark.io.encryption.enabled", "true")
+
+
@unittest.skipIf(
not _have_pandas or not _have_pyarrow,
_pandas_requirement_message or _pyarrow_requirement_message)
class PandasUDFTests(ReusedSQLTestCase):
+
def test_pandas_udf_basic(self):
from pyspark.rdd import PythonEvalType
from pyspark.sql.functions import pandas_udf, PandasUDFType
@@ -4654,6 +4803,24 @@ def random_udf(v):
random_udf = random_udf.asNondeterministic()
return random_udf
+ def test_pandas_udf_tokenize(self):
+ from pyspark.sql.functions import pandas_udf
+ tokenize = pandas_udf(lambda s: s.apply(lambda str: str.split(' ')),
+ ArrayType(StringType()))
+ self.assertEqual(tokenize.returnType, ArrayType(StringType()))
+ df = self.spark.createDataFrame([("hi boo",), ("bye boo",)], ["vals"])
+ result = df.select(tokenize("vals").alias("hi"))
+ self.assertEqual([Row(hi=[u'hi', u'boo']), Row(hi=[u'bye', u'boo'])], result.collect())
+
+ def test_pandas_udf_nested_arrays(self):
+ from pyspark.sql.functions import pandas_udf
+ tokenize = pandas_udf(lambda s: s.apply(lambda str: [str.split(' ')]),
+ ArrayType(ArrayType(StringType())))
+ self.assertEqual(tokenize.returnType, ArrayType(ArrayType(StringType())))
+ df = self.spark.createDataFrame([("hi boo",), ("bye boo",)], ["vals"])
+ result = df.select(tokenize("vals").alias("hi"))
+ self.assertEqual([Row(hi=[[u'hi', u'boo']]), Row(hi=[[u'bye', u'boo']])], result.collect())
+
def test_vectorized_udf_basic(self):
from pyspark.sql.functions import pandas_udf, col, array
df = self.spark.range(10).select(
@@ -5377,32 +5544,81 @@ def data(self):
.withColumn("v", explode(col('vs'))).drop('vs')
def test_supported_types(self):
- from pyspark.sql.functions import pandas_udf, PandasUDFType, array, col
- df = self.data.withColumn("arr", array(col("id")))
+ from decimal import Decimal
+ from distutils.version import LooseVersion
+ import pyarrow as pa
+ from pyspark.sql.functions import pandas_udf, PandasUDFType
- # Different forms of group map pandas UDF, results of these are the same
+ values = [
+ 1, 2, 3,
+ 4, 5, 1.1,
+ 2.2, Decimal(1.123),
+ [1, 2, 2], True, 'hello'
+ ]
+ output_fields = [
+ ('id', IntegerType()), ('byte', ByteType()), ('short', ShortType()),
+ ('int', IntegerType()), ('long', LongType()), ('float', FloatType()),
+ ('double', DoubleType()), ('decim', DecimalType(10, 3)),
+ ('array', ArrayType(IntegerType())), ('bool', BooleanType()), ('str', StringType())
+ ]
- output_schema = StructType(
- [StructField('id', LongType()),
- StructField('v', IntegerType()),
- StructField('arr', ArrayType(LongType())),
- StructField('v1', DoubleType()),
- StructField('v2', LongType())])
+ # TODO: Add BinaryType to variables above once minimum pyarrow version is 0.10.0
+ if LooseVersion(pa.__version__) >= LooseVersion("0.10.0"):
+ values.append(bytearray([0x01, 0x02]))
+ output_fields.append(('bin', BinaryType()))
+
+ output_schema = StructType([StructField(*x) for x in output_fields])
+ df = self.spark.createDataFrame([values], schema=output_schema)
+ # Different forms of group map pandas UDF, results of these are the same
udf1 = pandas_udf(
- lambda pdf: pdf.assign(v1=pdf.v * pdf.id * 1.0, v2=pdf.v + pdf.id),
+ lambda pdf: pdf.assign(
+ byte=pdf.byte * 2,
+ short=pdf.short * 2,
+ int=pdf.int * 2,
+ long=pdf.long * 2,
+ float=pdf.float * 2,
+ double=pdf.double * 2,
+ decim=pdf.decim * 2,
+ bool=False if pdf.bool else True,
+ str=pdf.str + 'there',
+ array=pdf.array,
+ ),
output_schema,
PandasUDFType.GROUPED_MAP
)
udf2 = pandas_udf(
- lambda _, pdf: pdf.assign(v1=pdf.v * pdf.id * 1.0, v2=pdf.v + pdf.id),
+ lambda _, pdf: pdf.assign(
+ byte=pdf.byte * 2,
+ short=pdf.short * 2,
+ int=pdf.int * 2,
+ long=pdf.long * 2,
+ float=pdf.float * 2,
+ double=pdf.double * 2,
+ decim=pdf.decim * 2,
+ bool=False if pdf.bool else True,
+ str=pdf.str + 'there',
+ array=pdf.array,
+ ),
output_schema,
PandasUDFType.GROUPED_MAP
)
udf3 = pandas_udf(
- lambda key, pdf: pdf.assign(id=key[0], v1=pdf.v * pdf.id * 1.0, v2=pdf.v + pdf.id),
+ lambda key, pdf: pdf.assign(
+ id=key[0],
+ byte=pdf.byte * 2,
+ short=pdf.short * 2,
+ int=pdf.int * 2,
+ long=pdf.long * 2,
+ float=pdf.float * 2,
+ double=pdf.double * 2,
+ decim=pdf.decim * 2,
+ bool=False if pdf.bool else True,
+ str=pdf.str + 'there',
+ array=pdf.array,
+ ),
output_schema,
PandasUDFType.GROUPED_MAP
)
@@ -5445,8 +5661,9 @@ def test_register_grouped_map_udf(self):
foo_udf = pandas_udf(lambda x: x, "id long", PandasUDFType.GROUPED_MAP)
with QuietTest(self.sc):
- with self.assertRaisesRegexp(ValueError, 'f must be either SQL_BATCHED_UDF or '
- 'SQL_SCALAR_PANDAS_UDF'):
+ with self.assertRaisesRegexp(
+ ValueError,
+ 'f.*SQL_BATCHED_UDF.*SQL_SCALAR_PANDAS_UDF.*SQL_GROUPED_AGG_PANDAS_UDF.*'):
self.spark.catalog.registerFunction("foo_udf", foo_udf)
def test_decorator(self):
@@ -5566,24 +5783,26 @@ def test_wrong_args(self):
pandas_udf(lambda x, y: x, DoubleType(), PandasUDFType.SCALAR))
def test_unsupported_types(self):
+ from distutils.version import LooseVersion
+ import pyarrow as pa
from pyspark.sql.functions import pandas_udf, PandasUDFType
- schema = StructType(
- [StructField("id", LongType(), True),
- StructField("map", MapType(StringType(), IntegerType()), True)])
- with QuietTest(self.sc):
- with self.assertRaisesRegexp(
- NotImplementedError,
- 'Invalid returnType.*grouped map Pandas UDF.*MapType'):
- pandas_udf(lambda x: x, schema, PandasUDFType.GROUPED_MAP)
- schema = StructType(
- [StructField("id", LongType(), True),
- StructField("arr_ts", ArrayType(TimestampType()), True)])
- with QuietTest(self.sc):
- with self.assertRaisesRegexp(
- NotImplementedError,
- 'Invalid returnType.*grouped map Pandas UDF.*ArrayType.*TimestampType'):
- pandas_udf(lambda x: x, schema, PandasUDFType.GROUPED_MAP)
+ common_err_msg = 'Invalid returnType.*grouped map Pandas UDF.*'
+ unsupported_types = [
+ StructField('map', MapType(StringType(), IntegerType())),
+ StructField('arr_ts', ArrayType(TimestampType())),
+ StructField('null', NullType()),
+ ]
+
+ # TODO: Remove this if-statement once minimum pyarrow version is 0.10.0
+ if LooseVersion(pa.__version__) < LooseVersion("0.10.0"):
+ unsupported_types.append(StructField('bin', BinaryType()))
+
+ for unsupported_type in unsupported_types:
+ schema = StructType([StructField('id', LongType(), True), unsupported_type])
+ with QuietTest(self.sc):
+ with self.assertRaisesRegexp(NotImplementedError, common_err_msg):
+ pandas_udf(lambda x: x, schema, PandasUDFType.GROUPED_MAP)
# Regression test for SPARK-23314
def test_timestamp_dst(self):
@@ -5755,14 +5974,22 @@ def invalid_positional_types(pdf):
with QuietTest(self.sc):
with self.assertRaisesRegexp(Exception, "KeyError: 'id'"):
grouped_df.apply(column_name_typo).collect()
- with self.assertRaisesRegexp(Exception, "No cast implemented"):
- grouped_df.apply(invalid_positional_types).collect()
+ from distutils.version import LooseVersion
+ import pyarrow as pa
+ if LooseVersion(pa.__version__) < LooseVersion("0.11.0"):
+ # TODO: see ARROW-1949. Remove when the minimum PyArrow version becomes 0.11.0.
+ with self.assertRaisesRegexp(Exception, "No cast implemented"):
+ grouped_df.apply(invalid_positional_types).collect()
+ else:
+ with self.assertRaisesRegexp(Exception, "an integer is required"):
+ grouped_df.apply(invalid_positional_types).collect()
def test_positional_assignment_conf(self):
import pandas as pd
from pyspark.sql.functions import pandas_udf, PandasUDFType
- with self.sql_conf({"spark.sql.execution.pandas.groupedMap.assignColumnsByPosition": True}):
+ with self.sql_conf({
+ "spark.sql.legacy.execution.pandas.groupedMap.assignColumnsByName": False}):
@pandas_udf("a string, b float", PandasUDFType.GROUPED_MAP)
def foo(_):
@@ -6259,6 +6486,21 @@ def test_invalid_args(self):
'mixture.*aggregate function.*group aggregate pandas UDF'):
df.groupby(df.id).agg(mean_udf(df.v), mean(df.v)).collect()
+ def test_register_vectorized_udf_basic(self):
+ from pyspark.sql.functions import pandas_udf
+ from pyspark.rdd import PythonEvalType
+
+ sum_pandas_udf = pandas_udf(
+ lambda v: v.sum(), "integer", PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF)
+
+ self.assertEqual(sum_pandas_udf.evalType, PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF)
+ group_agg_pandas_udf = self.spark.udf.register("sum_pandas_udf", sum_pandas_udf)
+ self.assertEqual(group_agg_pandas_udf.evalType, PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF)
+ q = "SELECT sum_pandas_udf(v1) FROM VALUES (3, 0), (2, 0), (1, 1) tbl(v1, v2) GROUP BY v2"
+ actual = sorted(map(lambda r: r[0], self.spark.sql(q).collect()))
+ expected = [1, 5]
+ self.assertEqual(actual, expected)
+
@unittest.skipIf(
not _have_pandas or not _have_pyarrow,
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 0b61707c8cc0a..1d24c40e5858e 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -752,7 +752,7 @@ def __eq__(self, other):
for v in [ArrayType, MapType, StructType])
-_FIXED_DECIMAL = re.compile("decimal\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)")
+_FIXED_DECIMAL = re.compile(r"decimal\(\s*(\d+)\s*,\s*(\d+)\s*\)")
def _parse_datatype_string(s):
@@ -1500,6 +1500,9 @@ def __contains__(self, item):
# let object acts like class
def __call__(self, *args):
"""create new Row object"""
+ if len(args) > len(self):
+ raise ValueError("Can not create Row with fields %s, expected %d values "
+ "but got %s" % (self, len(self), args))
return _create_row(self, args)
def __getitem__(self, item):
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index 9dbe49b831cef..58f4e0dff5ee5 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -298,6 +298,15 @@ def register(self, name, f, returnType=None):
>>> spark.sql("SELECT add_one(id) FROM range(3)").collect() # doctest: +SKIP
[Row(add_one(id)=1), Row(add_one(id)=2), Row(add_one(id)=3)]
+ >>> @pandas_udf("integer", PandasUDFType.GROUPED_AGG) # doctest: +SKIP
+ ... def sum_udf(v):
+ ... return v.sum()
+ ...
+ >>> _ = spark.udf.register("sum_udf", sum_udf) # doctest: +SKIP
+ >>> q = "SELECT sum_udf(v1) FROM VALUES (3, 0), (2, 0), (1, 1) tbl(v1, v2) GROUP BY v2"
+ >>> spark.sql(q).collect() # doctest: +SKIP
+ [Row(sum_udf(v1)=1), Row(sum_udf(v1)=5)]
+
.. note:: Registration for a user-defined function (case 2.) was added from
Spark 2.3.0.
"""
@@ -310,9 +319,11 @@ def register(self, name, f, returnType=None):
"Invalid returnType: data type can not be specified when f is"
"a user-defined function, but got %s." % returnType)
if f.evalType not in [PythonEvalType.SQL_BATCHED_UDF,
- PythonEvalType.SQL_SCALAR_PANDAS_UDF]:
+ PythonEvalType.SQL_SCALAR_PANDAS_UDF,
+ PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF]:
raise ValueError(
- "Invalid f: f must be either SQL_BATCHED_UDF or SQL_SCALAR_PANDAS_UDF")
+ "Invalid f: f must be SQL_BATCHED_UDF, SQL_SCALAR_PANDAS_UDF or "
+ "SQL_GROUPED_AGG_PANDAS_UDF")
register_udf = UserDefinedFunction(f.func, returnType=f.returnType, name=name,
evalType=f.evalType,
deterministic=f.deterministic)
diff --git a/python/pyspark/sql/window.py b/python/pyspark/sql/window.py
index d19ced954f04e..e76563dfaa9c8 100644
--- a/python/pyspark/sql/window.py
+++ b/python/pyspark/sql/window.py
@@ -16,11 +16,9 @@
#
import sys
-if sys.version >= '3':
- long = int
from pyspark import since, SparkContext
-from pyspark.sql.column import Column, _to_seq, _to_java_column
+from pyspark.sql.column import _to_seq, _to_java_column
__all__ = ["Window", "WindowSpec"]
@@ -126,45 +124,20 @@ def rangeBetween(start, end):
and "5" means the five off after the current row.
We recommend users use ``Window.unboundedPreceding``, ``Window.unboundedFollowing``,
- ``Window.currentRow``, ``pyspark.sql.functions.unboundedPreceding``,
- ``pyspark.sql.functions.unboundedFollowing`` and ``pyspark.sql.functions.currentRow``
- to specify special boundary values, rather than using integral values directly.
+ and ``Window.currentRow`` to specify special boundary values, rather than using integral
+ values directly.
:param start: boundary start, inclusive.
- The frame is unbounded if this is ``Window.unboundedPreceding``,
- a column returned by ``pyspark.sql.functions.unboundedPreceding``, or
+ The frame is unbounded if this is ``Window.unboundedPreceding``, or
any value less than or equal to max(-sys.maxsize, -9223372036854775808).
:param end: boundary end, inclusive.
- The frame is unbounded if this is ``Window.unboundedFollowing``,
- a column returned by ``pyspark.sql.functions.unboundedFollowing``, or
+ The frame is unbounded if this is ``Window.unboundedFollowing``, or
any value greater than or equal to min(sys.maxsize, 9223372036854775807).
-
- >>> from pyspark.sql import functions as F, SparkSession, Window
- >>> spark = SparkSession.builder.getOrCreate()
- >>> df = spark.createDataFrame(
- ... [(1, "a"), (1, "a"), (2, "a"), (1, "b"), (2, "b"), (3, "b")], ["id", "category"])
- >>> window = Window.orderBy("id").partitionBy("category").rangeBetween(
- ... F.currentRow(), F.lit(1))
- >>> df.withColumn("sum", F.sum("id").over(window)).show()
- +---+--------+---+
- | id|category|sum|
- +---+--------+---+
- | 1| b| 3|
- | 2| b| 5|
- | 3| b| 3|
- | 1| a| 4|
- | 1| a| 4|
- | 2| a| 2|
- +---+--------+---+
"""
- if isinstance(start, (int, long)) and isinstance(end, (int, long)):
- if start <= Window._PRECEDING_THRESHOLD:
- start = Window.unboundedPreceding
- if end >= Window._FOLLOWING_THRESHOLD:
- end = Window.unboundedFollowing
- elif isinstance(start, Column) and isinstance(end, Column):
- start = start._jc
- end = end._jc
+ if start <= Window._PRECEDING_THRESHOLD:
+ start = Window.unboundedPreceding
+ if end >= Window._FOLLOWING_THRESHOLD:
+ end = Window.unboundedFollowing
sc = SparkContext._active_spark_context
jspec = sc._jvm.org.apache.spark.sql.expressions.Window.rangeBetween(start, end)
return WindowSpec(jspec)
@@ -239,34 +212,27 @@ def rangeBetween(self, start, end):
and "5" means the five off after the current row.
We recommend users use ``Window.unboundedPreceding``, ``Window.unboundedFollowing``,
- ``Window.currentRow``, ``pyspark.sql.functions.unboundedPreceding``,
- ``pyspark.sql.functions.unboundedFollowing`` and ``pyspark.sql.functions.currentRow``
- to specify special boundary values, rather than using integral values directly.
+ and ``Window.currentRow`` to specify special boundary values, rather than using integral
+ values directly.
:param start: boundary start, inclusive.
- The frame is unbounded if this is ``Window.unboundedPreceding``,
- a column returned by ``pyspark.sql.functions.unboundedPreceding``, or
+ The frame is unbounded if this is ``Window.unboundedPreceding``, or
any value less than or equal to max(-sys.maxsize, -9223372036854775808).
:param end: boundary end, inclusive.
- The frame is unbounded if this is ``Window.unboundedFollowing``,
- a column returned by ``pyspark.sql.functions.unboundedFollowing``, or
+ The frame is unbounded if this is ``Window.unboundedFollowing``, or
any value greater than or equal to min(sys.maxsize, 9223372036854775807).
"""
- if isinstance(start, (int, long)) and isinstance(end, (int, long)):
- if start <= Window._PRECEDING_THRESHOLD:
- start = Window.unboundedPreceding
- if end >= Window._FOLLOWING_THRESHOLD:
- end = Window.unboundedFollowing
- elif isinstance(start, Column) and isinstance(end, Column):
- start = start._jc
- end = end._jc
+ if start <= Window._PRECEDING_THRESHOLD:
+ start = Window.unboundedPreceding
+ if end >= Window._FOLLOWING_THRESHOLD:
+ end = Window.unboundedFollowing
return WindowSpec(self._jspec.rangeBetween(start, end))
def _test():
import doctest
SparkContext('local[4]', 'PythonTest')
- (failure_count, test_count) = doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
+ (failure_count, test_count) = doctest.testmod()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/storagelevel.py b/python/pyspark/storagelevel.py
index ef012d27cb22f..7f29646c07432 100644
--- a/python/pyspark/storagelevel.py
+++ b/python/pyspark/storagelevel.py
@@ -58,8 +58,8 @@ def __str__(self):
StorageLevel.OFF_HEAP = StorageLevel(True, True, True, False, 1)
"""
-.. note:: The following four storage level constants are deprecated in 2.0, since the records \
-will always be serialized in Python.
+.. note:: The following four storage level constants are deprecated in 2.0, since the records
+ will always be serialized in Python.
"""
StorageLevel.MEMORY_ONLY_SER = StorageLevel.MEMORY_ONLY
""".. note:: Deprecated in 2.0, use ``StorageLevel.MEMORY_ONLY`` instead."""
diff --git a/python/pyspark/streaming/tests.py b/python/pyspark/streaming/tests.py
index 5cef621a28e6e..2f7fa83b575cd 100644
--- a/python/pyspark/streaming/tests.py
+++ b/python/pyspark/streaming/tests.py
@@ -1519,11 +1519,7 @@ def search_kafka_assembly_jar():
kafka_assembly_dir = os.path.join(SPARK_HOME, "external/kafka-0-8-assembly")
jars = search_jar(kafka_assembly_dir, "spark-streaming-kafka-0-8-assembly")
if not jars:
- raise Exception(
- ("Failed to find Spark Streaming kafka assembly jar in %s. " % kafka_assembly_dir) +
- "You need to build Spark with "
- "'build/sbt -Pkafka-0-8 assembly/package streaming-kafka-0-8-assembly/assembly' or "
- "'build/mvn -DskipTests -Pkafka-0-8 package' before running this test.")
+ return None
elif len(jars) > 1:
raise Exception(("Found multiple Spark Streaming Kafka assembly JARs: %s; please "
"remove all but one") % (", ".join(jars)))
@@ -1579,13 +1575,11 @@ def search_kinesis_asl_assembly_jar():
kafka_assembly_jar = search_kafka_assembly_jar()
flume_assembly_jar = search_flume_assembly_jar()
kinesis_asl_assembly_jar = search_kinesis_asl_assembly_jar()
+ jars_list = [j for j in (kafka_assembly_jar, flume_assembly_jar, kinesis_asl_assembly_jar) if j]
+ jars = ",".join(jars_list)
- if kinesis_asl_assembly_jar is None:
- kinesis_jar_present = False
- jars = "%s,%s" % (kafka_assembly_jar, flume_assembly_jar)
- else:
- kinesis_jar_present = True
- jars = "%s,%s,%s" % (kafka_assembly_jar, flume_assembly_jar, kinesis_asl_assembly_jar)
+ kinesis_jar_present = kinesis_asl_assembly_jar in jars_list
+ kafka08_jar_present = kafka_assembly_jar in jars_list
existing_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "pyspark-shell")
jars_args = "--jars %s" % jars
@@ -1601,12 +1595,16 @@ def search_kinesis_asl_assembly_jar():
"Skipped test_flume_stream (enable by setting environment variable %s=1"
% flume_test_environ_var)
- if are_kafka_tests_enabled:
+ if kafka08_jar_present:
testcases.append(KafkaStreamTests)
else:
- sys.stderr.write(
- "Skipped test_kafka_stream (enable by setting environment variable %s=1"
- % kafka_test_environ_var)
+ sys.stderr.write("Skipping kafka-0-8 tests as the optional kafka-0-8 profile was "
+ "not compiled into a JAR. To run these tests, "
+ "you need to build Spark with 'build/sbt -Pkafka-0-8 assembly/package "
+ "streaming-kafka-0-8-assembly/assembly' or "
+ "'build/mvn -DskipTests -Pkafka-0-8 package' before running this test. "
+ "Note that kafka-0-8 is not compatible with Scala version 2.12 and "
+ "higher.")
if kinesis_jar_present is True:
testcases.append(KinesisStreamTests)
diff --git a/python/pyspark/taskcontext.py b/python/pyspark/taskcontext.py
index b61643eb0a16e..98b505c9046be 100644
--- a/python/pyspark/taskcontext.py
+++ b/python/pyspark/taskcontext.py
@@ -147,8 +147,8 @@ def __init__(self):
@classmethod
def _getOrCreate(cls):
"""Internal function to get or create global BarrierTaskContext."""
- if cls._taskContext is None:
- cls._taskContext = BarrierTaskContext()
+ if not isinstance(cls._taskContext, BarrierTaskContext):
+ cls._taskContext = object.__new__(cls)
return cls._taskContext
@classmethod
diff --git a/python/pyspark/test_broadcast.py b/python/pyspark/test_broadcast.py
new file mode 100644
index 0000000000000..4b6dbf7b5020a
--- /dev/null
+++ b/python/pyspark/test_broadcast.py
@@ -0,0 +1,141 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import random
+import tempfile
+import unittest
+
+try:
+ import xmlrunner
+except ImportError:
+ xmlrunner = None
+
+from pyspark.broadcast import Broadcast
+from pyspark.conf import SparkConf
+from pyspark.context import SparkContext
+from pyspark.java_gateway import launch_gateway
+from pyspark.serializers import ChunkedStream
+
+
+class BroadcastTest(unittest.TestCase):
+
+ def tearDown(self):
+ if getattr(self, "sc", None) is not None:
+ self.sc.stop()
+ self.sc = None
+
+ def _test_encryption_helper(self, vs):
+ """
+ Creates a broadcast variables for each value in vs, and runs a simple job to make sure the
+ value is the same when it's read in the executors. Also makes sure there are no task
+ failures.
+ """
+ bs = [self.sc.broadcast(value=v) for v in vs]
+ exec_values = self.sc.parallelize(range(2)).map(lambda x: [b.value for b in bs]).collect()
+ for ev in exec_values:
+ self.assertEqual(ev, vs)
+ # make sure there are no task failures
+ status = self.sc.statusTracker()
+ for jid in status.getJobIdsForGroup():
+ for sid in status.getJobInfo(jid).stageIds:
+ stage_info = status.getStageInfo(sid)
+ self.assertEqual(0, stage_info.numFailedTasks)
+
+ def _test_multiple_broadcasts(self, *extra_confs):
+ """
+ Test broadcast variables make it OK to the executors. Tests multiple broadcast variables,
+ and also multiple jobs.
+ """
+ conf = SparkConf()
+ for key, value in extra_confs:
+ conf.set(key, value)
+ conf.setMaster("local-cluster[2,1,1024]")
+ self.sc = SparkContext(conf=conf)
+ self._test_encryption_helper([5])
+ self._test_encryption_helper([5, 10, 20])
+
+ def test_broadcast_with_encryption(self):
+ self._test_multiple_broadcasts(("spark.io.encryption.enabled", "true"))
+
+ def test_broadcast_no_encryption(self):
+ self._test_multiple_broadcasts()
+
+ def _test_broadcast_on_driver(self, *extra_confs):
+ conf = SparkConf()
+ for key, value in extra_confs:
+ conf.set(key, value)
+ conf.setMaster("local-cluster[2,1,1024]")
+ self.sc = SparkContext(conf=conf)
+ bs = self.sc.broadcast(value=5)
+ self.assertEqual(5, bs.value)
+
+ def test_broadcast_value_driver_no_encryption(self):
+ self._test_broadcast_on_driver()
+
+ def test_broadcast_value_driver_encryption(self):
+ self._test_broadcast_on_driver(("spark.io.encryption.enabled", "true"))
+
+
+class BroadcastFrameProtocolTest(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ gateway = launch_gateway(SparkConf())
+ cls._jvm = gateway.jvm
+ cls.longMessage = True
+ random.seed(42)
+
+ def _test_chunked_stream(self, data, py_buf_size):
+ # write data using the chunked protocol from python.
+ chunked_file = tempfile.NamedTemporaryFile(delete=False)
+ dechunked_file = tempfile.NamedTemporaryFile(delete=False)
+ dechunked_file.close()
+ try:
+ out = ChunkedStream(chunked_file, py_buf_size)
+ out.write(data)
+ out.close()
+ # now try to read it in java
+ jin = self._jvm.java.io.FileInputStream(chunked_file.name)
+ jout = self._jvm.java.io.FileOutputStream(dechunked_file.name)
+ self._jvm.DechunkedInputStream.dechunkAndCopyToOutput(jin, jout)
+ # java should have decoded it back to the original data
+ self.assertEqual(len(data), os.stat(dechunked_file.name).st_size)
+ with open(dechunked_file.name, "rb") as f:
+ byte = f.read(1)
+ idx = 0
+ while byte:
+ self.assertEqual(data[idx], bytearray(byte)[0], msg="idx = " + str(idx))
+ byte = f.read(1)
+ idx += 1
+ finally:
+ os.unlink(chunked_file.name)
+ os.unlink(dechunked_file.name)
+
+ def test_chunked_stream(self):
+ def random_bytes(n):
+ return bytearray(random.getrandbits(8) for _ in range(n))
+ for data_length in [1, 10, 100, 10000]:
+ for buffer_length in [1, 2, 5, 8192]:
+ self._test_chunked_stream(random_bytes(data_length), buffer_length)
+
+if __name__ == '__main__':
+ from pyspark.test_broadcast import *
+ if xmlrunner:
+ unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ else:
+ unittest.main(verbosity=2)
diff --git a/python/pyspark/test_serializers.py b/python/pyspark/test_serializers.py
new file mode 100644
index 0000000000000..5b43729f9ebb1
--- /dev/null
+++ b/python/pyspark/test_serializers.py
@@ -0,0 +1,90 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import io
+import math
+import struct
+import sys
+import unittest
+
+try:
+ import xmlrunner
+except ImportError:
+ xmlrunner = None
+
+from pyspark import serializers
+
+
+def read_int(b):
+ return struct.unpack("!i", b)[0]
+
+
+def write_int(i):
+ return struct.pack("!i", i)
+
+
+class SerializersTest(unittest.TestCase):
+
+ def test_chunked_stream(self):
+ original_bytes = bytearray(range(100))
+ for data_length in [1, 10, 100]:
+ for buffer_length in [1, 2, 3, 5, 20, 99, 100, 101, 500]:
+ dest = ByteArrayOutput()
+ stream_out = serializers.ChunkedStream(dest, buffer_length)
+ stream_out.write(original_bytes[:data_length])
+ stream_out.close()
+ num_chunks = int(math.ceil(float(data_length) / buffer_length))
+ # length for each chunk, and a final -1 at the very end
+ exp_size = (num_chunks + 1) * 4 + data_length
+ self.assertEqual(len(dest.buffer), exp_size)
+ dest_pos = 0
+ data_pos = 0
+ for chunk_idx in range(num_chunks):
+ chunk_length = read_int(dest.buffer[dest_pos:(dest_pos + 4)])
+ if chunk_idx == num_chunks - 1:
+ exp_length = data_length % buffer_length
+ if exp_length == 0:
+ exp_length = buffer_length
+ else:
+ exp_length = buffer_length
+ self.assertEqual(chunk_length, exp_length)
+ dest_pos += 4
+ dest_chunk = dest.buffer[dest_pos:dest_pos + chunk_length]
+ orig_chunk = original_bytes[data_pos:data_pos + chunk_length]
+ self.assertEqual(dest_chunk, orig_chunk)
+ dest_pos += chunk_length
+ data_pos += chunk_length
+ # ends with a -1
+ self.assertEqual(dest.buffer[-4:], write_int(-1))
+
+
+class ByteArrayOutput(object):
+ def __init__(self):
+ self.buffer = bytearray()
+
+ def write(self, b):
+ self.buffer += b
+
+ def close(self):
+ pass
+
+if __name__ == '__main__':
+ from pyspark.test_serializers import *
+ if xmlrunner:
+ unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ else:
+ unittest.main(verbosity=2)
diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py
index 8ac1df52fc597..26c91262fde32 100644
--- a/python/pyspark/tests.py
+++ b/python/pyspark/tests.py
@@ -61,6 +61,7 @@
from pyspark import keyword_only
from pyspark.conf import SparkConf
from pyspark.context import SparkContext
+from pyspark.java_gateway import _launch_gateway
from pyspark.rdd import RDD
from pyspark.files import SparkFiles
from pyspark.serializers import read_int, BatchedSerializer, MarshalSerializer, PickleSerializer, \
@@ -372,9 +373,16 @@ def tearDown(self):
class ReusedPySparkTestCase(unittest.TestCase):
+ @classmethod
+ def conf(cls):
+ """
+ Override this in subclasses to supply a more specific conf
+ """
+ return SparkConf()
+
@classmethod
def setUpClass(cls):
- cls.sc = SparkContext('local[4]', cls.__name__)
+ cls.sc = SparkContext('local[4]', cls.__name__, conf=cls.conf())
@classmethod
def tearDownClass(cls):
@@ -607,6 +615,21 @@ def context_barrier(x):
times = rdd.barrier().mapPartitions(f).map(context_barrier).collect()
self.assertTrue(max(times) - min(times) < 1)
+ def test_barrier_with_python_worker_reuse(self):
+ """
+ Verify that BarrierTaskContext.barrier() with reused python worker.
+ """
+ self.sc._conf.set("spark.python.work.reuse", "true")
+ rdd = self.sc.parallelize(range(4), 4)
+ # start a normal job first to start all worker
+ result = rdd.map(lambda x: x ** 2).collect()
+ self.assertEqual([0, 1, 4, 9], result)
+ # make sure `spark.python.work.reuse=true`
+ self.assertEqual(self.sc._conf.get("spark.python.work.reuse"), "true")
+
+ # worker will be reused in this barrier job
+ self.test_barrier()
+
def test_barrier_infos(self):
"""
Verify that BarrierTaskContext.getTaskInfos() returns a list of all task infos in the
@@ -1916,6 +1939,9 @@ def do_termination_test(self, terminator):
# daemon should accept connections
self.assertTrue(self.connect(port))
+ # wait worker process spawned from daemon exit.
+ time.sleep(1)
+
# request shutdown
terminator(daemon)
time.sleep(1)
@@ -2359,6 +2385,37 @@ def test_startTime(self):
with SparkContext() as sc:
self.assertGreater(sc.startTime, 0)
+ def test_forbid_insecure_gateway(self):
+ # By default, we fail immediately if you try to create a SparkContext
+ # with an insecure gateway
+ gateway = _launch_gateway(insecure=True)
+ log4j = gateway.jvm.org.apache.log4j
+ old_level = log4j.LogManager.getRootLogger().getLevel()
+ try:
+ log4j.LogManager.getRootLogger().setLevel(log4j.Level.FATAL)
+ with self.assertRaises(Exception) as context:
+ SparkContext(gateway=gateway)
+ self.assertIn("insecure Py4j gateway", str(context.exception))
+ self.assertIn("PYSPARK_ALLOW_INSECURE_GATEWAY", str(context.exception))
+ self.assertIn("removed in Spark 3.0", str(context.exception))
+ finally:
+ log4j.LogManager.getRootLogger().setLevel(old_level)
+
+ def test_allow_insecure_gateway_with_conf(self):
+ with SparkContext._lock:
+ SparkContext._gateway = None
+ SparkContext._jvm = None
+ gateway = _launch_gateway(insecure=True)
+ try:
+ os.environ["PYSPARK_ALLOW_INSECURE_GATEWAY"] = "1"
+ with SparkContext(gateway=gateway) as sc:
+ a = sc.accumulator(1)
+ rdd = sc.parallelize([1, 2, 3])
+ rdd.foreach(lambda x: a.add(x))
+ self.assertEqual(7, a.value)
+ finally:
+ os.environ.pop("PYSPARK_ALLOW_INSECURE_GATEWAY", None)
+
class ConfTests(unittest.TestCase):
def test_memory_conf(self):
diff --git a/python/pyspark/util.py b/python/pyspark/util.py
index f015542c8799d..f906f49595438 100644
--- a/python/pyspark/util.py
+++ b/python/pyspark/util.py
@@ -80,7 +80,7 @@ def majorMinorVersion(sparkVersion):
(2, 3)
"""
- m = re.search('^(\d+)\.(\d+)(\..*)?$', sparkVersion)
+ m = re.search(r'^(\d+)\.(\d+)(\..*)?$', sparkVersion)
if m is not None:
return (int(m.group(1)), int(m.group(2)))
else:
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index b9c2c4ced71d5..1d0ef78e0ce39 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-__version__ = "2.4.0.dev0"
+__version__ = "2.4.5.dev0"
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index e934da4d2eb6e..953b468e96519 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -22,7 +22,12 @@
import os
import sys
import time
-import resource
+# 'resource' is a Unix specific module.
+has_resource_module = True
+try:
+ import resource
+except ImportError:
+ has_resource_module = False
import socket
import traceback
@@ -97,8 +102,9 @@ def verify_result_length(*a):
def wrap_grouped_map_pandas_udf(f, return_type, argspec, runner_conf):
- assign_cols_by_pos = runner_conf.get(
- "spark.sql.execution.pandas.groupedMap.assignColumnsByPosition", False)
+ assign_cols_by_name = runner_conf.get(
+ "spark.sql.legacy.execution.pandas.groupedMap.assignColumnsByName", "true")
+ assign_cols_by_name = assign_cols_by_name.lower() == "true"
def wrapped(key_series, value_series):
import pandas as pd
@@ -119,7 +125,7 @@ def wrapped(key_series, value_series):
"Expected: {} Actual: {}".format(len(return_type), len(result.columns)))
# Assign result columns by schema name if user labeled with strings, else use position
- if not assign_cols_by_pos and any(isinstance(name, basestring) for name in result.columns):
+ if assign_cols_by_name and any(isinstance(name, basestring) for name in result.columns):
return [(result[field.name], to_arrow_type(field.dataType)) for field in return_type]
else:
return [(result[result.columns[i]], to_arrow_type(field.dataType))
@@ -267,9 +273,9 @@ def main(infile, outfile):
# set up memory limits
memory_limit_mb = int(os.environ.get('PYSPARK_EXECUTOR_MEMORY_MB', "-1"))
- total_memory = resource.RLIMIT_AS
- try:
- if memory_limit_mb > 0:
+ if memory_limit_mb > 0 and has_resource_module:
+ total_memory = resource.RLIMIT_AS
+ try:
(soft_limit, hard_limit) = resource.getrlimit(total_memory)
msg = "Current mem limits: {0} of max {1}\n".format(soft_limit, hard_limit)
print(msg, file=sys.stderr)
@@ -282,9 +288,9 @@ def main(infile, outfile):
print(msg, file=sys.stderr)
resource.setrlimit(total_memory, (new_limit, new_limit))
- except (resource.error, OSError, ValueError) as e:
- # not all systems support resource limits, so warn instead of failing
- print("WARN: Failed to set memory limit: {0}\n".format(e), file=sys.stderr)
+ except (resource.error, OSError, ValueError) as e:
+ # not all systems support resource limits, so warn instead of failing
+ print("WARN: Failed to set memory limit: {0}\n".format(e), file=sys.stderr)
# initialize global state
taskContext = None
@@ -324,16 +330,34 @@ def main(infile, outfile):
importlib.invalidate_caches()
# fetch names and values of broadcast variables
+ needs_broadcast_decryption_server = read_bool(infile)
num_broadcast_variables = read_int(infile)
+ if needs_broadcast_decryption_server:
+ # read the decrypted data from a server in the jvm
+ port = read_int(infile)
+ auth_secret = utf8_deserializer.loads(infile)
+ (broadcast_sock_file, _) = local_connect_and_auth(port, auth_secret)
+
for _ in range(num_broadcast_variables):
bid = read_long(infile)
if bid >= 0:
- path = utf8_deserializer.loads(infile)
- _broadcastRegistry[bid] = Broadcast(path=path)
+ if needs_broadcast_decryption_server:
+ read_bid = read_long(broadcast_sock_file)
+ assert(read_bid == bid)
+ _broadcastRegistry[bid] = \
+ Broadcast(sock_file=broadcast_sock_file)
+ else:
+ path = utf8_deserializer.loads(infile)
+ _broadcastRegistry[bid] = Broadcast(path=path)
+
else:
bid = - bid - 1
_broadcastRegistry.pop(bid)
+ if needs_broadcast_decryption_server:
+ broadcast_sock_file.write(b'1')
+ broadcast_sock_file.close()
+
_accumulatorRegistry.clear()
eval_type = read_int(infile)
if eval_type == PythonEvalType.NON_UDF:
diff --git a/python/run-tests.py b/python/run-tests.py
index 4c90926cfa350..c34e48aad1211 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -59,9 +59,7 @@ def print_red(text):
LOGGER = logging.getLogger()
# Find out where the assembly jars are located.
-# Later, add back 2.12 to this list:
-# for scala in ["2.11", "2.12"]:
-for scala in ["2.11"]:
+for scala in ["2.11", "2.12"]:
build_dir = os.path.join(SPARK_HOME, "assembly", "target", "scala-" + scala)
if os.path.isdir(build_dir):
SPARK_DIST_CLASSPATH = os.path.join(build_dir, "jars", "*")
@@ -119,7 +117,7 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
log_file.writelines(per_test_output)
per_test_output.seek(0)
for line in per_test_output:
- decoded_line = line.decode()
+ decoded_line = line.decode("utf-8", "replace")
if not re.match('[0-9]+', decoded_line):
print(decoded_line, end='')
per_test_output.close()
@@ -136,9 +134,9 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
per_test_output.seek(0)
# Here expects skipped test output from unittest when verbosity level is
# 2 (or --verbose option is enabled).
- decoded_lines = map(lambda line: line.decode(), iter(per_test_output))
+ decoded_lines = map(lambda line: line.decode("utf-8", "replace"), iter(per_test_output))
skipped_tests = list(filter(
- lambda line: re.search('test_.* \(pyspark\..*\) ... skipped ', line),
+ lambda line: re.search(r'test_.* \(pyspark\..*\) ... skipped ', line),
decoded_lines))
skipped_counts = len(skipped_tests)
if skipped_counts > 0:
@@ -162,7 +160,7 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
def get_default_python_executables():
- python_execs = [x for x in ["python2.7", "python3.4", "pypy"] if which(x)]
+ python_execs = [x for x in ["python2.7", "python3.6", "pypy"] if which(x)]
if "python2.7" not in python_execs:
LOGGER.warning("Not testing against `python2.7` because it could not be found; falling"
" back to `python` instead")
diff --git a/repl/pom.xml b/repl/pom.xml
index e8464a688336b..f77a3dcc243f2 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../pom.xml
diff --git a/repl/src/main/scala/org/apache/spark/repl/ExecutorClassLoader.scala b/repl/src/main/scala/org/apache/spark/repl/ExecutorClassLoader.scala
index 88eb0ad1da3d7..a4a11f09ad2b3 100644
--- a/repl/src/main/scala/org/apache/spark/repl/ExecutorClassLoader.scala
+++ b/repl/src/main/scala/org/apache/spark/repl/ExecutorClassLoader.scala
@@ -33,8 +33,11 @@ import org.apache.spark.util.ParentClassLoader
/**
* A ClassLoader that reads classes from a Hadoop FileSystem or Spark RPC endpoint, used to load
* classes defined by the interpreter when the REPL is used. Allows the user to specify if user
- * class path should be first. This class loader delegates getting/finding resources to parent
- * loader, which makes sense until REPL never provide resource dynamically.
+ * class path should be first.
+ * This class loader delegates getting/finding resources to parent loader, which makes sense because
+ * the REPL never produce resources dynamically. One exception is when getting a Class file as
+ * resource stream, in which case we will try to fetch the Class file in the same way as loading
+ * the class, so that dynamically generated Classes from the REPL can be picked up.
*
* Note: [[ClassLoader]] will preferentially load class from parent. Only when parent is null or
* the load failed, that it will call the overridden `findClass` function. To avoid the potential
@@ -71,6 +74,30 @@ class ExecutorClassLoader(
parentLoader.getResources(name)
}
+ override def getResourceAsStream(name: String): InputStream = {
+ if (userClassPathFirst) {
+ val res = getClassResourceAsStreamLocally(name)
+ if (res != null) res else parentLoader.getResourceAsStream(name)
+ } else {
+ val res = parentLoader.getResourceAsStream(name)
+ if (res != null) res else getClassResourceAsStreamLocally(name)
+ }
+ }
+
+ private def getClassResourceAsStreamLocally(name: String): InputStream = {
+ // Class files can be dynamically generated from the REPL. Allow this class loader to
+ // load such files for purposes other than loading the class.
+ try {
+ if (name.endsWith(".class")) fetchFn(name) else null
+ } catch {
+ // The helper functions referenced by fetchFn throw CNFE to indicate failure to fetch
+ // the class. It matches what IOException was supposed to be used for, and
+ // ClassLoader.getResourceAsStream() catches IOException and returns null in that case.
+ // So we follow that model and handle CNFE here.
+ case _: ClassNotFoundException => null
+ }
+ }
+
override def findClass(name: String): Class[_] = {
if (userClassPathFirst) {
findClassLocally(name).getOrElse(parentLoader.loadClass(name))
diff --git a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
index e5e2094368fb0..1ff93a179b6e4 100644
--- a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
@@ -208,6 +208,17 @@ class ExecutorClassLoaderSuite
intercept[java.lang.ClassNotFoundException] {
classLoader.loadClass("ReplFakeClassDoesNotExist").newInstance()
}
+
+ // classLoader.getResourceAsStream() should also be able to fetch the Class file
+ val fakeClassInputStream = classLoader.getResourceAsStream("ReplFakeClass2.class")
+ try {
+ val magic = new Array[Byte](4)
+ fakeClassInputStream.read(magic)
+ // first 4 bytes should match the magic number of Class file
+ assert(magic === Array[Byte](0xCA.toByte, 0xFE.toByte, 0xBA.toByte, 0xBE.toByte))
+ } finally {
+ if (fakeClassInputStream != null) fakeClassInputStream.close()
+ }
}
}
diff --git a/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
index 4f3df729177fb..a46cb6b3f4013 100644
--- a/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
@@ -260,4 +260,36 @@ class ReplSuite extends SparkFunSuite {
assertContains("!!2!!", output2)
}
+ test("SPARK-26633: ExecutorClassLoader.getResourceAsStream find REPL classes") {
+ val output = runInterpreterInPasteMode("local-cluster[1,1,1024]",
+ """
+ |case class TestClass(value: Int)
+ |
+ |sc.parallelize(1 to 1).map { _ =>
+ | val clz = classOf[TestClass]
+ | val name = clz.getName.replace('.', '/') + ".class";
+ | val stream = clz.getClassLoader.getResourceAsStream(name)
+ | if (stream == null) {
+ | "failed: stream is null"
+ | } else {
+ | val magic = new Array[Byte](4)
+ | try {
+ | stream.read(magic)
+ | // the magic number of a Java Class file
+ | val expected = Array[Byte](0xCA.toByte, 0xFE.toByte, 0xBA.toByte, 0xBE.toByte)
+ | if (magic sameElements expected) {
+ | "successful"
+ | } else {
+ | "failed: unexpected contents from stream"
+ | }
+ | } finally {
+ | stream.close()
+ | }
+ | }
+ |}.collect()
+ """.stripMargin)
+ assertDoesNotContain("failed", output)
+ assertContains("successful", output)
+ }
+
}
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index 920f0f6ebf2c8..56e1afc89e7c3 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../../pom.xml
@@ -29,7 +29,7 @@
Spark Project Kubernetes
kubernetes
- 3.0.0
+ 4.6.1
@@ -83,12 +83,6 @@
-
- com.squareup.okhttp3
- okhttp
- 3.8.1
-
-
org.mockito
mockito-core
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index 1b582fe53624a..c7338a721595f 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -225,6 +225,13 @@ private[spark] object Config extends Logging {
"Ensure that major Python version is either Python2 or Python3")
.createWithDefault("2")
+ val APP_RESOURCE_TYPE =
+ ConfigBuilder("spark.kubernetes.resource.type")
+ .doc("This sets the resource type internally")
+ .internal()
+ .stringConf
+ .createOptional
+
val KUBERNETES_AUTH_SUBMISSION_CONF_PREFIX =
"spark.kubernetes.authenticate.submission"
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
index 3aa35d419073f..cae6e7d5ad518 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
@@ -24,6 +24,7 @@ import org.apache.spark.SparkConf
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
import org.apache.spark.deploy.k8s.submit._
+import org.apache.spark.deploy.k8s.submit.KubernetesClientApplication._
import org.apache.spark.internal.config.ConfigEntry
@@ -220,10 +221,20 @@ private[spark] object KubernetesConf {
val executorVolumes = KubernetesVolumeUtils.parseVolumesWithPrefix(
sparkConf, KUBERNETES_EXECUTOR_VOLUMES_PREFIX).map(_.get)
+ // If no prefix is defined then we are in pure client mode
+ // (not the one used by cluster mode inside the container)
+ val appResourceNamePrefix = {
+ if (sparkConf.getOption(KUBERNETES_EXECUTOR_POD_NAME_PREFIX.key).isEmpty) {
+ getResourceNamePrefix(getAppName(sparkConf))
+ } else {
+ sparkConf.get(KUBERNETES_EXECUTOR_POD_NAME_PREFIX)
+ }
+ }
+
KubernetesConf(
sparkConf.clone(),
KubernetesExecutorSpecificConf(executorId, driverPod),
- sparkConf.get(KUBERNETES_EXECUTOR_POD_NAME_PREFIX),
+ appResourceNamePrefix,
appId,
executorLabels,
executorAnnotations,
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
index c37f713c56de1..d89995ba5e4f4 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
@@ -24,7 +24,7 @@ import org.apache.spark.SparkException
import org.apache.spark.deploy.k8s._
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
-import org.apache.spark.internal.config.{EXECUTOR_CLASS_PATH, EXECUTOR_JAVA_OPTIONS, EXECUTOR_MEMORY, EXECUTOR_MEMORY_OVERHEAD}
+import org.apache.spark.internal.config.{EXECUTOR_CLASS_PATH, EXECUTOR_JAVA_OPTIONS, EXECUTOR_MEMORY, EXECUTOR_MEMORY_OVERHEAD, PYSPARK_EXECUTOR_MEMORY}
import org.apache.spark.rpc.RpcEndpointAddress
import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
import org.apache.spark.util.Utils
@@ -58,6 +58,16 @@ private[spark] class BasicExecutorFeatureStep(
(kubernetesConf.get(MEMORY_OVERHEAD_FACTOR) * executorMemoryMiB).toInt,
MEMORY_OVERHEAD_MIN_MIB))
private val executorMemoryWithOverhead = executorMemoryMiB + memoryOverheadMiB
+ private val executorMemoryTotal = kubernetesConf.sparkConf
+ .getOption(APP_RESOURCE_TYPE.key).map{ res =>
+ val additionalPySparkMemory = res match {
+ case "python" =>
+ kubernetesConf.sparkConf
+ .get(PYSPARK_EXECUTOR_MEMORY).map(_.toInt).getOrElse(0)
+ case _ => 0
+ }
+ executorMemoryWithOverhead + additionalPySparkMemory
+ }.getOrElse(executorMemoryWithOverhead)
private val executorCores = kubernetesConf.sparkConf.getInt("spark.executor.cores", 1)
private val executorCoresRequest =
@@ -76,7 +86,7 @@ private[spark] class BasicExecutorFeatureStep(
// executorId
val hostname = name.substring(Math.max(0, name.length - 63))
val executorMemoryQuantity = new QuantityBuilder(false)
- .withAmount(s"${executorMemoryWithOverhead}Mi")
+ .withAmount(s"${executorMemoryTotal}Mi")
.build()
val executorCpuQuantity = new QuantityBuilder(false)
.withAmount(executorCoresRequest)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
index bb0e2b3128efd..026b7eb774c15 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
@@ -57,7 +57,9 @@ private[spark] class MountVolumesFeatureStep(
val volumeBuilder = spec.volumeConf match {
case KubernetesHostPathVolumeConf(hostPath) =>
new VolumeBuilder()
- .withHostPath(new HostPathVolumeSource(hostPath))
+ .withHostPath(new HostPathVolumeSourceBuilder()
+ .withPath(hostPath)
+ .build())
case KubernetesPVCVolumeConf(claimName) =>
new VolumeBuilder()
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/bindings/JavaDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/bindings/JavaDriverFeatureStep.scala
index f52ec9fdc677e..6f063b253cd73 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/bindings/JavaDriverFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/bindings/JavaDriverFeatureStep.scala
@@ -19,6 +19,7 @@ package org.apache.spark.deploy.k8s.features.bindings
import io.fabric8.kubernetes.api.model.{ContainerBuilder, HasMetadata}
import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverSpecificConf, SparkPod}
+import org.apache.spark.deploy.k8s.Config.APP_RESOURCE_TYPE
import org.apache.spark.deploy.k8s.Constants.SPARK_CONF_PATH
import org.apache.spark.deploy.k8s.features.KubernetesFeatureConfigStep
import org.apache.spark.launcher.SparkLauncher
@@ -38,7 +39,8 @@ private[spark] class JavaDriverFeatureStep(
.build()
SparkPod(pod.pod, withDriverArgs)
}
- override def getAdditionalPodSystemProperties(): Map[String, String] = Map.empty
+ override def getAdditionalPodSystemProperties(): Map[String, String] =
+ Map(APP_RESOURCE_TYPE.key -> "java")
override def getAdditionalKubernetesResources(): Seq[HasMetadata] = Seq.empty
}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/bindings/PythonDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/bindings/PythonDriverFeatureStep.scala
index 406944a953382..cf0c03b22bd7e 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/bindings/PythonDriverFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/bindings/PythonDriverFeatureStep.scala
@@ -21,6 +21,7 @@ import scala.collection.JavaConverters._
import io.fabric8.kubernetes.api.model.{ContainerBuilder, EnvVarBuilder, HasMetadata}
import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverSpecificConf, KubernetesUtils, SparkPod}
+import org.apache.spark.deploy.k8s.Config.APP_RESOURCE_TYPE
import org.apache.spark.deploy.k8s.Constants._
import org.apache.spark.deploy.k8s.features.KubernetesFeatureConfigStep
@@ -68,7 +69,8 @@ private[spark] class PythonDriverFeatureStep(
SparkPod(pod.pod, withPythonPrimaryContainer)
}
- override def getAdditionalPodSystemProperties(): Map[String, String] = Map.empty
+ override def getAdditionalPodSystemProperties(): Map[String, String] =
+ Map(APP_RESOURCE_TYPE.key -> "python")
override def getAdditionalKubernetesResources(): Seq[HasMetadata] = Seq.empty
}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/bindings/RDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/bindings/RDriverFeatureStep.scala
index 11b09b399618b..1a7ef52fefe70 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/bindings/RDriverFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/bindings/RDriverFeatureStep.scala
@@ -21,6 +21,7 @@ import scala.collection.JavaConverters._
import io.fabric8.kubernetes.api.model.{ContainerBuilder, EnvVarBuilder, HasMetadata}
import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverSpecificConf, KubernetesUtils, SparkPod}
+import org.apache.spark.deploy.k8s.Config.APP_RESOURCE_TYPE
import org.apache.spark.deploy.k8s.Constants._
import org.apache.spark.deploy.k8s.features.KubernetesFeatureConfigStep
@@ -54,7 +55,8 @@ private[spark] class RDriverFeatureStep(
SparkPod(pod.pod, withRPrimaryContainer)
}
- override def getAdditionalPodSystemProperties(): Map[String, String] = Map.empty
+ override def getAdditionalPodSystemProperties(): Map[String, String] =
+ Map(APP_RESOURCE_TYPE.key -> "r")
override def getAdditionalKubernetesResources(): Seq[HasMetadata] = Seq.empty
}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala
index 986c950ab365a..edeaa380194ac 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala
@@ -211,11 +211,8 @@ private[spark] class KubernetesClientApplication extends SparkApplication {
// considerably restrictive, e.g. must be no longer than 63 characters in length. So we generate
// a unique app ID (captured by spark.app.id) in the format below.
val kubernetesAppId = s"spark-${UUID.randomUUID().toString.replaceAll("-", "")}"
- val launchTime = System.currentTimeMillis()
val waitForAppCompletion = sparkConf.get(WAIT_FOR_APP_COMPLETION)
- val kubernetesResourceNamePrefix = {
- s"$appName-$launchTime".toLowerCase.replaceAll("\\.", "-")
- }
+ val kubernetesResourceNamePrefix = KubernetesClientApplication.getResourceNamePrefix(appName)
sparkConf.set(KUBERNETES_PYSPARK_PY_FILES, clientArguments.maybePyFiles.getOrElse(""))
val kubernetesConf = KubernetesConf.createDriverConf(
sparkConf,
@@ -254,3 +251,19 @@ private[spark] class KubernetesClientApplication extends SparkApplication {
}
}
}
+
+private[spark] object KubernetesClientApplication {
+
+ def getAppName(conf: SparkConf): String = conf.getOption("spark.app.name").getOrElse("spark")
+
+ def getResourceNamePrefix(appName: String): String = {
+ val launchTime = System.currentTimeMillis()
+ s"$appName-$launchTime"
+ .trim
+ .toLowerCase
+ .replaceAll("\\s+", "-")
+ .replaceAll("\\.", "-")
+ .replaceAll("[^a-z0-9\\-]", "")
+ .replaceAll("-+", "-")
+ }
+}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala
index 173ac541626a7..4a7d3d42d23db 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala
@@ -20,7 +20,7 @@ import java.util.concurrent.{CountDownLatch, TimeUnit}
import scala.collection.JavaConverters._
-import io.fabric8.kubernetes.api.model.{ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod, Time}
+import io.fabric8.kubernetes.api.model.{ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod}
import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher}
import io.fabric8.kubernetes.client.Watcher.Action
@@ -174,7 +174,7 @@ private[k8s] class LoggingPodStatusWatcherImpl(
}.getOrElse(Seq(("Container state", "N/A")))
}
- private def formatTime(time: Time): String = {
- if (time != null) time.getTime else "N/A"
+ private def formatTime(time: String): String = {
+ if (time != null || time != "") time else "N/A"
}
}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
index 77bb9c3fcc9f4..35d68e3ab8218 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
@@ -89,6 +89,8 @@ private[spark] class ExecutorPodsAllocator(
Utils.tryLogNonFatalError {
kubernetesClient
.pods()
+ .withLabel(SPARK_APP_ID_LABEL, applicationId)
+ .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)
.withLabel(SPARK_EXECUTOR_ID_LABEL, execId.toString)
.delete()
}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
index b28d93990313e..14814635732d4 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
@@ -99,8 +99,11 @@ private[spark] class ExecutorPodsLifecycleManager(
}
}
}
- logDebug(s"Removed executors with ids ${execIdsRemovedInThisRound.mkString(",")}" +
- s" from Spark that were either found to be deleted or non-existent in the cluster.")
+
+ if (execIdsRemovedInThisRound.nonEmpty) {
+ logDebug(s"Removed executors with ids ${execIdsRemovedInThisRound.mkString(",")}" +
+ s" from Spark that were either found to be deleted or non-existent in the cluster.")
+ }
}
private def onFinalNonDeletedState(
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
index fa6dc2c479bbf..bdd41343e8af0 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
@@ -18,9 +18,11 @@ package org.apache.spark.scheduler.cluster.k8s
import java.util.concurrent.ExecutorService
-import io.fabric8.kubernetes.client.KubernetesClient
import scala.concurrent.{ExecutionContext, Future}
+import io.fabric8.kubernetes.client.KubernetesClient
+
+import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
import org.apache.spark.rpc.{RpcAddress, RpcEnv}
import org.apache.spark.scheduler.{ExecutorLossReason, TaskSchedulerImpl}
@@ -39,8 +41,8 @@ private[spark] class KubernetesClusterSchedulerBackend(
pollEvents: ExecutorPodsPollingSnapshotSource)
extends CoarseGrainedSchedulerBackend(scheduler, rpcEnv) {
- private implicit val requestExecutorContext = ExecutionContext.fromExecutorService(
- requestExecutorsService)
+ private implicit val requestExecutorContext =
+ ExecutionContext.fromExecutorService(requestExecutorsService)
protected override val minRegisteredRatio =
if (conf.getOption("spark.scheduler.minRegisteredResourcesRatio").isEmpty) {
@@ -56,6 +58,17 @@ private[spark] class KubernetesClusterSchedulerBackend(
removeExecutor(executorId, reason)
}
+ /**
+ * Get an application ID associated with the job.
+ * This returns the string value of spark.app.id if set, otherwise
+ * the locally-generated ID from the superclass.
+ *
+ * @return The application ID
+ */
+ override def applicationId(): String = {
+ conf.getOption("spark.app.id").map(_.toString).getOrElse(super.applicationId)
+ }
+
override def start(): Unit = {
super.start()
if (!Utils.isDynamicAllocationEnabled(conf)) {
@@ -83,7 +96,8 @@ private[spark] class KubernetesClusterSchedulerBackend(
}
Utils.tryLogNonFatalError {
- kubernetesClient.pods()
+ kubernetesClient
+ .pods()
.withLabel(SPARK_APP_ID_LABEL, applicationId())
.withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)
.delete()
@@ -114,7 +128,8 @@ private[spark] class KubernetesClusterSchedulerBackend(
}
override def doKillExecutors(executorIds: Seq[String]): Future[Boolean] = Future[Boolean] {
- kubernetesClient.pods()
+ kubernetesClient
+ .pods()
.withLabel(SPARK_APP_ID_LABEL, applicationId())
.withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)
.withLabelIn(SPARK_EXECUTOR_ID_LABEL, executorIds: _*)
@@ -127,7 +142,7 @@ private[spark] class KubernetesClusterSchedulerBackend(
}
private class KubernetesDriverEndpoint(rpcEnv: RpcEnv, sparkProperties: Seq[(String, String)])
- extends DriverEndpoint(rpcEnv, sparkProperties) {
+ extends DriverEndpoint(rpcEnv, sparkProperties) {
override def onDisconnected(rpcAddress: RpcAddress): Unit = {
// Don't do anything besides disabling the executor - allow the Kubernetes API events to
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
index d98e113554648..0968cce971c31 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
@@ -57,7 +57,6 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
MAIN_CLASS,
APP_ARGS)
-
test("Check the pod respects all configurations from the user.") {
val sparkConf = new SparkConf()
.set(KUBERNETES_DRIVER_POD_NAME, "spark-driver-pod")
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
index 95d373f791649..63b237b9dfe46 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
@@ -75,6 +75,7 @@ class BasicExecutorFeatureStepSuite
.set("spark.driver.host", DRIVER_HOSTNAME)
.set("spark.driver.port", DRIVER_PORT.toString)
.set(IMAGE_PULL_SECRETS, TEST_IMAGE_PULL_SECRETS.mkString(","))
+ .set("spark.kubernetes.resource.type", "java")
}
test("basic executor pod has reasonable defaults") {
@@ -161,6 +162,29 @@ class BasicExecutorFeatureStepSuite
checkOwnerReferences(executor.pod, DRIVER_POD_UID)
}
+ test("test executor pyspark memory") {
+ val conf = baseConf.clone()
+ conf.set("spark.kubernetes.resource.type", "python")
+ conf.set(org.apache.spark.internal.config.PYSPARK_EXECUTOR_MEMORY, 42L)
+
+ val step = new BasicExecutorFeatureStep(
+ KubernetesConf(
+ conf,
+ KubernetesExecutorSpecificConf("1", Some(DRIVER_POD)),
+ RESOURCE_NAME_PREFIX,
+ APP_ID,
+ LABELS,
+ ANNOTATIONS,
+ Map.empty,
+ Map.empty,
+ Map.empty,
+ Nil,
+ Seq.empty[String]))
+ val executor = step.configurePod(SparkPod.initialPod())
+ // This is checking that basic executor + executorMemory = 1408 + 42 = 1450
+ assert(executor.container.getResources.getRequests.get("memory").getAmount === "1450Mi")
+ }
+
// There is always exactly one controller reference, and it points to the driver pod.
private def checkOwnerReferences(executor: Pod, driverPodUid: String): Unit = {
assert(executor.getMetadata.getOwnerReferences.size() === 1)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/bindings/JavaDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/bindings/JavaDriverFeatureStepSuite.scala
index 18874afe6e53a..bf552aeb8b901 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/bindings/JavaDriverFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/bindings/JavaDriverFeatureStepSuite.scala
@@ -56,6 +56,5 @@ class JavaDriverFeatureStepSuite extends SparkFunSuite {
"--properties-file", SPARK_CONF_PATH,
"--class", "test-class",
"spark-internal", "5 7"))
-
}
}
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala
index c6b667ed85e8c..2e883623a4b1c 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala
@@ -82,7 +82,7 @@ object ExecutorLifecycleTestUtils {
def deletedExecutor(executorId: Long): Pod = {
new PodBuilder(podWithAttachedContainerForId(executorId))
.editOrNewMetadata()
- .withNewDeletionTimestamp("523012521")
+ .withDeletionTimestamp("523012521")
.endMetadata()
.build()
}
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala
index e847f8590d353..ee964cb3cb28b 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala
@@ -138,7 +138,15 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
snapshotsStore.notifySubscribers()
snapshotsStore.replaceSnapshot(Seq.empty[Pod])
waitForExecutorPodsClock.setTime(podCreationTimeout + 1)
- when(podOperations.withLabel(SPARK_EXECUTOR_ID_LABEL, "1")).thenReturn(labeledPods)
+ when(podOperations
+ .withLabel(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID))
+ .thenReturn(podOperations)
+ when(podOperations
+ withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE))
+ .thenReturn(podOperations)
+ when(podOperations
+ .withLabel(SPARK_EXECUTOR_ID_LABEL, "1"))
+ .thenReturn(labeledPods)
snapshotsStore.notifySubscribers()
verify(labeledPods).delete()
verify(podOperations).create(podWithAttachedContainerForId(2))
@@ -167,13 +175,23 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
executorSpecificConf.executorId,
TEST_SPARK_APP_ID,
Some(driverPod))
- k8sConf.sparkConf.getAll.toMap == conf.getAll.toMap &&
+
+ // Set prefixes to a common string since KUBERNETES_EXECUTOR_POD_NAME_PREFIX
+ // has not be set for the tests and thus KubernetesConf will use a random
+ // string for the prefix, based on the app name, and this comparison here will fail.
+ val k8sConfCopy = k8sConf
+ .copy(appResourceNamePrefix = "")
+ .copy(sparkConf = conf)
+ val expectedK8sConfCopy = expectedK8sConf
+ .copy(appResourceNamePrefix = "")
+ .copy(sparkConf = conf)
+
+ k8sConf.sparkConf.getAll.toMap == conf.getAll.toMap &&
// Since KubernetesConf.createExecutorConf clones the SparkConf object, force
// deep equality comparison for the SparkConf object and use object equality
// comparison on all other fields.
- k8sConf.copy(sparkConf = conf) == expectedK8sConf.copy(sparkConf = conf)
+ k8sConfCopy == expectedK8sConfCopy
}
}
})
-
}
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala
index 52e7a12dbaf06..fbff1d7a24c82 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala
@@ -37,6 +37,7 @@ class KubernetesClusterSchedulerBackendSuite extends SparkFunSuite with BeforeAn
private val requestExecutorsService = new DeterministicScheduler()
private val sparkConf = new SparkConf(false)
.set("spark.executor.instances", "3")
+ .set("spark.app.id", TEST_SPARK_APP_ID)
@Mock
private var sc: SparkContext = _
@@ -82,8 +83,10 @@ class KubernetesClusterSchedulerBackendSuite extends SparkFunSuite with BeforeAn
when(taskScheduler.sc).thenReturn(sc)
when(sc.conf).thenReturn(sparkConf)
driverEndpoint = ArgumentCaptor.forClass(classOf[RpcEndpoint])
- when(rpcEnv.setupEndpoint(
- mockitoEq(CoarseGrainedSchedulerBackend.ENDPOINT_NAME), driverEndpoint.capture()))
+ when(
+ rpcEnv.setupEndpoint(
+ mockitoEq(CoarseGrainedSchedulerBackend.ENDPOINT_NAME),
+ driverEndpoint.capture()))
.thenReturn(driverEndpointRef)
when(kubernetesClient.pods()).thenReturn(podOperations)
schedulerBackendUnderTest = new KubernetesClusterSchedulerBackend(
@@ -95,9 +98,7 @@ class KubernetesClusterSchedulerBackendSuite extends SparkFunSuite with BeforeAn
podAllocator,
lifecycleEventHandler,
watchEvents,
- pollEvents) {
- override def applicationId(): String = TEST_SPARK_APP_ID
- }
+ pollEvents)
}
test("Start all components") {
@@ -122,8 +123,7 @@ class KubernetesClusterSchedulerBackendSuite extends SparkFunSuite with BeforeAn
test("Remove executor") {
schedulerBackendUnderTest.start()
- schedulerBackendUnderTest.doRemoveExecutor(
- "1", ExecutorKilled)
+ schedulerBackendUnderTest.doRemoveExecutor("1", ExecutorKilled)
verify(driverEndpointRef).send(RemoveExecutor("1", ExecutorKilled))
}
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
index 071aa2020dd85..385f5b95ae3f8 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
@@ -15,10 +15,11 @@
# limitations under the License.
#
-FROM openjdk:8-alpine
+FROM openjdk:8-jdk-slim
ARG spark_jars=jars
ARG img_path=kubernetes/dockerfiles
+ARG k8s_tests=kubernetes/tests
# Before building the docker image, first build and make a Spark distribution following
# the instructions in http://spark.apache.org/docs/latest/building-spark.html.
@@ -28,21 +29,24 @@ ARG img_path=kubernetes/dockerfiles
# docker build -t spark:latest -f kubernetes/dockerfiles/spark/Dockerfile .
RUN set -ex && \
- apk upgrade --no-cache && \
- apk add --no-cache bash tini libc6-compat linux-pam && \
+ apt-get update && \
+ ln -s /lib /lib64 && \
+ apt install -y bash tini libc6 libpam-modules libnss3 && \
mkdir -p /opt/spark && \
mkdir -p /opt/spark/work-dir && \
touch /opt/spark/RELEASE && \
rm /bin/sh && \
ln -sv /bin/bash /bin/sh && \
echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd
+ chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \
+ rm -rf /var/cache/apt/*
COPY ${spark_jars} /opt/spark/jars
COPY bin /opt/spark/bin
COPY sbin /opt/spark/sbin
COPY ${img_path}/spark/entrypoint.sh /opt/
COPY examples /opt/spark/examples
+COPY ${k8s_tests} /opt/spark/tests
COPY data /opt/spark/data
ENV SPARK_HOME /opt/spark
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
index e627883ba782e..4dd7729dfe3f8 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
@@ -19,10 +19,10 @@ ARG base_img
FROM $base_img
WORKDIR /
RUN mkdir ${SPARK_HOME}/R
-COPY R ${SPARK_HOME}/R
-RUN apk add --no-cache R R-dev
+RUN apt install -y r-base r-base-dev && rm -rf /var/cache/apt/*
+COPY R ${SPARK_HOME}/R
ENV R_HOME /usr/lib/R
WORKDIR /opt/spark/work-dir
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile
index 72bb9620b45de..c2644cf54e5d0 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile
@@ -19,20 +19,18 @@ ARG base_img
FROM $base_img
WORKDIR /
RUN mkdir ${SPARK_HOME}/python
-COPY python/lib ${SPARK_HOME}/python/lib
# TODO: Investigate running both pip and pip3 via virtualenvs
-RUN apk add --no-cache python && \
- apk add --no-cache python3 && \
- python -m ensurepip && \
- python3 -m ensurepip && \
+RUN apt install -y python python-pip && \
+ apt install -y python3 python3-pip && \
# We remove ensurepip since it adds no functionality since pip is
# installed on the image and it just takes up 1.6MB on the image
rm -r /usr/lib/python*/ensurepip && \
pip install --upgrade pip setuptools && \
# You may install with python3 packages by using pip3.6
# Removed the .cache to save space
- rm -r /root/.cache
+ rm -r /root/.cache && rm -rf /var/cache/apt/*
+COPY python/lib ${SPARK_HOME}/python/lib
ENV PYTHONPATH ${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-*.zip
WORKDIR /opt/spark/work-dir
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
index 216e8fe31becb..ba5d17b6b1c74 100755
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
@@ -45,7 +45,7 @@ case "$SPARK_K8S_CMD" in
;;
*)
echo "Non-spark-on-k8s command provided, proceeding in pass-through mode..."
- exec /sbin/tini -s -- "$@"
+ exec /usr/bin/tini -s -- "$@"
;;
esac
@@ -130,4 +130,4 @@ case "$SPARK_K8S_CMD" in
esac
# Execute the container CMD under tini for better hygiene
-exec /sbin/tini -s -- "${CMD[@]}"
+exec /usr/bin/tini -s -- "${CMD[@]}"
diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
index b28b8b82ca016..1b9de488a837e 100755
--- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
+++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
@@ -28,6 +28,12 @@ NAMESPACE=
SERVICE_ACCOUNT=
INCLUDE_TAGS="k8s"
EXCLUDE_TAGS=
+MVN="$TEST_ROOT_DIR/build/mvn"
+
+SCALA_VERSION=$("$MVN" help:evaluate -Dexpression=scala.binary.version 2>/dev/null\
+ | grep -v "INFO"\
+ | grep -v "WARNING"\
+ | tail -n 1)
# Parse arguments
while (( "$#" )); do
@@ -103,4 +109,4 @@ then
properties=( ${properties[@]} -Dtest.exclude.tags=$EXCLUDE_TAGS )
fi
-$TEST_ROOT_DIR/build/mvn integration-test -f $TEST_ROOT_DIR/pom.xml -pl resource-managers/kubernetes/integration-tests -am -Pkubernetes -Phadoop-2.7 ${properties[@]}
+$TEST_ROOT_DIR/build/mvn integration-test -f $TEST_ROOT_DIR/pom.xml -pl resource-managers/kubernetes/integration-tests -am -Pscala-$SCALA_VERSION -Pkubernetes -Pkubernetes-integration-tests -Phadoop-2.7 ${properties[@]}
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 614705c1ed668..f5bb1f8629124 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../../pom.xml
@@ -29,7 +29,7 @@
1.3.0
1.4.0
3.0.0
+ 4.6.1
3.2.2
1.0
kubernetes-integration-tests
@@ -126,7 +126,7 @@
${project.build.directory}/surefire-reports
.
SparkTestSuite.txt
- -ea -Xmx3g -XX:ReservedCodeCacheSize=512m ${extraScalaTestArgs}
+ -ea -Xmx3g -XX:ReservedCodeCacheSize=1g ${extraScalaTestArgs}
file:src/test/resources/log4j.properties
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
index 4e749c40563dc..3c1d9ea54ab2c 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
@@ -52,6 +52,9 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite =>
}
test("Run SparkPi with an argument.", k8sTestTag) {
+ // This additional configuration with snappy is for SPARK-26995
+ sparkAppConf
+ .set("spark.io.compression.codec", "snappy")
runSparkPiAndVerifyCompletion(appArgs = Array("5"))
}
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index 896a83a5badbb..e6840ce818c1f 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -23,7 +23,10 @@ import java.util.regex.Pattern
import com.google.common.io.PatternFilenameFilter
import io.fabric8.kubernetes.api.model.Pod
+import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher}
+import io.fabric8.kubernetes.client.Watcher.Action
import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, Tag}
+import org.scalatest.Matchers
import org.scalatest.concurrent.{Eventually, PatienceConfiguration}
import org.scalatest.time.{Minutes, Seconds, Span}
import scala.collection.JavaConverters._
@@ -31,10 +34,12 @@ import scala.collection.JavaConverters._
import org.apache.spark.SparkFunSuite
import org.apache.spark.deploy.k8s.integrationtest.TestConfig._
import org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBackend, IntegrationTestBackendFactory}
+import org.apache.spark.internal.Logging
private[spark] class KubernetesSuite extends SparkFunSuite
with BeforeAndAfterAll with BeforeAndAfter with BasicTestsSuite with SecretsTestsSuite
- with PythonTestsSuite with ClientModeTestsSuite {
+ with PythonTestsSuite with ClientModeTestsSuite
+ with Logging with Eventually with Matchers {
import KubernetesSuite._
@@ -50,6 +55,17 @@ private[spark] class KubernetesSuite extends SparkFunSuite
protected var containerLocalSparkDistroExamplesJar: String = _
protected var appLocator: String = _
+ // Default memory limit is 1024M + 384M (minimum overhead constant)
+ private val baseMemory = s"${1024 + 384}Mi"
+ protected val memOverheadConstant = 0.8
+ private val standardNonJVMMemory = s"${(1024 + 0.4*1024).toInt}Mi"
+ protected val additionalMemory = 200
+ // 209715200 is 200Mi
+ protected val additionalMemoryInBytes = 209715200
+ private val extraDriverTotalMemory = s"${(1024 + memOverheadConstant*1024).toInt}Mi"
+ private val extraExecTotalMemory =
+ s"${(1024 + memOverheadConstant*1024 + additionalMemory).toInt}Mi"
+
override def beforeAll(): Unit = {
// The scalatest-maven-plugin gives system properties that are referenced but not set null
// values. We need to remove the null-value properties before initializing the test backend.
@@ -207,17 +223,28 @@ private[spark] class KubernetesSuite extends SparkFunSuite
.getItems
.get(0)
driverPodChecker(driverPod)
-
- val executorPods = kubernetesTestComponents.kubernetesClient
+ val execPods = scala.collection.mutable.Map[String, Pod]()
+ val execWatcher = kubernetesTestComponents.kubernetesClient
.pods()
.withLabel("spark-app-locator", appLocator)
.withLabel("spark-role", "executor")
- .list()
- .getItems
- executorPods.asScala.foreach { pod =>
- executorPodChecker(pod)
- }
-
+ .watch(new Watcher[Pod] {
+ logInfo("Beginning watch of executors")
+ override def onClose(cause: KubernetesClientException): Unit =
+ logInfo("Ending watch of executors")
+ override def eventReceived(action: Watcher.Action, resource: Pod): Unit = {
+ val name = resource.getMetadata.getName
+ action match {
+ case Action.ADDED | Action.MODIFIED =>
+ execPods(name) = resource
+ case Action.DELETED | Action.ERROR =>
+ execPods.remove(name)
+ }
+ }
+ })
+ Eventually.eventually(TIMEOUT, INTERVAL) { execPods.values.nonEmpty should be (true) }
+ execWatcher.close()
+ execPods.values.foreach(executorPodChecker(_))
Eventually.eventually(TIMEOUT, INTERVAL) {
expectedLogOnCompletion.foreach { e =>
assert(kubernetesTestComponents.kubernetesClient
@@ -228,11 +255,12 @@ private[spark] class KubernetesSuite extends SparkFunSuite
}
}
}
-
protected def doBasicDriverPodCheck(driverPod: Pod): Unit = {
assert(driverPod.getMetadata.getName === driverPodName)
assert(driverPod.getSpec.getContainers.get(0).getImage === image)
assert(driverPod.getSpec.getContainers.get(0).getName === "spark-kubernetes-driver")
+ assert(driverPod.getSpec.getContainers.get(0).getResources.getRequests.get("memory").getAmount
+ === baseMemory)
}
@@ -240,28 +268,48 @@ private[spark] class KubernetesSuite extends SparkFunSuite
assert(driverPod.getMetadata.getName === driverPodName)
assert(driverPod.getSpec.getContainers.get(0).getImage === pyImage)
assert(driverPod.getSpec.getContainers.get(0).getName === "spark-kubernetes-driver")
+ assert(driverPod.getSpec.getContainers.get(0).getResources.getRequests.get("memory").getAmount
+ === standardNonJVMMemory)
}
protected def doBasicDriverRPodCheck(driverPod: Pod): Unit = {
assert(driverPod.getMetadata.getName === driverPodName)
assert(driverPod.getSpec.getContainers.get(0).getImage === rImage)
assert(driverPod.getSpec.getContainers.get(0).getName === "spark-kubernetes-driver")
+ assert(driverPod.getSpec.getContainers.get(0).getResources.getRequests.get("memory").getAmount
+ === standardNonJVMMemory)
}
protected def doBasicExecutorPodCheck(executorPod: Pod): Unit = {
assert(executorPod.getSpec.getContainers.get(0).getImage === image)
assert(executorPod.getSpec.getContainers.get(0).getName === "executor")
+ assert(executorPod.getSpec.getContainers.get(0).getResources.getRequests.get("memory").getAmount
+ === baseMemory)
}
protected def doBasicExecutorPyPodCheck(executorPod: Pod): Unit = {
assert(executorPod.getSpec.getContainers.get(0).getImage === pyImage)
assert(executorPod.getSpec.getContainers.get(0).getName === "executor")
+ assert(executorPod.getSpec.getContainers.get(0).getResources.getRequests.get("memory").getAmount
+ === standardNonJVMMemory)
}
protected def doBasicExecutorRPodCheck(executorPod: Pod): Unit = {
assert(executorPod.getSpec.getContainers.get(0).getImage === rImage)
assert(executorPod.getSpec.getContainers.get(0).getName === "executor")
+ assert(executorPod.getSpec.getContainers.get(0).getResources.getRequests.get("memory").getAmount
+ === standardNonJVMMemory)
+ }
+
+ protected def doDriverMemoryCheck(driverPod: Pod): Unit = {
+ assert(driverPod.getSpec.getContainers.get(0).getResources.getRequests.get("memory").getAmount
+ === extraDriverTotalMemory)
+ }
+
+ protected def doExecutorMemoryCheck(executorPod: Pod): Unit = {
+ assert(executorPod.getSpec.getContainers.get(0).getResources.getRequests.get("memory").getAmount
+ === extraExecTotalMemory)
}
protected def checkCustomSettings(pod: Pod): Unit = {
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
index b602fdf39731f..5615d6173eebd 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
@@ -62,7 +62,6 @@ private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesCl
new SparkAppConf()
.set("spark.master", s"k8s://${kubernetesClient.getMasterUrl}")
.set("spark.kubernetes.namespace", namespace)
- .set("spark.executor.memory", "500m")
.set("spark.executor.cores", "1")
.set("spark.executors.instances", "1")
.set("spark.app.name", "spark-test-app")
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala
index 1ebb30094dcde..06b73107ec236 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PythonTestsSuite.scala
@@ -23,9 +23,11 @@ private[spark] trait PythonTestsSuite { k8sSuite: KubernetesSuite =>
import PythonTestsSuite._
import KubernetesSuite.k8sTestTag
+ private val pySparkDockerImage =
+ s"${getTestImageRepo}/spark-py:${getTestImageTag}"
test("Run PySpark on simple pi.py example", k8sTestTag) {
sparkAppConf
- .set("spark.kubernetes.container.image", s"${getTestImageRepo}/spark-py:${getTestImageTag}")
+ .set("spark.kubernetes.container.image", pySparkDockerImage)
runSparkApplicationAndVerifyCompletion(
appResource = PYSPARK_PI,
mainClass = "",
@@ -39,7 +41,7 @@ private[spark] trait PythonTestsSuite { k8sSuite: KubernetesSuite =>
test("Run PySpark with Python2 to test a pyfiles example", k8sTestTag) {
sparkAppConf
- .set("spark.kubernetes.container.image", s"${getTestImageRepo}/spark-py:${getTestImageTag}")
+ .set("spark.kubernetes.container.image", pySparkDockerImage)
.set("spark.kubernetes.pyspark.pythonVersion", "2")
runSparkApplicationAndVerifyCompletion(
appResource = PYSPARK_FILES,
@@ -57,7 +59,7 @@ private[spark] trait PythonTestsSuite { k8sSuite: KubernetesSuite =>
test("Run PySpark with Python3 to test a pyfiles example", k8sTestTag) {
sparkAppConf
- .set("spark.kubernetes.container.image", s"${getTestImageRepo}/spark-py:${getTestImageTag}")
+ .set("spark.kubernetes.container.image", pySparkDockerImage)
.set("spark.kubernetes.pyspark.pythonVersion", "3")
runSparkApplicationAndVerifyCompletion(
appResource = PYSPARK_FILES,
@@ -72,12 +74,32 @@ private[spark] trait PythonTestsSuite { k8sSuite: KubernetesSuite =>
isJVM = false,
pyFiles = Some(PYSPARK_CONTAINER_TESTS))
}
+
+ test("Run PySpark with memory customization", k8sTestTag) {
+ sparkAppConf
+ .set("spark.kubernetes.container.image", pySparkDockerImage)
+ .set("spark.kubernetes.pyspark.pythonVersion", "3")
+ .set("spark.kubernetes.memoryOverheadFactor", s"$memOverheadConstant")
+ .set("spark.executor.pyspark.memory", s"${additionalMemory}m")
+ runSparkApplicationAndVerifyCompletion(
+ appResource = PYSPARK_MEMORY_CHECK,
+ mainClass = "",
+ expectedLogOnCompletion = Seq(
+ "PySpark Worker Memory Check is: True"),
+ appArgs = Array(s"$additionalMemoryInBytes"),
+ driverPodChecker = doDriverMemoryCheck,
+ executorPodChecker = doExecutorMemoryCheck,
+ appLocator = appLocator,
+ isJVM = false,
+ pyFiles = Some(PYSPARK_CONTAINER_TESTS))
+ }
}
private[spark] object PythonTestsSuite {
val CONTAINER_LOCAL_PYSPARK: String = "local:///opt/spark/examples/src/main/python/"
val PYSPARK_PI: String = CONTAINER_LOCAL_PYSPARK + "pi.py"
- val PYSPARK_FILES: String = CONTAINER_LOCAL_PYSPARK + "pyfiles.py"
- val PYSPARK_CONTAINER_TESTS: String = CONTAINER_LOCAL_PYSPARK + "py_container_checks.py"
+ val TEST_LOCAL_PYSPARK: String = "local:///opt/spark/tests/"
+ val PYSPARK_FILES: String = TEST_LOCAL_PYSPARK + "pyfiles.py"
+ val PYSPARK_CONTAINER_TESTS: String = TEST_LOCAL_PYSPARK + "py_container_checks.py"
+ val PYSPARK_MEMORY_CHECK: String = TEST_LOCAL_PYSPARK + "worker_memory_check.py"
}
-
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SecretsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SecretsTestsSuite.scala
index 7b05c1355ca24..b18a6aebda497 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SecretsTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SecretsTestsSuite.scala
@@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s.integrationtest
import scala.collection.JavaConverters._
-import io.fabric8.kubernetes.api.model.{Pod, Secret, SecretBuilder}
+import io.fabric8.kubernetes.api.model.{Pod, SecretBuilder}
import org.apache.commons.codec.binary.Base64
import org.apache.commons.io.output.ByteArrayOutputStream
import org.scalatest.concurrent.Eventually
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
index 6494cbc18f33e..78ef44be7fb78 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
@@ -16,7 +16,6 @@
*/
package org.apache.spark.deploy.k8s.integrationtest.backend.minikube
-import java.io.File
import java.nio.file.Paths
import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient}
@@ -26,8 +25,18 @@ import org.apache.spark.internal.Logging
// TODO support windows
private[spark] object Minikube extends Logging {
-
private val MINIKUBE_STARTUP_TIMEOUT_SECONDS = 60
+ private val HOST_PREFIX = "host:"
+ private val KUBELET_PREFIX = "kubelet:"
+ private val APISERVER_PREFIX = "apiserver:"
+ private val KUBECTL_PREFIX = "kubectl:"
+ private val MINIKUBE_VM_PREFIX = "minikubeVM: "
+ private val MINIKUBE_PREFIX = "minikube: "
+ private val MINIKUBE_PATH = ".minikube"
+
+ def logVersion(): Unit = {
+ logInfo(executeMinikube("version").mkString("\n"))
+ }
def getMinikubeIp: String = {
val outputs = executeMinikube("ip")
@@ -38,12 +47,21 @@ private[spark] object Minikube extends Logging {
def getMinikubeStatus: MinikubeStatus.Value = {
val statusString = executeMinikube("status")
- .filter(line => line.contains("minikubeVM: ") || line.contains("minikube:"))
- .head
- .replaceFirst("minikubeVM: ", "")
- .replaceFirst("minikube: ", "")
- MinikubeStatus.unapply(statusString)
+ logInfo(s"Minikube status command output:\n$statusString")
+ // up to minikube version v0.30.0 use this to check for minikube status
+ val oldMinikube = statusString
+ .filter(line => line.contains(MINIKUBE_VM_PREFIX) || line.contains(MINIKUBE_PREFIX))
+
+ if (oldMinikube.isEmpty) {
+ getIfNewMinikubeStatus(statusString)
+ } else {
+ val finalStatusString = oldMinikube
+ .head
+ .replaceFirst(MINIKUBE_VM_PREFIX, "")
+ .replaceFirst(MINIKUBE_PREFIX, "")
+ MinikubeStatus.unapply(finalStatusString)
.getOrElse(throw new IllegalStateException(s"Unknown status $statusString"))
+ }
}
def getKubernetesClient: DefaultKubernetesClient = {
@@ -52,13 +70,46 @@ private[spark] object Minikube extends Logging {
val kubernetesConf = new ConfigBuilder()
.withApiVersion("v1")
.withMasterUrl(kubernetesMaster)
- .withCaCertFile(Paths.get(userHome, ".minikube", "ca.crt").toFile.getAbsolutePath)
- .withClientCertFile(Paths.get(userHome, ".minikube", "apiserver.crt").toFile.getAbsolutePath)
- .withClientKeyFile(Paths.get(userHome, ".minikube", "apiserver.key").toFile.getAbsolutePath)
+ .withCaCertFile(
+ Paths.get(userHome, MINIKUBE_PATH, "ca.crt").toFile.getAbsolutePath)
+ .withClientCertFile(
+ Paths.get(userHome, MINIKUBE_PATH, "apiserver.crt").toFile.getAbsolutePath)
+ .withClientKeyFile(
+ Paths.get(userHome, MINIKUBE_PATH, "apiserver.key").toFile.getAbsolutePath)
.build()
new DefaultKubernetesClient(kubernetesConf)
}
+ // Covers minikube status output after Minikube V0.30.
+ private def getIfNewMinikubeStatus(statusString: Seq[String]): MinikubeStatus.Value = {
+ val hostString = statusString.find(_.contains(s"$HOST_PREFIX "))
+ val kubeletString = statusString.find(_.contains(s"$KUBELET_PREFIX "))
+ val apiserverString = statusString.find(_.contains(s"$APISERVER_PREFIX "))
+ val kubectlString = statusString.find(_.contains(s"$KUBECTL_PREFIX "))
+
+ if (hostString.isEmpty || kubeletString.isEmpty
+ || apiserverString.isEmpty || kubectlString.isEmpty) {
+ MinikubeStatus.NONE
+ } else {
+ val status1 = hostString.get.replaceFirst(s"$HOST_PREFIX ", "")
+ val status2 = kubeletString.get.replaceFirst(s"$KUBELET_PREFIX ", "")
+ val status3 = apiserverString.get.replaceFirst(s"$APISERVER_PREFIX ", "")
+ val status4 = kubectlString.get.replaceFirst(s"$KUBECTL_PREFIX ", "")
+ if (!status4.contains("Correctly Configured:")) {
+ MinikubeStatus.NONE
+ } else {
+ val stats = List(status1, status2, status3)
+ .map(MinikubeStatus.unapply)
+ .map(_.getOrElse(throw new IllegalStateException(s"Unknown status $statusString")))
+ if (stats.exists(_ != MinikubeStatus.RUNNING)) {
+ MinikubeStatus.NONE
+ } else {
+ MinikubeStatus.RUNNING
+ }
+ }
+ }
+ }
+
private def executeMinikube(action: String, args: String*): Seq[String] = {
ProcessUtils.executeProcess(
Array("bash", "-c", s"minikube $action") ++ args, MINIKUBE_STARTUP_TIMEOUT_SECONDS)
diff --git a/examples/src/main/python/py_container_checks.py b/resource-managers/kubernetes/integration-tests/tests/py_container_checks.py
similarity index 100%
rename from examples/src/main/python/py_container_checks.py
rename to resource-managers/kubernetes/integration-tests/tests/py_container_checks.py
diff --git a/examples/src/main/python/pyfiles.py b/resource-managers/kubernetes/integration-tests/tests/pyfiles.py
similarity index 100%
rename from examples/src/main/python/pyfiles.py
rename to resource-managers/kubernetes/integration-tests/tests/pyfiles.py
diff --git a/resource-managers/kubernetes/integration-tests/tests/worker_memory_check.py b/resource-managers/kubernetes/integration-tests/tests/worker_memory_check.py
new file mode 100644
index 0000000000000..d312a29f388e4
--- /dev/null
+++ b/resource-managers/kubernetes/integration-tests/tests/worker_memory_check.py
@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import print_function
+
+import resource
+import sys
+
+from pyspark.sql import SparkSession
+
+
+if __name__ == "__main__":
+ """
+ Usage: worker_memory_check [Memory_in_Mi]
+ """
+ spark = SparkSession \
+ .builder \
+ .appName("PyMemoryTest") \
+ .getOrCreate()
+ sc = spark.sparkContext
+ if len(sys.argv) < 2:
+ print("Usage: worker_memory_check [Memory_in_Mi]", file=sys.stderr)
+ sys.exit(-1)
+
+ def f(x):
+ rLimit = resource.getrlimit(resource.RLIMIT_AS)
+ print("RLimit is " + str(rLimit))
+ return rLimit
+ resourceValue = sc.parallelize([1]).map(f).collect()[0][0]
+ print("Resource Value is " + str(resourceValue))
+ truthCheck = (resourceValue == int(sys.argv[1]))
+ print("PySpark Worker Memory Check is: " + str(truthCheck))
+ spark.stop()
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index 3995d0afeb5f4..80ab9639d819e 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
index 7d80eedcc43ce..9dbe9fde31fe4 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
@@ -129,7 +129,6 @@ private[spark] class MesosClusterScheduler(
private val queuedCapacity = conf.getInt("spark.mesos.maxDrivers", 200)
private val retainedDrivers = conf.getInt("spark.mesos.retainedDrivers", 200)
private val maxRetryWaitTime = conf.getInt("spark.mesos.cluster.retry.wait.max", 60) // 1 minute
- private val useFetchCache = conf.getBoolean("spark.mesos.fetchCache.enable", false)
private val schedulerState = engineFactory.createEngine("scheduler")
private val stateLock = new Object()
// Keyed by submission id
@@ -202,7 +201,7 @@ private[spark] class MesosClusterScheduler(
} else if (removeFromPendingRetryDrivers(submissionId)) {
k.success = true
k.message = "Removed driver while it's being retried"
- } else if (finishedDrivers.exists(_.driverDescription.submissionId.equals(submissionId))) {
+ } else if (finishedDrivers.exists(_.driverDescription.submissionId == submissionId)) {
k.success = false
k.message = "Driver already terminated"
} else {
@@ -222,21 +221,21 @@ private[spark] class MesosClusterScheduler(
}
s.submissionId = submissionId
stateLock.synchronized {
- if (queuedDrivers.exists(_.submissionId.equals(submissionId))) {
+ if (queuedDrivers.exists(_.submissionId == submissionId)) {
s.success = true
s.driverState = "QUEUED"
} else if (launchedDrivers.contains(submissionId)) {
s.success = true
s.driverState = "RUNNING"
launchedDrivers(submissionId).mesosTaskStatus.foreach(state => s.message = state.toString)
- } else if (finishedDrivers.exists(_.driverDescription.submissionId.equals(submissionId))) {
+ } else if (finishedDrivers.exists(_.driverDescription.submissionId == submissionId)) {
s.success = true
s.driverState = "FINISHED"
finishedDrivers
.find(d => d.driverDescription.submissionId.equals(submissionId)).get.mesosTaskStatus
.foreach(state => s.message = state.toString)
- } else if (pendingRetryDrivers.exists(_.submissionId.equals(submissionId))) {
- val status = pendingRetryDrivers.find(_.submissionId.equals(submissionId))
+ } else if (pendingRetryDrivers.exists(_.submissionId == submissionId)) {
+ val status = pendingRetryDrivers.find(_.submissionId == submissionId)
.get.retryState.get.lastFailureStatus
s.success = true
s.driverState = "RETRYING"
@@ -254,13 +253,13 @@ private[spark] class MesosClusterScheduler(
*/
def getDriverState(submissionId: String): Option[MesosDriverState] = {
stateLock.synchronized {
- queuedDrivers.find(_.submissionId.equals(submissionId))
+ queuedDrivers.find(_.submissionId == submissionId)
.map(d => new MesosDriverState("QUEUED", d))
.orElse(launchedDrivers.get(submissionId)
.map(d => new MesosDriverState("RUNNING", d.driverDescription, Some(d))))
- .orElse(finishedDrivers.find(_.driverDescription.submissionId.equals(submissionId))
+ .orElse(finishedDrivers.find(_.driverDescription.submissionId == submissionId)
.map(d => new MesosDriverState("FINISHED", d.driverDescription, Some(d))))
- .orElse(pendingRetryDrivers.find(_.submissionId.equals(submissionId))
+ .orElse(pendingRetryDrivers.find(_.submissionId == submissionId)
.map(d => new MesosDriverState("RETRYING", d)))
}
}
@@ -430,6 +429,9 @@ private[spark] class MesosClusterScheduler(
}
private def getDriverUris(desc: MesosDriverDescription): List[CommandInfo.URI] = {
+ val useFetchCache = desc.conf.getBoolean("spark.mesos.fetcherCache.enable", false) ||
+ conf.getBoolean("spark.mesos.fetcherCache.enable", false)
+
val confUris = List(conf.getOption("spark.mesos.uris"),
desc.conf.getOption("spark.mesos.uris"),
desc.conf.getOption("spark.submit.pyFiles")).flatMap(
@@ -769,6 +771,10 @@ private[spark] class MesosClusterScheduler(
val state = launchedDrivers(subId)
// Check if the driver is supervise enabled and can be relaunched.
if (state.driverDescription.supervise && shouldRelaunch(status.getState)) {
+ if (isTaskOutdated(taskId, state)) {
+ // Prevent outdated task from overwriting a more recent status
+ return
+ }
removeFromLaunchedDrivers(subId)
state.finishDate = Some(new Date())
val retryState: Option[MesosClusterRetryState] = state.driverDescription.retryState
@@ -789,6 +795,16 @@ private[spark] class MesosClusterScheduler(
}
}
+ /**
+ * Check if the task is outdated i.e. has already been launched or is pending
+ * If neither, the taskId is outdated and should be ignored
+ * This is to avoid scenarios where an outdated status update arrives
+ * after a supervised driver has already been relaunched
+ */
+ private def isTaskOutdated(taskId: String, state: MesosClusterSubmissionState): Boolean =
+ taskId != state.taskId.getValue &&
+ !pendingRetryDrivers.exists(_.submissionId == state.driverDescription.submissionId)
+
private def retireDriver(
submissionId: String,
state: MesosClusterSubmissionState) = {
@@ -814,7 +830,7 @@ private[spark] class MesosClusterScheduler(
status: Int): Unit = {}
private def removeFromQueuedDrivers(subId: String): Boolean = {
- val index = queuedDrivers.indexWhere(_.submissionId.equals(subId))
+ val index = queuedDrivers.indexWhere(_.submissionId == subId)
if (index != -1) {
queuedDrivers.remove(index)
queuedDriversState.expunge(subId)
@@ -834,7 +850,7 @@ private[spark] class MesosClusterScheduler(
}
private def removeFromPendingRetryDrivers(subId: String): Boolean = {
- val index = pendingRetryDrivers.indexWhere(_.submissionId.equals(subId))
+ val index = pendingRetryDrivers.indexWhere(_.submissionId == subId)
if (index != -1) {
pendingRetryDrivers.remove(index)
pendingRetryDriversState.expunge(subId)
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
index 178de30f0f381..fb74c72952ce7 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
@@ -33,6 +33,7 @@ import org.apache.mesos.SchedulerDriver
import org.apache.spark.{SecurityManager, SparkConf, SparkContext, SparkException, TaskState}
import org.apache.spark.deploy.mesos.config._
import org.apache.spark.internal.config
+import org.apache.spark.internal.config.EXECUTOR_HEARTBEAT_INTERVAL
import org.apache.spark.launcher.{LauncherBackend, SparkAppHandle}
import org.apache.spark.network.netty.SparkTransportConf
import org.apache.spark.network.shuffle.mesos.MesosExternalShuffleClient
@@ -272,27 +273,19 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
.getOrElse {
throw new SparkException("Executor Spark home `spark.mesos.executor.home` is not set!")
}
- val runScript = new File(executorSparkHome, "./bin/spark-class").getPath
- command.setValue(
- "%s \"%s\" org.apache.spark.executor.CoarseGrainedExecutorBackend"
- .format(prefixEnv, runScript) +
- s" --driver-url $driverURL" +
- s" --executor-id $taskId" +
- s" --hostname ${executorHostname(offer)}" +
- s" --cores $numCores" +
- s" --app-id $appId")
+ val executable = new File(executorSparkHome, "./bin/spark-class").getPath
+ val runScript = "%s \"%s\" org.apache.spark.executor.CoarseGrainedExecutorBackend"
+ .format(prefixEnv, executable)
+
+ command.setValue(buildExecutorCommand(runScript, taskId, numCores, offer))
} else {
// Grab everything to the first '.'. We'll use that and '*' to
// glob the directory "correctly".
val basename = uri.get.split('/').last.split('.').head
- command.setValue(
- s"cd $basename*; $prefixEnv " +
- "./bin/spark-class org.apache.spark.executor.CoarseGrainedExecutorBackend" +
- s" --driver-url $driverURL" +
- s" --executor-id $taskId" +
- s" --hostname ${executorHostname(offer)}" +
- s" --cores $numCores" +
- s" --app-id $appId")
+ val runScript = s"cd $basename*; $prefixEnv " +
+ "./bin/spark-class org.apache.spark.executor.CoarseGrainedExecutorBackend"
+
+ command.setValue(buildExecutorCommand(runScript, taskId, numCores, offer))
command.addUris(CommandInfo.URI.newBuilder().setValue(uri.get).setCache(useFetcherCache))
}
@@ -301,6 +294,28 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
command.build()
}
+ private def buildExecutorCommand(
+ runScript: String, taskId: String, numCores: Int, offer: Offer): String = {
+
+ val sb = new StringBuilder()
+ .append(runScript)
+ .append(" --driver-url ")
+ .append(driverURL)
+ .append(" --executor-id ")
+ .append(taskId)
+ .append(" --cores ")
+ .append(numCores)
+ .append(" --app-id ")
+ .append(appId)
+
+ if (sc.conf.get(NETWORK_NAME).isEmpty) {
+ sb.append(" --hostname ")
+ sb.append(offer.getHostname)
+ }
+
+ sb.toString()
+ }
+
protected def driverURL: String = {
if (conf.contains("spark.testing")) {
"driverURL"
@@ -635,7 +650,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
externalShufflePort,
sc.conf.getTimeAsMs("spark.storage.blockManagerSlaveTimeoutMs",
s"${sc.conf.getTimeAsSeconds("spark.network.timeout", "120s")}s"),
- sc.conf.getTimeAsMs("spark.executor.heartbeatInterval", "10s"))
+ sc.conf.get(EXECUTOR_HEARTBEAT_INTERVAL))
slave.shuffleRegistered = true
}
@@ -780,15 +795,6 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
slaves.values.map(_.taskIDs.size).sum
}
- private def executorHostname(offer: Offer): String = {
- if (sc.conf.get(NETWORK_NAME).isDefined) {
- // The agent's IP is not visible in a CNI container, so we bind to 0.0.0.0
- "0.0.0.0"
- } else {
- offer.getHostname
- }
- }
-
override def fetchHadoopDelegationTokens(): Option[Array[Byte]] = {
if (UserGroupInformation.isSecurityEnabled) {
Some(hadoopDelegationTokenManager.getTokens())
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala
index b4364a5e2eb3a..def33255bd811 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala
@@ -153,7 +153,9 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging {
.getOrElse(List.empty)
if (containerType == ContainerInfo.Type.DOCKER) {
- containerInfo.setDocker(dockerInfo(image, forcePullImage, portMaps, params))
+ containerInfo.setDocker(
+ dockerInfo(image, forcePullImage, portMaps, params, conf.get(NETWORK_NAME))
+ )
} else {
containerInfo.setMesos(mesosInfo(image, forcePullImage))
}
@@ -263,13 +265,24 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging {
image: String,
forcePullImage: Boolean,
portMaps: List[ContainerInfo.DockerInfo.PortMapping],
- params: List[Parameter]): DockerInfo = {
+ params: List[Parameter],
+ networkName: Option[String]): DockerInfo = {
val dockerBuilder = ContainerInfo.DockerInfo.newBuilder()
.setImage(image)
.setForcePullImage(forcePullImage)
portMaps.foreach(dockerBuilder.addPortMappings(_))
params.foreach(dockerBuilder.addParameters(_))
+ networkName.foreach { net =>
+ val network = Parameter.newBuilder()
+ .setKey("net")
+ .setValue(net)
+ .build()
+
+ dockerBuilder.setNetwork(DockerInfo.Network.USER)
+ dockerBuilder.addParameters(network)
+ }
+
dockerBuilder.build
}
diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala
index e534b9d7e3ed9..ae001e2997fdc 100644
--- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala
+++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala
@@ -21,7 +21,7 @@ import java.util.{Collection, Collections, Date}
import scala.collection.JavaConverters._
-import org.apache.mesos.Protos.{Environment, Secret, TaskState => MesosTaskState, _}
+import org.apache.mesos.Protos.{TaskState => MesosTaskState, _}
import org.apache.mesos.Protos.Value.{Scalar, Type}
import org.apache.mesos.SchedulerDriver
import org.mockito.{ArgumentCaptor, Matchers}
@@ -146,14 +146,14 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
assert(scheduler.getResource(resources, "cpus") == 1.5)
assert(scheduler.getResource(resources, "mem") == 1200)
val resourcesSeq: Seq[Resource] = resources.asScala
- val cpus = resourcesSeq.filter(_.getName.equals("cpus")).toList
+ val cpus = resourcesSeq.filter(_.getName == "cpus").toList
assert(cpus.size == 2)
- assert(cpus.exists(_.getRole().equals("role2")))
- assert(cpus.exists(_.getRole().equals("*")))
- val mem = resourcesSeq.filter(_.getName.equals("mem")).toList
+ assert(cpus.exists(_.getRole() == "role2"))
+ assert(cpus.exists(_.getRole() == "*"))
+ val mem = resourcesSeq.filter(_.getName == "mem").toList
assert(mem.size == 2)
- assert(mem.exists(_.getRole().equals("role2")))
- assert(mem.exists(_.getRole().equals("*")))
+ assert(mem.exists(_.getRole() == "role2"))
+ assert(mem.exists(_.getRole() == "*"))
verify(driver, times(1)).launchTasks(
Matchers.eq(Collections.singleton(offer.getId)),
@@ -254,6 +254,79 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
assert(networkInfos.get(0).getLabels.getLabels(1).getValue == "val2")
}
+ test("supports setting fetcher cache on the dispatcher") {
+ setScheduler(Map("spark.mesos.fetcherCache.enable" -> "true"))
+
+ val mem = 1000
+ val cpu = 1
+
+ val response = scheduler.submitDriver(
+ new MesosDriverDescription("d1", "jar", mem, cpu, true,
+ command,
+ Map("spark.mesos.executor.home" -> "test",
+ "spark.app.name" -> "test"),
+ "s1",
+ new Date()))
+
+ assert(response.success)
+
+ val offer = Utils.createOffer("o1", "s1", mem, cpu)
+ scheduler.resourceOffers(driver, List(offer).asJava)
+
+ val launchedTasks = Utils.verifyTaskLaunched(driver, "o1")
+ val uris = launchedTasks.head.getCommand.getUrisList
+ assert(uris.asScala.forall(_.getCache))
+ }
+
+ test("supports setting fetcher cache in the submission") {
+ setScheduler(Map())
+
+ val mem = 1000
+ val cpu = 1
+
+ val response = scheduler.submitDriver(
+ new MesosDriverDescription("d1", "jar", mem, cpu, true,
+ command,
+ Map("spark.mesos.executor.home" -> "test",
+ "spark.app.name" -> "test",
+ "spark.mesos.fetcherCache.enable" -> "true"),
+ "s1",
+ new Date()))
+
+ assert(response.success)
+
+ val offer = Utils.createOffer("o1", "s1", mem, cpu)
+ scheduler.resourceOffers(driver, List(offer).asJava)
+
+ val launchedTasks = Utils.verifyTaskLaunched(driver, "o1")
+ val uris = launchedTasks.head.getCommand.getUrisList
+ assert(uris.asScala.forall(_.getCache))
+ }
+
+ test("supports disabling fetcher cache") {
+ setScheduler(Map("spark.mesos.fetcherCache.enable" -> "false"))
+
+ val mem = 1000
+ val cpu = 1
+
+ val response = scheduler.submitDriver(
+ new MesosDriverDescription("d1", "jar", mem, cpu, true,
+ command,
+ Map("spark.mesos.executor.home" -> "test",
+ "spark.app.name" -> "test"),
+ "s1",
+ new Date()))
+
+ assert(response.success)
+
+ val offer = Utils.createOffer("o1", "s1", mem, cpu)
+ scheduler.resourceOffers(driver, List(offer).asJava)
+
+ val launchedTasks = Utils.verifyTaskLaunched(driver, "o1")
+ val uris = launchedTasks.head.getCommand.getUrisList
+ assert(uris.asScala.forall(!_.getCache))
+ }
+
test("accept/decline offers with driver constraints") {
setScheduler()
@@ -375,6 +448,86 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
assert(state.finishedDrivers.size == 1)
}
+ test("SPARK-27347: do not restart outdated supervised drivers") {
+ // Covers scenario where:
+ // - agent goes down
+ // - supervised job is relaunched on another agent
+ // - first agent re-registers and sends status update: TASK_FAILED
+ // - job should NOT be relaunched again
+ val conf = new SparkConf()
+ conf.setMaster("mesos://localhost:5050")
+ conf.setAppName("SparkMesosDriverRetries")
+ setScheduler(conf.getAll.toMap)
+
+ val mem = 1000
+ val cpu = 1
+ val offers = List(
+ Utils.createOffer("o1", "s1", mem, cpu, None),
+ Utils.createOffer("o2", "s2", mem, cpu, None),
+ Utils.createOffer("o3", "s1", mem, cpu, None))
+
+ val response = scheduler.submitDriver(
+ new MesosDriverDescription("d1", "jar", 100, 1, true, command,
+ Map(("spark.mesos.executor.home", "test"), ("spark.app.name", "test")), "sub1", new Date()))
+ assert(response.success)
+
+ // Offer a resource to launch the submitted driver
+ scheduler.resourceOffers(driver, Collections.singletonList(offers.head))
+ var state = scheduler.getSchedulerState()
+ assert(state.launchedDrivers.size == 1)
+
+ // Signal agent lost with status with TASK_LOST
+ val agent1 = SlaveID.newBuilder().setValue("s1").build()
+ var taskStatus = TaskStatus.newBuilder()
+ .setTaskId(TaskID.newBuilder().setValue(response.submissionId).build())
+ .setSlaveId(agent1)
+ .setReason(TaskStatus.Reason.REASON_SLAVE_REMOVED)
+ .setState(MesosTaskState.TASK_LOST)
+ .build()
+
+ scheduler.statusUpdate(driver, taskStatus)
+ state = scheduler.getSchedulerState()
+ assert(state.pendingRetryDrivers.size == 1)
+ assert(state.pendingRetryDrivers.head.submissionId == taskStatus.getTaskId.getValue)
+ assert(state.launchedDrivers.isEmpty)
+
+ // Offer new resource to retry driver on a new agent
+ Thread.sleep(1500) // sleep to cover nextRetry's default wait time of 1s
+ scheduler.resourceOffers(driver, Collections.singletonList(offers(1)))
+
+ val agent2 = SlaveID.newBuilder().setValue("s2").build()
+ taskStatus = TaskStatus.newBuilder()
+ .setTaskId(TaskID.newBuilder().setValue(response.submissionId).build())
+ .setSlaveId(agent2)
+ .setState(MesosTaskState.TASK_RUNNING)
+ .build()
+
+ scheduler.statusUpdate(driver, taskStatus)
+ state = scheduler.getSchedulerState()
+ assert(state.pendingRetryDrivers.isEmpty)
+ assert(state.launchedDrivers.size == 1)
+ assert(state.launchedDrivers.head.taskId.getValue.endsWith("-retry-1"))
+ assert(state.launchedDrivers.head.taskId.getValue != taskStatus.getTaskId.getValue)
+
+ // Agent1 comes back online and sends status update: TASK_FAILED
+ taskStatus = TaskStatus.newBuilder()
+ .setTaskId(TaskID.newBuilder().setValue(response.submissionId).build())
+ .setSlaveId(agent1)
+ .setState(MesosTaskState.TASK_FAILED)
+ .setMessage("Abnormal executor termination")
+ .setReason(TaskStatus.Reason.REASON_EXECUTOR_TERMINATED)
+ .build()
+
+ scheduler.statusUpdate(driver, taskStatus)
+ scheduler.resourceOffers(driver, Collections.singletonList(offers.last))
+
+ // Assert driver does not restart 2nd time
+ state = scheduler.getSchedulerState()
+ assert(state.pendingRetryDrivers.isEmpty)
+ assert(state.launchedDrivers.size == 1)
+ assert(state.launchedDrivers.head.taskId.getValue.endsWith("-retry-1"))
+ }
+
test("Declines offer with refuse seconds = 120.") {
setScheduler()
diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
index da33d85d8fb2e..d33f65edec848 100644
--- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
+++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackendSuite.scala
@@ -588,6 +588,30 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
assert(networkInfos.get(0).getLabels.getLabels(1).getValue == "val2")
}
+ test("SPARK-28778 '--hostname' shouldn't be set for executor when virtual network is enabled") {
+ setBackend()
+ val (mem, cpu) = (backend.executorMemory(sc), 4)
+ val offer = createOffer("o1", "s1", mem, cpu)
+
+ assert(backend.createCommand(offer, cpu, "test").getValue.contains("--hostname"))
+ sc.stop()
+
+ setBackend(Map("spark.executor.uri" -> "hdfs://test/executor.jar"))
+ assert(backend.createCommand(offer, cpu, "test").getValue.contains("--hostname"))
+ sc.stop()
+
+ setBackend(Map("spark.mesos.network.name" -> "test"))
+ assert(!backend.createCommand(offer, cpu, "test").getValue.contains("--hostname"))
+ sc.stop()
+
+ setBackend(Map(
+ "spark.mesos.network.name" -> "test",
+ "spark.executor.uri" -> "hdfs://test/executor.jar"
+ ))
+ assert(!backend.createCommand(offer, cpu, "test").getValue.contains("--hostname"))
+ sc.stop()
+ }
+
test("supports spark.scheduler.minRegisteredResourcesRatio") {
val expectedCores = 1
setBackend(Map(
diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala
index 31f84310485a0..1ead4b1ed7c7e 100644
--- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala
+++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosFineGrainedSchedulerBackendSuite.scala
@@ -106,7 +106,7 @@ class MesosFineGrainedSchedulerBackendSuite
// uri is null.
val (executorInfo, _) = mesosSchedulerBackend.createExecutorInfo(resources, "test-id")
val executorResources = executorInfo.getResourcesList
- val cpus = executorResources.asScala.find(_.getName.equals("cpus")).get.getScalar.getValue
+ val cpus = executorResources.asScala.find(_.getName == "cpus").get.getScalar.getValue
assert(cpus === mesosExecutorCores)
}
diff --git a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala
index 442c43960ec1f..4e2573b6f56b5 100644
--- a/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala
+++ b/resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtilSuite.scala
@@ -17,6 +17,8 @@
package org.apache.spark.scheduler.cluster.mesos
+import org.apache.mesos.Protos.ContainerInfo.DockerInfo
+
import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.deploy.mesos.config
@@ -31,7 +33,7 @@ class MesosSchedulerBackendUtilSuite extends SparkFunSuite {
conf)
val params = containerInfo.getDocker.getParametersList
- assert(params.size() == 0)
+ assert(params.size() === 0)
}
test("ContainerInfo parses docker parameters") {
@@ -42,12 +44,28 @@ class MesosSchedulerBackendUtilSuite extends SparkFunSuite {
val containerInfo = MesosSchedulerBackendUtil.buildContainerInfo(
conf)
val params = containerInfo.getDocker.getParametersList
- assert(params.size() == 3)
- assert(params.get(0).getKey == "a")
- assert(params.get(0).getValue == "1")
- assert(params.get(1).getKey == "b")
- assert(params.get(1).getValue == "2")
- assert(params.get(2).getKey == "c")
- assert(params.get(2).getValue == "3")
+ assert(params.size() === 3)
+ assert(params.get(0).getKey === "a")
+ assert(params.get(0).getValue === "1")
+ assert(params.get(1).getKey === "b")
+ assert(params.get(1).getValue === "2")
+ assert(params.get(2).getKey === "c")
+ assert(params.get(2).getValue === "3")
+ }
+
+ test("SPARK-28778 ContainerInfo respects Docker network configuration") {
+ val networkName = "test"
+ val conf = new SparkConf()
+ conf.set("spark.mesos.containerizer", "docker")
+ conf.set("spark.mesos.executor.docker.image", "image")
+ conf.set("spark.mesos.network.name", networkName)
+
+ val containerInfo = MesosSchedulerBackendUtil.buildContainerInfo(conf)
+
+ assert(containerInfo.getDocker.getNetwork === DockerInfo.Network.USER)
+ val params = containerInfo.getDocker.getParametersList
+ assert(params.size() === 1)
+ assert(params.get(0).getKey === "net")
+ assert(params.get(0).getValue === networkName)
}
}
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index 37e25ceecb883..0b4f96c438ff0 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 8f94e3f731007..f9a009fdcbe8b 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -451,7 +451,8 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments) extends
val ms = MetricsSystem.createMetricsSystem("applicationMaster", sparkConf, securityMgr)
val prefix = _sparkConf.get(YARN_METRICS_NAMESPACE).getOrElse(appId)
ms.registerSource(new ApplicationMasterSource(prefix, allocator))
- ms.start()
+ // do not register static sources in this case as per SPARK-25277
+ ms.start(false)
metricsSystem = Some(ms)
reporterThread = launchReporterThread()
}
@@ -521,88 +522,94 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments) extends
reporterThread.join()
}
- private def launchReporterThread(): Thread = {
- // The number of failures in a row until Reporter thread give up
+ private def allocationThreadImpl(): Unit = {
+ // The number of failures in a row until the allocation thread gives up.
val reporterMaxFailures = sparkConf.get(MAX_REPORTER_THREAD_FAILURES)
-
- val t = new Thread {
- override def run() {
- var failureCount = 0
- while (!finished) {
- try {
- if (allocator.getNumExecutorsFailed >= maxNumExecutorFailures) {
- finish(FinalApplicationStatus.FAILED,
- ApplicationMaster.EXIT_MAX_EXECUTOR_FAILURES,
- s"Max number of executor failures ($maxNumExecutorFailures) reached")
- } else if (allocator.isAllNodeBlacklisted) {
- finish(FinalApplicationStatus.FAILED,
- ApplicationMaster.EXIT_MAX_EXECUTOR_FAILURES,
- "Due to executor failures all available nodes are blacklisted")
- } else {
- logDebug("Sending progress")
- allocator.allocateResources()
- }
- failureCount = 0
- } catch {
- case i: InterruptedException => // do nothing
- case e: ApplicationAttemptNotFoundException =>
- failureCount += 1
- logError("Exception from Reporter thread.", e)
- finish(FinalApplicationStatus.FAILED, ApplicationMaster.EXIT_REPORTER_FAILURE,
- e.getMessage)
- case e: Throwable =>
- failureCount += 1
- if (!NonFatal(e) || failureCount >= reporterMaxFailures) {
- finish(FinalApplicationStatus.FAILED,
- ApplicationMaster.EXIT_REPORTER_FAILURE, "Exception was thrown " +
- s"$failureCount time(s) from Reporter thread.")
- } else {
- logWarning(s"Reporter thread fails $failureCount time(s) in a row.", e)
- }
+ var failureCount = 0
+ while (!finished) {
+ try {
+ if (allocator.getNumExecutorsFailed >= maxNumExecutorFailures) {
+ finish(FinalApplicationStatus.FAILED,
+ ApplicationMaster.EXIT_MAX_EXECUTOR_FAILURES,
+ s"Max number of executor failures ($maxNumExecutorFailures) reached")
+ } else if (allocator.isAllNodeBlacklisted) {
+ finish(FinalApplicationStatus.FAILED,
+ ApplicationMaster.EXIT_MAX_EXECUTOR_FAILURES,
+ "Due to executor failures all available nodes are blacklisted")
+ } else {
+ logDebug("Sending progress")
+ allocator.allocateResources()
+ }
+ failureCount = 0
+ } catch {
+ case i: InterruptedException => // do nothing
+ case e: ApplicationAttemptNotFoundException =>
+ failureCount += 1
+ logError("Exception from Reporter thread.", e)
+ finish(FinalApplicationStatus.FAILED, ApplicationMaster.EXIT_REPORTER_FAILURE,
+ e.getMessage)
+ case e: Throwable =>
+ failureCount += 1
+ if (!NonFatal(e) || failureCount >= reporterMaxFailures) {
+ finish(FinalApplicationStatus.FAILED,
+ ApplicationMaster.EXIT_REPORTER_FAILURE, "Exception was thrown " +
+ s"$failureCount time(s) from Reporter thread.")
+ } else {
+ logWarning(s"Reporter thread fails $failureCount time(s) in a row.", e)
}
- try {
- val numPendingAllocate = allocator.getPendingAllocate.size
- var sleepStart = 0L
- var sleepInterval = 200L // ms
- allocatorLock.synchronized {
- sleepInterval =
- if (numPendingAllocate > 0 || allocator.getNumPendingLossReasonRequests > 0) {
- val currentAllocationInterval =
- math.min(heartbeatInterval, nextAllocationInterval)
- nextAllocationInterval = currentAllocationInterval * 2 // avoid overflow
- currentAllocationInterval
- } else {
- nextAllocationInterval = initialAllocationInterval
- heartbeatInterval
- }
- sleepStart = System.currentTimeMillis()
- allocatorLock.wait(sleepInterval)
- }
- val sleepDuration = System.currentTimeMillis() - sleepStart
- if (sleepDuration < sleepInterval) {
- // log when sleep is interrupted
- logDebug(s"Number of pending allocations is $numPendingAllocate. " +
- s"Slept for $sleepDuration/$sleepInterval ms.")
- // if sleep was less than the minimum interval, sleep for the rest of it
- val toSleep = math.max(0, initialAllocationInterval - sleepDuration)
- if (toSleep > 0) {
- logDebug(s"Going back to sleep for $toSleep ms")
- // use Thread.sleep instead of allocatorLock.wait. there is no need to be woken up
- // by the methods that signal allocatorLock because this is just finishing the min
- // sleep interval, which should happen even if this is signalled again.
- Thread.sleep(toSleep)
- }
+ }
+ try {
+ val numPendingAllocate = allocator.getPendingAllocate.size
+ var sleepStart = 0L
+ var sleepInterval = 200L // ms
+ allocatorLock.synchronized {
+ sleepInterval =
+ if (numPendingAllocate > 0 || allocator.getNumPendingLossReasonRequests > 0) {
+ val currentAllocationInterval =
+ math.min(heartbeatInterval, nextAllocationInterval)
+ nextAllocationInterval = currentAllocationInterval * 2 // avoid overflow
+ currentAllocationInterval
} else {
- logDebug(s"Number of pending allocations is $numPendingAllocate. " +
- s"Slept for $sleepDuration/$sleepInterval.")
+ nextAllocationInterval = initialAllocationInterval
+ heartbeatInterval
}
- } catch {
- case e: InterruptedException =>
+ sleepStart = System.currentTimeMillis()
+ allocatorLock.wait(sleepInterval)
+ }
+ val sleepDuration = System.currentTimeMillis() - sleepStart
+ if (sleepDuration < sleepInterval) {
+ // log when sleep is interrupted
+ logDebug(s"Number of pending allocations is $numPendingAllocate. " +
+ s"Slept for $sleepDuration/$sleepInterval ms.")
+ // if sleep was less than the minimum interval, sleep for the rest of it
+ val toSleep = math.max(0, initialAllocationInterval - sleepDuration)
+ if (toSleep > 0) {
+ logDebug(s"Going back to sleep for $toSleep ms")
+ // use Thread.sleep instead of allocatorLock.wait. there is no need to be woken up
+ // by the methods that signal allocatorLock because this is just finishing the min
+ // sleep interval, which should happen even if this is signalled again.
+ Thread.sleep(toSleep)
}
+ } else {
+ logDebug(s"Number of pending allocations is $numPendingAllocate. " +
+ s"Slept for $sleepDuration/$sleepInterval.")
+ }
+ } catch {
+ case e: InterruptedException =>
+ }
+ }
+ }
+
+ private def launchReporterThread(): Thread = {
+ val t = new Thread {
+ override def run(): Unit = {
+ try {
+ allocationThreadImpl()
+ } finally {
+ allocator.stop()
}
}
}
- // setting to daemon status, though this is usually not a good idea.
t.setDaemon(true)
t.setName("Reporter")
t.start()
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 4a85898ef880b..76b31f17ab6b0 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException
import org.apache.hadoop.yarn.util.Records
import org.apache.spark.{SecurityManager, SparkConf, SparkException}
+import org.apache.spark.api.python.PythonUtils
import org.apache.spark.deploy.{SparkApplication, SparkHadoopUtil}
import org.apache.spark.deploy.yarn.config._
import org.apache.spark.deploy.yarn.security.YARNHadoopDelegationTokenManager
@@ -1165,7 +1166,7 @@ private[spark] class Client(
val pyArchivesFile = new File(pyLibPath, "pyspark.zip")
require(pyArchivesFile.exists(),
s"$pyArchivesFile not found; cannot run pyspark application in YARN mode.")
- val py4jFile = new File(pyLibPath, "py4j-0.10.7-src.zip")
+ val py4jFile = new File(pyLibPath, PythonUtils.PY4J_ZIP_NAME)
require(py4jFile.exists(),
s"$py4jFile not found; cannot run pyspark application in YARN mode.")
Seq(pyArchivesFile.getAbsolutePath(), py4jFile.getAbsolutePath())
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
index 8a7551de7c088..96bc1c7889500 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -287,20 +287,20 @@ private[yarn] class YarnAllocator(
s"pending: $numPendingAllocate, running: ${runningExecutors.size}, " +
s"executorsStarting: ${numExecutorsStarting.get}")
+ // Split the pending container request into three groups: locality matched list, locality
+ // unmatched list and non-locality list. Take the locality matched container request into
+ // consideration of container placement, treat as allocated containers.
+ // For locality unmatched and locality free container requests, cancel these container
+ // requests, since required locality preference has been changed, recalculating using
+ // container placement strategy.
+ val (localRequests, staleRequests, anyHostRequests) = splitPendingAllocationsByLocality(
+ hostToLocalTaskCounts, pendingAllocate)
+
if (missing > 0) {
logInfo(s"Will request $missing executor container(s), each with " +
s"${resource.getVirtualCores} core(s) and " +
s"${resource.getMemory} MB memory (including $memoryOverhead MB of overhead)")
- // Split the pending container request into three groups: locality matched list, locality
- // unmatched list and non-locality list. Take the locality matched container request into
- // consideration of container placement, treat as allocated containers.
- // For locality unmatched and locality free container requests, cancel these container
- // requests, since required locality preference has been changed, recalculating using
- // container placement strategy.
- val (localRequests, staleRequests, anyHostRequests) = splitPendingAllocationsByLocality(
- hostToLocalTaskCounts, pendingAllocate)
-
// cancel "stale" requests for locations that are no longer needed
staleRequests.foreach { stale =>
amClient.removeContainerRequest(stale)
@@ -360,17 +360,19 @@ private[yarn] class YarnAllocator(
val numToCancel = math.min(numPendingAllocate, -missing)
logInfo(s"Canceling requests for $numToCancel executor container(s) to have a new desired " +
s"total $targetNumExecutors executors.")
-
- val matchingRequests = amClient.getMatchingRequests(RM_REQUEST_PRIORITY, ANY_HOST, resource)
- if (!matchingRequests.isEmpty) {
- matchingRequests.iterator().next().asScala
- .take(numToCancel).foreach(amClient.removeContainerRequest)
- } else {
- logWarning("Expected to find pending requests, but found none.")
- }
+ // cancel pending allocate requests by taking locality preference into account
+ val cancelRequests = (staleRequests ++ anyHostRequests ++ localRequests).take(numToCancel)
+ cancelRequests.foreach(amClient.removeContainerRequest)
}
}
+ def stop(): Unit = {
+ // Forcefully shut down the launcher pool, in case this is being called in the middle of
+ // container allocation. This will prevent queued executors from being started - and
+ // potentially interrupt active ExecutorRunnable instaces too.
+ launcherPool.shutdownNow()
+ }
+
private def hostStr(request: ContainerRequest): String = {
Option(request.getNodes) match {
case Some(nodes) => nodes.asScala.mkString(",")
@@ -407,12 +409,40 @@ private[yarn] class YarnAllocator(
containersToUse, remainingAfterHostMatches)
}
- // Match remaining by rack
+ // Match remaining by rack. Because YARN's RackResolver swallows thread interrupts
+ // (see SPARK-27094), which can cause this code to miss interrupts from the AM, use
+ // a separate thread to perform the operation.
val remainingAfterRackMatches = new ArrayBuffer[Container]
- for (allocatedContainer <- remainingAfterHostMatches) {
- val rack = resolver.resolve(conf, allocatedContainer.getNodeId.getHost)
- matchContainerToRequest(allocatedContainer, rack, containersToUse,
- remainingAfterRackMatches)
+ if (remainingAfterHostMatches.nonEmpty) {
+ var exception: Option[Throwable] = None
+ val thread = new Thread("spark-rack-resolver") {
+ override def run(): Unit = {
+ try {
+ for (allocatedContainer <- remainingAfterHostMatches) {
+ val rack = resolver.resolve(conf, allocatedContainer.getNodeId.getHost)
+ matchContainerToRequest(allocatedContainer, rack, containersToUse,
+ remainingAfterRackMatches)
+ }
+ } catch {
+ case e: Throwable =>
+ exception = Some(e)
+ }
+ }
+ }
+ thread.setDaemon(true)
+ thread.start()
+
+ try {
+ thread.join()
+ } catch {
+ case e: InterruptedException =>
+ thread.interrupt()
+ throw e
+ }
+
+ if (exception.isDefined) {
+ throw exception.get
+ }
}
// Assign remaining that are neither node-local nor rack-local
@@ -583,13 +613,23 @@ private[yarn] class YarnAllocator(
(true, memLimitExceededLogMessage(
completedContainer.getDiagnostics,
PMEM_EXCEEDED_PATTERN))
- case _ =>
- // all the failures which not covered above, like:
- // disk failure, kill by app master or resource manager, ...
- allocatorBlacklistTracker.handleResourceAllocationFailure(hostOpt)
- (true, "Container marked as failed: " + containerId + onHostStr +
- ". Exit status: " + completedContainer.getExitStatus +
- ". Diagnostics: " + completedContainer.getDiagnostics)
+ case other_exit_status =>
+ // SPARK-26269: follow YARN's blacklisting behaviour(see https://github
+ // .com/apache/hadoop/blob/228156cfd1b474988bc4fedfbf7edddc87db41e3/had
+ // oop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/ap
+ // ache/hadoop/yarn/util/Apps.java#L273 for details)
+ if (NOT_APP_AND_SYSTEM_FAULT_EXIT_STATUS.contains(other_exit_status)) {
+ (false, s"Container marked as failed: $containerId$onHostStr" +
+ s". Exit status: ${completedContainer.getExitStatus}" +
+ s". Diagnostics: ${completedContainer.getDiagnostics}.")
+ } else {
+ // completed container from a bad node
+ allocatorBlacklistTracker.handleResourceAllocationFailure(hostOpt)
+ (true, s"Container from a bad node: $containerId$onHostStr" +
+ s". Exit status: ${completedContainer.getExitStatus}" +
+ s". Diagnostics: ${completedContainer.getDiagnostics}.")
+ }
+
}
if (exitCausedByApp) {
@@ -727,4 +767,11 @@ private object YarnAllocator {
"Consider boosting spark.yarn.executor.memoryOverhead or " +
"disabling yarn.nodemanager.vmem-check-enabled because of YARN-4714."
}
+ val NOT_APP_AND_SYSTEM_FAULT_EXIT_STATUS = Set(
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER,
+ ContainerExitStatus.KILLED_BY_APPMASTER,
+ ContainerExitStatus.KILLED_AFTER_APP_COMPLETION,
+ ContainerExitStatus.ABORTED,
+ ContainerExitStatus.DISKS_FAILED
+ )
}
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTracker.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTracker.scala
index ceac7cda5f8be..268976b629507 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTracker.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTracker.scala
@@ -120,7 +120,9 @@ private[spark] class YarnAllocatorBlacklistTracker(
if (removals.nonEmpty) {
logInfo(s"removing nodes from YARN application master's blacklist: $removals")
}
- amClient.updateBlacklist(additions.asJava, removals.asJava)
+ if (additions.nonEmpty || removals.nonEmpty) {
+ amClient.updateBlacklist(additions.asJava, removals.asJava)
+ }
currentBlacklistedYarnNodes = nodesToBlacklist
}
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/AMCredentialRenewer.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/AMCredentialRenewer.scala
index bc8d47dbd54c6..51ef7d3a60362 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/AMCredentialRenewer.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/AMCredentialRenewer.scala
@@ -86,6 +86,16 @@ private[yarn] class AMCredentialRenewer(
val originalCreds = UserGroupInformation.getCurrentUser().getCredentials()
val ugi = doLogin()
+ ugi.doAs(new PrivilegedExceptionAction[Unit]() {
+ override def run(): Unit = {
+ startInternal(ugi, originalCreds)
+ }
+ })
+
+ ugi
+ }
+
+ private def startInternal(ugi: UserGroupInformation, originalCreds: Credentials): Unit = {
val tgtRenewalTask = new Runnable() {
override def run(): Unit = {
ugi.checkTGTAndReloginFromKeytab()
@@ -104,8 +114,6 @@ private[yarn] class AMCredentialRenewer(
val existing = ugi.getCredentials()
existing.mergeAll(originalCreds)
ugi.addCredentials(existing)
-
- ugi
}
def stop(): Unit = {
@@ -136,8 +144,8 @@ private[yarn] class AMCredentialRenewer(
// This shouldn't really happen, since the driver should register way before tokens expire
// (or the AM should time out the application).
logWarning("Delegation tokens close to expiration but no driver has registered yet.")
- SparkHadoopUtil.get.addDelegationTokens(tokens, sparkConf)
}
+ SparkHadoopUtil.get.addDelegationTokens(tokens, sparkConf)
} catch {
case e: Exception =>
val delay = TimeUnit.SECONDS.toMillis(sparkConf.get(CREDENTIALS_RENEWAL_RETRY_WAIT))
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackend.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackend.scala
index 63bea3e7a5003..0d8e3839d6005 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackend.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackend.scala
@@ -173,7 +173,9 @@ private[spark] abstract class YarnSchedulerBackend(
logInfo(s"Add WebUI Filter. $filterName, $filterParams, $proxyBase")
conf.set("spark.ui.filters", filterName)
filterParams.foreach { case (k, v) => conf.set(s"spark.$filterName.param.$k", v) }
- scheduler.sc.ui.foreach { ui => JettyUtils.addFilters(ui.getHandlers, conf) }
+ scheduler.sc.ui.foreach { ui =>
+ ui.getDelegatingHandlers.foreach(_.addFilter(filterName, filterName, filterParams))
+ }
}
}
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala
index 3a7913122dd83..48ce1780aeb9e 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala
@@ -169,7 +169,7 @@ abstract class BaseYarnClusterSuite
val handle = launcher.startApplication()
try {
- eventually(timeout(2 minutes), interval(1 second)) {
+ eventually(timeout(3.minutes), interval(1.second)) {
assert(handle.getState().isFinal())
}
} finally {
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
index 7fa597167f3f0..26013a109c42b 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
@@ -191,7 +191,7 @@ class ClientSuite extends SparkFunSuite with Matchers {
appContext.getQueue should be ("staging-queue")
appContext.getAMContainerSpec should be (containerLaunchContext)
appContext.getApplicationType should be ("SPARK")
- appContext.getClass.getMethods.filter(_.getName.equals("getApplicationTags")).foreach{ method =>
+ appContext.getClass.getMethods.filter(_.getName == "getApplicationTags").foreach { method =>
val tags = method.invoke(appContext).asInstanceOf[java.util.Set[String]]
tags should contain allOf ("tag1", "dup", "tag2", "multi word")
tags.asScala.count(_.nonEmpty) should be (4)
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTrackerSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTrackerSuite.scala
index aeac68e6ed330..201910731e934 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTrackerSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorBlacklistTrackerSuite.scala
@@ -87,7 +87,7 @@ class YarnAllocatorBlacklistTrackerSuite extends SparkFunSuite with Matchers
// expired blacklisted nodes (simulating a resource request)
yarnBlacklistTracker.setSchedulerBlacklistedNodes(Set("host1", "host2"))
// no change is communicated to YARN regarding the blacklisting
- verify(amClientMock).updateBlacklist(Collections.emptyList(), Collections.emptyList())
+ verify(amClientMock, times(0)).updateBlacklist(Collections.emptyList(), Collections.emptyList())
}
test("combining scheduler and allocation blacklist") {
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
index 3f783baed110d..2fb892ecbc33a 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
@@ -17,6 +17,8 @@
package org.apache.spark.deploy.yarn
+import java.util.Collections
+
import scala.collection.JavaConverters._
import org.apache.hadoop.conf.Configuration
@@ -24,6 +26,7 @@ import org.apache.hadoop.yarn.api.records._
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.mockito.ArgumentCaptor
import org.mockito.Mockito._
import org.scalatest.{BeforeAndAfterEach, Matchers}
@@ -86,7 +89,8 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
def createAllocator(
maxExecutors: Int = 5,
- rmClient: AMRMClient[ContainerRequest] = rmClient): YarnAllocator = {
+ rmClient: AMRMClient[ContainerRequest] = rmClient,
+ additionalConfigs: Map[String, String] = Map()): YarnAllocator = {
val args = Array(
"--jar", "somejar.jar",
"--class", "SomeClass")
@@ -95,6 +99,11 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
.set("spark.executor.instances", maxExecutors.toString)
.set("spark.executor.cores", "5")
.set("spark.executor.memory", "2048")
+
+ for ((name, value) <- additionalConfigs) {
+ sparkConfClone.set(name, value)
+ }
+
new YarnAllocator(
"not used",
mock(classOf[RpcEndpointRef]),
@@ -108,14 +117,29 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
clock)
}
- def createContainer(host: String): Container = {
- // When YARN 2.6+ is required, avoid deprecation by using version with long second arg
- val containerId = ContainerId.newInstance(appAttemptId, containerNum)
+ def createContainer(
+ host: String,
+ containerNumber: Int = containerNum,
+ resource: Resource = containerResource): Container = {
+ val containerId: ContainerId = ContainerId.newContainerId(appAttemptId, containerNum)
containerNum += 1
val nodeId = NodeId.newInstance(host, 1000)
Container.newInstance(containerId, nodeId, "", containerResource, RM_REQUEST_PRIORITY, null)
}
+ def createContainers(hosts: Seq[String], containerIds: Seq[Int]): Seq[Container] = {
+ hosts.zip(containerIds).map{case (host, id) => createContainer(host, id)}
+ }
+
+ def createContainerStatus(
+ containerId: ContainerId,
+ exitStatus: Int,
+ containerState: ContainerState = ContainerState.COMPLETE,
+ diagnostics: String = "diagnostics"): ContainerStatus = {
+ ContainerStatus.newInstance(containerId, containerState, diagnostics, exitStatus)
+ }
+
+
test("single container allocated") {
// request a single container and receive it
val handler = createAllocator(1)
@@ -400,4 +424,55 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
clock.advance(50 * 1000L)
handler.getNumExecutorsFailed should be (0)
}
+
+ test("SPARK-26269: YarnAllocator should have same blacklist behaviour with YARN") {
+ val rmClientSpy = spy(rmClient)
+ val maxExecutors = 11
+
+ val handler = createAllocator(
+ maxExecutors,
+ rmClientSpy,
+ Map(
+ "spark.yarn.blacklist.executor.launch.blacklisting.enabled" -> "true",
+ "spark.blacklist.application.maxFailedExecutorsPerNode" -> "0"))
+ handler.updateResourceRequests()
+
+ val hosts = (0 until maxExecutors).map(i => s"host$i")
+ val ids = 0 to maxExecutors
+ val containers = createContainers(hosts, ids)
+
+ val nonBlacklistedStatuses = Seq(
+ ContainerExitStatus.SUCCESS,
+ ContainerExitStatus.PREEMPTED,
+ ContainerExitStatus.KILLED_EXCEEDED_VMEM,
+ ContainerExitStatus.KILLED_EXCEEDED_PMEM,
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER,
+ ContainerExitStatus.KILLED_BY_APPMASTER,
+ ContainerExitStatus.KILLED_AFTER_APP_COMPLETION,
+ ContainerExitStatus.ABORTED,
+ ContainerExitStatus.DISKS_FAILED)
+
+ val nonBlacklistedContainerStatuses = nonBlacklistedStatuses.zipWithIndex.map {
+ case (exitStatus, idx) => createContainerStatus(containers(idx).getId, exitStatus)
+ }
+
+ val BLACKLISTED_EXIT_CODE = 1
+ val blacklistedStatuses = Seq(ContainerExitStatus.INVALID, BLACKLISTED_EXIT_CODE)
+
+ val blacklistedContainerStatuses = blacklistedStatuses.zip(9 until maxExecutors).map {
+ case (exitStatus, idx) => createContainerStatus(containers(idx).getId, exitStatus)
+ }
+
+ handler.handleAllocatedContainers(containers.slice(0, 9))
+ handler.processCompletedContainers(nonBlacklistedContainerStatuses)
+ verify(rmClientSpy, never())
+ .updateBlacklist(hosts.slice(0, 9).asJava, Collections.emptyList())
+
+ handler.handleAllocatedContainers(containers.slice(9, 11))
+ handler.processCompletedContainers(blacklistedContainerStatuses)
+ verify(rmClientSpy)
+ .updateBlacklist(hosts.slice(9, 10).asJava, Collections.emptyList())
+ verify(rmClientSpy)
+ .updateBlacklist(hosts.slice(10, 11).asJava, Collections.emptyList())
+ }
}
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala
index 58d11e96942e1..2c34b4e3781da 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala
@@ -33,6 +33,7 @@ import org.scalatest.Matchers
import org.scalatest.concurrent.Eventually._
import org.apache.spark._
+import org.apache.spark.api.python.PythonUtils
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.deploy.yarn.config._
import org.apache.spark.internal.Logging
@@ -201,7 +202,7 @@ class YarnClusterSuite extends BaseYarnClusterSuite {
.startApplication()
try {
- eventually(timeout(30 seconds), interval(100 millis)) {
+ eventually(timeout(3.minutes), interval(100.milliseconds)) {
handle.getState() should be (SparkAppHandle.State.RUNNING)
}
@@ -209,7 +210,7 @@ class YarnClusterSuite extends BaseYarnClusterSuite {
handle.getAppId() should startWith ("application_")
handle.stop()
- eventually(timeout(30 seconds), interval(100 millis)) {
+ eventually(timeout(3.minutes), interval(100.milliseconds)) {
handle.getState() should be (SparkAppHandle.State.KILLED)
}
} finally {
@@ -265,7 +266,7 @@ class YarnClusterSuite extends BaseYarnClusterSuite {
// needed locations.
val sparkHome = sys.props("spark.test.home")
val pythonPath = Seq(
- s"$sparkHome/python/lib/py4j-0.10.7-src.zip",
+ s"$sparkHome/python/lib/${PythonUtils.PY4J_ZIP_NAME}",
s"$sparkHome/python")
val extraEnvVars = Map(
"PYSPARK_ARCHIVES_PATH" -> pythonPath.map("local:" + _).mkString(File.pathSeparator),
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala
index 7fac57ff68abc..bd2cf97426637 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.scheduler.cluster
+import java.util.concurrent.atomic.AtomicReference
+
import scala.language.reflectiveCalls
import org.mockito.Mockito.when
@@ -27,15 +29,35 @@ import org.apache.spark.serializer.JavaSerializer
class YarnSchedulerBackendSuite extends SparkFunSuite with MockitoSugar with LocalSparkContext {
+ private var yarnSchedulerBackend: YarnSchedulerBackend = _
+
+ override def afterEach(): Unit = {
+ try {
+ if (yarnSchedulerBackend != null) {
+ yarnSchedulerBackend.stop()
+ }
+ } finally {
+ super.afterEach()
+ }
+ }
+
test("RequestExecutors reflects node blacklist and is serializable") {
sc = new SparkContext("local", "YarnSchedulerBackendSuite")
- val sched = mock[TaskSchedulerImpl]
- when(sched.sc).thenReturn(sc)
- val yarnSchedulerBackend = new YarnSchedulerBackend(sched, sc) {
+ // Subclassing the TaskSchedulerImpl here instead of using Mockito. For details see SPARK-26891.
+ val sched = new TaskSchedulerImpl(sc) {
+ val blacklistedNodes = new AtomicReference[Set[String]]()
+
+ def setNodeBlacklist(nodeBlacklist: Set[String]): Unit = blacklistedNodes.set(nodeBlacklist)
+
+ override def nodeBlacklist(): Set[String] = blacklistedNodes.get()
+ }
+
+ val yarnSchedulerBackendExtended = new YarnSchedulerBackend(sched, sc) {
def setHostToLocalTaskCount(hostToLocalTaskCount: Map[String, Int]): Unit = {
this.hostToLocalTaskCount = hostToLocalTaskCount
}
}
+ yarnSchedulerBackend = yarnSchedulerBackendExtended
val ser = new JavaSerializer(sc.conf).newInstance()
for {
blacklist <- IndexedSeq(Set[String](), Set("a", "b", "c"))
@@ -45,16 +67,15 @@ class YarnSchedulerBackendSuite extends SparkFunSuite with MockitoSugar with Loc
Map("a" -> 1, "b" -> 2)
)
} {
- yarnSchedulerBackend.setHostToLocalTaskCount(hostToLocalCount)
- when(sched.nodeBlacklist()).thenReturn(blacklist)
- val req = yarnSchedulerBackend.prepareRequestExecutors(numRequested)
+ yarnSchedulerBackendExtended.setHostToLocalTaskCount(hostToLocalCount)
+ sched.setNodeBlacklist(blacklist)
+ val req = yarnSchedulerBackendExtended.prepareRequestExecutors(numRequested)
assert(req.requestedTotal === numRequested)
assert(req.nodeBlacklist === blacklist)
assert(req.hostToLocalTaskCount.keySet.intersect(blacklist).isEmpty)
// Serialize to make sure serialization doesn't throw an error
ser.serialize(req)
}
- sc.stop()
}
}
diff --git a/sbin/start-slave.sh b/sbin/start-slave.sh
index 8c268b8859155..4e57f749a7fb3 100755
--- a/sbin/start-slave.sh
+++ b/sbin/start-slave.sh
@@ -40,7 +40,7 @@ fi
CLASS="org.apache.spark.deploy.worker.Worker"
if [[ $# -lt 1 ]] || [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
- echo "Usage: ./sbin/start-slave.sh [options] "
+ echo "Usage: ./sbin/start-slave.sh [options]"
pattern="Usage:"
pattern+="\|Using Spark's default log4j profile:"
pattern+="\|Registered signal handlers for"
diff --git a/scalastyle-config.xml b/scalastyle-config.xml
index da5c3f29c32dc..866e93869eeb0 100644
--- a/scalastyle-config.xml
+++ b/scalastyle-config.xml
@@ -234,7 +234,7 @@ This file is divided into 3 sections:
scala.collection.JavaConverters._ and use .asScala / .asJava methods
-
+
org\.apache\.commons\.lang\. Use Commons Lang 3 classes (package org.apache.commons.lang3.*) instead
of Commons Lang 2 (package org.apache.commons.lang.*)
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 7d23637e28342..0864dd309394a 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 94283f59011a8..0792a7b7eff54 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -690,6 +690,7 @@ namedWindow
windowSpec
: name=identifier #windowRef
+ | '('name=identifier')' #windowRef
| '('
( CLUSTER BY partition+=expression (',' partition+=expression)*
| ((PARTITION | DISTRIBUTE) BY partition+=expression (',' partition+=expression)*)?
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
index ab13ac9cc5483..20bdd874fd5a0 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
@@ -79,7 +79,7 @@ public ExpressionInfo(
assert name != null;
assert arguments != null;
assert examples != null;
- assert examples.isEmpty() || examples.startsWith("\n Examples:");
+ assert examples.isEmpty() || examples.contains(" Examples:");
assert note != null;
assert since != null;
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
index 9e7b15d339eeb..d3a8da4f3145b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
@@ -17,20 +17,30 @@
package org.apache.spark.sql.catalyst.expressions;
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.KryoSerializable;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
import org.apache.spark.sql.catalyst.util.ArrayData;
import org.apache.spark.sql.types.*;
import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.array.ByteArrayMethods;
import org.apache.spark.unsafe.bitset.BitSetMethods;
import org.apache.spark.unsafe.hash.Murmur3_x86_32;
-import org.apache.spark.unsafe.memory.MemoryBlock;
import org.apache.spark.unsafe.types.CalendarInterval;
import org.apache.spark.unsafe.types.UTF8String;
+import static org.apache.spark.unsafe.Platform.BYTE_ARRAY_OFFSET;
+
/**
* An Unsafe implementation of Array which is backed by raw memory instead of Java objects.
*
@@ -53,7 +63,7 @@
* Instances of `UnsafeArrayData` act as pointers to row data stored in this format.
*/
-public final class UnsafeArrayData extends ArrayData {
+public final class UnsafeArrayData extends ArrayData implements Externalizable, KryoSerializable {
public static int calculateHeaderPortionInBytes(int numFields) {
return (int)calculateHeaderPortionInBytes((long)numFields);
@@ -241,8 +251,7 @@ public UTF8String getUTF8String(int ordinal) {
final long offsetAndSize = getLong(ordinal);
final int offset = (int) (offsetAndSize >> 32);
final int size = (int) offsetAndSize;
- MemoryBlock mb = MemoryBlock.allocateFromObject(baseObject, baseOffset + offset, size);
- return new UTF8String(mb);
+ return UTF8String.fromAddress(baseObject, baseOffset + offset, size);
}
@Override
@@ -525,4 +534,40 @@ public static UnsafeArrayData fromPrimitiveArray(float[] arr) {
public static UnsafeArrayData fromPrimitiveArray(double[] arr) {
return fromPrimitiveArray(arr, Platform.DOUBLE_ARRAY_OFFSET, arr.length, 8);
}
+
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+ byte[] bytes = UnsafeDataUtils.getBytes(baseObject, baseOffset, sizeInBytes);
+ out.writeInt(bytes.length);
+ out.writeInt(this.numElements);
+ out.write(bytes);
+ }
+
+ @Override
+ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ this.baseOffset = BYTE_ARRAY_OFFSET;
+ this.sizeInBytes = in.readInt();
+ this.numElements = in.readInt();
+ this.elementOffset = baseOffset + calculateHeaderPortionInBytes(this.numElements);
+ this.baseObject = new byte[sizeInBytes];
+ in.readFully((byte[]) baseObject);
+ }
+
+ @Override
+ public void write(Kryo kryo, Output output) {
+ byte[] bytes = UnsafeDataUtils.getBytes(baseObject, baseOffset, sizeInBytes);
+ output.writeInt(bytes.length);
+ output.writeInt(this.numElements);
+ output.write(bytes);
+ }
+
+ @Override
+ public void read(Kryo kryo, Input input) {
+ this.baseOffset = BYTE_ARRAY_OFFSET;
+ this.sizeInBytes = input.readInt();
+ this.numElements = input.readInt();
+ this.elementOffset = baseOffset + calculateHeaderPortionInBytes(this.numElements);
+ this.baseObject = new byte[sizeInBytes];
+ input.read((byte[]) baseObject);
+ }
}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java
new file mode 100644
index 0000000000000..9b600192ac250
--- /dev/null
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.catalyst.expressions;
+
+import org.apache.spark.unsafe.Platform;
+
+/**
+ * General utilities available for unsafe data
+ */
+final class UnsafeDataUtils {
+
+ private UnsafeDataUtils() {
+ }
+
+ public static byte[] getBytes(Object baseObject, long baseOffset, int sizeInBytes) {
+ if (baseObject instanceof byte[]
+ && baseOffset == Platform.BYTE_ARRAY_OFFSET
+ && (((byte[]) baseObject).length == sizeInBytes)) {
+ return (byte[]) baseObject;
+ }
+ byte[] bytes = new byte[sizeInBytes];
+ Platform.copyMemory(baseObject, baseOffset, bytes, Platform.BYTE_ARRAY_OFFSET,
+ sizeInBytes);
+ return bytes;
+ }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeMapData.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeMapData.java
index f17441dfccb6d..e07ac2fcf81b9 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeMapData.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeMapData.java
@@ -17,11 +17,22 @@
package org.apache.spark.sql.catalyst.expressions;
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
import java.nio.ByteBuffer;
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.KryoSerializable;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
import org.apache.spark.sql.catalyst.util.MapData;
import org.apache.spark.unsafe.Platform;
+import static org.apache.spark.unsafe.Platform.BYTE_ARRAY_OFFSET;
+
/**
* An Unsafe implementation of Map which is backed by raw memory instead of Java objects.
*
@@ -30,7 +41,7 @@
* [unsafe key array numBytes] [unsafe key array] [unsafe value array]
*/
// TODO: Use a more efficient format which doesn't depend on unsafe array.
-public final class UnsafeMapData extends MapData {
+public final class UnsafeMapData extends MapData implements Externalizable, KryoSerializable {
private Object baseObject;
private long baseOffset;
@@ -120,4 +131,36 @@ public UnsafeMapData copy() {
mapCopy.pointTo(mapDataCopy, Platform.BYTE_ARRAY_OFFSET, sizeInBytes);
return mapCopy;
}
+
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+ byte[] bytes = UnsafeDataUtils.getBytes(baseObject, baseOffset, sizeInBytes);
+ out.writeInt(bytes.length);
+ out.write(bytes);
+ }
+
+ @Override
+ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ this.baseOffset = BYTE_ARRAY_OFFSET;
+ this.sizeInBytes = in.readInt();
+ this.baseObject = new byte[sizeInBytes];
+ in.readFully((byte[]) baseObject);
+ pointTo(baseObject, baseOffset, sizeInBytes);
+ }
+
+ @Override
+ public void write(Kryo kryo, Output output) {
+ byte[] bytes = UnsafeDataUtils.getBytes(baseObject, baseOffset, sizeInBytes);
+ output.writeInt(bytes.length);
+ output.write(bytes);
+ }
+
+ @Override
+ public void read(Kryo kryo, Input input) {
+ this.baseOffset = BYTE_ARRAY_OFFSET;
+ this.sizeInBytes = input.readInt();
+ this.baseObject = new byte[sizeInBytes];
+ input.read((byte[]) baseObject);
+ pointTo(baseObject, baseOffset, sizeInBytes);
+ }
}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
index 469b0e60cc9a2..ee2b67a2d1885 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
@@ -37,7 +37,6 @@
import org.apache.spark.unsafe.array.ByteArrayMethods;
import org.apache.spark.unsafe.bitset.BitSetMethods;
import org.apache.spark.unsafe.hash.Murmur3_x86_32;
-import org.apache.spark.unsafe.memory.MemoryBlock;
import org.apache.spark.unsafe.types.CalendarInterval;
import org.apache.spark.unsafe.types.UTF8String;
@@ -417,8 +416,7 @@ public UTF8String getUTF8String(int ordinal) {
final long offsetAndSize = getLong(ordinal);
final int offset = (int) (offsetAndSize >> 32);
final int size = (int) offsetAndSize;
- MemoryBlock mb = MemoryBlock.allocateFromObject(baseObject, baseOffset + offset, size);
- return new UTF8String(mb);
+ return UTF8String.fromAddress(baseObject, baseOffset + offset, size);
}
@Override
@@ -588,14 +586,7 @@ public boolean equals(Object other) {
* Returns the underlying bytes for this UnsafeRow.
*/
public byte[] getBytes() {
- if (baseObject instanceof byte[] && baseOffset == Platform.BYTE_ARRAY_OFFSET
- && (((byte[]) baseObject).length == sizeInBytes)) {
- return (byte[]) baseObject;
- } else {
- byte[] bytes = new byte[sizeInBytes];
- Platform.copyMemory(baseObject, baseOffset, bytes, Platform.BYTE_ARRAY_OFFSET, sizeInBytes);
- return bytes;
- }
+ return UnsafeDataUtils.getBytes(baseObject, baseOffset, sizeInBytes);
}
// This is for debugging
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH64.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH64.java
index 8e9c0a2e9dc81..eb5051b284073 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH64.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH64.java
@@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.catalyst.expressions;
-import org.apache.spark.unsafe.memory.MemoryBlock;
+import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.types.UTF8String;
// scalastyle: off
@@ -72,13 +72,13 @@ public static long hashLong(long input, long seed) {
return fmix(hash);
}
- public long hashUnsafeWordsBlock(MemoryBlock mb) {
- return hashUnsafeWordsBlock(mb, seed);
+ public long hashUnsafeWords(Object base, long offset, int length) {
+ return hashUnsafeWords(base, offset, length, seed);
}
- public static long hashUnsafeWordsBlock(MemoryBlock mb, long seed) {
- assert (mb.size() % 8 == 0) : "lengthInBytes must be a multiple of 8 (word-aligned)";
- long hash = hashBytesByWordsBlock(mb, seed);
+ public static long hashUnsafeWords(Object base, long offset, int length, long seed) {
+ assert (length % 8 == 0) : "lengthInBytes must be a multiple of 8 (word-aligned)";
+ long hash = hashBytesByWords(base, offset, length, seed);
return fmix(hash);
}
@@ -86,22 +86,20 @@ public long hashUnsafeBytes(Object base, long offset, int length) {
return hashUnsafeBytes(base, offset, length, seed);
}
- public static long hashUnsafeBytesBlock(MemoryBlock mb, long seed) {
- long offset = 0;
- long length = mb.size();
+ public static long hashUnsafeBytes(Object base, long offset, int length, long seed) {
assert (length >= 0) : "lengthInBytes cannot be negative";
- long hash = hashBytesByWordsBlock(mb, seed);
+ long hash = hashBytesByWords(base, offset, length, seed);
long end = offset + length;
offset += length & -8;
if (offset + 4L <= end) {
- hash ^= (mb.getInt(offset) & 0xFFFFFFFFL) * PRIME64_1;
+ hash ^= (Platform.getInt(base, offset) & 0xFFFFFFFFL) * PRIME64_1;
hash = Long.rotateLeft(hash, 23) * PRIME64_2 + PRIME64_3;
offset += 4L;
}
while (offset < end) {
- hash ^= (mb.getByte(offset) & 0xFFL) * PRIME64_5;
+ hash ^= (Platform.getByte(base, offset) & 0xFFL) * PRIME64_5;
hash = Long.rotateLeft(hash, 11) * PRIME64_1;
offset++;
}
@@ -109,11 +107,7 @@ public static long hashUnsafeBytesBlock(MemoryBlock mb, long seed) {
}
public static long hashUTF8String(UTF8String str, long seed) {
- return hashUnsafeBytesBlock(str.getMemoryBlock(), seed);
- }
-
- public static long hashUnsafeBytes(Object base, long offset, int length, long seed) {
- return hashUnsafeBytesBlock(MemoryBlock.allocateFromObject(base, offset, length), seed);
+ return hashUnsafeBytes(str.getBaseObject(), str.getBaseOffset(), str.numBytes(), seed);
}
private static long fmix(long hash) {
@@ -125,31 +119,30 @@ private static long fmix(long hash) {
return hash;
}
- private static long hashBytesByWordsBlock(MemoryBlock mb, long seed) {
- long offset = 0;
- long length = mb.size();
+ private static long hashBytesByWords(Object base, long offset, int length, long seed) {
+ long end = offset + length;
long hash;
if (length >= 32) {
- long limit = length - 32;
+ long limit = end - 32;
long v1 = seed + PRIME64_1 + PRIME64_2;
long v2 = seed + PRIME64_2;
long v3 = seed;
long v4 = seed - PRIME64_1;
do {
- v1 += mb.getLong(offset) * PRIME64_2;
+ v1 += Platform.getLong(base, offset) * PRIME64_2;
v1 = Long.rotateLeft(v1, 31);
v1 *= PRIME64_1;
- v2 += mb.getLong(offset + 8) * PRIME64_2;
+ v2 += Platform.getLong(base, offset + 8) * PRIME64_2;
v2 = Long.rotateLeft(v2, 31);
v2 *= PRIME64_1;
- v3 += mb.getLong(offset + 16) * PRIME64_2;
+ v3 += Platform.getLong(base, offset + 16) * PRIME64_2;
v3 = Long.rotateLeft(v3, 31);
v3 *= PRIME64_1;
- v4 += mb.getLong(offset + 24) * PRIME64_2;
+ v4 += Platform.getLong(base, offset + 24) * PRIME64_2;
v4 = Long.rotateLeft(v4, 31);
v4 *= PRIME64_1;
@@ -190,9 +183,9 @@ private static long hashBytesByWordsBlock(MemoryBlock mb, long seed) {
hash += length;
- long limit = length - 8;
+ long limit = end - 8;
while (offset <= limit) {
- long k1 = mb.getLong(offset);
+ long k1 = Platform.getLong(base, offset);
hash ^= Long.rotateLeft(k1 * PRIME64_2, 31) * PRIME64_1;
hash = Long.rotateLeft(hash, 27) * PRIME64_1 + PRIME64_4;
offset += 8L;
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UTF8StringBuilder.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UTF8StringBuilder.java
index f8000d78cd1b6..f0f66bae245fd 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UTF8StringBuilder.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UTF8StringBuilder.java
@@ -19,8 +19,6 @@
import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.array.ByteArrayMethods;
-import org.apache.spark.unsafe.memory.ByteArrayMemoryBlock;
-import org.apache.spark.unsafe.memory.MemoryBlock;
import org.apache.spark.unsafe.types.UTF8String;
/**
@@ -31,34 +29,43 @@ public class UTF8StringBuilder {
private static final int ARRAY_MAX = ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH;
- private ByteArrayMemoryBlock buffer;
- private int length = 0;
+ private byte[] buffer;
+ private int cursor = Platform.BYTE_ARRAY_OFFSET;
public UTF8StringBuilder() {
// Since initial buffer size is 16 in `StringBuilder`, we set the same size here
- this.buffer = new ByteArrayMemoryBlock(16);
+ this.buffer = new byte[16];
}
// Grows the buffer by at least `neededSize`
private void grow(int neededSize) {
- if (neededSize > ARRAY_MAX - length) {
+ if (neededSize > ARRAY_MAX - totalSize()) {
throw new UnsupportedOperationException(
"Cannot grow internal buffer by size " + neededSize + " because the size after growing " +
"exceeds size limitation " + ARRAY_MAX);
}
- final int requestedSize = length + neededSize;
- if (buffer.size() < requestedSize) {
- int newLength = requestedSize < ARRAY_MAX / 2 ? requestedSize * 2 : ARRAY_MAX;
- final ByteArrayMemoryBlock tmp = new ByteArrayMemoryBlock(newLength);
- MemoryBlock.copyMemory(buffer, tmp, length);
+ final int length = totalSize() + neededSize;
+ if (buffer.length < length) {
+ int newLength = length < ARRAY_MAX / 2 ? length * 2 : ARRAY_MAX;
+ final byte[] tmp = new byte[newLength];
+ Platform.copyMemory(
+ buffer,
+ Platform.BYTE_ARRAY_OFFSET,
+ tmp,
+ Platform.BYTE_ARRAY_OFFSET,
+ totalSize());
buffer = tmp;
}
}
+ private int totalSize() {
+ return cursor - Platform.BYTE_ARRAY_OFFSET;
+ }
+
public void append(UTF8String value) {
grow(value.numBytes());
- value.writeToMemory(buffer.getByteArray(), length + Platform.BYTE_ARRAY_OFFSET);
- length += value.numBytes();
+ value.writeToMemory(buffer, cursor);
+ cursor += value.numBytes();
}
public void append(String value) {
@@ -66,6 +73,6 @@ public void append(String value) {
}
public UTF8String build() {
- return UTF8String.fromBytes(buffer.getByteArray(), 0, length);
+ return UTF8String.fromBytes(buffer, 0, totalSize());
}
}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriter.java
index 71c49d8ed0177..3960d6d520476 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriter.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriter.java
@@ -185,13 +185,13 @@ public void write(int ordinal, Decimal input, int precision, int scale) {
// grow the global buffer before writing data.
holder.grow(16);
+ // always zero-out the 16-byte buffer
+ Platform.putLong(getBuffer(), cursor(), 0L);
+ Platform.putLong(getBuffer(), cursor() + 8, 0L);
+
// Make sure Decimal object has the same scale as DecimalType.
// Note that we may pass in null Decimal object to set null for it.
if (input == null || !input.changePrecision(precision, scale)) {
- // zero-out the bytes
- Platform.putLong(getBuffer(), cursor(), 0L);
- Platform.putLong(getBuffer(), cursor() + 8, 0L);
-
BitSetMethods.set(getBuffer(), startingOffset, ordinal);
// keep the offset for future update
setOffsetAndSize(ordinal, 0);
@@ -200,8 +200,6 @@ public void write(int ordinal, Decimal input, int precision, int scale) {
final int numBytes = bytes.length;
assert numBytes <= 16;
- zeroOutPaddingBytes(numBytes);
-
// Write the bytes to the variable length portion.
Platform.copyMemory(
bytes, Platform.BYTE_ARRAY_OFFSET, getBuffer(), cursor(), numBytes);
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/execution/RecordBinaryComparator.java b/sql/catalyst/src/main/java/org/apache/spark/sql/execution/RecordBinaryComparator.java
index 40c2cc806e87a..1f243406c77e0 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/execution/RecordBinaryComparator.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/execution/RecordBinaryComparator.java
@@ -20,8 +20,13 @@
import org.apache.spark.unsafe.Platform;
import org.apache.spark.util.collection.unsafe.sort.RecordComparator;
+import java.nio.ByteOrder;
+
public final class RecordBinaryComparator extends RecordComparator {
+ private static final boolean LITTLE_ENDIAN =
+ ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
+
@Override
public int compare(
Object leftObj, long leftOff, int leftLen, Object rightObj, long rightOff, int rightLen) {
@@ -38,10 +43,10 @@ public int compare(
// check if stars align and we can get both offsets to be aligned
if ((leftOff % 8) == (rightOff % 8)) {
while ((leftOff + i) % 8 != 0 && i < leftLen) {
- final int v1 = Platform.getByte(leftObj, leftOff + i) & 0xff;
- final int v2 = Platform.getByte(rightObj, rightOff + i) & 0xff;
+ final int v1 = Platform.getByte(leftObj, leftOff + i);
+ final int v2 = Platform.getByte(rightObj, rightOff + i);
if (v1 != v2) {
- return v1 > v2 ? 1 : -1;
+ return (v1 & 0xff) > (v2 & 0xff) ? 1 : -1;
}
i += 1;
}
@@ -49,10 +54,17 @@ public int compare(
// for architectures that support unaligned accesses, chew it up 8 bytes at a time
if (Platform.unaligned() || (((leftOff + i) % 8 == 0) && ((rightOff + i) % 8 == 0))) {
while (i <= leftLen - 8) {
- final long v1 = Platform.getLong(leftObj, leftOff + i);
- final long v2 = Platform.getLong(rightObj, rightOff + i);
+ long v1 = Platform.getLong(leftObj, leftOff + i);
+ long v2 = Platform.getLong(rightObj, rightOff + i);
if (v1 != v2) {
- return v1 > v2 ? 1 : -1;
+ if (LITTLE_ENDIAN) {
+ // if read as little-endian, we have to reverse bytes so that the long comparison result
+ // is equivalent to byte-by-byte comparison result.
+ // See discussion in https://github.com/apache/spark/pull/26548#issuecomment-554645859
+ v1 = Long.reverseBytes(v1);
+ v2 = Long.reverseBytes(v2);
+ }
+ return Long.compareUnsigned(v1, v2);
}
i += 8;
}
@@ -60,10 +72,10 @@ public int compare(
// this will finish off the unaligned comparisons, or do the entire aligned comparison
// whichever is needed.
while (i < leftLen) {
- final int v1 = Platform.getByte(leftObj, leftOff + i) & 0xff;
- final int v2 = Platform.getByte(rightObj, rightOff + i) & 0xff;
+ final int v1 = Platform.getByte(leftObj, leftOff + i);
+ final int v2 = Platform.getByte(rightObj, rightOff + i);
if (v1 != v2) {
- return v1 > v2 ? 1 : -1;
+ return (v1 & 0xff) > (v2 & 0xff) ? 1 : -1;
}
i += 1;
}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java
index 1b2f5eee5ccdd..56995b4188301 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java
@@ -20,7 +20,6 @@
import java.io.IOException;
import java.util.function.Supplier;
-import scala.collection.AbstractIterator;
import scala.collection.Iterator;
import scala.math.Ordering;
@@ -53,6 +52,12 @@ public final class UnsafeExternalRowSorter {
private final PrefixComputer prefixComputer;
private final UnsafeExternalSorter sorter;
+ // This flag makes sure the cleanupResource() has been called. After the cleanup work,
+ // iterator.next should always return false. Downstream operator triggers the resource
+ // cleanup while they found there's no need to keep the iterator any more.
+ // See more details in SPARK-21492.
+ private boolean isReleased = false;
+
public abstract static class PrefixComputer {
public static class Prefix {
@@ -159,11 +164,12 @@ public long getSortTimeNanos() {
return sorter.getSortTimeNanos();
}
- private void cleanupResources() {
+ public void cleanupResources() {
+ isReleased = true;
sorter.cleanupResources();
}
- public Iterator sort() throws IOException {
+ public Iterator sort() throws IOException {
try {
final UnsafeSorterIterator sortedIterator = sorter.getSortedIterator();
if (!sortedIterator.hasNext()) {
@@ -171,31 +177,32 @@ public Iterator sort() throws IOException {
// here in order to prevent memory leaks.
cleanupResources();
}
- return new AbstractIterator() {
+ return new RowIterator() {
private final int numFields = schema.length();
private UnsafeRow row = new UnsafeRow(numFields);
@Override
- public boolean hasNext() {
- return sortedIterator.hasNext();
- }
-
- @Override
- public UnsafeRow next() {
+ public boolean advanceNext() {
try {
- sortedIterator.loadNext();
- row.pointTo(
- sortedIterator.getBaseObject(),
- sortedIterator.getBaseOffset(),
- sortedIterator.getRecordLength());
- if (!hasNext()) {
- UnsafeRow copy = row.copy(); // so that we don't have dangling pointers to freed page
- row = null; // so that we don't keep references to the base object
- cleanupResources();
- return copy;
+ if (!isReleased && sortedIterator.hasNext()) {
+ sortedIterator.loadNext();
+ row.pointTo(
+ sortedIterator.getBaseObject(),
+ sortedIterator.getBaseOffset(),
+ sortedIterator.getRecordLength());
+ // Here is the initial bug fix in SPARK-9364: the bug fix of use-after-free bug
+ // when returning the last row from an iterator. For example, in
+ // [[GroupedIterator]], we still use the last row after traversing the iterator
+ // in `fetchNextGroupIterator`
+ if (!sortedIterator.hasNext()) {
+ row = row.copy(); // so that we don't have dangling pointers to freed page
+ cleanupResources();
+ }
+ return true;
} else {
- return row;
+ row = null; // so that we don't keep references to the base object
+ return false;
}
} catch (IOException e) {
cleanupResources();
@@ -205,14 +212,18 @@ public UnsafeRow next() {
}
throw new RuntimeException("Exception should have been re-thrown in next()");
}
- };
+
+ @Override
+ public UnsafeRow getRow() { return row; }
+
+ }.toScala();
} catch (IOException e) {
cleanupResources();
throw e;
}
}
- public Iterator sort(Iterator inputIterator) throws IOException {
+ public Iterator sort(Iterator inputIterator) throws IOException {
while (inputIterator.hasNext()) {
insertRow(inputIterator.next());
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
index 3ecc137c8cd7f..7a226d72f5977 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
@@ -271,10 +271,9 @@ object JavaTypeInference {
case c if listType.isAssignableFrom(typeToken) =>
val et = elementType(typeToken)
- MapObjects(
+ UnresolvedMapObjects(
p => deserializerFor(et, Some(p)),
getPath,
- inferDataType(et)._1,
customCollectionCls = Some(c))
case _ if mapType.isAssignableFrom(typeToken) =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 0238d57de2446..858c33ac725b2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -19,8 +19,11 @@ package org.apache.spark.sql.catalyst
import java.lang.reflect.Constructor
+import scala.util.Properties
+
import org.apache.commons.lang3.reflect.ConstructorUtils
+import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.analysis.{GetColumnByOrdinal, UnresolvedAttribute, UnresolvedExtractValue}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.objects._
@@ -37,6 +40,9 @@ import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
trait DefinedByConstructorParams
+private[catalyst] object ScalaSubtypeLock
+
+
/**
* A default version of ScalaReflection that uses the runtime universe.
*/
@@ -65,19 +71,32 @@ object ScalaReflection extends ScalaReflection {
*/
def dataTypeFor[T : TypeTag]: DataType = dataTypeFor(localTypeOf[T])
+ /**
+ * Synchronize to prevent concurrent usage of `<:<` operator.
+ * This operator is not thread safe in any current version of scala; i.e.
+ * (2.11.12, 2.12.10, 2.13.0-M5).
+ *
+ * See https://github.com/scala/bug/issues/10766
+ */
+ private[catalyst] def isSubtype(tpe1: `Type`, tpe2: `Type`): Boolean = {
+ ScalaSubtypeLock.synchronized {
+ tpe1 <:< tpe2
+ }
+ }
+
private def dataTypeFor(tpe: `Type`): DataType = cleanUpReflectionObjects {
tpe.dealias match {
- case t if t <:< definitions.NullTpe => NullType
- case t if t <:< definitions.IntTpe => IntegerType
- case t if t <:< definitions.LongTpe => LongType
- case t if t <:< definitions.DoubleTpe => DoubleType
- case t if t <:< definitions.FloatTpe => FloatType
- case t if t <:< definitions.ShortTpe => ShortType
- case t if t <:< definitions.ByteTpe => ByteType
- case t if t <:< definitions.BooleanTpe => BooleanType
- case t if t <:< localTypeOf[Array[Byte]] => BinaryType
- case t if t <:< localTypeOf[CalendarInterval] => CalendarIntervalType
- case t if t <:< localTypeOf[Decimal] => DecimalType.SYSTEM_DEFAULT
+ case t if isSubtype(t, definitions.NullTpe) => NullType
+ case t if isSubtype(t, definitions.IntTpe) => IntegerType
+ case t if isSubtype(t, definitions.LongTpe) => LongType
+ case t if isSubtype(t, definitions.DoubleTpe) => DoubleType
+ case t if isSubtype(t, definitions.FloatTpe) => FloatType
+ case t if isSubtype(t, definitions.ShortTpe) => ShortType
+ case t if isSubtype(t, definitions.ByteTpe) => ByteType
+ case t if isSubtype(t, definitions.BooleanTpe) => BooleanType
+ case t if isSubtype(t, localTypeOf[Array[Byte]]) => BinaryType
+ case t if isSubtype(t, localTypeOf[CalendarInterval]) => CalendarIntervalType
+ case t if isSubtype(t, localTypeOf[Decimal]) => DecimalType.SYSTEM_DEFAULT
case _ =>
val className = getClassNameFromType(tpe)
className match {
@@ -100,13 +119,13 @@ object ScalaReflection extends ScalaReflection {
*/
private def arrayClassFor(tpe: `Type`): ObjectType = cleanUpReflectionObjects {
val cls = tpe.dealias match {
- case t if t <:< definitions.IntTpe => classOf[Array[Int]]
- case t if t <:< definitions.LongTpe => classOf[Array[Long]]
- case t if t <:< definitions.DoubleTpe => classOf[Array[Double]]
- case t if t <:< definitions.FloatTpe => classOf[Array[Float]]
- case t if t <:< definitions.ShortTpe => classOf[Array[Short]]
- case t if t <:< definitions.ByteTpe => classOf[Array[Byte]]
- case t if t <:< definitions.BooleanTpe => classOf[Array[Boolean]]
+ case t if isSubtype(t, definitions.IntTpe) => classOf[Array[Int]]
+ case t if isSubtype(t, definitions.LongTpe) => classOf[Array[Long]]
+ case t if isSubtype(t, definitions.DoubleTpe) => classOf[Array[Double]]
+ case t if isSubtype(t, definitions.FloatTpe) => classOf[Array[Float]]
+ case t if isSubtype(t, definitions.ShortTpe) => classOf[Array[Short]]
+ case t if isSubtype(t, definitions.ByteTpe) => classOf[Array[Byte]]
+ case t if isSubtype(t, definitions.BooleanTpe) => classOf[Array[Boolean]]
case other =>
// There is probably a better way to do this, but I couldn't find it...
val elementType = dataTypeFor(other).asInstanceOf[ObjectType].cls
@@ -207,48 +226,48 @@ object ScalaReflection extends ScalaReflection {
tpe.dealias match {
case t if !dataTypeFor(t).isInstanceOf[ObjectType] => getPath
- case t if t <:< localTypeOf[Option[_]] =>
+ case t if isSubtype(t, localTypeOf[Option[_]]) =>
val TypeRef(_, _, Seq(optType)) = t
val className = getClassNameFromType(optType)
val newTypePath = s"""- option value class: "$className"""" +: walkedTypePath
WrapOption(deserializerFor(optType, path, newTypePath), dataTypeFor(optType))
- case t if t <:< localTypeOf[java.lang.Integer] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Integer]) =>
val boxedType = classOf[java.lang.Integer]
val objectType = ObjectType(boxedType)
StaticInvoke(boxedType, objectType, "valueOf", getPath :: Nil, returnNullable = false)
- case t if t <:< localTypeOf[java.lang.Long] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Long]) =>
val boxedType = classOf[java.lang.Long]
val objectType = ObjectType(boxedType)
StaticInvoke(boxedType, objectType, "valueOf", getPath :: Nil, returnNullable = false)
- case t if t <:< localTypeOf[java.lang.Double] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Double]) =>
val boxedType = classOf[java.lang.Double]
val objectType = ObjectType(boxedType)
StaticInvoke(boxedType, objectType, "valueOf", getPath :: Nil, returnNullable = false)
- case t if t <:< localTypeOf[java.lang.Float] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Float]) =>
val boxedType = classOf[java.lang.Float]
val objectType = ObjectType(boxedType)
StaticInvoke(boxedType, objectType, "valueOf", getPath :: Nil, returnNullable = false)
- case t if t <:< localTypeOf[java.lang.Short] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Short]) =>
val boxedType = classOf[java.lang.Short]
val objectType = ObjectType(boxedType)
StaticInvoke(boxedType, objectType, "valueOf", getPath :: Nil, returnNullable = false)
- case t if t <:< localTypeOf[java.lang.Byte] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Byte]) =>
val boxedType = classOf[java.lang.Byte]
val objectType = ObjectType(boxedType)
StaticInvoke(boxedType, objectType, "valueOf", getPath :: Nil, returnNullable = false)
- case t if t <:< localTypeOf[java.lang.Boolean] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Boolean]) =>
val boxedType = classOf[java.lang.Boolean]
val objectType = ObjectType(boxedType)
StaticInvoke(boxedType, objectType, "valueOf", getPath :: Nil, returnNullable = false)
- case t if t <:< localTypeOf[java.sql.Date] =>
+ case t if isSubtype(t, localTypeOf[java.sql.Date]) =>
StaticInvoke(
DateTimeUtils.getClass,
ObjectType(classOf[java.sql.Date]),
@@ -256,7 +275,7 @@ object ScalaReflection extends ScalaReflection {
getPath :: Nil,
returnNullable = false)
- case t if t <:< localTypeOf[java.sql.Timestamp] =>
+ case t if isSubtype(t, localTypeOf[java.sql.Timestamp]) =>
StaticInvoke(
DateTimeUtils.getClass,
ObjectType(classOf[java.sql.Timestamp]),
@@ -264,25 +283,25 @@ object ScalaReflection extends ScalaReflection {
getPath :: Nil,
returnNullable = false)
- case t if t <:< localTypeOf[java.lang.String] =>
+ case t if isSubtype(t, localTypeOf[java.lang.String]) =>
Invoke(getPath, "toString", ObjectType(classOf[String]), returnNullable = false)
- case t if t <:< localTypeOf[java.math.BigDecimal] =>
+ case t if isSubtype(t, localTypeOf[java.math.BigDecimal]) =>
Invoke(getPath, "toJavaBigDecimal", ObjectType(classOf[java.math.BigDecimal]),
returnNullable = false)
- case t if t <:< localTypeOf[BigDecimal] =>
+ case t if isSubtype(t, localTypeOf[BigDecimal]) =>
Invoke(getPath, "toBigDecimal", ObjectType(classOf[BigDecimal]), returnNullable = false)
- case t if t <:< localTypeOf[java.math.BigInteger] =>
+ case t if isSubtype(t, localTypeOf[java.math.BigInteger]) =>
Invoke(getPath, "toJavaBigInteger", ObjectType(classOf[java.math.BigInteger]),
returnNullable = false)
- case t if t <:< localTypeOf[scala.math.BigInt] =>
+ case t if isSubtype(t, localTypeOf[scala.math.BigInt]) =>
Invoke(getPath, "toScalaBigInt", ObjectType(classOf[scala.math.BigInt]),
returnNullable = false)
- case t if t <:< localTypeOf[Array[_]] =>
+ case t if isSubtype(t, localTypeOf[Array[_]]) =>
val TypeRef(_, _, Seq(elementType)) = t
val Schema(dataType, elementNullable) = schemaFor(elementType)
val className = getClassNameFromType(elementType)
@@ -306,13 +325,13 @@ object ScalaReflection extends ScalaReflection {
Invoke(arrayData, "array", arrayCls, returnNullable = false)
} else {
val primitiveMethod = elementType match {
- case t if t <:< definitions.IntTpe => "toIntArray"
- case t if t <:< definitions.LongTpe => "toLongArray"
- case t if t <:< definitions.DoubleTpe => "toDoubleArray"
- case t if t <:< definitions.FloatTpe => "toFloatArray"
- case t if t <:< definitions.ShortTpe => "toShortArray"
- case t if t <:< definitions.ByteTpe => "toByteArray"
- case t if t <:< definitions.BooleanTpe => "toBooleanArray"
+ case t if isSubtype(t, definitions.IntTpe) => "toIntArray"
+ case t if isSubtype(t, definitions.LongTpe) => "toLongArray"
+ case t if isSubtype(t, definitions.DoubleTpe) => "toDoubleArray"
+ case t if isSubtype(t, definitions.FloatTpe) => "toFloatArray"
+ case t if isSubtype(t, definitions.ShortTpe) => "toShortArray"
+ case t if isSubtype(t, definitions.ByteTpe) => "toByteArray"
+ case t if isSubtype(t, definitions.BooleanTpe) => "toBooleanArray"
case other => throw new IllegalStateException("expect primitive array element type " +
"but got " + other)
}
@@ -321,8 +340,8 @@ object ScalaReflection extends ScalaReflection {
// We serialize a `Set` to Catalyst array. When we deserialize a Catalyst array
// to a `Set`, if there are duplicated elements, the elements will be de-duplicated.
- case t if t <:< localTypeOf[Seq[_]] ||
- t <:< localTypeOf[scala.collection.Set[_]] =>
+ case t if isSubtype(t, localTypeOf[Seq[_]]) ||
+ isSubtype(t, localTypeOf[scala.collection.Set[_]]) =>
val TypeRef(_, _, Seq(elementType)) = t
val Schema(dataType, elementNullable) = schemaFor(elementType)
val className = getClassNameFromType(elementType)
@@ -341,14 +360,14 @@ object ScalaReflection extends ScalaReflection {
val companion = t.dealias.typeSymbol.companion.typeSignature
val cls = companion.member(TermName("newBuilder")) match {
- case NoSymbol if t <:< localTypeOf[Seq[_]] => classOf[Seq[_]]
- case NoSymbol if t <:< localTypeOf[scala.collection.Set[_]] =>
+ case NoSymbol if isSubtype(t, localTypeOf[Seq[_]]) => classOf[Seq[_]]
+ case NoSymbol if isSubtype(t, localTypeOf[scala.collection.Set[_]]) =>
classOf[scala.collection.Set[_]]
case _ => mirror.runtimeClass(t.typeSymbol.asClass)
}
UnresolvedMapObjects(mapFunction, getPath, Some(cls))
- case t if t <:< localTypeOf[Map[_, _]] =>
+ case t if isSubtype(t, localTypeOf[Map[_, _]]) =>
// TODO: add walked type path for map
val TypeRef(_, _, Seq(keyType, valueType)) = t
@@ -483,7 +502,7 @@ object ScalaReflection extends ScalaReflection {
tpe.dealias match {
case _ if !inputObject.dataType.isInstanceOf[ObjectType] => inputObject
- case t if t <:< localTypeOf[Option[_]] =>
+ case t if isSubtype(t, localTypeOf[Option[_]]) =>
val TypeRef(_, _, Seq(optType)) = t
val className = getClassNameFromType(optType)
val newPath = s"""- option value class: "$className"""" +: walkedTypePath
@@ -493,15 +512,15 @@ object ScalaReflection extends ScalaReflection {
// Since List[_] also belongs to localTypeOf[Product], we put this case before
// "case t if definedByConstructorParams(t)" to make sure it will match to the
// case "localTypeOf[Seq[_]]"
- case t if t <:< localTypeOf[Seq[_]] =>
+ case t if isSubtype(t, localTypeOf[Seq[_]]) =>
val TypeRef(_, _, Seq(elementType)) = t
toCatalystArray(inputObject, elementType)
- case t if t <:< localTypeOf[Array[_]] =>
+ case t if isSubtype(t, localTypeOf[Array[_]]) =>
val TypeRef(_, _, Seq(elementType)) = t
toCatalystArray(inputObject, elementType)
- case t if t <:< localTypeOf[Map[_, _]] =>
+ case t if isSubtype(t, localTypeOf[Map[_, _]]) =>
val TypeRef(_, _, Seq(keyType, valueType)) = t
val keyClsName = getClassNameFromType(keyType)
val valueClsName = getClassNameFromType(valueType)
@@ -517,7 +536,7 @@ object ScalaReflection extends ScalaReflection {
serializerFor(_, valueType, valuePath, seenTypeSet),
valueNullable = !valueType.typeSymbol.asClass.isPrimitive)
- case t if t <:< localTypeOf[scala.collection.Set[_]] =>
+ case t if isSubtype(t, localTypeOf[scala.collection.Set[_]]) =>
val TypeRef(_, _, Seq(elementType)) = t
// There's no corresponding Catalyst type for `Set`, we serialize a `Set` to Catalyst array.
@@ -530,7 +549,7 @@ object ScalaReflection extends ScalaReflection {
toCatalystArray(newInput, elementType)
- case t if t <:< localTypeOf[String] =>
+ case t if isSubtype(t, localTypeOf[String]) =>
StaticInvoke(
classOf[UTF8String],
StringType,
@@ -538,7 +557,7 @@ object ScalaReflection extends ScalaReflection {
inputObject :: Nil,
returnNullable = false)
- case t if t <:< localTypeOf[java.sql.Timestamp] =>
+ case t if isSubtype(t, localTypeOf[java.sql.Timestamp]) =>
StaticInvoke(
DateTimeUtils.getClass,
TimestampType,
@@ -546,7 +565,7 @@ object ScalaReflection extends ScalaReflection {
inputObject :: Nil,
returnNullable = false)
- case t if t <:< localTypeOf[java.sql.Date] =>
+ case t if isSubtype(t, localTypeOf[java.sql.Date]) =>
StaticInvoke(
DateTimeUtils.getClass,
DateType,
@@ -554,7 +573,7 @@ object ScalaReflection extends ScalaReflection {
inputObject :: Nil,
returnNullable = false)
- case t if t <:< localTypeOf[BigDecimal] =>
+ case t if isSubtype(t, localTypeOf[BigDecimal]) =>
StaticInvoke(
Decimal.getClass,
DecimalType.SYSTEM_DEFAULT,
@@ -562,7 +581,7 @@ object ScalaReflection extends ScalaReflection {
inputObject :: Nil,
returnNullable = false)
- case t if t <:< localTypeOf[java.math.BigDecimal] =>
+ case t if isSubtype(t, localTypeOf[java.math.BigDecimal]) =>
StaticInvoke(
Decimal.getClass,
DecimalType.SYSTEM_DEFAULT,
@@ -570,7 +589,7 @@ object ScalaReflection extends ScalaReflection {
inputObject :: Nil,
returnNullable = false)
- case t if t <:< localTypeOf[java.math.BigInteger] =>
+ case t if isSubtype(t, localTypeOf[java.math.BigInteger]) =>
StaticInvoke(
Decimal.getClass,
DecimalType.BigIntDecimal,
@@ -578,7 +597,7 @@ object ScalaReflection extends ScalaReflection {
inputObject :: Nil,
returnNullable = false)
- case t if t <:< localTypeOf[scala.math.BigInt] =>
+ case t if isSubtype(t, localTypeOf[scala.math.BigInt]) =>
StaticInvoke(
Decimal.getClass,
DecimalType.BigIntDecimal,
@@ -586,19 +605,19 @@ object ScalaReflection extends ScalaReflection {
inputObject :: Nil,
returnNullable = false)
- case t if t <:< localTypeOf[java.lang.Integer] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Integer]) =>
Invoke(inputObject, "intValue", IntegerType)
- case t if t <:< localTypeOf[java.lang.Long] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Long]) =>
Invoke(inputObject, "longValue", LongType)
- case t if t <:< localTypeOf[java.lang.Double] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Double]) =>
Invoke(inputObject, "doubleValue", DoubleType)
- case t if t <:< localTypeOf[java.lang.Float] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Float]) =>
Invoke(inputObject, "floatValue", FloatType)
- case t if t <:< localTypeOf[java.lang.Short] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Short]) =>
Invoke(inputObject, "shortValue", ShortType)
- case t if t <:< localTypeOf[java.lang.Byte] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Byte]) =>
Invoke(inputObject, "byteValue", ByteType)
- case t if t <:< localTypeOf[java.lang.Boolean] =>
+ case t if isSubtype(t, localTypeOf[java.lang.Boolean]) =>
Invoke(inputObject, "booleanValue", BooleanType)
case t if t.typeSymbol.annotations.exists(_.tree.tpe =:= typeOf[SQLUserDefinedType]) =>
@@ -655,7 +674,7 @@ object ScalaReflection extends ScalaReflection {
*/
def optionOfProductType(tpe: `Type`): Boolean = cleanUpReflectionObjects {
tpe.dealias match {
- case t if t <:< localTypeOf[Option[_]] =>
+ case t if isSubtype(t, localTypeOf[Option[_]]) =>
val TypeRef(_, _, Seq(optType)) = t
definedByConstructorParams(optType)
case _ => false
@@ -721,7 +740,7 @@ object ScalaReflection extends ScalaReflection {
tpe.dealias match {
// this must be the first case, since all objects in scala are instances of Null, therefore
// Null type would wrongly match the first of them, which is Option as of now
- case t if t <:< definitions.NullTpe => Schema(NullType, nullable = true)
+ case t if isSubtype(t, definitions.NullTpe) => Schema(NullType, nullable = true)
case t if t.typeSymbol.annotations.exists(_.tree.tpe =:= typeOf[SQLUserDefinedType]) =>
val udt = getClassFromType(t).getAnnotation(classOf[SQLUserDefinedType]).udt().newInstance()
Schema(udt, nullable = true)
@@ -729,52 +748,56 @@ object ScalaReflection extends ScalaReflection {
val udt = UDTRegistration.getUDTFor(getClassNameFromType(t)).get.newInstance()
.asInstanceOf[UserDefinedType[_]]
Schema(udt, nullable = true)
- case t if t <:< localTypeOf[Option[_]] =>
+ case t if isSubtype(t, localTypeOf[Option[_]]) =>
val TypeRef(_, _, Seq(optType)) = t
Schema(schemaFor(optType).dataType, nullable = true)
- case t if t <:< localTypeOf[Array[Byte]] => Schema(BinaryType, nullable = true)
- case t if t <:< localTypeOf[Array[_]] =>
+ case t if isSubtype(t, localTypeOf[Array[Byte]]) => Schema(BinaryType, nullable = true)
+ case t if isSubtype(t, localTypeOf[Array[_]]) =>
val TypeRef(_, _, Seq(elementType)) = t
val Schema(dataType, nullable) = schemaFor(elementType)
Schema(ArrayType(dataType, containsNull = nullable), nullable = true)
- case t if t <:< localTypeOf[Seq[_]] =>
+ case t if isSubtype(t, localTypeOf[Seq[_]]) =>
val TypeRef(_, _, Seq(elementType)) = t
val Schema(dataType, nullable) = schemaFor(elementType)
Schema(ArrayType(dataType, containsNull = nullable), nullable = true)
- case t if t <:< localTypeOf[Map[_, _]] =>
+ case t if isSubtype(t, localTypeOf[Map[_, _]]) =>
val TypeRef(_, _, Seq(keyType, valueType)) = t
val Schema(valueDataType, valueNullable) = schemaFor(valueType)
Schema(MapType(schemaFor(keyType).dataType,
valueDataType, valueContainsNull = valueNullable), nullable = true)
- case t if t <:< localTypeOf[Set[_]] =>
+ case t if isSubtype(t, localTypeOf[Set[_]]) =>
val TypeRef(_, _, Seq(elementType)) = t
val Schema(dataType, nullable) = schemaFor(elementType)
Schema(ArrayType(dataType, containsNull = nullable), nullable = true)
- case t if t <:< localTypeOf[String] => Schema(StringType, nullable = true)
- case t if t <:< localTypeOf[java.sql.Timestamp] => Schema(TimestampType, nullable = true)
- case t if t <:< localTypeOf[java.sql.Date] => Schema(DateType, nullable = true)
- case t if t <:< localTypeOf[BigDecimal] => Schema(DecimalType.SYSTEM_DEFAULT, nullable = true)
- case t if t <:< localTypeOf[java.math.BigDecimal] =>
+ case t if isSubtype(t, localTypeOf[String]) => Schema(StringType, nullable = true)
+ case t if isSubtype(t, localTypeOf[java.sql.Timestamp]) =>
+ Schema(TimestampType, nullable = true)
+ case t if isSubtype(t, localTypeOf[java.sql.Date]) =>
+ Schema(DateType, nullable = true)
+ case t if isSubtype(t, localTypeOf[BigDecimal]) =>
Schema(DecimalType.SYSTEM_DEFAULT, nullable = true)
- case t if t <:< localTypeOf[java.math.BigInteger] =>
+ case t if isSubtype(t, localTypeOf[java.math.BigDecimal]) =>
+ Schema(DecimalType.SYSTEM_DEFAULT, nullable = true)
+ case t if isSubtype(t, localTypeOf[java.math.BigInteger]) =>
Schema(DecimalType.BigIntDecimal, nullable = true)
- case t if t <:< localTypeOf[scala.math.BigInt] =>
+ case t if isSubtype(t, localTypeOf[scala.math.BigInt]) =>
Schema(DecimalType.BigIntDecimal, nullable = true)
- case t if t <:< localTypeOf[Decimal] => Schema(DecimalType.SYSTEM_DEFAULT, nullable = true)
- case t if t <:< localTypeOf[java.lang.Integer] => Schema(IntegerType, nullable = true)
- case t if t <:< localTypeOf[java.lang.Long] => Schema(LongType, nullable = true)
- case t if t <:< localTypeOf[java.lang.Double] => Schema(DoubleType, nullable = true)
- case t if t <:< localTypeOf[java.lang.Float] => Schema(FloatType, nullable = true)
- case t if t <:< localTypeOf[java.lang.Short] => Schema(ShortType, nullable = true)
- case t if t <:< localTypeOf[java.lang.Byte] => Schema(ByteType, nullable = true)
- case t if t <:< localTypeOf[java.lang.Boolean] => Schema(BooleanType, nullable = true)
- case t if t <:< definitions.IntTpe => Schema(IntegerType, nullable = false)
- case t if t <:< definitions.LongTpe => Schema(LongType, nullable = false)
- case t if t <:< definitions.DoubleTpe => Schema(DoubleType, nullable = false)
- case t if t <:< definitions.FloatTpe => Schema(FloatType, nullable = false)
- case t if t <:< definitions.ShortTpe => Schema(ShortType, nullable = false)
- case t if t <:< definitions.ByteTpe => Schema(ByteType, nullable = false)
- case t if t <:< definitions.BooleanTpe => Schema(BooleanType, nullable = false)
+ case t if isSubtype(t, localTypeOf[Decimal]) =>
+ Schema(DecimalType.SYSTEM_DEFAULT, nullable = true)
+ case t if isSubtype(t, localTypeOf[java.lang.Integer]) => Schema(IntegerType, nullable = true)
+ case t if isSubtype(t, localTypeOf[java.lang.Long]) => Schema(LongType, nullable = true)
+ case t if isSubtype(t, localTypeOf[java.lang.Double]) => Schema(DoubleType, nullable = true)
+ case t if isSubtype(t, localTypeOf[java.lang.Float]) => Schema(FloatType, nullable = true)
+ case t if isSubtype(t, localTypeOf[java.lang.Short]) => Schema(ShortType, nullable = true)
+ case t if isSubtype(t, localTypeOf[java.lang.Byte]) => Schema(ByteType, nullable = true)
+ case t if isSubtype(t, localTypeOf[java.lang.Boolean]) => Schema(BooleanType, nullable = true)
+ case t if isSubtype(t, definitions.IntTpe) => Schema(IntegerType, nullable = false)
+ case t if isSubtype(t, definitions.LongTpe) => Schema(LongType, nullable = false)
+ case t if isSubtype(t, definitions.DoubleTpe) => Schema(DoubleType, nullable = false)
+ case t if isSubtype(t, definitions.FloatTpe) => Schema(FloatType, nullable = false)
+ case t if isSubtype(t, definitions.ShortTpe) => Schema(ShortType, nullable = false)
+ case t if isSubtype(t, definitions.ByteTpe) => Schema(ByteType, nullable = false)
+ case t if isSubtype(t, definitions.BooleanTpe) => Schema(BooleanType, nullable = false)
case t if definedByConstructorParams(t) =>
val params = getConstructorParameters(t)
Schema(StructType(
@@ -802,9 +825,9 @@ object ScalaReflection extends ScalaReflection {
def definedByConstructorParams(tpe: Type): Boolean = cleanUpReflectionObjects {
tpe.dealias match {
// `Option` is a `Product`, but we don't wanna treat `Option[Int]` as a struct type.
- case t if t <:< localTypeOf[Option[_]] => definedByConstructorParams(t.typeArgs.head)
- case _ => tpe.dealias <:< localTypeOf[Product] ||
- tpe.dealias <:< localTypeOf[DefinedByConstructorParams]
+ case t if isSubtype(t, localTypeOf[Option[_]]) => definedByConstructorParams(t.typeArgs.head)
+ case _ => isSubtype(tpe.dealias, localTypeOf[Product]) ||
+ isSubtype(tpe.dealias, localTypeOf[DefinedByConstructorParams])
}
}
@@ -879,7 +902,7 @@ object ScalaReflection extends ScalaReflection {
* Support for generating catalyst schemas for scala objects. Note that unlike its companion
* object, this trait able to work in both the runtime and the compile time (macro) universe.
*/
-trait ScalaReflection {
+trait ScalaReflection extends Logging {
/** The universe we work in (runtime or macro) */
val universe: scala.reflect.api.Universe
@@ -932,6 +955,23 @@ trait ScalaReflection {
tpe.dealias.erasure.typeSymbol.asClass.fullName
}
+ /**
+ * Returns the nullability of the input parameter types of the scala function object.
+ *
+ * Note that this only works with Scala 2.11, and the information returned may be inaccurate if
+ * used with a different Scala version.
+ */
+ def getParameterTypeNullability(func: AnyRef): Seq[Boolean] = {
+ if (!Properties.versionString.contains("2.11")) {
+ logWarning(s"Scala ${Properties.versionString} cannot get type nullability correctly via " +
+ "reflection, thus Spark cannot add proper input null check for UDF. To avoid this " +
+ "problem, use the typed UDF interfaces instead.")
+ }
+ val methods = func.getClass.getMethods.filter(m => m.getName == "apply" && !m.isBridge)
+ assert(methods.length == 1)
+ methods.head.getParameterTypes.map(!_.isPrimitive)
+ }
+
/**
* Returns the parameter names and types for the primary constructor of this type.
*
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 580133dd971b1..f8b95139cab33 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -189,8 +189,6 @@ class Analyzer(
TypeCoercion.typeCoercionRules(conf) ++
extendedResolutionRules : _*),
Batch("Post-Hoc Resolution", Once, postHocResolutionRules: _*),
- Batch("View", Once,
- AliasViewChild(conf)),
Batch("Nondeterministic", Once,
PullOutNondeterministic),
Batch("UDF", Once,
@@ -554,8 +552,10 @@ class Analyzer(
Cast(value, pivotColumn.dataType, Some(conf.sessionLocalTimeZone)).eval(EmptyRow)
}
// Group-by expressions coming from SQL are implicit and need to be deduced.
- val groupByExprs = groupByExprsOpt.getOrElse(
- (child.outputSet -- aggregates.flatMap(_.references) -- pivotColumn.references).toSeq)
+ val groupByExprs = groupByExprsOpt.getOrElse {
+ val pivotColAndAggRefs = pivotColumn.references ++ AttributeSet(aggregates)
+ child.output.filterNot(pivotColAndAggRefs.contains)
+ }
val singleAgg = aggregates.size == 1
def outputName(value: Expression, aggregate: Expression): String = {
val stringValue = value match {
@@ -792,6 +792,18 @@ class Analyzer(
val newOutput = oldVersion.generatorOutput.map(_.newInstance())
(oldVersion, oldVersion.copy(generatorOutput = newOutput))
+ case oldVersion: Expand
+ if oldVersion.producedAttributes.intersect(conflictingAttributes).nonEmpty =>
+ val producedAttributes = oldVersion.producedAttributes
+ val newOutput = oldVersion.output.map { attr =>
+ if (producedAttributes.contains(attr)) {
+ attr.newInstance()
+ } else {
+ attr
+ }
+ }
+ (oldVersion, oldVersion.copy(output = newOutput))
+
case oldVersion @ Window(windowExpressions, _, _, child)
if AttributeSet(windowExpressions.map(_.toAttribute)).intersect(conflictingAttributes)
.nonEmpty =>
@@ -822,7 +834,8 @@ class Analyzer(
}
private def dedupAttr(attr: Attribute, attrMap: AttributeMap[Attribute]): Attribute = {
- attrMap.get(attr).getOrElse(attr).withQualifier(attr.qualifier)
+ val exprId = attrMap.getOrElse(attr, attr).exprId
+ attr.withExprId(exprId)
}
/**
@@ -868,7 +881,7 @@ class Analyzer(
private def dedupOuterReferencesInSubquery(
plan: LogicalPlan,
attrMap: AttributeMap[Attribute]): LogicalPlan = {
- plan resolveOperatorsDown { case currentFragment =>
+ plan transformDown { case currentFragment =>
currentFragment transformExpressions {
case OuterReference(a: Attribute) =>
OuterReference(dedupAttr(a, attrMap))
@@ -1045,7 +1058,7 @@ class Analyzer(
// support CURRENT_DATE and CURRENT_TIMESTAMP
val literalFunctions = Seq(CurrentDate(), CurrentTimestamp())
val name = nameParts.head
- val func = literalFunctions.find(e => resolver(e.prettyName, name))
+ val func = literalFunctions.find(e => caseInsensitiveResolution(e.prettyName, name))
func.map(wrapper)
}
@@ -2149,34 +2162,27 @@ class Analyzer(
case p => p transformExpressionsUp {
- case udf@ScalaUDF(func, _, inputs, _, _, _, _, nullableTypes) =>
- if (nullableTypes.isEmpty) {
- // If no nullability info is available, do nothing. No fields will be specially
- // checked for null in the plan. If nullability info is incorrect, the results
- // of the UDF could be wrong.
- udf
- } else {
- // Otherwise, add special handling of null for fields that can't accept null.
- // The result of operations like this, when passed null, is generally to return null.
- assert(nullableTypes.length == inputs.length)
-
- // TODO: skip null handling for not-nullable primitive inputs after we can completely
- // trust the `nullable` information.
- val inputsNullCheck = nullableTypes.zip(inputs)
- .filter { case (nullable, _) => !nullable }
- .map { case (_, expr) => IsNull(expr) }
- .reduceLeftOption[Expression]((e1, e2) => Or(e1, e2))
- // Once we add an `If` check above the udf, it is safe to mark those checked inputs
- // as not nullable (i.e., wrap them with `KnownNotNull`), because the null-returning
- // branch of `If` will be called if any of these checked inputs is null. Thus we can
- // prevent this rule from being applied repeatedly.
- val newInputs = nullableTypes.zip(inputs).map { case (nullable, expr) =>
- if (nullable) expr else KnownNotNull(expr)
- }
- inputsNullCheck
- .map(If(_, Literal.create(null, udf.dataType), udf.copy(children = newInputs)))
- .getOrElse(udf)
- }
+ case udf @ ScalaUDF(_, _, inputs, inputsNullSafe, _, _, _, _)
+ if inputsNullSafe.contains(false) =>
+ // Otherwise, add special handling of null for fields that can't accept null.
+ // The result of operations like this, when passed null, is generally to return null.
+ assert(inputsNullSafe.length == inputs.length)
+
+ // TODO: skip null handling for not-nullable primitive inputs after we can completely
+ // trust the `nullable` information.
+ val inputsNullCheck = inputsNullSafe.zip(inputs)
+ .filter { case (nullSafe, _) => !nullSafe }
+ .map { case (_, expr) => IsNull(expr) }
+ .reduceLeftOption[Expression]((e1, e2) => Or(e1, e2))
+ // Once we add an `If` check above the udf, it is safe to mark those checked inputs
+ // as null-safe (i.e., set `inputsNullSafe` all `true`), because the null-returning
+ // branch of `If` will be called if any of these checked inputs is null. Thus we can
+ // prevent this rule from being applied repeatedly.
+ val newInputsNullSafe = inputsNullSafe.map(_ => true)
+ inputsNullCheck
+ .map(If(_, Literal.create(null, udf.dataType),
+ udf.copy(inputsNullSafe = newInputsNullSafe)))
+ .getOrElse(udf)
}
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 6a91d556b2f3e..6fb66ca6819a2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
import org.apache.spark.sql.catalyst.optimizer.BooleanSimplification
import org.apache.spark.sql.catalyst.plans._
import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
/**
@@ -300,6 +301,48 @@ trait CheckAnalysis extends PredicateHelper {
}
}
+ // If the view output doesn't have the same number of columns neither with the child
+ // output, nor with the query column names, throw an AnalysisException.
+ // If the view's child output can't up cast to the view output,
+ // throw an AnalysisException, too.
+ case v @ View(desc, output, child) if child.resolved && output != child.output =>
+ val queryColumnNames = desc.viewQueryColumnNames
+ val queryOutput = if (queryColumnNames.nonEmpty) {
+ if (output.length != queryColumnNames.length) {
+ // If the view output doesn't have the same number of columns with the query column
+ // names, throw an AnalysisException.
+ throw new AnalysisException(
+ s"The view output ${output.mkString("[", ",", "]")} doesn't have the same" +
+ "number of columns with the query column names " +
+ s"${queryColumnNames.mkString("[", ",", "]")}")
+ }
+ val resolver = SQLConf.get.resolver
+ queryColumnNames.map { colName =>
+ child.output.find { attr =>
+ resolver(attr.name, colName)
+ }.getOrElse(throw new AnalysisException(
+ s"Attribute with name '$colName' is not found in " +
+ s"'${child.output.map(_.name).mkString("(", ",", ")")}'"))
+ }
+ } else {
+ child.output
+ }
+
+ output.zip(queryOutput).foreach {
+ case (attr, originAttr) if !attr.dataType.sameType(originAttr.dataType) =>
+ // The dataType of the output attributes may be not the same with that of the view
+ // output, so we should cast the attribute to the dataType of the view output
+ // attribute. Will throw an AnalysisException if the cast can't be performed or
+ // might truncate.
+ if (Cast.mayTruncate(originAttr.dataType, attr.dataType) ||
+ !Cast.canCast(originAttr.dataType, attr.dataType)) {
+ throw new AnalysisException(s"Cannot up cast ${originAttr.sql} from " +
+ s"${originAttr.dataType.catalogString} to ${attr.dataType.catalogString} " +
+ "as it may truncate\n")
+ }
+ case _ =>
+ }
+
case _ => // Fallbacks to the following checks
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
index e511f8064e28a..82692334544e2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
@@ -290,11 +290,13 @@ object DecimalPrecision extends TypeCoercionRule {
// potentially loosing 11 digits of the fractional part. Using only the precision needed
// by the Literal, instead, the result would be DECIMAL(38 + 1 + 1, 18), which would
// become DECIMAL(38, 16), safely having a much lower precision loss.
- case (l: Literal, r) if r.dataType.isInstanceOf[DecimalType]
- && l.dataType.isInstanceOf[IntegralType] =>
+ case (l: Literal, r) if r.dataType.isInstanceOf[DecimalType] &&
+ l.dataType.isInstanceOf[IntegralType] &&
+ SQLConf.get.literalPickMinimumPrecision =>
b.makeCopy(Array(Cast(l, DecimalType.fromLiteral(l)), r))
- case (l, r: Literal) if l.dataType.isInstanceOf[DecimalType]
- && r.dataType.isInstanceOf[IntegralType] =>
+ case (l, r: Literal) if l.dataType.isInstanceOf[DecimalType] &&
+ r.dataType.isInstanceOf[IntegralType] &&
+ SQLConf.get.literalPickMinimumPrecision =>
b.makeCopy(Array(l, Cast(r, DecimalType.fromLiteral(r))))
// Promote integers inside a binary expression with fixed-precision decimals to decimals,
// and fixed-precision decimals in an expression with floats / doubles to doubles
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 77860e1584f42..efd760c785b40 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -299,15 +299,6 @@ object FunctionRegistry {
expression[CollectList]("collect_list"),
expression[CollectSet]("collect_set"),
expression[CountMinSketchAgg]("count_min_sketch"),
- expression[RegrCount]("regr_count"),
- expression[RegrSXX]("regr_sxx"),
- expression[RegrSYY]("regr_syy"),
- expression[RegrAvgX]("regr_avgx"),
- expression[RegrAvgY]("regr_avgy"),
- expression[RegrSXY]("regr_sxy"),
- expression[RegrSlope]("regr_slope"),
- expression[RegrR2]("regr_r2"),
- expression[RegrIntercept]("regr_intercept"),
// string functions
expression[Ascii]("ascii"),
@@ -423,7 +414,6 @@ object FunctionRegistry {
expression[MapFromArrays]("map_from_arrays"),
expression[MapKeys]("map_keys"),
expression[MapValues]("map_values"),
- expression[MapEntries]("map_entries"),
expression[MapFromEntries]("map_from_entries"),
expression[MapConcat]("map_concat"),
expression[Size]("size"),
@@ -442,13 +432,9 @@ object FunctionRegistry {
expression[ArrayRemove]("array_remove"),
expression[ArrayDistinct]("array_distinct"),
expression[ArrayTransform]("transform"),
- expression[MapFilter]("map_filter"),
expression[ArrayFilter]("filter"),
expression[ArrayExists]("exists"),
expression[ArrayAggregate]("aggregate"),
- expression[TransformValues]("transform_values"),
- expression[TransformKeys]("transform_keys"),
- expression[MapZipWith]("map_zip_with"),
expression[ZipWith]("zip_with"),
CreateStruct.registryEntry,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index 288b6358fbff1..1659156b86956 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -54,13 +54,12 @@ object TypeCoercion {
BooleanEquality ::
FunctionArgumentConversion ::
ConcatCoercion(conf) ::
- MapZipWithCoercion ::
EltCoercion(conf) ::
CaseWhenCoercion ::
IfCoercion ::
StackCoercion ::
Division ::
- new ImplicitTypeCasts(conf) ::
+ ImplicitTypeCasts ::
DateTimeOperations ::
WindowFrameCoercion ::
Nil
@@ -763,30 +762,6 @@ object TypeCoercion {
}
}
- /**
- * Coerces key types of two different [[MapType]] arguments of the [[MapZipWith]] expression
- * to a common type.
- */
- object MapZipWithCoercion extends TypeCoercionRule {
- override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
- // Lambda function isn't resolved when the rule is executed.
- case m @ MapZipWith(left, right, function) if m.arguments.forall(a => a.resolved &&
- MapType.acceptsType(a.dataType)) && !m.leftKeyType.sameType(m.rightKeyType) =>
- findWiderTypeForTwo(m.leftKeyType, m.rightKeyType) match {
- case Some(finalKeyType) if !Cast.forceNullable(m.leftKeyType, finalKeyType) &&
- !Cast.forceNullable(m.rightKeyType, finalKeyType) =>
- val newLeft = castIfNotSameType(
- left,
- MapType(finalKeyType, m.leftValueType, m.leftValueContainsNull))
- val newRight = castIfNotSameType(
- right,
- MapType(finalKeyType, m.rightValueType, m.rightValueContainsNull))
- MapZipWith(newLeft, newRight, function)
- case _ => m
- }
- }
- }
-
/**
* Coerces the types of [[Elt]] children to expected ones.
*
@@ -841,33 +816,12 @@ object TypeCoercion {
/**
* Casts types according to the expected input types for [[Expression]]s.
*/
- class ImplicitTypeCasts(conf: SQLConf) extends TypeCoercionRule {
-
- private def rejectTzInString = conf.getConf(SQLConf.REJECT_TIMEZONE_IN_STRING)
-
+ object ImplicitTypeCasts extends TypeCoercionRule {
override protected def coerceTypes(
plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
// Skip nodes who's children have not been resolved yet.
case e if !e.childrenResolved => e
- // Special rules for `from/to_utc_timestamp`. These 2 functions assume the input timestamp
- // string is in a specific timezone, so the string itself should not contain timezone.
- // TODO: We should move the type coercion logic to expressions instead of a central
- // place to put all the rules.
- case e: FromUTCTimestamp if e.left.dataType == StringType =>
- if (rejectTzInString) {
- e.copy(left = StringToTimestampWithoutTimezone(e.left))
- } else {
- e.copy(left = Cast(e.left, TimestampType))
- }
-
- case e: ToUTCTimestamp if e.left.dataType == StringType =>
- if (rejectTzInString) {
- e.copy(left = StringToTimestampWithoutTimezone(e.left))
- } else {
- e.copy(left = Cast(e.left, TimestampType))
- }
-
case b @ BinaryOperator(left, right) if left.dataType != right.dataType =>
findTightestCommonType(left.dataType, right.dataType).map { commonType =>
if (b.inputType.acceptsType(commonType)) {
@@ -884,7 +838,7 @@ object TypeCoercion {
case e: ImplicitCastInputTypes if e.inputTypes.nonEmpty =>
val children: Seq[Expression] = e.children.zip(e.inputTypes).map { case (in, expected) =>
// If we cannot do the implicit cast, just use the original input.
- ImplicitTypeCasts.implicitCast(in, expected).getOrElse(in)
+ implicitCast(in, expected).getOrElse(in)
}
e.withNewChildren(children)
@@ -900,9 +854,6 @@ object TypeCoercion {
}
e.withNewChildren(children)
}
- }
-
- object ImplicitTypeCasts {
/**
* Given an expected data type, try to cast the expression and return the cast expression.
@@ -974,6 +925,25 @@ object TypeCoercion {
if !Cast.forceNullable(fromType, toType) =>
implicitCast(fromType, toType).map(ArrayType(_, false)).orNull
+ // Implicit cast between Map types.
+ // Follows the same semantics of implicit casting between two array types.
+ // Refer to documentation above. Make sure that both key and values
+ // can not be null after the implicit cast operation by calling forceNullable
+ // method.
+ case (MapType(fromKeyType, fromValueType, fn), MapType(toKeyType, toValueType, tn))
+ if !Cast.forceNullable(fromKeyType, toKeyType) && Cast.resolvableNullability(fn, tn) =>
+ if (Cast.forceNullable(fromValueType, toValueType) && !tn) {
+ null
+ } else {
+ val newKeyType = implicitCast(fromKeyType, toKeyType).orNull
+ val newValueType = implicitCast(fromValueType, toValueType).orNull
+ if (newKeyType != null && newValueType != null) {
+ MapType(newKeyType, newValueType, tn)
+ } else {
+ null
+ }
+ }
+
case _ => null
}
Option(ret)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/higherOrderFunctions.scala
index dd08190e1e8a3..c8c75808df88d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/higherOrderFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/higherOrderFunctions.scala
@@ -148,13 +148,14 @@ case class ResolveLambdaVariables(conf: SQLConf) extends Rule[LogicalPlan] {
val lambdaMap = l.arguments.map(v => canonicalizer(v.name) -> v).toMap
l.mapChildren(resolve(_, parentLambdaMap ++ lambdaMap))
- case u @ UnresolvedAttribute(name +: nestedFields) =>
+ case u @ UnresolvedNamedLambdaVariable(name +: nestedFields) =>
parentLambdaMap.get(canonicalizer(name)) match {
case Some(lambda) =>
nestedFields.foldLeft(lambda: Expression) { (expr, fieldName) =>
ExtractValue(expr, Literal(fieldName), conf.resolver)
}
- case None => u
+ case None =>
+ UnresolvedAttribute(u.nameParts)
}
case _ =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index c1ec736c32ed4..36cad3cf74785 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -112,6 +112,7 @@ case class UnresolvedAttribute(nameParts: Seq[String]) extends Attribute with Un
override def withQualifier(newQualifier: Seq[String]): UnresolvedAttribute = this
override def withName(newName: String): UnresolvedAttribute = UnresolvedAttribute.quoted(newName)
override def withMetadata(newMetadata: Metadata): Attribute = this
+ override def withExprId(newExprId: ExprId): UnresolvedAttribute = this
override def toString: String = s"'$name"
@@ -407,7 +408,10 @@ case class ResolvedStar(expressions: Seq[NamedExpression]) extends Star with Une
* can be key of Map, index of Array, field name of Struct.
*/
case class UnresolvedExtractValue(child: Expression, extraction: Expression)
- extends UnaryExpression with Unevaluable {
+ extends BinaryExpression with Unevaluable {
+
+ override def left: Expression = child
+ override def right: Expression = extraction
override def dataType: DataType = throw new UnresolvedException(this, "dataType")
override def foldable: Boolean = throw new UnresolvedException(this, "foldable")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala
index af74693000c44..6a94f5100ab5d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/view.scala
@@ -17,8 +17,7 @@
package org.apache.spark.sql.catalyst.analysis
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Cast}
+import org.apache.spark.sql.catalyst.expressions.Alias
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, View}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.internal.SQLConf
@@ -28,7 +27,12 @@ import org.apache.spark.sql.internal.SQLConf
*/
/**
- * Make sure that a view's child plan produces the view's output attributes. We try to wrap the
+ * This rule has two goals:
+ *
+ * 1. Removes [[View]] operators from the plan. The operator is respected till the end of analysis
+ * stage because we want to see which part of an analyzed logical plan is generated from a view.
+ *
+ * 2. Make sure that a view's child plan produces the view's output attributes. We try to wrap the
* child by:
* 1. Generate the `queryOutput` by:
* 1.1. If the query column names are defined, map the column names to attributes in the child
@@ -41,27 +45,29 @@ import org.apache.spark.sql.internal.SQLConf
* 2. Map the `queryOutput` to view output by index, if the corresponding attributes don't match,
* try to up cast and alias the attribute in `queryOutput` to the attribute in the view output.
* 3. Add a Project over the child, with the new output generated by the previous steps.
- * If the view output doesn't have the same number of columns neither with the child output, nor
- * with the query column names, throw an AnalysisException.
+ *
+ * Once reaches this rule, it means `CheckAnalysis` did necessary checks on number of columns
+ * between the view output and the child output or the query column names. `CheckAnalysis` also
+ * checked the cast from the view's child to the Project is up-cast.
*
* This should be only done after the batch of Resolution, because the view attributes are not
* completely resolved during the batch of Resolution.
*/
-case class AliasViewChild(conf: SQLConf) extends Rule[LogicalPlan] with CastSupport {
- override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp {
+object EliminateView extends Rule[LogicalPlan] with CastSupport {
+ override def conf: SQLConf = SQLConf.get
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
+ // The child has the different output attributes with the View operator. Adds a Project over
+ // the child of the view.
case v @ View(desc, output, child) if child.resolved && output != child.output =>
val resolver = conf.resolver
val queryColumnNames = desc.viewQueryColumnNames
val queryOutput = if (queryColumnNames.nonEmpty) {
- // If the view output doesn't have the same number of columns with the query column names,
- // throw an AnalysisException.
- if (output.length != queryColumnNames.length) {
- throw new AnalysisException(
- s"The view output ${output.mkString("[", ",", "]")} doesn't have the same number of " +
- s"columns with the query column names ${queryColumnNames.mkString("[", ",", "]")}")
- }
+ // Find the attribute that has the expected attribute name from an attribute list, the names
+ // are compared using conf.resolver.
+ // `CheckAnalysis` already guarantees the expected attribute can be found for sure.
desc.viewQueryColumnNames.map { colName =>
- findAttributeByName(colName, child.output, resolver)
+ child.output.find(attr => resolver(attr.name, colName)).get
}
} else {
// For view created before Spark 2.2.0, the view text is already fully qualified, the plan
@@ -70,52 +76,17 @@ case class AliasViewChild(conf: SQLConf) extends Rule[LogicalPlan] with CastSupp
}
// Map the attributes in the query output to the attributes in the view output by index.
val newOutput = output.zip(queryOutput).map {
- case (attr, originAttr) if attr != originAttr =>
- // The dataType of the output attributes may be not the same with that of the view
- // output, so we should cast the attribute to the dataType of the view output attribute.
- // Will throw an AnalysisException if the cast can't perform or might truncate.
- if (Cast.mayTruncate(originAttr.dataType, attr.dataType)) {
- throw new AnalysisException(s"Cannot up cast ${originAttr.sql} from " +
- s"${originAttr.dataType.catalogString} to ${attr.dataType.catalogString} as it " +
- s"may truncate\n")
- } else {
- Alias(cast(originAttr, attr.dataType), attr.name)(exprId = attr.exprId,
- qualifier = attr.qualifier, explicitMetadata = Some(attr.metadata))
- }
+ case (attr, originAttr) if !attr.semanticEquals(originAttr) =>
+ // `CheckAnalysis` already guarantees that the cast is a up-cast for sure.
+ Alias(cast(originAttr, attr.dataType), attr.name)(exprId = attr.exprId,
+ qualifier = attr.qualifier, explicitMetadata = Some(attr.metadata))
case (_, originAttr) => originAttr
}
- v.copy(child = Project(newOutput, child))
- }
+ Project(newOutput, child)
- /**
- * Find the attribute that has the expected attribute name from an attribute list, the names
- * are compared using conf.resolver.
- * If the expected attribute is not found, throw an AnalysisException.
- */
- private def findAttributeByName(
- name: String,
- attrs: Seq[Attribute],
- resolver: Resolver): Attribute = {
- attrs.find { attr =>
- resolver(attr.name, name)
- }.getOrElse(throw new AnalysisException(
- s"Attribute with name '$name' is not found in " +
- s"'${attrs.map(_.name).mkString("(", ",", ")")}'"))
- }
-}
-
-/**
- * Removes [[View]] operators from the plan. The operator is respected till the end of analysis
- * stage because we want to see which part of an analyzed logical plan is generated from a view.
- */
-object EliminateView extends Rule[LogicalPlan] {
- def apply(plan: LogicalPlan): LogicalPlan = plan transform {
// The child should have the same output attributes with the View operator, so we simply
// remove the View operator.
- case View(_, output, child) =>
- assert(output == child.output,
- s"The output of the child ${child.output.mkString("[", ",", "]")} is different from the " +
- s"view output ${output.mkString("[", ",", "]")}")
+ case View(_, _, child) =>
child
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index afb0f009db05c..5987a2019ef41 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -218,6 +218,11 @@ class SessionCatalog(
if (dbName == DEFAULT_DATABASE) {
throw new AnalysisException(s"Can not drop default database")
}
+ if (cascade && databaseExists(dbName)) {
+ listTables(dbName).foreach { t =>
+ invalidateCachedTable(QualifiedTableName(dbName, t.table))
+ }
+ }
externalCatalog.dropDatabase(dbName, ignoreIfNotExists, cascade)
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index 30ded13410f7c..6453264083457 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, AttributeReference, Cast, ExprId, Literal}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.EstimationUtils
-import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
+import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateFormatter, DateTimeUtils, TimestampFormatter}
import org.apache.spark.sql.catalyst.util.quoteIdentifier
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
@@ -413,7 +413,8 @@ case class CatalogColumnStat(
nullCount: Option[BigInt] = None,
avgLen: Option[Long] = None,
maxLen: Option[Long] = None,
- histogram: Option[Histogram] = None) {
+ histogram: Option[Histogram] = None,
+ version: Int = CatalogColumnStat.VERSION) {
/**
* Returns a map from string to string that can be used to serialize the column stats.
@@ -427,7 +428,7 @@ case class CatalogColumnStat(
*/
def toMap(colName: String): Map[String, String] = {
val map = new scala.collection.mutable.HashMap[String, String]
- map.put(s"${colName}.${CatalogColumnStat.KEY_VERSION}", "1")
+ map.put(s"${colName}.${CatalogColumnStat.KEY_VERSION}", CatalogColumnStat.VERSION.toString)
distinctCount.foreach { v =>
map.put(s"${colName}.${CatalogColumnStat.KEY_DISTINCT_COUNT}", v.toString)
}
@@ -450,12 +451,13 @@ case class CatalogColumnStat(
dataType: DataType): ColumnStat =
ColumnStat(
distinctCount = distinctCount,
- min = min.map(CatalogColumnStat.fromExternalString(_, colName, dataType)),
- max = max.map(CatalogColumnStat.fromExternalString(_, colName, dataType)),
+ min = min.map(CatalogColumnStat.fromExternalString(_, colName, dataType, version)),
+ max = max.map(CatalogColumnStat.fromExternalString(_, colName, dataType, version)),
nullCount = nullCount,
avgLen = avgLen,
maxLen = maxLen,
- histogram = histogram)
+ histogram = histogram,
+ version = version)
}
object CatalogColumnStat extends Logging {
@@ -470,14 +472,23 @@ object CatalogColumnStat extends Logging {
private val KEY_MAX_LEN = "maxLen"
private val KEY_HISTOGRAM = "histogram"
+ val VERSION = 1
+
+ private def getTimestampFormatter(): TimestampFormatter = {
+ TimestampFormatter(format = "yyyy-MM-dd HH:mm:ss.SSSSSS", timeZone = DateTimeUtils.TimeZoneUTC)
+ }
+
/**
* Converts from string representation of data type to the corresponding Catalyst data type.
*/
- def fromExternalString(s: String, name: String, dataType: DataType): Any = {
+ def fromExternalString(s: String, name: String, dataType: DataType, version: Int): Any = {
dataType match {
case BooleanType => s.toBoolean
- case DateType => DateTimeUtils.fromJavaDate(java.sql.Date.valueOf(s))
- case TimestampType => DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf(s))
+ case DateType if version == 1 => DateTimeUtils.fromJavaDate(java.sql.Date.valueOf(s))
+ case DateType => DateFormatter().parse(s)
+ case TimestampType if version == 1 =>
+ DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf(s))
+ case TimestampType => getTimestampFormatter().parse(s)
case ByteType => s.toByte
case ShortType => s.toShort
case IntegerType => s.toInt
@@ -530,7 +541,8 @@ object CatalogColumnStat extends Logging {
nullCount = map.get(s"${colName}.${KEY_NULL_COUNT}").map(v => BigInt(v.toLong)),
avgLen = map.get(s"${colName}.${KEY_AVG_LEN}").map(_.toLong),
maxLen = map.get(s"${colName}.${KEY_MAX_LEN}").map(_.toLong),
- histogram = map.get(s"${colName}.${KEY_HISTOGRAM}").map(HistogramSerializer.deserialize)
+ histogram = map.get(s"${colName}.${KEY_HISTOGRAM}").map(HistogramSerializer.deserialize),
+ version = map(s"${colName}.${KEY_VERSION}").toInt
))
} catch {
case NonFatal(e) =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
index 3340789398f9c..13f72bd3eb87f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
@@ -108,12 +108,12 @@ object RowEncoder {
returnNullable = false)
case d: DecimalType =>
- StaticInvoke(
+ CheckOverflow(StaticInvoke(
Decimal.getClass,
d,
"fromDecimal",
inputObject :: Nil,
- returnNullable = false)
+ returnNullable = false), d)
case StringType =>
StaticInvoke(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala
index fe6db8b344d3d..4d218b936b3a2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala
@@ -26,6 +26,7 @@ package org.apache.spark.sql.catalyst.expressions
*
* The following rules are applied:
* - Names and nullability hints for [[org.apache.spark.sql.types.DataType]]s are stripped.
+ * - Names for [[GetStructField]] are stripped.
* - Commutative and associative operations ([[Add]] and [[Multiply]]) have their children ordered
* by `hashCode`.
* - [[EqualTo]] and [[EqualNullSafe]] are reordered by `hashCode`.
@@ -37,10 +38,11 @@ object Canonicalize {
expressionReorder(ignoreNamesTypes(e))
}
- /** Remove names and nullability from types. */
+ /** Remove names and nullability from types, and names from `GetStructField`. */
private[expressions] def ignoreNamesTypes(e: Expression): Expression = e match {
case a: AttributeReference =>
AttributeReference("none", a.dataType.asNullable)(exprId = a.exprId)
+ case GetStructField(child, ordinal, Some(_)) => GetStructField(child, ordinal, None)
case _ => e
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 0053503501047..780db658a71a8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -131,7 +131,7 @@ object Cast {
private def illegalNumericPrecedence(from: DataType, to: DataType): Boolean = {
val fromPrecedence = TypeCoercion.numericPrecedence.indexOf(from)
val toPrecedence = TypeCoercion.numericPrecedence.indexOf(to)
- toPrecedence > 0 && fromPrecedence > toPrecedence
+ toPrecedence >= 0 && fromPrecedence > toPrecedence
}
/**
@@ -183,7 +183,7 @@ object Cast {
case _ => false
}
- private def resolvableNullability(from: Boolean, to: Boolean) = !from || to
+ def resolvableNullability(from: Boolean, to: Boolean): Boolean = !from || to
}
/**
@@ -607,6 +607,12 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
// We can return what the children return. Same thing should happen in the codegen path.
if (DataType.equalsStructurally(from, to)) {
identity
+ } else if (from == NullType) {
+ // According to `canCast`, NullType can be casted to any type.
+ // For primitive types, we don't reach here because the guard of `nullSafeEval`.
+ // But for nested types like struct, we might reach here for nested null type field.
+ // We won't call the returned function actually, but returns a placeholder.
+ _ => throw new SparkException(s"should not directly cast from NullType to $to.")
} else {
to match {
case dt if dt == from => identity[Any]
@@ -924,27 +930,36 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
}
private[this] def changePrecision(d: ExprValue, decimalType: DecimalType,
- evPrim: ExprValue, evNull: ExprValue): Block =
- code"""
- if ($d.changePrecision(${decimalType.precision}, ${decimalType.scale})) {
- $evPrim = $d;
- } else {
- $evNull = true;
- }
- """
+ evPrim: ExprValue, evNull: ExprValue, canNullSafeCast: Boolean): Block = {
+ if (canNullSafeCast) {
+ code"""
+ |$d.changePrecision(${decimalType.precision}, ${decimalType.scale});
+ |$evPrim = $d;
+ """.stripMargin
+ } else {
+ code"""
+ |if ($d.changePrecision(${decimalType.precision}, ${decimalType.scale})) {
+ | $evPrim = $d;
+ |} else {
+ | $evNull = true;
+ |}
+ """.stripMargin
+ }
+ }
private[this] def castToDecimalCode(
from: DataType,
target: DecimalType,
ctx: CodegenContext): CastFunction = {
val tmp = ctx.freshVariable("tmpDecimal", classOf[Decimal])
+ val canNullSafeCast = Cast.canNullSafeCastToDecimal(from, target)
from match {
case StringType =>
(c, evPrim, evNull) =>
code"""
try {
Decimal $tmp = Decimal.apply(new java.math.BigDecimal($c.toString()));
- ${changePrecision(tmp, target, evPrim, evNull)}
+ ${changePrecision(tmp, target, evPrim, evNull, canNullSafeCast)}
} catch (java.lang.NumberFormatException e) {
$evNull = true;
}
@@ -953,7 +968,7 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
(c, evPrim, evNull) =>
code"""
Decimal $tmp = $c ? Decimal.apply(1) : Decimal.apply(0);
- ${changePrecision(tmp, target, evPrim, evNull)}
+ ${changePrecision(tmp, target, evPrim, evNull, canNullSafeCast)}
"""
case DateType =>
// date can't cast to decimal in Hive
@@ -964,19 +979,19 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
code"""
Decimal $tmp = Decimal.apply(
scala.math.BigDecimal.valueOf(${timestampToDoubleCode(c)}));
- ${changePrecision(tmp, target, evPrim, evNull)}
+ ${changePrecision(tmp, target, evPrim, evNull, canNullSafeCast)}
"""
case DecimalType() =>
(c, evPrim, evNull) =>
code"""
Decimal $tmp = $c.clone();
- ${changePrecision(tmp, target, evPrim, evNull)}
+ ${changePrecision(tmp, target, evPrim, evNull, canNullSafeCast)}
"""
case x: IntegralType =>
(c, evPrim, evNull) =>
code"""
Decimal $tmp = Decimal.apply((long) $c);
- ${changePrecision(tmp, target, evPrim, evNull)}
+ ${changePrecision(tmp, target, evPrim, evNull, canNullSafeCast)}
"""
case x: FractionalType =>
// All other numeric types can be represented precisely as Doubles
@@ -984,7 +999,7 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
code"""
try {
Decimal $tmp = Decimal.apply(scala.math.BigDecimal.valueOf((double) $c));
- ${changePrecision(tmp, target, evPrim, evNull)}
+ ${changePrecision(tmp, target, evPrim, evNull, canNullSafeCast)}
} catch (java.lang.NumberFormatException e) {
$evNull = true;
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
index 8954fe8a58e6e..fae90caebf96c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.SparkException
-import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
+import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, ScalaReflection}
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
import org.apache.spark.sql.types.DataType
@@ -31,6 +31,9 @@ import org.apache.spark.sql.types.DataType
* null. Use boxed type or [[Option]] if you wanna do the null-handling yourself.
* @param dataType Return type of function.
* @param children The input expressions of this UDF.
+ * @param inputsNullSafe Whether the inputs are of non-primitive types or not nullable. Null values
+ * of Scala primitive types will be converted to the type's default value and
+ * lead to wrong results, thus need special handling before calling the UDF.
* @param inputTypes The expected input types of this UDF, used to perform type coercion. If we do
* not want to perform coercion, simply use "Nil". Note that it would've been
* better to use Option of Seq[DataType] so we can use "None" as the case for no
@@ -39,17 +42,16 @@ import org.apache.spark.sql.types.DataType
* @param nullable True if the UDF can return null value.
* @param udfDeterministic True if the UDF is deterministic. Deterministic UDF returns same result
* each time it is invoked with a particular input.
- * @param nullableTypes which of the inputTypes are nullable (i.e. not primitive)
*/
case class ScalaUDF(
function: AnyRef,
dataType: DataType,
children: Seq[Expression],
+ inputsNullSafe: Seq[Boolean],
inputTypes: Seq[DataType] = Nil,
udfName: Option[String] = None,
nullable: Boolean = true,
- udfDeterministic: Boolean = true,
- nullableTypes: Seq[Boolean] = Nil)
+ udfDeterministic: Boolean = true)
extends Expression with ImplicitCastInputTypes with NonSQLExpression with UserDefinedExpression {
// The constructor for SPARK 2.1 and 2.2
@@ -60,8 +62,8 @@ case class ScalaUDF(
inputTypes: Seq[DataType],
udfName: Option[String]) = {
this(
- function, dataType, children, inputTypes, udfName, nullable = true,
- udfDeterministic = true, nullableTypes = Nil)
+ function, dataType, children, ScalaReflection.getParameterTypeNullability(function),
+ inputTypes, udfName, nullable = true, udfDeterministic = true)
}
override lazy val deterministic: Boolean = udfDeterministic && children.forall(_.deterministic)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
index 8e48856d4607c..9aae678deb4bc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql.catalyst.expressions
-import org.apache.commons.lang3.StringUtils
-
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckFailure
@@ -104,20 +102,7 @@ object TimeWindow {
* precision.
*/
private def getIntervalInMicroSeconds(interval: String): Long = {
- if (StringUtils.isBlank(interval)) {
- throw new IllegalArgumentException(
- "The window duration, slide duration and start time cannot be null or blank.")
- }
- val intervalString = if (interval.startsWith("interval")) {
- interval
- } else {
- "interval " + interval
- }
- val cal = CalendarInterval.fromString(intervalString)
- if (cal == null) {
- throw new IllegalArgumentException(
- s"The provided interval ($interval) did not correspond to a valid interval string.")
- }
+ val cal = CalendarInterval.fromCaseInsensitiveString(interval)
if (cal.months > 0) {
throw new IllegalArgumentException(
s"Intervals greater than a month is not supported ($interval).")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala
index 40582d0abd762..8cab8e4856997 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala
@@ -52,7 +52,7 @@ abstract class CountLike extends DeclarativeAggregate {
usage = """
_FUNC_(*) - Returns the total number of retrieved rows, including rows containing null.
- _FUNC_(expr) - Returns the number of rows for which the supplied expression is non-null.
+ _FUNC_(expr[, expr...]) - Returns the number of rows for which the supplied expression(s) are all non-null.
_FUNC_(DISTINCT expr[, expr...]) - Returns the number of rows for which the supplied expression(s) are unique and non-null.
""")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala
index e1d16a2cd38b0..56c2ee6b53fe5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala
@@ -128,12 +128,10 @@ case class AggregateExpression(
override def nullable: Boolean = aggregateFunction.nullable
override def references: AttributeSet = {
- val childReferences = mode match {
- case Partial | Complete => aggregateFunction.references.toSeq
- case PartialMerge | Final => aggregateFunction.aggBufferAttributes
+ mode match {
+ case Partial | Complete => aggregateFunction.references
+ case PartialMerge | Final => AttributeSet(aggregateFunction.aggBufferAttributes)
}
-
- AttributeSet(childReferences)
}
override def toString: String = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/regression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/regression.scala
deleted file mode 100644
index d8f4505588ff2..0000000000000
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/regression.scala
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.expressions.aggregate
-
-import org.apache.spark.sql.catalyst.dsl.expressions._
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.types.{AbstractDataType, DoubleType}
-
-/**
- * Base trait for all regression functions.
- */
-trait RegrLike extends AggregateFunction with ImplicitCastInputTypes {
- def y: Expression
- def x: Expression
-
- override def children: Seq[Expression] = Seq(y, x)
- override def inputTypes: Seq[AbstractDataType] = Seq(DoubleType, DoubleType)
-
- protected def updateIfNotNull(exprs: Seq[Expression]): Seq[Expression] = {
- assert(aggBufferAttributes.length == exprs.length)
- val nullableChildren = children.filter(_.nullable)
- if (nullableChildren.isEmpty) {
- exprs
- } else {
- exprs.zip(aggBufferAttributes).map { case (e, a) =>
- If(nullableChildren.map(IsNull).reduce(Or), a, e)
- }
- }
- }
-}
-
-
-@ExpressionDescription(
- usage = "_FUNC_(y, x) - Returns the number of non-null pairs.",
- since = "2.4.0")
-case class RegrCount(y: Expression, x: Expression)
- extends CountLike with RegrLike {
-
- override lazy val updateExpressions: Seq[Expression] = updateIfNotNull(Seq(count + 1L))
-
- override def prettyName: String = "regr_count"
-}
-
-
-@ExpressionDescription(
- usage = "_FUNC_(y, x) - Returns SUM(x*x)-SUM(x)*SUM(x)/N. Any pair with a NULL is ignored.",
- since = "2.4.0")
-case class RegrSXX(y: Expression, x: Expression)
- extends CentralMomentAgg(x) with RegrLike {
-
- override protected def momentOrder = 2
-
- override lazy val updateExpressions: Seq[Expression] = updateIfNotNull(updateExpressionsDef)
-
- override val evaluateExpression: Expression = {
- If(n === Literal(0.0), Literal.create(null, DoubleType), m2)
- }
-
- override def prettyName: String = "regr_sxx"
-}
-
-
-@ExpressionDescription(
- usage = "_FUNC_(y, x) - Returns SUM(y*y)-SUM(y)*SUM(y)/N. Any pair with a NULL is ignored.",
- since = "2.4.0")
-case class RegrSYY(y: Expression, x: Expression)
- extends CentralMomentAgg(y) with RegrLike {
-
- override protected def momentOrder = 2
-
- override lazy val updateExpressions: Seq[Expression] = updateIfNotNull(updateExpressionsDef)
-
- override val evaluateExpression: Expression = {
- If(n === Literal(0.0), Literal.create(null, DoubleType), m2)
- }
-
- override def prettyName: String = "regr_syy"
-}
-
-
-@ExpressionDescription(
- usage = "_FUNC_(y, x) - Returns the average of x. Any pair with a NULL is ignored.",
- since = "2.4.0")
-case class RegrAvgX(y: Expression, x: Expression)
- extends AverageLike(x) with RegrLike {
-
- override lazy val updateExpressions: Seq[Expression] = updateIfNotNull(updateExpressionsDef)
-
- override def prettyName: String = "regr_avgx"
-}
-
-
-@ExpressionDescription(
- usage = "_FUNC_(y, x) - Returns the average of y. Any pair with a NULL is ignored.",
- since = "2.4.0")
-case class RegrAvgY(y: Expression, x: Expression)
- extends AverageLike(y) with RegrLike {
-
- override lazy val updateExpressions: Seq[Expression] = updateIfNotNull(updateExpressionsDef)
-
- override def prettyName: String = "regr_avgy"
-}
-
-// scalastyle:off line.size.limit
-@ExpressionDescription(
- usage = "_FUNC_(y, x) - Returns the covariance of y and x multiplied for the number of items in the dataset. Any pair with a NULL is ignored.",
- since = "2.4.0")
-// scalastyle:on line.size.limit
-case class RegrSXY(y: Expression, x: Expression)
- extends Covariance(y, x) with RegrLike {
-
- override lazy val updateExpressions: Seq[Expression] = updateIfNotNull(updateExpressionsDef)
-
- override val evaluateExpression: Expression = {
- If(n === Literal(0.0), Literal.create(null, DoubleType), ck)
- }
-
- override def prettyName: String = "regr_sxy"
-}
-
-
-// scalastyle:off line.size.limit
-@ExpressionDescription(
- usage = "_FUNC_(y, x) - Returns the slope of the linear regression line. Any pair with a NULL is ignored.",
- since = "2.4.0")
-// scalastyle:on line.size.limit
-case class RegrSlope(y: Expression, x: Expression)
- extends PearsonCorrelation(y, x) with RegrLike {
-
- override lazy val updateExpressions: Seq[Expression] = updateIfNotNull(updateExpressionsDef)
-
- override val evaluateExpression: Expression = {
- If(n < Literal(2.0) || yMk === Literal(0.0), Literal.create(null, DoubleType), ck / yMk)
- }
-
- override def prettyName: String = "regr_slope"
-}
-
-
-// scalastyle:off line.size.limit
-@ExpressionDescription(
- usage = "_FUNC_(y, x) - Returns the coefficient of determination (also called R-squared or goodness of fit) for the regression line. Any pair with a NULL is ignored.",
- since = "2.4.0")
-// scalastyle:on line.size.limit
-case class RegrR2(y: Expression, x: Expression)
- extends PearsonCorrelation(y, x) with RegrLike {
-
- override lazy val updateExpressions: Seq[Expression] = updateIfNotNull(updateExpressionsDef)
-
- override val evaluateExpression: Expression = {
- If(n < Literal(2.0) || yMk === Literal(0.0), Literal.create(null, DoubleType),
- If(xMk === Literal(0.0), Literal(1.0), ck * ck / yMk / xMk))
- }
-
- override def prettyName: String = "regr_r2"
-}
-
-
-// scalastyle:off line.size.limit
-@ExpressionDescription(
- usage = "_FUNC_(y, x) - Returns the y-intercept of the linear regression line. Any pair with a NULL is ignored.",
- since = "2.4.0")
-// scalastyle:on line.size.limit
-case class RegrIntercept(y: Expression, x: Expression)
- extends PearsonCorrelation(y, x) with RegrLike {
-
- override lazy val updateExpressions: Seq[Expression] = updateIfNotNull(updateExpressionsDef)
-
- override val evaluateExpression: Expression = {
- If(n === Literal(0.0) || yMk === Literal(0.0), Literal.create(null, DoubleType),
- xAvg - (ck / yMk) * yAvg)
- }
-
- override def prettyName: String = "regr_intercept"
-}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
index cc24e397cc14a..c766bd8e56bb6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
@@ -95,7 +95,7 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet
examples = """
Examples:
> SELECT 3 _FUNC_ 5;
- 2
+ 6
""")
case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithmetic {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index d5857e060a2c4..56a7d5910b193 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -1293,20 +1293,27 @@ object CodeGenerator extends Logging {
case e: InternalCompilerException =>
val msg = s"failed to compile: $e"
logError(msg, e)
- val maxLines = SQLConf.get.loggingMaxLinesForCodegen
- logInfo(s"\n${CodeFormatter.format(code, maxLines)}")
+ logGeneratedCode(code)
throw new InternalCompilerException(msg, e)
case e: CompileException =>
val msg = s"failed to compile: $e"
logError(msg, e)
- val maxLines = SQLConf.get.loggingMaxLinesForCodegen
- logInfo(s"\n${CodeFormatter.format(code, maxLines)}")
+ logGeneratedCode(code)
throw new CompileException(msg, e.getLocation)
}
(evaluator.getClazz().newInstance().asInstanceOf[GeneratedClass], maxCodeSize)
}
+ private def logGeneratedCode(code: CodeAndComment): Unit = {
+ val maxLines = SQLConf.get.loggingMaxLinesForCodegen
+ if (Utils.isTesting) {
+ logError(s"\n${CodeFormatter.format(code, maxLines)}")
+ } else {
+ logInfo(s"\n${CodeFormatter.format(code, maxLines)}")
+ }
+ }
+
/**
* Returns the max bytecode size of the generated functions by inspecting janino private fields.
* Also, this method updates the metrics information.
@@ -1351,7 +1358,11 @@ object CodeGenerator extends Logging {
}
}.flatten
- codeSizes.max
+ if (codeSizes.nonEmpty) {
+ codeSizes.max
+ } else {
+ 0
+ }
}
/**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala
index febf7b0c96c2a..070570d8f20b2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala
@@ -55,7 +55,7 @@ object GenerateUnsafeRowJoiner extends CodeGenerator[(StructType, StructType), U
def create(schema1: StructType, schema2: StructType): UnsafeRowJoiner = {
val ctx = new CodegenContext
- val offset = Platform.BYTE_ARRAY_OFFSET
+ val offset = "Platform.BYTE_ARRAY_OFFSET"
val getLong = "Platform.getLong"
val putLong = "Platform.putLong"
@@ -92,7 +92,7 @@ object GenerateUnsafeRowJoiner extends CodeGenerator[(StructType, StructType), U
s"$getLong(obj2, offset2 + ${(i - bitset1Words) * 8})"
}
}
- s"$putLong(buf, ${offset + i * 8}, $bits);\n"
+ s"$putLong(buf, $offset + ${i * 8}, $bits);\n"
}
val copyBitsets = ctx.splitExpressions(
@@ -102,12 +102,12 @@ object GenerateUnsafeRowJoiner extends CodeGenerator[(StructType, StructType), U
("java.lang.Object", "obj2") :: ("long", "offset2") :: Nil)
// --------------------- copy fixed length portion from row 1 ----------------------- //
- var cursor = offset + outputBitsetWords * 8
+ var cursor = outputBitsetWords * 8
val copyFixedLengthRow1 = s"""
|// Copy fixed length data for row1
|Platform.copyMemory(
| obj1, offset1 + ${bitset1Words * 8},
- | buf, $cursor,
+ | buf, $offset + $cursor,
| ${schema1.size * 8});
""".stripMargin
cursor += schema1.size * 8
@@ -117,7 +117,7 @@ object GenerateUnsafeRowJoiner extends CodeGenerator[(StructType, StructType), U
|// Copy fixed length data for row2
|Platform.copyMemory(
| obj2, offset2 + ${bitset2Words * 8},
- | buf, $cursor,
+ | buf, $offset + $cursor,
| ${schema2.size * 8});
""".stripMargin
cursor += schema2.size * 8
@@ -129,7 +129,7 @@ object GenerateUnsafeRowJoiner extends CodeGenerator[(StructType, StructType), U
|long numBytesVariableRow1 = row1.getSizeInBytes() - $numBytesBitsetAndFixedRow1;
|Platform.copyMemory(
| obj1, offset1 + ${(bitset1Words + schema1.size) * 8},
- | buf, $cursor,
+ | buf, $offset + $cursor,
| numBytesVariableRow1);
""".stripMargin
@@ -140,7 +140,7 @@ object GenerateUnsafeRowJoiner extends CodeGenerator[(StructType, StructType), U
|long numBytesVariableRow2 = row2.getSizeInBytes() - $numBytesBitsetAndFixedRow2;
|Platform.copyMemory(
| obj2, offset2 + ${(bitset2Words + schema2.size) * 8},
- | buf, $cursor + numBytesVariableRow1,
+ | buf, $offset + $cursor + numBytesVariableRow1,
| numBytesVariableRow2);
""".stripMargin
@@ -161,7 +161,7 @@ object GenerateUnsafeRowJoiner extends CodeGenerator[(StructType, StructType), U
} else {
s"(${(outputBitsetWords - bitset2Words + schema1.size) * 8}L + numBytesVariableRow1)"
}
- val cursor = offset + outputBitsetWords * 8 + i * 8
+ val cursor = outputBitsetWords * 8 + i * 8
// UnsafeRow is a little underspecified, so in what follows we'll treat UnsafeRowWriter's
// output as a de-facto specification for the internal layout of data.
//
@@ -198,9 +198,9 @@ object GenerateUnsafeRowJoiner extends CodeGenerator[(StructType, StructType), U
// Thus it is safe to perform `existingOffset != 0` checks here in the place of
// more expensive null-bit checks.
s"""
- |existingOffset = $getLong(buf, $cursor);
+ |existingOffset = $getLong(buf, $offset + $cursor);
|if (existingOffset != 0) {
- | $putLong(buf, $cursor, existingOffset + ($shift << 32));
+ | $putLong(buf, $offset + $cursor, existingOffset + ($shift << 32));
|}
""".stripMargin
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index ea6fcccddfd49..785369673aa63 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -159,9 +159,9 @@ case class MapKeys(child: Expression)
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array(2, 3, 4));
- [[1, 2], [2, 3], [3, 4]]
+ [{"0":1,"1":2},{"0":2,"1":3},{"0":3,"1":4}]
> SELECT _FUNC_(array(1, 2), array(2, 3), array(3, 4));
- [[1, 2, 3], [2, 3, 4]]
+ [{"0":1,"1":2,"2":3},{"0":2,"1":3,"2":4}]
""",
since = "2.4.0")
case class ArraysZip(children: Seq[Expression]) extends Expression with ExpectsInputTypes {
@@ -340,174 +340,6 @@ case class MapValues(child: Expression)
override def prettyName: String = "map_values"
}
-/**
- * Returns an unordered array of all entries in the given map.
- */
-@ExpressionDescription(
- usage = "_FUNC_(map) - Returns an unordered array of all entries in the given map.",
- examples = """
- Examples:
- > SELECT _FUNC_(map(1, 'a', 2, 'b'));
- [(1,"a"),(2,"b")]
- """,
- since = "2.4.0")
-case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInputTypes {
-
- override def inputTypes: Seq[AbstractDataType] = Seq(MapType)
-
- @transient private lazy val childDataType: MapType = child.dataType.asInstanceOf[MapType]
-
- override def dataType: DataType = {
- ArrayType(
- StructType(
- StructField("key", childDataType.keyType, false) ::
- StructField("value", childDataType.valueType, childDataType.valueContainsNull) ::
- Nil),
- false)
- }
-
- override protected def nullSafeEval(input: Any): Any = {
- val childMap = input.asInstanceOf[MapData]
- val keys = childMap.keyArray()
- val values = childMap.valueArray()
- val length = childMap.numElements()
- val resultData = new Array[AnyRef](length)
- var i = 0
- while (i < length) {
- val key = keys.get(i, childDataType.keyType)
- val value = values.get(i, childDataType.valueType)
- val row = new GenericInternalRow(Array[Any](key, value))
- resultData.update(i, row)
- i += 1
- }
- new GenericArrayData(resultData)
- }
-
- override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
- nullSafeCodeGen(ctx, ev, c => {
- val arrayData = ctx.freshName("arrayData")
- val numElements = ctx.freshName("numElements")
- val keys = ctx.freshName("keys")
- val values = ctx.freshName("values")
- val isKeyPrimitive = CodeGenerator.isPrimitiveType(childDataType.keyType)
- val isValuePrimitive = CodeGenerator.isPrimitiveType(childDataType.valueType)
-
- val wordSize = UnsafeRow.WORD_SIZE
- val structSize = UnsafeRow.calculateBitSetWidthInBytes(2) + wordSize * 2
- val (isPrimitive, elementSize) = if (isKeyPrimitive && isValuePrimitive) {
- (true, structSize + wordSize)
- } else {
- (false, -1)
- }
-
- val allocation =
- s"""
- |ArrayData $arrayData = ArrayData.allocateArrayData(
- | $elementSize, $numElements, " $prettyName failed.");
- """.stripMargin
-
- val code = if (isPrimitive) {
- val genCodeForPrimitive = genCodeForPrimitiveElements(
- ctx, arrayData, keys, values, ev.value, numElements, structSize)
- s"""
- |if ($arrayData instanceof UnsafeArrayData) {
- | $genCodeForPrimitive
- |} else {
- | ${genCodeForAnyElements(ctx, arrayData, keys, values, ev.value, numElements)}
- |}
- """.stripMargin
- } else {
- s"${genCodeForAnyElements(ctx, arrayData, keys, values, ev.value, numElements)}"
- }
-
- s"""
- |final int $numElements = $c.numElements();
- |final ArrayData $keys = $c.keyArray();
- |final ArrayData $values = $c.valueArray();
- |$allocation
- |$code
- """.stripMargin
- })
- }
-
- private def getKey(varName: String, index: String) =
- CodeGenerator.getValue(varName, childDataType.keyType, index)
-
- private def getValue(varName: String, index: String) =
- CodeGenerator.getValue(varName, childDataType.valueType, index)
-
- private def genCodeForPrimitiveElements(
- ctx: CodegenContext,
- arrayData: String,
- keys: String,
- values: String,
- resultArrayData: String,
- numElements: String,
- structSize: Int): String = {
- val unsafeArrayData = ctx.freshName("unsafeArrayData")
- val baseObject = ctx.freshName("baseObject")
- val unsafeRow = ctx.freshName("unsafeRow")
- val structsOffset = ctx.freshName("structsOffset")
- val offset = ctx.freshName("offset")
- val z = ctx.freshName("z")
- val calculateHeader = "UnsafeArrayData.calculateHeaderPortionInBytes"
-
- val baseOffset = Platform.BYTE_ARRAY_OFFSET
- val wordSize = UnsafeRow.WORD_SIZE
- val structSizeAsLong = s"${structSize}L"
-
- val setKey = CodeGenerator.setColumn(unsafeRow, childDataType.keyType, 0, getKey(keys, z))
-
- val valueAssignmentChecked = CodeGenerator.createArrayAssignment(
- unsafeRow, childDataType.valueType, values, "1", z, childDataType.valueContainsNull)
-
- s"""
- |UnsafeArrayData $unsafeArrayData = (UnsafeArrayData)$arrayData;
- |Object $baseObject = $unsafeArrayData.getBaseObject();
- |final int $structsOffset = $calculateHeader($numElements) + $numElements * $wordSize;
- |UnsafeRow $unsafeRow = new UnsafeRow(2);
- |for (int $z = 0; $z < $numElements; $z++) {
- | long $offset = $structsOffset + $z * $structSizeAsLong;
- | $unsafeArrayData.setLong($z, ($offset << 32) + $structSizeAsLong);
- | $unsafeRow.pointTo($baseObject, $baseOffset + $offset, $structSize);
- | $setKey;
- | $valueAssignmentChecked
- |}
- |$resultArrayData = $arrayData;
- """.stripMargin
- }
-
- private def genCodeForAnyElements(
- ctx: CodegenContext,
- arrayData: String,
- keys: String,
- values: String,
- resultArrayData: String,
- numElements: String): String = {
- val z = ctx.freshName("z")
- val isValuePrimitive = CodeGenerator.isPrimitiveType(childDataType.valueType)
- val getValueWithCheck = if (childDataType.valueContainsNull && isValuePrimitive) {
- s"$values.isNullAt($z) ? null : (Object)${getValue(values, z)}"
- } else {
- getValue(values, z)
- }
-
- val rowClass = classOf[GenericInternalRow].getName
- val genericArrayDataClass = classOf[GenericArrayData].getName
- val genericArrayData = ctx.freshName("genericArrayData")
- val rowObject = s"new $rowClass(new Object[]{${getKey(keys, z)}, $getValueWithCheck})"
- s"""
- |$genericArrayDataClass $genericArrayData = ($genericArrayDataClass)$arrayData;
- |for (int $z = 0; $z < $numElements; $z++) {
- | $genericArrayData.update($z, $rowObject);
- |}
- |$resultArrayData = $arrayData;
- """.stripMargin
- }
-
- override def prettyName: String = "map_entries"
-}
-
/**
* Returns the union of all the given maps.
*/
@@ -516,7 +348,7 @@ case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInp
examples = """
Examples:
> SELECT _FUNC_(map(1, 'a', 2, 'b'), map(2, 'c', 3, 'd'));
- [[1 -> "a"], [2 -> "b"], [2 -> "c"], [3 -> "d"]]
+ {1:"a",2:"c",3:"d"}
""", since = "2.4.0")
case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
@@ -834,7 +666,7 @@ case class MapFromEntries(child: Expression) extends UnaryExpression {
val keyArrayData = ctx.freshName("keyArrayData")
val valueArrayData = ctx.freshName("valueArrayData")
- val baseOffset = Platform.BYTE_ARRAY_OFFSET
+ val baseOffset = "Platform.BYTE_ARRAY_OFFSET"
val keySize = dataType.keyType.defaultSize
val valueSize = dataType.valueType.defaultSize
val kByteSize = s"UnsafeArrayData.calculateSizeOfUnderlyingByteArray($numEntries, $keySize)"
@@ -864,8 +696,8 @@ case class MapFromEntries(child: Expression) extends UnaryExpression {
| final byte[] $data = new byte[(int)$byteArraySize];
| UnsafeMapData $unsafeMapData = new UnsafeMapData();
| Platform.putLong($data, $baseOffset, $keySectionSize);
- | Platform.putLong($data, ${baseOffset + 8}, $numEntries);
- | Platform.putLong($data, ${baseOffset + 8} + $keySectionSize, $numEntries);
+ | Platform.putLong($data, $baseOffset + 8, $numEntries);
+ | Platform.putLong($data, $baseOffset + 8 + $keySectionSize, $numEntries);
| $unsafeMapData.pointTo($data, $baseOffset, (int)$byteArraySize);
| ArrayData $keyArrayData = $unsafeMapData.keyArray();
| ArrayData $valueArrayData = $unsafeMapData.valueArray();
@@ -1171,9 +1003,9 @@ case class ArraySort(child: Expression) extends UnaryExpression with ArraySortLi
examples = """
Examples:
> SELECT _FUNC_(array(1, 20, 3, 5));
- [3, 1, 5, 20]
+ [3,1,5,20]
> SELECT _FUNC_(array(1, 20, null, 3));
- [20, null, 3, 1]
+ [20,null,3,1]
""",
note = "The function is non-deterministic.",
since = "2.4.0")
@@ -1256,7 +1088,7 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None)
> SELECT _FUNC_('Spark SQL');
LQS krapS
> SELECT _FUNC_(array(2, 1, 4, 3));
- [3, 4, 1, 2]
+ [3,4,1,2]
""",
since = "1.5.0",
note = "Reverse logic for arrays is available since 2.4.0."
@@ -1268,11 +1100,15 @@ case class Reverse(child: Expression) extends UnaryExpression with ImplicitCastI
override def dataType: DataType = child.dataType
- @transient private lazy val elementType: DataType = dataType.asInstanceOf[ArrayType].elementType
+ override def nullSafeEval(input: Any): Any = doReverse(input)
- override def nullSafeEval(input: Any): Any = input match {
- case a: ArrayData => new GenericArrayData(a.toObjectArray(elementType).reverse)
- case s: UTF8String => s.reverse()
+ @transient private lazy val doReverse: Any => Any = dataType match {
+ case ArrayType(elementType, _) =>
+ input => {
+ val arrayData = input.asInstanceOf[ArrayData]
+ new GenericArrayData(arrayData.toObjectArray(elementType).reverse)
+ }
+ case StringType => _.asInstanceOf[UTF8String].reverse()
}
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -1294,6 +1130,7 @@ case class Reverse(child: Expression) extends UnaryExpression with ImplicitCastI
val i = ctx.freshName("i")
val j = ctx.freshName("j")
+ val elementType = dataType.asInstanceOf[ArrayType].elementType
val initialization = CodeGenerator.createArrayData(
arrayData, elementType, numElements, s" $prettyName failed.")
val assignment = CodeGenerator.createArrayAssignment(
@@ -1331,23 +1168,27 @@ case class ArrayContains(left: Expression, right: Expression)
@transient private lazy val ordering: Ordering[Any] =
TypeUtils.getInterpretedOrdering(right.dataType)
- override def inputTypes: Seq[AbstractDataType] = right.dataType match {
- case NullType => Seq.empty
- case _ => left.dataType match {
- case n @ ArrayType(element, _) => Seq(n, element)
+ override def inputTypes: Seq[AbstractDataType] = {
+ (left.dataType, right.dataType) match {
+ case (_, NullType) => Seq.empty
+ case (ArrayType(e1, hasNull), e2) =>
+ TypeCoercion.findTightestCommonType(e1, e2) match {
+ case Some(dt) => Seq(ArrayType(dt, hasNull), dt)
+ case _ => Seq.empty
+ }
case _ => Seq.empty
}
}
override def checkInputDataTypes(): TypeCheckResult = {
- if (right.dataType == NullType) {
- TypeCheckResult.TypeCheckFailure("Null typed values cannot be used as arguments")
- } else if (!left.dataType.isInstanceOf[ArrayType]
- || !left.dataType.asInstanceOf[ArrayType].elementType.sameType(right.dataType)) {
- TypeCheckResult.TypeCheckFailure(
- "Arguments must be an array followed by a value of same type as the array members")
- } else {
- TypeUtils.checkForOrderingExpr(right.dataType, s"function $prettyName")
+ (left.dataType, right.dataType) match {
+ case (_, NullType) =>
+ TypeCheckResult.TypeCheckFailure("Null typed values cannot be used as arguments")
+ case (ArrayType(e1, _), e2) if e1.sameType(e2) =>
+ TypeUtils.checkForOrderingExpr(e2, s"function $prettyName")
+ case _ => TypeCheckResult.TypeCheckFailure(s"Input to function $prettyName should have " +
+ s"been ${ArrayType.simpleString} followed by a value with same element type, but it's " +
+ s"[${left.dataType.catalogString}, ${right.dataType.catalogString}].")
}
}
@@ -1630,7 +1471,7 @@ case class ArraysOverlap(left: Expression, right: Expression)
*/
// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "_FUNC_(x, start, length) - Subsets array x starting from index start (or starting from the end if start is negative) with the specified length.",
+ usage = "_FUNC_(x, start, length) - Subsets array x starting from index start (array indices start at 1, or starting from the end if start is negative) with the specified length.",
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3, 4), 2, 2);
@@ -2062,18 +1903,23 @@ case class ArrayPosition(left: Expression, right: Expression)
override def dataType: DataType = LongType
override def inputTypes: Seq[AbstractDataType] = {
- val elementType = left.dataType match {
- case t: ArrayType => t.elementType
- case _ => AnyDataType
+ (left.dataType, right.dataType) match {
+ case (ArrayType(e1, hasNull), e2) =>
+ TypeCoercion.findTightestCommonType(e1, e2) match {
+ case Some(dt) => Seq(ArrayType(dt, hasNull), dt)
+ case _ => Seq.empty
+ }
+ case _ => Seq.empty
}
- Seq(ArrayType, elementType)
}
override def checkInputDataTypes(): TypeCheckResult = {
- super.checkInputDataTypes() match {
- case f: TypeCheckResult.TypeCheckFailure => f
- case TypeCheckResult.TypeCheckSuccess =>
- TypeUtils.checkForOrderingExpr(right.dataType, s"function $prettyName")
+ (left.dataType, right.dataType) match {
+ case (ArrayType(e1, _), e2) if e1.sameType(e2) =>
+ TypeUtils.checkForOrderingExpr(e2, s"function $prettyName")
+ case _ => TypeCheckResult.TypeCheckFailure(s"Input to function $prettyName should have " +
+ s"been ${ArrayType.simpleString} followed by a value with same element type, but it's " +
+ s"[${left.dataType.catalogString}, ${right.dataType.catalogString}].")
}
}
@@ -2123,7 +1969,7 @@ case class ArrayPosition(left: Expression, right: Expression)
> SELECT _FUNC_(array(1, 2, 3), 2);
2
> SELECT _FUNC_(map(1, 'a', 2, 'b'), 2);
- "b"
+ b
""",
since = "2.4.0")
case class ElementAt(left: Expression, right: Expression) extends GetMapValueUtil {
@@ -2140,29 +1986,44 @@ case class ElementAt(left: Expression, right: Expression) extends GetMapValueUti
}
override def inputTypes: Seq[AbstractDataType] = {
- Seq(TypeCollection(ArrayType, MapType),
- left.dataType match {
- case _: ArrayType => IntegerType
- case _: MapType => mapKeyType
- case _ => AnyDataType // no match for a wrong 'left' expression type
- }
- )
+ (left.dataType, right.dataType) match {
+ case (arr: ArrayType, e2: IntegralType) if (e2 != LongType) =>
+ Seq(arr, IntegerType)
+ case (MapType(keyType, valueType, hasNull), e2) =>
+ TypeCoercion.findTightestCommonType(keyType, e2) match {
+ case Some(dt) => Seq(MapType(dt, valueType, hasNull), dt)
+ case _ => Seq.empty
+ }
+ case (l, r) => Seq.empty
+
+ }
}
override def checkInputDataTypes(): TypeCheckResult = {
- super.checkInputDataTypes() match {
- case f: TypeCheckResult.TypeCheckFailure => f
- case TypeCheckResult.TypeCheckSuccess if left.dataType.isInstanceOf[MapType] =>
- TypeUtils.checkForOrderingExpr(mapKeyType, s"function $prettyName")
- case TypeCheckResult.TypeCheckSuccess => TypeCheckResult.TypeCheckSuccess
+ (left.dataType, right.dataType) match {
+ case (_: ArrayType, e2) if e2 != IntegerType =>
+ TypeCheckResult.TypeCheckFailure(s"Input to function $prettyName should have " +
+ s"been ${ArrayType.simpleString} followed by a ${IntegerType.simpleString}, but it's " +
+ s"[${left.dataType.catalogString}, ${right.dataType.catalogString}].")
+ case (MapType(e1, _, _), e2) if (!e2.sameType(e1)) =>
+ TypeCheckResult.TypeCheckFailure(s"Input to function $prettyName should have " +
+ s"been ${MapType.simpleString} followed by a value of same key type, but it's " +
+ s"[${left.dataType.catalogString}, ${right.dataType.catalogString}].")
+ case (e1, _) if (!e1.isInstanceOf[MapType] && !e1.isInstanceOf[ArrayType]) =>
+ TypeCheckResult.TypeCheckFailure(s"The first argument to function $prettyName should " +
+ s"have been ${ArrayType.simpleString} or ${MapType.simpleString} type, but its " +
+ s"${left.dataType.catalogString} type.")
+ case _ => TypeCheckResult.TypeCheckSuccess
}
}
override def nullable: Boolean = true
- override def nullSafeEval(value: Any, ordinal: Any): Any = {
- left.dataType match {
- case _: ArrayType =>
+ override def nullSafeEval(value: Any, ordinal: Any): Any = doElementAt(value, ordinal)
+
+ @transient private lazy val doElementAt: (Any, Any) => Any = left.dataType match {
+ case _: ArrayType =>
+ (value, ordinal) => {
val array = value.asInstanceOf[ArrayData]
val index = ordinal.asInstanceOf[Int]
if (array.numElements() < math.abs(index)) {
@@ -2181,9 +2042,9 @@ case class ElementAt(left: Expression, right: Expression) extends GetMapValueUti
array.get(idx, dataType)
}
}
- case _: MapType =>
- getValueEval(value, ordinal, mapKeyType, ordering)
- }
+ }
+ case _: MapType =>
+ (value, ordinal) => getValueEval(value, ordinal, mapKeyType, ordering)
}
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -2238,8 +2099,9 @@ case class ElementAt(left: Expression, right: Expression) extends GetMapValueUti
> SELECT _FUNC_('Spark', 'SQL');
SparkSQL
> SELECT _FUNC_(array(1, 2, 3), array(4, 5), array(6));
- | [1,2,3,4,5,6]
- """)
+ [1,2,3,4,5,6]
+ """,
+ note = "Concat logic for arrays is available since 2.4.0.")
case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
private def allowedTypes: Seq[AbstractDataType] = Seq(StringType, BinaryType, ArrayType)
@@ -2273,33 +2135,41 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio
override def foldable: Boolean = children.forall(_.foldable)
- override def eval(input: InternalRow): Any = dataType match {
+ override def eval(input: InternalRow): Any = doConcat(input)
+
+ @transient private lazy val doConcat: InternalRow => Any = dataType match {
case BinaryType =>
- val inputs = children.map(_.eval(input).asInstanceOf[Array[Byte]])
- ByteArray.concat(inputs: _*)
+ input => {
+ val inputs = children.map(_.eval(input).asInstanceOf[Array[Byte]])
+ ByteArray.concat(inputs: _*)
+ }
case StringType =>
- val inputs = children.map(_.eval(input).asInstanceOf[UTF8String])
- UTF8String.concat(inputs : _*)
+ input => {
+ val inputs = children.map(_.eval(input).asInstanceOf[UTF8String])
+ UTF8String.concat(inputs: _*)
+ }
case ArrayType(elementType, _) =>
- val inputs = children.toStream.map(_.eval(input))
- if (inputs.contains(null)) {
- null
- } else {
- val arrayData = inputs.map(_.asInstanceOf[ArrayData])
- val numberOfElements = arrayData.foldLeft(0L)((sum, ad) => sum + ad.numElements())
- if (numberOfElements > ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH) {
- throw new RuntimeException(s"Unsuccessful try to concat arrays with $numberOfElements" +
- " elements due to exceeding the array size limit " +
- ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH + ".")
- }
- val finalData = new Array[AnyRef](numberOfElements.toInt)
- var position = 0
- for(ad <- arrayData) {
- val arr = ad.toObjectArray(elementType)
- Array.copy(arr, 0, finalData, position, arr.length)
- position += arr.length
+ input => {
+ val inputs = children.toStream.map(_.eval(input))
+ if (inputs.contains(null)) {
+ null
+ } else {
+ val arrayData = inputs.map(_.asInstanceOf[ArrayData])
+ val numberOfElements = arrayData.foldLeft(0L)((sum, ad) => sum + ad.numElements())
+ if (numberOfElements > ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH) {
+ throw new RuntimeException(s"Unsuccessful try to concat arrays with $numberOfElements" +
+ " elements due to exceeding the array size limit " +
+ ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH + ".")
+ }
+ val finalData = new Array[AnyRef](numberOfElements.toInt)
+ var position = 0
+ for (ad <- arrayData) {
+ val arr = ad.toObjectArray(elementType)
+ Array.copy(arr, 0, finalData, position, arr.length)
+ position += arr.length
+ }
+ new GenericArrayData(finalData)
}
- new GenericArrayData(finalData)
}
}
@@ -2427,7 +2297,7 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio
usage = "_FUNC_(arrayOfArrays) - Transforms an array of arrays into a single array.",
examples = """
Examples:
- > SELECT _FUNC_(array(array(1, 2), array(3, 4));
+ > SELECT _FUNC_(array(array(1, 2), array(3, 4)));
[1,2,3,4]
""",
since = "2.4.0")
@@ -2556,11 +2426,11 @@ case class Flatten(child: Expression) extends UnaryExpression {
examples = """
Examples:
> SELECT _FUNC_(1, 5);
- [1, 2, 3, 4, 5]
+ [1,2,3,4,5]
> SELECT _FUNC_(5, 1);
- [5, 4, 3, 2, 1]
+ [5,4,3,2,1]
> SELECT _FUNC_(to_date('2018-01-01'), to_date('2018-03-01'), interval 1 month);
- [2018-01-01, 2018-02-01, 2018-03-01]
+ [2018-01-01,2018-02-01,2018-03-01]
""",
since = "2.4.0"
)
@@ -2934,7 +2804,7 @@ object Sequence {
examples = """
Examples:
> SELECT _FUNC_('123', 2);
- ['123', '123']
+ ["123","123"]
""",
since = "2.4.0")
case class ArrayRepeat(left: Expression, right: Expression)
@@ -3063,11 +2933,24 @@ case class ArrayRemove(left: Expression, right: Expression)
override def dataType: DataType = left.dataType
override def inputTypes: Seq[AbstractDataType] = {
- val elementType = left.dataType match {
- case t: ArrayType => t.elementType
- case _ => AnyDataType
+ (left.dataType, right.dataType) match {
+ case (ArrayType(e1, hasNull), e2) =>
+ TypeCoercion.findTightestCommonType(e1, e2) match {
+ case Some(dt) => Seq(ArrayType(dt, hasNull), dt)
+ case _ => Seq.empty
+ }
+ case _ => Seq.empty
+ }
+ }
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ (left.dataType, right.dataType) match {
+ case (ArrayType(e1, _), e2) if e1.sameType(e2) =>
+ TypeUtils.checkForOrderingExpr(e2, s"function $prettyName")
+ case _ => TypeCheckResult.TypeCheckFailure(s"Input to function $prettyName should have " +
+ s"been ${ArrayType.simpleString} followed by a value with same element type, but it's " +
+ s"[${left.dataType.catalogString}, ${right.dataType.catalogString}].")
}
- Seq(ArrayType, elementType)
}
private def elementType: DataType = left.dataType.asInstanceOf[ArrayType].elementType
@@ -3075,14 +2958,6 @@ case class ArrayRemove(left: Expression, right: Expression)
@transient private lazy val ordering: Ordering[Any] =
TypeUtils.getInterpretedOrdering(right.dataType)
- override def checkInputDataTypes(): TypeCheckResult = {
- super.checkInputDataTypes() match {
- case f: TypeCheckResult.TypeCheckFailure => f
- case TypeCheckResult.TypeCheckSuccess =>
- TypeUtils.checkForOrderingExpr(right.dataType, s"function $prettyName")
- }
- }
-
override def nullSafeEval(arr: Any, value: Any): Any = {
val newArray = new Array[Any](arr.asInstanceOf[ArrayData].numElements())
var pos = 0
@@ -3276,29 +3151,29 @@ case class ArrayDistinct(child: Expression)
(data: Array[AnyRef]) => new GenericArrayData(data.distinct.asInstanceOf[Array[Any]])
} else {
(data: Array[AnyRef]) => {
- var foundNullElement = false
- var pos = 0
+ val arrayBuffer = new scala.collection.mutable.ArrayBuffer[AnyRef]
+ var alreadyStoredNull = false
for (i <- 0 until data.length) {
- if (data(i) == null) {
- if (!foundNullElement) {
- foundNullElement = true
- pos = pos + 1
- }
- } else {
+ if (data(i) != null) {
+ var found = false
var j = 0
- var done = false
- while (j <= i && !done) {
- if (data(j) != null && ordering.equiv(data(j), data(i))) {
- done = true
- }
- j = j + 1
+ while (!found && j < arrayBuffer.size) {
+ val va = arrayBuffer(j)
+ found = (va != null) && ordering.equiv(va, data(i))
+ j += 1
+ }
+ if (!found) {
+ arrayBuffer += data(i)
}
- if (i == j - 1) {
- pos = pos + 1
+ } else {
+ // De-duplicate the null values.
+ if (!alreadyStoredNull) {
+ arrayBuffer += data(i)
+ alreadyStoredNull = true
}
}
}
- new GenericArrayData(data.slice(0, pos))
+ new GenericArrayData(arrayBuffer)
}
}
@@ -3421,7 +3296,7 @@ object ArrayBinaryLike {
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
- array(1, 2, 3, 5)
+ [1,2,3,5]
""",
since = "2.4.0")
case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLike
@@ -3632,7 +3507,7 @@ object ArrayUnion {
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
- array(1, 3)
+ [1,3]
""",
since = "2.4.0")
case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBinaryLike
@@ -3873,7 +3748,7 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
- array(2)
+ [2]
""",
since = "2.4.0")
case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryLike
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index 077a6dc93bd17..16b4a1a5e5d2a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -124,7 +124,7 @@ private [sql] object GenArrayData {
val unsafeArraySizeInBytes =
UnsafeArrayData.calculateHeaderPortionInBytes(numElements) +
ByteArrayMethods.roundNumberOfBytesToNearestWord(elementType.defaultSize * numElements)
- val baseOffset = Platform.BYTE_ARRAY_OFFSET
+ val baseOffset = "Platform.BYTE_ARRAY_OFFSET"
val primitiveValueTypeName = CodeGenerator.primitiveTypeName(elementType)
val assignments = elementsCode.zipWithIndex.map { case (eval, i) =>
@@ -248,7 +248,7 @@ case class CreateMap(children: Seq[Expression]) extends Expression {
in keys should not be null""",
examples = """
Examples:
- > SELECT _FUNC_([1.0, 3.0], ['2', '4']);
+ > SELECT _FUNC_(array(1.0, 3.0), array('2', '4'));
{1.0:"2",3.0:"4"}
""", since = "2.4.0")
case class MapFromArrays(left: Expression, right: Expression)
@@ -379,10 +379,7 @@ trait CreateNamedStructLike extends Expression {
}
override def checkInputDataTypes(): TypeCheckResult = {
- if (children.length < 1) {
- TypeCheckResult.TypeCheckFailure(
- s"input to function $prettyName requires at least one argument")
- } else if (children.size % 2 != 0) {
+ if (children.size % 2 != 0) {
TypeCheckResult.TypeCheckFailure(s"$prettyName expects an even number of arguments.")
} else {
val invalidNames = nameExprs.filterNot(e => e.foldable && e.dataType == StringType)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index f95798d64db19..8b3f53228f5f1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -542,6 +542,12 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa
// scalastyle:off line.size.limit
@ExpressionDescription(
usage = "_FUNC_(timestamp, fmt) - Converts `timestamp` to a value of string in the format specified by the date format `fmt`.",
+ arguments = """
+ Arguments:
+ * timestamp - A date/timestamp or string to be converted to the given format.
+ * fmt - Date/time format pattern to follow. See `java.text.SimpleDateFormat` for valid date
+ and time format patterns.
+ """,
examples = """
Examples:
> SELECT _FUNC_('2016-04-08', 'y');
@@ -583,7 +589,14 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
* Deterministic version of [[UnixTimestamp]], must have at least one parameter.
*/
@ExpressionDescription(
- usage = "_FUNC_(expr[, pattern]) - Returns the UNIX timestamp of the given time.",
+ usage = "_FUNC_(timeExp[, format]) - Returns the UNIX timestamp of the given time.",
+ arguments = """
+ Arguments:
+ * timeExp - A date/timestamp or string which is returned as a UNIX timestamp.
+ * format - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
+ Default value is "yyyy-MM-dd HH:mm:ss". See `java.text.SimpleDateFormat`
+ for valid date and time format patterns.
+ """,
examples = """
Examples:
> SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd');
@@ -622,7 +635,14 @@ case class ToUnixTimestamp(
* second parameter.
*/
@ExpressionDescription(
- usage = "_FUNC_([expr[, pattern]]) - Returns the UNIX timestamp of current or specified time.",
+ usage = "_FUNC_([timeExp[, format]]) - Returns the UNIX timestamp of current or specified time.",
+ arguments = """
+ Arguments:
+ * timeExp - A date/timestamp or string. If not provided, this defaults to current time.
+ * format - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
+ Default value is "yyyy-MM-dd HH:mm:ss". See `java.text.SimpleDateFormat`
+ for valid date and time format patterns.
+ """,
examples = """
Examples:
> SELECT _FUNC_();
@@ -777,6 +797,12 @@ abstract class UnixTime
*/
@ExpressionDescription(
usage = "_FUNC_(unix_time, format) - Returns `unix_time` in the specified `format`.",
+ arguments = """
+ Arguments:
+ * unix_time - UNIX Timestamp to be converted to the provided format.
+ * format - Date/time format pattern to follow. See `java.text.SimpleDateFormat`
+ for valid date and time format patterns.
+ """,
examples = """
Examples:
> SELECT _FUNC_(0, 'yyyy-MM-dd HH:mm:ss');
@@ -1018,51 +1044,18 @@ case class TimeAdd(start: Expression, interval: Expression, timeZoneId: Option[S
}
/**
- * A special expression used to convert the string input of `to/from_utc_timestamp` to timestamp,
- * which requires the timestamp string to not have timezone information, otherwise null is returned.
- */
-case class StringToTimestampWithoutTimezone(child: Expression, timeZoneId: Option[String] = None)
- extends UnaryExpression with TimeZoneAwareExpression with ExpectsInputTypes {
-
- override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
- copy(timeZoneId = Option(timeZoneId))
-
- override def inputTypes: Seq[AbstractDataType] = Seq(StringType)
- override def dataType: DataType = TimestampType
- override def nullable: Boolean = true
- override def toString: String = child.toString
- override def sql: String = child.sql
-
- override def nullSafeEval(input: Any): Any = {
- DateTimeUtils.stringToTimestamp(
- input.asInstanceOf[UTF8String], timeZone, rejectTzInString = true).orNull
- }
-
- override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
- val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
- val tz = ctx.addReferenceObj("timeZone", timeZone)
- val longOpt = ctx.freshName("longOpt")
- val eval = child.genCode(ctx)
- val code = code"""
- |${eval.code}
- |${CodeGenerator.JAVA_BOOLEAN} ${ev.isNull} = true;
- |${CodeGenerator.JAVA_LONG} ${ev.value} = ${CodeGenerator.defaultValue(TimestampType)};
- |if (!${eval.isNull}) {
- | scala.Option $longOpt = $dtu.stringToTimestamp(${eval.value}, $tz, true);
- | if ($longOpt.isDefined()) {
- | ${ev.value} = ((Long) $longOpt.get()).longValue();
- | ${ev.isNull} = false;
- | }
- |}
- """.stripMargin
- ev.copy(code = code)
- }
-}
-
-/**
- * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders
- * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield
- * '2017-07-14 03:40:00.0'.
+ * This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
+ * takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in UTC, and
+ * renders that timestamp as a timestamp in the given time zone.
+ *
+ * However, timestamp in Spark represents number of microseconds from the Unix epoch, which is not
+ * timezone-agnostic. So in Spark this function just shift the timestamp value from UTC timezone to
+ * the given timezone.
+ *
+ * This function may return confusing result if the input is a string with timezone, e.g.
+ * '2018-03-13T06:18:23+00:00'. The reason is that, Spark firstly cast the string to timestamp
+ * according to the timezone in the string, and finally display the result by converting the
+ * timestamp to string according to the session local timezone.
*/
// scalastyle:off line.size.limit
@ExpressionDescription(
@@ -1257,9 +1250,18 @@ case class MonthsBetween(
}
/**
- * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time zone,
- * and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield
- * '2017-07-14 01:40:00.0'.
+ * This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
+ * takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in the given
+ * timezone, and renders that timestamp as a timestamp in UTC.
+ *
+ * However, timestamp in Spark represents number of microseconds from the Unix epoch, which is not
+ * timezone-agnostic. So in Spark this function just shift the timestamp value from the given
+ * timezone to UTC timezone.
+ *
+ * This function may return confusing result if the input is a string with timezone, e.g.
+ * '2018-03-13T06:18:23+00:00'. The reason is that, Spark firstly cast the string to timestamp
+ * according to the timezone in the string, and finally display the result by converting the
+ * timestamp to string according to the session local timezone.
*/
// scalastyle:off line.size.limit
@ExpressionDescription(
@@ -1328,6 +1330,12 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
a date. Returns null with invalid input. By default, it follows casting rules to a date if
the `fmt` is omitted.
""",
+ arguments = """
+ Arguments:
+ * date_str - A string to be parsed to date.
+ * fmt - Date format pattern to follow. See `java.text.SimpleDateFormat` for valid
+ date and time format patterns.
+ """,
examples = """
Examples:
> SELECT _FUNC_('2009-07-30 04:17:52');
@@ -1366,10 +1374,16 @@ case class ParseToDate(left: Expression, format: Option[Expression], child: Expr
*/
@ExpressionDescription(
usage = """
- _FUNC_(timestamp[, fmt]) - Parses the `timestamp` expression with the `fmt` expression to
- a timestamp. Returns null with invalid input. By default, it follows casting rules to
+ _FUNC_(timestamp_str[, fmt]) - Parses the `timestamp_str` expression with the `fmt` expression
+ to a timestamp. Returns null with invalid input. By default, it follows casting rules to
a timestamp if the `fmt` is omitted.
""",
+ arguments = """
+ Arguments:
+ * timestamp_str - A string to be parsed to timestamp.
+ * fmt - Timestamp format pattern to follow. See `java.text.SimpleDateFormat` for valid
+ date and time format patterns.
+ """,
examples = """
Examples:
> SELECT _FUNC_('2016-12-31 00:12:00');
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/grouping.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/grouping.scala
index 3be761c8676c9..b8ff455d27b8d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/grouping.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/grouping.scala
@@ -38,14 +38,65 @@ trait GroupingSet extends Expression with CodegenFallback {
override def eval(input: InternalRow): Any = throw new UnsupportedOperationException
}
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+ usage = """
+ _FUNC_([col1[, col2 ..]]) - create a multi-dimensional cube using the specified columns
+ so that we can run aggregation on them.
+ """,
+ examples = """
+ Examples:
+ > SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY _FUNC_(name, age);
+ NULL 2 1
+ NULL NULL 2
+ Alice 2 1
+ Bob 5 1
+ NULL 5 1
+ Bob NULL 1
+ Alice NULL 1
+ """,
+ since = "2.0.0")
+// scalastyle:on line.size.limit
case class Cube(groupByExprs: Seq[Expression]) extends GroupingSet {}
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+ usage = """
+ _FUNC_([col1[, col2 ..]]) - create a multi-dimensional rollup using the specified columns
+ so that we can run aggregation on them.
+ """,
+ examples = """
+ Examples:
+ > SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY _FUNC_(name, age);
+ NULL NULL 2
+ Alice 2 1
+ Bob 5 1
+ Bob NULL 1
+ Alice NULL 1
+ """,
+ since = "2.0.0")
+// scalastyle:on line.size.limit
case class Rollup(groupByExprs: Seq[Expression]) extends GroupingSet {}
/**
* Indicates whether a specified column expression in a GROUP BY list is aggregated or not.
* GROUPING returns 1 for aggregated or 0 for not aggregated in the result set.
*/
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+ usage = """
+ _FUNC_(col) - indicates whether a specified column in a GROUP BY is aggregated or
+ not, returns 1 for aggregated or 0 for not aggregated in the result set.",
+ """,
+ examples = """
+ Examples:
+ > SELECT name, _FUNC_(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name);
+ Alice 0 2
+ NULL 1 7
+ Bob 0 5
+ """,
+ since = "2.0.0")
+// scalastyle:on line.size.limit
case class Grouping(child: Expression) extends Expression with Unevaluable {
override def references: AttributeSet = AttributeSet(VirtualColumn.groupingIdAttribute :: Nil)
override def children: Seq[Expression] = child :: Nil
@@ -58,6 +109,29 @@ case class Grouping(child: Expression) extends Expression with Unevaluable {
*
* If groupByExprs is empty, it means all grouping expressions in GroupingSets.
*/
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+ usage = """
+ _FUNC_([col1[, col2 ..]]) - returns the level of grouping, equals to
+ `(grouping(c1) << (n-1)) + (grouping(c2) << (n-2)) + ... + grouping(cn)`
+ """,
+ examples = """
+ Examples:
+ > SELECT name, _FUNC_(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height);
+ NULL 2 2 165.0
+ Alice 0 2 165.0
+ NULL 2 5 180.0
+ NULL 3 7 172.5
+ Bob 0 5 180.0
+ Bob 1 5 180.0
+ Alice 1 2 165.0
+ """,
+ note = """
+ Input columns should match with grouping columns exactly, or empty (means all the grouping
+ columns).
+ """,
+ since = "2.0.0")
+// scalastyle:on line.size.limit
case class GroupingID(groupByExprs: Seq[Expression]) extends Expression with Unevaluable {
override def references: AttributeSet = AttributeSet(VirtualColumn.groupingIdAttribute :: Nil)
override def children: Seq[Expression] = groupByExprs
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
index a754e87a17968..742a4f87a9c04 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
@@ -33,7 +33,6 @@ import org.apache.spark.sql.catalyst.util.{ArrayData, MapData}
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.Platform
import org.apache.spark.unsafe.hash.Murmur3_x86_32
-import org.apache.spark.unsafe.memory.MemoryBlock
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -362,7 +361,10 @@ abstract class HashExpression[E] extends Expression {
}
protected def genHashString(input: String, result: String): String = {
- s"$result = $hasherClassName.hashUTF8String($input, $result);"
+ val baseObject = s"$input.getBaseObject()"
+ val baseOffset = s"$input.getBaseOffset()"
+ val numBytes = s"$input.numBytes()"
+ s"$result = $hasherClassName.hashUnsafeBytes($baseObject, $baseOffset, $numBytes, $result);"
}
protected def genHashForMap(
@@ -469,8 +471,6 @@ abstract class InterpretedHashFunction {
protected def hashUnsafeBytes(base: AnyRef, offset: Long, length: Int, seed: Long): Long
- protected def hashUnsafeBytesBlock(base: MemoryBlock, seed: Long): Long
-
/**
* Computes hash of a given `value` of type `dataType`. The caller needs to check the validity
* of input `value`.
@@ -496,7 +496,8 @@ abstract class InterpretedHashFunction {
case c: CalendarInterval => hashInt(c.months, hashLong(c.microseconds, seed))
case a: Array[Byte] =>
hashUnsafeBytes(a, Platform.BYTE_ARRAY_OFFSET, a.length, seed)
- case s: UTF8String => hashUnsafeBytesBlock(s.getMemoryBlock(), seed)
+ case s: UTF8String =>
+ hashUnsafeBytes(s.getBaseObject, s.getBaseOffset, s.numBytes(), seed)
case array: ArrayData =>
val elementType = dataType match {
@@ -583,15 +584,9 @@ object Murmur3HashFunction extends InterpretedHashFunction {
Murmur3_x86_32.hashLong(l, seed.toInt)
}
- override protected def hashUnsafeBytes(
- base: AnyRef, offset: Long, len: Int, seed: Long): Long = {
+ override protected def hashUnsafeBytes(base: AnyRef, offset: Long, len: Int, seed: Long): Long = {
Murmur3_x86_32.hashUnsafeBytes(base, offset, len, seed.toInt)
}
-
- override protected def hashUnsafeBytesBlock(
- base: MemoryBlock, seed: Long): Long = {
- Murmur3_x86_32.hashUnsafeBytesBlock(base, seed.toInt)
- }
}
/**
@@ -616,14 +611,9 @@ object XxHash64Function extends InterpretedHashFunction {
override protected def hashLong(l: Long, seed: Long): Long = XXH64.hashLong(l, seed)
- override protected def hashUnsafeBytes(
- base: AnyRef, offset: Long, len: Int, seed: Long): Long = {
+ override protected def hashUnsafeBytes(base: AnyRef, offset: Long, len: Int, seed: Long): Long = {
XXH64.hashUnsafeBytes(base, offset, len, seed)
}
-
- override protected def hashUnsafeBytesBlock(base: MemoryBlock, seed: Long): Long = {
- XXH64.hashUnsafeBytesBlock(base, seed)
- }
}
/**
@@ -730,7 +720,10 @@ case class HiveHash(children: Seq[Expression]) extends HashExpression[Int] {
"""
override protected def genHashString(input: String, result: String): String = {
- s"$result = $hasherClassName.hashUTF8String($input);"
+ val baseObject = s"$input.getBaseObject()"
+ val baseOffset = s"$input.getBaseOffset()"
+ val numBytes = s"$input.numBytes()"
+ s"$result = $hasherClassName.hashUnsafeBytes($baseObject, $baseOffset, $numBytes);"
}
override protected def genHashForArray(
@@ -824,14 +817,10 @@ object HiveHashFunction extends InterpretedHashFunction {
HiveHasher.hashLong(l)
}
- override protected def hashUnsafeBytes(
- base: AnyRef, offset: Long, len: Int, seed: Long): Long = {
+ override protected def hashUnsafeBytes(base: AnyRef, offset: Long, len: Int, seed: Long): Long = {
HiveHasher.hashUnsafeBytes(base, offset, len)
}
- override protected def hashUnsafeBytesBlock(
- base: MemoryBlock, seed: Long): Long = HiveHasher.hashUnsafeBytesBlock(base)
-
private val HIVE_DECIMAL_MAX_PRECISION = 38
private val HIVE_DECIMAL_MAX_SCALE = 38
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
index 2bb6b20b944d4..17cd2a73d5236 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
@@ -22,12 +22,34 @@ import java.util.concurrent.atomic.AtomicReference
import scala.collection.mutable
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion, UnresolvedAttribute}
+import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion, UnresolvedAttribute, UnresolvedException}
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.array.ByteArrayMethods
+/**
+ * A placeholder of lambda variables to prevent unexpected resolution of [[LambdaFunction]].
+ */
+case class UnresolvedNamedLambdaVariable(nameParts: Seq[String])
+ extends LeafExpression with NamedExpression with Unevaluable {
+
+ override def name: String =
+ nameParts.map(n => if (n.contains(".")) s"`$n`" else n).mkString(".")
+
+ override def exprId: ExprId = throw new UnresolvedException(this, "exprId")
+ override def dataType: DataType = throw new UnresolvedException(this, "dataType")
+ override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
+ override def qualifier: Seq[String] = throw new UnresolvedException(this, "qualifier")
+ override def toAttribute: Attribute = throw new UnresolvedException(this, "toAttribute")
+ override def newInstance(): NamedExpression = throw new UnresolvedException(this, "newInstance")
+ override lazy val resolved = false
+
+ override def toString: String = s"lambda '$name"
+
+ override def sql: String = name
+}
+
/**
* A named lambda variable.
*/
@@ -79,7 +101,7 @@ case class LambdaFunction(
object LambdaFunction {
val identity: LambdaFunction = {
- val id = UnresolvedAttribute.quoted("id")
+ val id = UnresolvedNamedLambdaVariable(Seq("id"))
LambdaFunction(id, Seq(id))
}
}
@@ -209,9 +231,9 @@ trait MapBasedSimpleHigherOrderFunction extends SimpleHigherOrderFunction {
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), x -> x + 1);
- array(2, 3, 4)
+ [2,3,4]
> SELECT _FUNC_(array(1, 2, 3), (x, i) -> x + i);
- array(1, 3, 5)
+ [1,3,5]
""",
since = "2.4.0")
case class ArrayTransform(
@@ -260,56 +282,6 @@ case class ArrayTransform(
override def prettyName: String = "transform"
}
-/**
- * Filters entries in a map using the provided function.
- */
-@ExpressionDescription(
-usage = "_FUNC_(expr, func) - Filters entries in a map using the function.",
-examples = """
- Examples:
- > SELECT _FUNC_(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v);
- [1 -> 0, 3 -> -1]
- """,
-since = "2.4.0")
-case class MapFilter(
- argument: Expression,
- function: Expression)
- extends MapBasedSimpleHigherOrderFunction with CodegenFallback {
-
- @transient lazy val (keyVar, valueVar) = {
- val args = function.asInstanceOf[LambdaFunction].arguments
- (args.head.asInstanceOf[NamedLambdaVariable], args.tail.head.asInstanceOf[NamedLambdaVariable])
- }
-
- @transient lazy val MapType(keyType, valueType, valueContainsNull) = argument.dataType
-
- override def bind(f: (Expression, Seq[(DataType, Boolean)]) => LambdaFunction): MapFilter = {
- copy(function = f(function, (keyType, false) :: (valueType, valueContainsNull) :: Nil))
- }
-
- override def nullSafeEval(inputRow: InternalRow, argumentValue: Any): Any = {
- val m = argumentValue.asInstanceOf[MapData]
- val f = functionForEval
- val retKeys = new mutable.ListBuffer[Any]
- val retValues = new mutable.ListBuffer[Any]
- m.foreach(keyType, valueType, (k, v) => {
- keyVar.value.set(k)
- valueVar.value.set(v)
- if (f.eval(inputRow).asInstanceOf[Boolean]) {
- retKeys += k
- retValues += v
- }
- })
- ArrayBasedMapData(retKeys.toArray, retValues.toArray)
- }
-
- override def dataType: DataType = argument.dataType
-
- override def functionType: AbstractDataType = BooleanType
-
- override def prettyName: String = "map_filter"
-}
-
/**
* Filters the input array using the given lambda function.
*/
@@ -318,7 +290,7 @@ case class MapFilter(
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 1);
- array(1, 3)
+ [1,3]
""",
since = "2.4.0")
case class ArrayFilter(
@@ -491,297 +463,17 @@ case class ArrayAggregate(
override def prettyName: String = "aggregate"
}
-/**
- * Transform Keys for every entry of the map by applying the transform_keys function.
- * Returns map with transformed key entries
- */
-@ExpressionDescription(
- usage = "_FUNC_(expr, func) - Transforms elements in a map using the function.",
- examples = """
- Examples:
- > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1);
- map(array(2, 3, 4), array(1, 2, 3))
- > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
- map(array(2, 4, 6), array(1, 2, 3))
- """,
- since = "2.4.0")
-case class TransformKeys(
- argument: Expression,
- function: Expression)
- extends MapBasedSimpleHigherOrderFunction with CodegenFallback {
-
- @transient lazy val MapType(keyType, valueType, valueContainsNull) = argument.dataType
-
- override def dataType: DataType = MapType(function.dataType, valueType, valueContainsNull)
-
- override def bind(f: (Expression, Seq[(DataType, Boolean)]) => LambdaFunction): TransformKeys = {
- copy(function = f(function, (keyType, false) :: (valueType, valueContainsNull) :: Nil))
- }
-
- @transient lazy val LambdaFunction(
- _, (keyVar: NamedLambdaVariable) :: (valueVar: NamedLambdaVariable) :: Nil, _) = function
-
-
- override def nullSafeEval(inputRow: InternalRow, argumentValue: Any): Any = {
- val map = argumentValue.asInstanceOf[MapData]
- val resultKeys = new GenericArrayData(new Array[Any](map.numElements))
- var i = 0
- while (i < map.numElements) {
- keyVar.value.set(map.keyArray().get(i, keyVar.dataType))
- valueVar.value.set(map.valueArray().get(i, valueVar.dataType))
- val result = functionForEval.eval(inputRow)
- if (result == null) {
- throw new RuntimeException("Cannot use null as map key!")
- }
- resultKeys.update(i, result)
- i += 1
- }
- new ArrayBasedMapData(resultKeys, map.valueArray())
- }
-
- override def prettyName: String = "transform_keys"
-}
-
-/**
- * Returns a map that applies the function to each value of the map.
- */
-@ExpressionDescription(
- usage = "_FUNC_(expr, func) - Transforms values in the map using the function.",
- examples = """
- Examples:
- > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1);
- map(array(1, 2, 3), array(2, 3, 4))
- > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v);
- map(array(1, 2, 3), array(2, 4, 6))
- """,
- since = "2.4.0")
-case class TransformValues(
- argument: Expression,
- function: Expression)
- extends MapBasedSimpleHigherOrderFunction with CodegenFallback {
-
- @transient lazy val MapType(keyType, valueType, valueContainsNull) = argument.dataType
-
- override def dataType: DataType = MapType(keyType, function.dataType, function.nullable)
-
- override def bind(f: (Expression, Seq[(DataType, Boolean)]) => LambdaFunction)
- : TransformValues = {
- copy(function = f(function, (keyType, false) :: (valueType, valueContainsNull) :: Nil))
- }
-
- @transient lazy val LambdaFunction(
- _, (keyVar: NamedLambdaVariable) :: (valueVar: NamedLambdaVariable) :: Nil, _) = function
-
- override def nullSafeEval(inputRow: InternalRow, argumentValue: Any): Any = {
- val map = argumentValue.asInstanceOf[MapData]
- val resultValues = new GenericArrayData(new Array[Any](map.numElements))
- var i = 0
- while (i < map.numElements) {
- keyVar.value.set(map.keyArray().get(i, keyVar.dataType))
- valueVar.value.set(map.valueArray().get(i, valueVar.dataType))
- resultValues.update(i, functionForEval.eval(inputRow))
- i += 1
- }
- new ArrayBasedMapData(map.keyArray(), resultValues)
- }
-
- override def prettyName: String = "transform_values"
-}
-
-/**
- * Merges two given maps into a single map by applying function to the pair of values with
- * the same key.
- */
-@ExpressionDescription(
- usage =
- """
- _FUNC_(map1, map2, function) - Merges two given maps into a single map by applying
- function to the pair of values with the same key. For keys only presented in one map,
- NULL will be passed as the value for the missing key. If an input map contains duplicated
- keys, only the first entry of the duplicated key is passed into the lambda function.
- """,
- examples = """
- Examples:
- > SELECT _FUNC_(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2));
- {1:"ax",2:"by"}
- """,
- since = "2.4.0")
-case class MapZipWith(left: Expression, right: Expression, function: Expression)
- extends HigherOrderFunction with CodegenFallback {
-
- def functionForEval: Expression = functionsForEval.head
-
- @transient lazy val MapType(leftKeyType, leftValueType, leftValueContainsNull) = left.dataType
-
- @transient lazy val MapType(rightKeyType, rightValueType, rightValueContainsNull) = right.dataType
-
- @transient lazy val keyType =
- TypeCoercion.findCommonTypeDifferentOnlyInNullFlags(leftKeyType, rightKeyType).get
-
- @transient lazy val ordering = TypeUtils.getInterpretedOrdering(keyType)
-
- override def arguments: Seq[Expression] = left :: right :: Nil
-
- override def argumentTypes: Seq[AbstractDataType] = MapType :: MapType :: Nil
-
- override def functions: Seq[Expression] = function :: Nil
-
- override def functionTypes: Seq[AbstractDataType] = AnyDataType :: Nil
-
- override def dataType: DataType = MapType(keyType, function.dataType, function.nullable)
-
- override def bind(f: (Expression, Seq[(DataType, Boolean)]) => LambdaFunction): MapZipWith = {
- val arguments = Seq((keyType, false), (leftValueType, true), (rightValueType, true))
- copy(function = f(function, arguments))
- }
-
- override def checkArgumentDataTypes(): TypeCheckResult = {
- super.checkArgumentDataTypes() match {
- case TypeCheckResult.TypeCheckSuccess =>
- if (leftKeyType.sameType(rightKeyType)) {
- TypeUtils.checkForOrderingExpr(leftKeyType, s"function $prettyName")
- } else {
- TypeCheckResult.TypeCheckFailure(s"The input to function $prettyName should have " +
- s"been two ${MapType.simpleString}s with compatible key types, but the key types are " +
- s"[${leftKeyType.catalogString}, ${rightKeyType.catalogString}].")
- }
- case failure => failure
- }
- }
-
- override def checkInputDataTypes(): TypeCheckResult = checkArgumentDataTypes()
-
- override def eval(input: InternalRow): Any = {
- val value1 = left.eval(input)
- if (value1 == null) {
- null
- } else {
- val value2 = right.eval(input)
- if (value2 == null) {
- null
- } else {
- nullSafeEval(input, value1, value2)
- }
- }
- }
-
- @transient lazy val LambdaFunction(_, Seq(
- keyVar: NamedLambdaVariable,
- value1Var: NamedLambdaVariable,
- value2Var: NamedLambdaVariable),
- _) = function
-
- /**
- * The function accepts two key arrays and returns a collection of keys with indexes
- * to value arrays. Indexes are represented as an array of two items. This is a small
- * optimization leveraging mutability of arrays.
- */
- @transient private lazy val getKeysWithValueIndexes:
- (ArrayData, ArrayData) => mutable.Iterable[(Any, Array[Option[Int]])] = {
- if (TypeUtils.typeWithProperEquals(keyType)) {
- getKeysWithIndexesFast
- } else {
- getKeysWithIndexesBruteForce
- }
- }
-
- private def assertSizeOfArrayBuffer(size: Int): Unit = {
- if (size > ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH) {
- throw new RuntimeException(s"Unsuccessful try to zip maps with $size " +
- s"unique keys due to exceeding the array size limit " +
- s"${ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH}.")
- }
- }
-
- private def getKeysWithIndexesFast(keys1: ArrayData, keys2: ArrayData) = {
- val hashMap = new mutable.LinkedHashMap[Any, Array[Option[Int]]]
- for((z, array) <- Array((0, keys1), (1, keys2))) {
- var i = 0
- while (i < array.numElements()) {
- val key = array.get(i, keyType)
- hashMap.get(key) match {
- case Some(indexes) =>
- if (indexes(z).isEmpty) {
- indexes(z) = Some(i)
- }
- case None =>
- val indexes = Array[Option[Int]](None, None)
- indexes(z) = Some(i)
- hashMap.put(key, indexes)
- }
- i += 1
- }
- }
- hashMap
- }
-
- private def getKeysWithIndexesBruteForce(keys1: ArrayData, keys2: ArrayData) = {
- val arrayBuffer = new mutable.ArrayBuffer[(Any, Array[Option[Int]])]
- for((z, array) <- Array((0, keys1), (1, keys2))) {
- var i = 0
- while (i < array.numElements()) {
- val key = array.get(i, keyType)
- var found = false
- var j = 0
- while (!found && j < arrayBuffer.size) {
- val (bufferKey, indexes) = arrayBuffer(j)
- if (ordering.equiv(bufferKey, key)) {
- found = true
- if(indexes(z).isEmpty) {
- indexes(z) = Some(i)
- }
- }
- j += 1
- }
- if (!found) {
- assertSizeOfArrayBuffer(arrayBuffer.size)
- val indexes = Array[Option[Int]](None, None)
- indexes(z) = Some(i)
- arrayBuffer += Tuple2(key, indexes)
- }
- i += 1
- }
- }
- arrayBuffer
- }
-
- private def nullSafeEval(inputRow: InternalRow, value1: Any, value2: Any): Any = {
- val mapData1 = value1.asInstanceOf[MapData]
- val mapData2 = value2.asInstanceOf[MapData]
- val keysWithIndexes = getKeysWithValueIndexes(mapData1.keyArray(), mapData2.keyArray())
- val size = keysWithIndexes.size
- val keys = new GenericArrayData(new Array[Any](size))
- val values = new GenericArrayData(new Array[Any](size))
- val valueData1 = mapData1.valueArray()
- val valueData2 = mapData2.valueArray()
- var i = 0
- for ((key, Array(index1, index2)) <- keysWithIndexes) {
- val v1 = index1.map(valueData1.get(_, leftValueType)).getOrElse(null)
- val v2 = index2.map(valueData2.get(_, rightValueType)).getOrElse(null)
- keyVar.value.set(key)
- value1Var.value.set(v1)
- value2Var.value.set(v2)
- keys.update(i, key)
- values.update(i, functionForEval.eval(inputRow))
- i += 1
- }
- new ArrayBasedMapData(keys, values)
- }
-
- override def prettyName: String = "map_zip_with"
-}
-
// scalastyle:off line.size.limit
@ExpressionDescription(
usage = "_FUNC_(left, right, func) - Merges the two given arrays, element-wise, into a single array using function. If one array is shorter, nulls are appended at the end to match the length of the longer array, before applying function.",
examples = """
Examples:
> SELECT _FUNC_(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x));
- array(('a', 1), ('b', 2), ('c', 3))
- > SELECT _FUNC_(array(1, 2), array(3, 4), (x, y) -> x + y));
- array(4, 6)
+ [{"y":"a","x":1},{"y":"b","x":2},{"y":"c","x":3}]
+ > SELECT _FUNC_(array(1, 2), array(3, 4), (x, y) -> x + y);
+ [4,6]
> SELECT _FUNC_(array('a', 'b', 'c'), array('d', 'e', 'f'), (x, y) -> concat(x, y));
- array('ad', 'be', 'cf')
+ ["ad","be","cf"]
""",
since = "2.4.0")
// scalastyle:on line.size.limit
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
index bd9090a07471b..4cd1a0916115e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
@@ -472,11 +472,6 @@ case class JsonTuple(children: Seq[Expression])
parser.getCurrentToken match {
// if the user requests a string field it needs to be returned without enclosing
// quotes which is accomplished via JsonGenerator.writeRaw instead of JsonGenerator.write
- case JsonToken.VALUE_STRING if parser.hasTextCharacters =>
- // slight optimization to avoid allocating a String instance, though the characters
- // still have to be decoded... Jackson doesn't have a way to access the raw bytes
- generator.writeRaw(parser.getTextCharacters, parser.getTextOffset, parser.getTextLength)
-
case JsonToken.VALUE_STRING =>
// the normal String case, pass it through to the output without enclosing quotes
generator.writeRaw(parser.getText)
@@ -744,14 +739,27 @@ case class StructsToJson(
""",
since = "2.4.0")
case class SchemaOfJson(child: Expression)
- extends UnaryExpression with String2StringExpression with CodegenFallback {
+ extends UnaryExpression with CodegenFallback {
+
+ override def dataType: DataType = StringType
+
+ override def nullable: Boolean = false
private val jsonOptions = new JSONOptions(Map.empty, "UTC")
private val jsonFactory = new JsonFactory()
jsonOptions.setJacksonOptions(jsonFactory)
- override def convert(v: UTF8String): UTF8String = {
- val dt = Utils.tryWithResource(CreateJacksonParser.utf8String(jsonFactory, v)) { parser =>
+ @transient
+ private lazy val json = child.eval().asInstanceOf[UTF8String]
+
+ override def checkInputDataTypes(): TypeCheckResult = child match {
+ case Literal(s, StringType) if s != null => super.checkInputDataTypes()
+ case _ => TypeCheckResult.TypeCheckFailure(
+ s"The input json should be a string literal and not null; however, got ${child.sql}.")
+ }
+
+ override def eval(v: InternalRow): Any = {
+ val dt = Utils.tryWithResource(CreateJacksonParser.utf8String(jsonFactory, json)) { parser =>
parser.nextToken()
inferField(parser, jsonOptions)
}
@@ -765,7 +773,7 @@ object JsonExprUtils {
def evalSchemaExpr(exp: Expression): DataType = exp match {
case Literal(s, StringType) => DataType.fromDDL(s.toString)
case e @ SchemaOfJson(_: Literal) =>
- val ddlSchema = e.eval().asInstanceOf[UTF8String]
+ val ddlSchema = e.eval(EmptyRow).asInstanceOf[UTF8String]
DataType.fromDDL(ddlSchema.toString)
case e => throw new AnalysisException(
"Schema should be specified in DDL format as a string literal" +
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 584a2946bd564..049ea77691395 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -115,6 +115,7 @@ abstract class Attribute extends LeafExpression with NamedExpression with NullIn
def withQualifier(newQualifier: Seq[String]): Attribute
def withName(newName: String): Attribute
def withMetadata(newMetadata: Metadata): Attribute
+ def withExprId(newExprId: ExprId): Attribute
override def toAttribute: Attribute = this
def newInstance(): Attribute
@@ -299,7 +300,7 @@ case class AttributeReference(
}
}
- def withExprId(newExprId: ExprId): AttributeReference = {
+ override def withExprId(newExprId: ExprId): AttributeReference = {
if (exprId == newExprId) {
this
} else {
@@ -362,6 +363,8 @@ case class PrettyAttribute(
throw new UnsupportedOperationException
override def qualifier: Seq[String] = throw new UnsupportedOperationException
override def exprId: ExprId = throw new UnsupportedOperationException
+ override def withExprId(newExprId: ExprId): Attribute =
+ throw new UnsupportedOperationException
override def nullable: Boolean = true
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
index b683d2a7e9ef3..293d28e93039a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala
@@ -42,7 +42,8 @@ import org.apache.spark.sql.types._
Examples:
> SELECT _FUNC_(NULL, 1, NULL);
1
- """)
+ """,
+ since = "1.0.0")
// scalastyle:on line.size.limit
case class Coalesce(children: Seq[Expression]) extends ComplexTypeMergingExpression {
@@ -127,7 +128,8 @@ case class Coalesce(children: Seq[Expression]) extends ComplexTypeMergingExpress
Examples:
> SELECT _FUNC_(NULL, array('2'));
["2"]
- """)
+ """,
+ since = "2.0.0")
case class IfNull(left: Expression, right: Expression, child: Expression)
extends RuntimeReplaceable {
@@ -146,7 +148,8 @@ case class IfNull(left: Expression, right: Expression, child: Expression)
Examples:
> SELECT _FUNC_(2, 2);
NULL
- """)
+ """,
+ since = "2.0.0")
case class NullIf(left: Expression, right: Expression, child: Expression)
extends RuntimeReplaceable {
@@ -165,7 +168,8 @@ case class NullIf(left: Expression, right: Expression, child: Expression)
Examples:
> SELECT _FUNC_(NULL, array('2'));
["2"]
- """)
+ """,
+ since = "2.0.0")
case class Nvl(left: Expression, right: Expression, child: Expression) extends RuntimeReplaceable {
def this(left: Expression, right: Expression) = {
@@ -184,7 +188,8 @@ case class Nvl(left: Expression, right: Expression, child: Expression) extends R
Examples:
> SELECT _FUNC_(NULL, 2, 1);
1
- """)
+ """,
+ since = "2.0.0")
// scalastyle:on line.size.limit
case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression, child: Expression)
extends RuntimeReplaceable {
@@ -207,7 +212,8 @@ case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression, child:
Examples:
> SELECT _FUNC_(cast('NaN' as double));
true
- """)
+ """,
+ since = "1.5.0")
case class IsNaN(child: Expression) extends UnaryExpression
with Predicate with ImplicitCastInputTypes {
@@ -249,7 +255,8 @@ case class IsNaN(child: Expression) extends UnaryExpression
Examples:
> SELECT _FUNC_(cast('NaN' as double), 123);
123.0
- """)
+ """,
+ since = "1.5.0")
case class NaNvl(left: Expression, right: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -309,7 +316,8 @@ case class NaNvl(left: Expression, right: Expression)
Examples:
> SELECT _FUNC_(1);
false
- """)
+ """,
+ since = "1.0.0")
case class IsNull(child: Expression) extends UnaryExpression with Predicate {
override def nullable: Boolean = false
@@ -335,7 +343,8 @@ case class IsNull(child: Expression) extends UnaryExpression with Predicate {
Examples:
> SELECT _FUNC_(1);
true
- """)
+ """,
+ since = "1.0.0")
case class IsNotNull(child: Expression) extends UnaryExpression with Predicate {
override def nullable: Boolean = false
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index 149bd79278a54..365fcad7d3626 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -129,6 +129,13 @@ case class Not(child: Expression)
override def inputTypes: Seq[DataType] = Seq(BooleanType)
+ // +---------+-----------+
+ // | CHILD | NOT CHILD |
+ // +---------+-----------+
+ // | TRUE | FALSE |
+ // | FALSE | TRUE |
+ // | UNKNOWN | UNKNOWN |
+ // +---------+-----------+
protected override def nullSafeEval(input: Any): Any = !input.asInstanceOf[Boolean]
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -362,31 +369,26 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with
}
@transient lazy val set: Set[Any] = child.dataType match {
- case _: AtomicType => hset
+ case t: AtomicType if !t.isInstanceOf[BinaryType] => hset
case _: NullType => hset
case _ =>
// for structs use interpreted ordering to be able to compare UnsafeRows with non-UnsafeRows
- TreeSet.empty(TypeUtils.getInterpretedOrdering(child.dataType)) ++ hset
+ TreeSet.empty(TypeUtils.getInterpretedOrdering(child.dataType)) ++ (hset - null)
}
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
- val setTerm = ctx.addReferenceObj("set", set)
- val childGen = child.genCode(ctx)
- val setIsNull = if (hasNull) {
- s"${ev.isNull} = !${ev.value};"
- } else {
- ""
- }
- ev.copy(code =
- code"""
- |${childGen.code}
- |${CodeGenerator.JAVA_BOOLEAN} ${ev.isNull} = ${childGen.isNull};
- |${CodeGenerator.JAVA_BOOLEAN} ${ev.value} = false;
- |if (!${ev.isNull}) {
- | ${ev.value} = $setTerm.contains(${childGen.value});
- | $setIsNull
- |}
- """.stripMargin)
+ nullSafeCodeGen(ctx, ev, c => {
+ val setTerm = ctx.addReferenceObj("set", set)
+ val setIsNull = if (hasNull) {
+ s"${ev.isNull} = !${ev.value};"
+ } else {
+ ""
+ }
+ s"""
+ |${ev.value} = $setTerm.contains($c);
+ |$setIsNull
+ """.stripMargin
+ })
}
override def sql: String = {
@@ -406,6 +408,13 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with
override def sqlOperator: String = "AND"
+ // +---------+---------+---------+---------+
+ // | AND | TRUE | FALSE | UNKNOWN |
+ // +---------+---------+---------+---------+
+ // | TRUE | TRUE | FALSE | UNKNOWN |
+ // | FALSE | FALSE | FALSE | FALSE |
+ // | UNKNOWN | UNKNOWN | FALSE | UNKNOWN |
+ // +---------+---------+---------+---------+
override def eval(input: InternalRow): Any = {
val input1 = left.eval(input)
if (input1 == false) {
@@ -469,6 +478,13 @@ case class Or(left: Expression, right: Expression) extends BinaryOperator with P
override def sqlOperator: String = "OR"
+ // +---------+---------+---------+---------+
+ // | OR | TRUE | FALSE | UNKNOWN |
+ // +---------+---------+---------+---------+
+ // | TRUE | TRUE | TRUE | TRUE |
+ // | FALSE | TRUE | FALSE | UNKNOWN |
+ // | UNKNOWN | TRUE | UNKNOWN | UNKNOWN |
+ // +---------+---------+---------+---------+
override def eval(input: InternalRow): Any = {
val input1 = left.eval(input)
if (input1 == true) {
@@ -592,6 +608,13 @@ case class EqualTo(left: Expression, right: Expression)
override def symbol: String = "="
+ // +---------+---------+---------+---------+
+ // | = | TRUE | FALSE | UNKNOWN |
+ // +---------+---------+---------+---------+
+ // | TRUE | TRUE | FALSE | UNKNOWN |
+ // | FALSE | FALSE | TRUE | UNKNOWN |
+ // | UNKNOWN | UNKNOWN | UNKNOWN | UNKNOWN |
+ // +---------+---------+---------+---------+
protected override def nullSafeEval(left: Any, right: Any): Any = ordering.equiv(left, right)
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -629,6 +652,13 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp
override def nullable: Boolean = false
+ // +---------+---------+---------+---------+
+ // | <=> | TRUE | FALSE | UNKNOWN |
+ // +---------+---------+---------+---------+
+ // | TRUE | TRUE | FALSE | FALSE |
+ // | FALSE | FALSE | TRUE | FALSE |
+ // | UNKNOWN | FALSE | FALSE | TRUE |
+ // +---------+---------+---------+---------+
override def eval(input: InternalRow): Any = {
val input1 = left.eval(input)
val input2 = right.eval(input)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
index b70c34141b97d..c02d2f0b69a31 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
@@ -78,7 +78,8 @@ trait ExpressionWithRandomSeed {
> SELECT _FUNC_(null);
0.8446490682263027
""",
- note = "The function is non-deterministic in general case.")
+ note = "The function is non-deterministic in general case.",
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class Rand(child: Expression) extends RDG with ExpressionWithRandomSeed {
@@ -118,7 +119,8 @@ object Rand {
> SELECT _FUNC_(null);
1.1164209726833079
""",
- note = "The function is non-deterministic in general case.")
+ note = "The function is non-deterministic in general case.",
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class Randn(child: Expression) extends RDG with ExpressionWithRandomSeed {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
index bf0c35fe61018..e80543cc91874 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
@@ -101,7 +101,8 @@ abstract class StringRegexExpression extends BinaryExpression
""",
note = """
Use RLIKE to match with standard regular expressions.
- """)
+ """,
+ since = "1.0.0")
case class Like(left: Expression, right: Expression) extends StringRegexExpression {
override def escape(v: String): String = StringUtils.escapeLikeRegex(v)
@@ -179,7 +180,8 @@ case class Like(left: Expression, right: Expression) extends StringRegexExpressi
""",
note = """
Use LIKE to match with simple string pattern.
- """)
+ """,
+ since = "1.0.0")
case class RLike(left: Expression, right: Expression) extends StringRegexExpression {
override def escape(v: String): String = v
@@ -237,7 +239,8 @@ case class RLike(left: Expression, right: Expression) extends StringRegexExpress
Examples:
> SELECT _FUNC_('oneAtwoBthreeC', '[ABC]');
["one","two","three",""]
- """)
+ """,
+ since = "1.5.0")
case class StringSplit(str: Expression, pattern: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -274,7 +277,8 @@ case class StringSplit(str: Expression, pattern: Expression)
Examples:
> SELECT _FUNC_('100-200', '(\\d+)', 'num');
num-num
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -373,7 +377,8 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio
Examples:
> SELECT _FUNC_('100-200', '(\\d+)-(\\d+)', 1);
100
- """)
+ """,
+ since = "1.5.0")
case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
def this(s: Expression, r: Expression) = this(s, r, Literal(1))
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 14faa62bde7d0..006d2f6f45a35 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -50,7 +50,8 @@ import org.apache.spark.unsafe.types.{ByteArray, UTF8String}
Examples:
> SELECT _FUNC_(' ', 'Spark', 'SQL');
Spark SQL
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class ConcatWs(children: Seq[Expression])
extends Expression with ImplicitCastInputTypes {
@@ -203,7 +204,8 @@ case class ConcatWs(children: Seq[Expression])
Examples:
> SELECT _FUNC_(1, 'scala', 'java');
scala
- """)
+ """,
+ since = "2.0.0")
// scalastyle:on line.size.limit
case class Elt(children: Seq[Expression]) extends Expression {
@@ -326,7 +328,8 @@ trait String2StringExpression extends ImplicitCastInputTypes {
Examples:
> SELECT _FUNC_('SparkSql');
SPARKSQL
- """)
+ """,
+ since = "1.0.1")
case class Upper(child: Expression)
extends UnaryExpression with String2StringExpression {
@@ -346,7 +349,8 @@ case class Upper(child: Expression)
Examples:
> SELECT _FUNC_('SparkSql');
sparksql
- """)
+ """,
+ since = "1.0.1")
case class Lower(child: Expression) extends UnaryExpression with String2StringExpression {
override def convert(v: UTF8String): UTF8String = v.toLowerCase
@@ -417,7 +421,8 @@ case class EndsWith(left: Expression, right: Expression) extends StringPredicate
Examples:
> SELECT _FUNC_('ABCabc', 'abc', 'DEF');
ABCDEF
- """)
+ """,
+ since = "2.3.0")
// scalastyle:on line.size.limit
case class StringReplace(srcExpr: Expression, searchExpr: Expression, replaceExpr: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -475,7 +480,8 @@ object StringTranslate {
Examples:
> SELECT _FUNC_('AaBbCc', 'abc', '123');
A1B2C3
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replaceExpr: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -539,7 +545,8 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac
Examples:
> SELECT _FUNC_('ab','abc,b,ab,c,def');
3
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class FindInSet(left: Expression, right: Expression) extends BinaryExpression
with ImplicitCastInputTypes {
@@ -621,7 +628,8 @@ object StringTrim {
parkSQLS
> SELECT _FUNC_(TRAILING 'SL' FROM 'SSparkSQLS');
SSparkSQ
- """)
+ """,
+ since = "1.5.0")
case class StringTrim(
srcStr: Expression,
trimStr: Option[Expression] = None)
@@ -720,7 +728,8 @@ object StringTrimLeft {
SparkSQL
> SELECT _FUNC_('Sp', 'SSparkSQLS');
arkSQLS
- """)
+ """,
+ since = "1.5.0")
case class StringTrimLeft(
srcStr: Expression,
trimStr: Option[Expression] = None)
@@ -821,7 +830,8 @@ object StringTrimRight {
SparkSQL
> SELECT _FUNC_('LQSa', 'SSparkSQLS');
SSpark
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class StringTrimRight(
srcStr: Expression,
@@ -901,7 +911,8 @@ case class StringTrimRight(
Examples:
> SELECT _FUNC_('SparkSQL', 'SQL');
6
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class StringInstr(str: Expression, substr: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -942,7 +953,8 @@ case class StringInstr(str: Expression, substr: Expression)
Examples:
> SELECT _FUNC_('www.apache.org', '.', 2);
www.apache
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -981,7 +993,8 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr:
7
> SELECT POSITION('bar' IN 'foobarbar');
4
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class StringLocate(substr: Expression, str: Expression, start: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -1066,7 +1079,8 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression)
???hi
> SELECT _FUNC_('hi', 1, '??');
h
- """)
+ """,
+ since = "1.5.0")
case class StringLPad(str: Expression, len: Expression, pad: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -1099,7 +1113,8 @@ case class StringLPad(str: Expression, len: Expression, pad: Expression)
hi???
> SELECT _FUNC_('hi', 1, '??');
h
- """)
+ """,
+ since = "1.5.0")
case class StringRPad(str: Expression, len: Expression, pad: Expression)
extends TernaryExpression with ImplicitCastInputTypes {
@@ -1144,7 +1159,8 @@ object ParseUrl {
query=1
> SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY', 'query')
1
- """)
+ """,
+ since = "2.0.0")
case class ParseUrl(children: Seq[Expression])
extends Expression with ExpectsInputTypes with CodegenFallback {
@@ -1295,7 +1311,8 @@ case class ParseUrl(children: Seq[Expression])
Examples:
> SELECT _FUNC_("Hello World %d %s", 100, "days");
Hello World 100 days
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class FormatString(children: Expression*) extends Expression with ImplicitCastInputTypes {
@@ -1382,7 +1399,8 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC
Examples:
> SELECT _FUNC_('sPark sql');
Spark Sql
- """)
+ """,
+ since = "1.5.0")
case class InitCap(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def inputTypes: Seq[DataType] = Seq(StringType)
@@ -1405,7 +1423,8 @@ case class InitCap(child: Expression) extends UnaryExpression with ImplicitCastI
Examples:
> SELECT _FUNC_('123', 2);
123123
- """)
+ """,
+ since = "1.5.0")
case class StringRepeat(str: Expression, times: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -1434,7 +1453,8 @@ case class StringRepeat(str: Expression, times: Expression)
Examples:
> SELECT concat(_FUNC_(2), '1');
1
- """)
+ """,
+ since = "1.5.0")
case class StringSpace(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes {
@@ -1471,7 +1491,8 @@ case class StringSpace(child: Expression)
SQL
> SELECT _FUNC_('Spark SQL', 5, 1);
k
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class Substring(str: Expression, pos: Expression, len: Expression)
extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
@@ -1517,7 +1538,8 @@ case class Substring(str: Expression, pos: Expression, len: Expression)
Examples:
> SELECT _FUNC_('Spark SQL', 3);
SQL
- """)
+ """,
+ since = "2.3.0")
// scalastyle:on line.size.limit
case class Right(str: Expression, len: Expression, child: Expression) extends RuntimeReplaceable {
def this(str: Expression, len: Expression) = {
@@ -1539,7 +1561,8 @@ case class Right(str: Expression, len: Expression, child: Expression) extends Ru
Examples:
> SELECT _FUNC_('Spark SQL', 3);
Spa
- """)
+ """,
+ since = "2.3.0")
// scalastyle:on line.size.limit
case class Left(str: Expression, len: Expression, child: Expression) extends RuntimeReplaceable {
def this(str: Expression, len: Expression) = {
@@ -1565,7 +1588,8 @@ case class Left(str: Expression, len: Expression, child: Expression) extends Run
10
> SELECT CHARACTER_LENGTH('Spark SQL ');
10
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class Length(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = IntegerType
@@ -1593,7 +1617,8 @@ case class Length(child: Expression) extends UnaryExpression with ImplicitCastIn
Examples:
> SELECT _FUNC_('Spark SQL');
72
- """)
+ """,
+ since = "2.3.0")
case class BitLength(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = IntegerType
override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(StringType, BinaryType))
@@ -1623,7 +1648,8 @@ case class BitLength(child: Expression) extends UnaryExpression with ImplicitCas
Examples:
> SELECT _FUNC_('Spark SQL');
9
- """)
+ """,
+ since = "2.3.0")
case class OctetLength(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = IntegerType
override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(StringType, BinaryType))
@@ -1652,7 +1678,8 @@ case class OctetLength(child: Expression) extends UnaryExpression with ImplicitC
Examples:
> SELECT _FUNC_('kitten', 'sitting');
3
- """)
+ """,
+ since = "1.5.0")
case class Levenshtein(left: Expression, right: Expression) extends BinaryExpression
with ImplicitCastInputTypes {
@@ -1677,7 +1704,8 @@ case class Levenshtein(left: Expression, right: Expression) extends BinaryExpres
Examples:
> SELECT _FUNC_('Miller');
M460
- """)
+ """,
+ since = "1.5.0")
case class SoundEx(child: Expression) extends UnaryExpression with ExpectsInputTypes {
override def dataType: DataType = StringType
@@ -1702,7 +1730,8 @@ case class SoundEx(child: Expression) extends UnaryExpression with ExpectsInputT
50
> SELECT _FUNC_(2);
50
- """)
+ """,
+ since = "1.5.0")
case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = IntegerType
@@ -1742,7 +1771,8 @@ case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInp
Examples:
> SELECT _FUNC_(65);
A
- """)
+ """,
+ since = "2.3.0")
// scalastyle:on line.size.limit
case class Chr(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
@@ -1785,7 +1815,8 @@ case class Chr(child: Expression) extends UnaryExpression with ImplicitCastInput
Examples:
> SELECT _FUNC_('Spark SQL');
U3BhcmsgU1FM
- """)
+ """,
+ since = "1.5.0")
case class Base64(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = StringType
@@ -1814,7 +1845,8 @@ case class Base64(child: Expression) extends UnaryExpression with ImplicitCastIn
Examples:
> SELECT _FUNC_('U3BhcmsgU1FM');
Spark SQL
- """)
+ """,
+ since = "1.5.0")
case class UnBase64(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
override def dataType: DataType = BinaryType
@@ -1843,7 +1875,8 @@ case class UnBase64(child: Expression) extends UnaryExpression with ImplicitCast
Examples:
> SELECT _FUNC_(encode('abc', 'utf-8'), 'utf-8');
abc
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class Decode(bin: Expression, charset: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -1882,7 +1915,8 @@ case class Decode(bin: Expression, charset: Expression)
Examples:
> SELECT _FUNC_('abc', 'utf-8');
abc
- """)
+ """,
+ since = "1.5.0")
// scalastyle:on line.size.limit
case class Encode(value: Expression, charset: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -1926,7 +1960,8 @@ case class Encode(value: Expression, charset: Expression)
12,332.1235
> SELECT _FUNC_(12332.123456, '##################.###');
12332.123
- """)
+ """,
+ since = "1.5.0")
case class FormatNumber(x: Expression, d: Expression)
extends BinaryExpression with ExpectsInputTypes {
@@ -2096,7 +2131,8 @@ case class FormatNumber(x: Expression, d: Expression)
Examples:
> SELECT _FUNC_('Hi there! Good morning.');
[["Hi","there"],["Good","morning"]]
- """)
+ """,
+ since = "2.0.0")
case class Sentences(
str: Expression,
language: Expression = Literal(""),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
index 9b86d865622dc..a379f8642b007 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.core._
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils, MapData}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimestampParser
import org.apache.spark.sql.types._
/**
@@ -74,6 +75,8 @@ private[sql] class JacksonGenerator(
private val lineSeparator: String = options.lineSeparatorInWrite
+ @transient private lazy val timestampParser = new TimestampParser(options.timestampFormat)
+
private def makeWriter(dataType: DataType): ValueWriter = dataType match {
case NullType =>
(row: SpecializedGetters, ordinal: Int) =>
@@ -113,8 +116,7 @@ private[sql] class JacksonGenerator(
case TimestampType =>
(row: SpecializedGetters, ordinal: Int) =>
- val timestampString =
- options.timestampFormat.format(DateTimeUtils.toJavaTimestamp(row.getLong(ordinal)))
+ val timestampString = timestampParser.format(row.getLong(ordinal))
gen.writeString(timestampString)
case DateType =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index 984979ac5e9b4..616de86aa6824 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -29,6 +29,7 @@ import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimestampParser
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.Utils
@@ -53,6 +54,8 @@ class JacksonParser(
private val factory = new JsonFactory()
options.setJacksonOptions(factory)
+ @transient private lazy val timestampParser = new TimestampParser(options.timestampFormat)
+
/**
* Create a converter which converts the JSON documents held by the `JsonParser`
* to a value according to a desired schema. This is a wrapper for the method
@@ -213,15 +216,12 @@ class JacksonParser(
(parser: JsonParser) => parseJsonToken[java.lang.Long](parser, dataType) {
case VALUE_STRING =>
val stringValue = parser.getText
- // This one will lose microseconds parts.
- // See https://issues.apache.org/jira/browse/SPARK-10681.
Long.box {
- Try(options.timestampFormat.parse(stringValue).getTime * 1000L)
- .getOrElse {
- // If it fails to parse, then tries the way used in 2.0 and 1.x for backwards
- // compatibility.
- DateTimeUtils.stringToTime(stringValue).getTime * 1000L
- }
+ Try(timestampParser.parse(stringValue)).getOrElse {
+ // If it fails to parse, then tries the way used in 2.0 and 1.x for backwards
+ // compatibility.
+ DateTimeUtils.stringToTime(stringValue).getTime * 1000L
+ }
}
case VALUE_NUMBER_INT =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
index 064ca68b7a628..01634a9d852c6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
@@ -48,6 +48,7 @@ object CostBasedJoinReorder extends Rule[LogicalPlan] with PredicateHelper {
if projectList.forall(_.isInstanceOf[Attribute]) =>
reorder(p, p.output)
}
+
// After reordering is finished, convert OrderedJoin back to Join
result transformDown {
case OrderedJoin(left, right, jt, cond) => Join(left, right, jt, cond)
@@ -175,11 +176,20 @@ object JoinReorderDP extends PredicateHelper with Logging {
assert(topOutputSet == p.outputSet)
// Keep the same order of final output attributes.
p.copy(projectList = output)
+ case finalPlan if !sameOutput(finalPlan, output) =>
+ Project(output, finalPlan)
case finalPlan =>
finalPlan
}
}
+ private def sameOutput(plan: LogicalPlan, expectedOutput: Seq[Attribute]): Boolean = {
+ val thisOutput = plan.output
+ thisOutput.length == expectedOutput.length && thisOutput.zip(expectedOutput).forall {
+ case (a1, a2) => a1.semanticEquals(a2)
+ }
+ }
+
/** Find all possible plans at the next level, based on existing levels. */
private def searchLevel(
existingLevels: Seq[JoinPlanMap],
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index e4b4f1ecbe21f..d90f3affe69dd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -165,7 +165,10 @@ abstract class Optimizer(sessionCatalog: SessionCatalog)
Batch("LocalRelation", fixedPoint,
ConvertToLocalRelation,
PropagateEmptyRelation) :+
- // The following batch should be executed after batch "Join Reorder" and "LocalRelation".
+ Batch("Extract PythonUDF From JoinCondition", Once,
+ PullOutPythonUDFInJoinCondition) :+
+ // The following batch should be executed after batch "Join Reorder" "LocalRelation" and
+ // "Extract PythonUDF From JoinCondition".
Batch("Check Cartesian Products", Once,
CheckCartesianProducts) :+
Batch("RewriteSubquery", Once,
@@ -202,7 +205,8 @@ abstract class Optimizer(sessionCatalog: SessionCatalog)
ReplaceDistinctWithAggregate.ruleName ::
PullupCorrelatedPredicates.ruleName ::
RewriteCorrelatedScalarSubquery.ruleName ::
- RewritePredicateSubquery.ruleName :: Nil
+ RewritePredicateSubquery.ruleName ::
+ PullOutPythonUDFInJoinCondition.ruleName :: Nil
/**
* Optimize all the subqueries inside expression.
@@ -486,6 +490,10 @@ object PushProjectionThroughUnion extends Rule[LogicalPlan] with PredicateHelper
private def pushToRight[A <: Expression](e: A, rewrites: AttributeMap[Attribute]) = {
val result = e transform {
case a: Attribute => rewrites(a)
+ } match {
+ // Make sure exprId is unique in each child of Union.
+ case Alias(child, alias) => Alias(child, alias)()
+ case other => other
}
// We must promise the compiler that we did not discard the names in the case of project
@@ -1351,9 +1359,9 @@ object ConvertToLocalRelation extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
case Project(projectList, LocalRelation(output, data, isStreaming))
if !projectList.exists(hasUnevaluableExpr) =>
- val projection = new InterpretedProjection(projectList, output)
+ val projection = new InterpretedMutableProjection(projectList, output)
projection.initialize(0)
- LocalRelation(projectList.map(_.toAttribute), data.map(projection), isStreaming)
+ LocalRelation(projectList.map(_.toAttribute), data.map(projection(_).copy()), isStreaming)
case Limit(IntegerLiteral(limit), LocalRelation(output, data, isStreaming)) =>
LocalRelation(output, data.take(limit), isStreaming)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceExceptWithFilter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceExceptWithFilter.scala
index efd3944eba7f5..4996d24dfd298 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceExceptWithFilter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceExceptWithFilter.scala
@@ -36,7 +36,8 @@ import org.apache.spark.sql.catalyst.rules.Rule
* Note:
* Before flipping the filter condition of the right node, we should:
* 1. Combine all it's [[Filter]].
- * 2. Apply InferFiltersFromConstraints rule (to take into account of NULL values in the condition).
+ * 2. Update the attribute references to the left node;
+ * 3. Add a Coalesce(condition, False) (to take into account of NULL values in the condition).
*/
object ReplaceExceptWithFilter extends Rule[LogicalPlan] {
@@ -47,23 +48,28 @@ object ReplaceExceptWithFilter extends Rule[LogicalPlan] {
plan.transform {
case e @ Except(left, right, false) if isEligible(left, right) =>
- val newCondition = transformCondition(left, skipProject(right))
- newCondition.map { c =>
- Distinct(Filter(Not(c), left))
- }.getOrElse {
+ val filterCondition = combineFilters(skipProject(right)).asInstanceOf[Filter].condition
+ if (filterCondition.deterministic) {
+ transformCondition(left, filterCondition).map { c =>
+ Distinct(Filter(Not(c), left))
+ }.getOrElse {
+ e
+ }
+ } else {
e
}
}
}
- private def transformCondition(left: LogicalPlan, right: LogicalPlan): Option[Expression] = {
- val filterCondition =
- InferFiltersFromConstraints(combineFilters(right)).asInstanceOf[Filter].condition
-
- val attributeNameMap: Map[String, Attribute] = left.output.map(x => (x.name, x)).toMap
-
- if (filterCondition.references.forall(r => attributeNameMap.contains(r.name))) {
- Some(filterCondition.transform { case a: AttributeReference => attributeNameMap(a.name) })
+ private def transformCondition(plan: LogicalPlan, condition: Expression): Option[Expression] = {
+ val attributeNameMap: Map[String, Attribute] = plan.output.map(x => (x.name, x)).toMap
+ if (condition.references.forall(r => attributeNameMap.contains(r.name))) {
+ val rewrittenCondition = condition.transform {
+ case a: AttributeReference => attributeNameMap(a.name)
+ }
+ // We need to consider as False when the condition is NULL, otherwise we do not return those
+ // rows containing NULL which are instead filtered in the Except right plan
+ Some(Coalesce(Seq(rewrittenCondition, Literal.FalseLiteral)))
} else {
None
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
index 4448ace7105a4..b9468007cac61 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
@@ -95,7 +95,7 @@ import org.apache.spark.sql.types.IntegerType
*
* This rule duplicates the input data by two or more times (# distinct groups + an optional
* non-distinct group). This will put quite a bit of memory pressure of the used aggregate and
- * exchange operators. Keeping the number of distinct groups as low a possible should be priority,
+ * exchange operators. Keeping the number of distinct groups as low as possible should be priority,
* we could improve this in the current rule by applying more advanced expression canonicalization
* techniques.
*/
@@ -241,7 +241,7 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
groupByAttrs ++ distinctAggChildAttrs ++ Seq(gid) ++ regularAggChildAttrMap.map(_._2),
a.child)
- // Construct the first aggregate operator. This de-duplicates the all the children of
+ // Construct the first aggregate operator. This de-duplicates all the children of
// distinct operators, and applies the regular aggregate operators.
val firstAggregateGroupBy = groupByAttrs ++ distinctAggChildAttrs :+ gid
val firstAggregate = Aggregate(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index 5629b72894225..468a950fb1087 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -263,23 +263,50 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper {
case TrueLiteral Or _ => TrueLiteral
case _ Or TrueLiteral => TrueLiteral
- case a And b if Not(a).semanticEquals(b) => FalseLiteral
- case a Or b if Not(a).semanticEquals(b) => TrueLiteral
- case a And b if a.semanticEquals(Not(b)) => FalseLiteral
- case a Or b if a.semanticEquals(Not(b)) => TrueLiteral
+ case a And b if Not(a).semanticEquals(b) =>
+ If(IsNull(a), Literal.create(null, a.dataType), FalseLiteral)
+ case a And b if a.semanticEquals(Not(b)) =>
+ If(IsNull(b), Literal.create(null, b.dataType), FalseLiteral)
+
+ case a Or b if Not(a).semanticEquals(b) =>
+ If(IsNull(a), Literal.create(null, a.dataType), TrueLiteral)
+ case a Or b if a.semanticEquals(Not(b)) =>
+ If(IsNull(b), Literal.create(null, b.dataType), TrueLiteral)
case a And b if a.semanticEquals(b) => a
case a Or b if a.semanticEquals(b) => a
- case a And (b Or c) if Not(a).semanticEquals(b) => And(a, c)
- case a And (b Or c) if Not(a).semanticEquals(c) => And(a, b)
- case (a Or b) And c if a.semanticEquals(Not(c)) => And(b, c)
- case (a Or b) And c if b.semanticEquals(Not(c)) => And(a, c)
-
- case a Or (b And c) if Not(a).semanticEquals(b) => Or(a, c)
- case a Or (b And c) if Not(a).semanticEquals(c) => Or(a, b)
- case (a And b) Or c if a.semanticEquals(Not(c)) => Or(b, c)
- case (a And b) Or c if b.semanticEquals(Not(c)) => Or(a, c)
+ // The following optimizations are applicable only when the operands are not nullable,
+ // since the three-value logic of AND and OR are different in NULL handling.
+ // See the chart:
+ // +---------+---------+---------+---------+
+ // | operand | operand | OR | AND |
+ // +---------+---------+---------+---------+
+ // | TRUE | TRUE | TRUE | TRUE |
+ // | TRUE | FALSE | TRUE | FALSE |
+ // | FALSE | FALSE | FALSE | FALSE |
+ // | UNKNOWN | TRUE | TRUE | UNKNOWN |
+ // | UNKNOWN | FALSE | UNKNOWN | FALSE |
+ // | UNKNOWN | UNKNOWN | UNKNOWN | UNKNOWN |
+ // +---------+---------+---------+---------+
+
+ // (NULL And (NULL Or FALSE)) = NULL, but (NULL And FALSE) = FALSE. Thus, a can't be nullable.
+ case a And (b Or c) if !a.nullable && Not(a).semanticEquals(b) => And(a, c)
+ // (NULL And (FALSE Or NULL)) = NULL, but (NULL And FALSE) = FALSE. Thus, a can't be nullable.
+ case a And (b Or c) if !a.nullable && Not(a).semanticEquals(c) => And(a, b)
+ // ((NULL Or FALSE) And NULL) = NULL, but (FALSE And NULL) = FALSE. Thus, c can't be nullable.
+ case (a Or b) And c if !c.nullable && a.semanticEquals(Not(c)) => And(b, c)
+ // ((FALSE Or NULL) And NULL) = NULL, but (FALSE And NULL) = FALSE. Thus, c can't be nullable.
+ case (a Or b) And c if !c.nullable && b.semanticEquals(Not(c)) => And(a, c)
+
+ // (NULL Or (NULL And TRUE)) = NULL, but (NULL Or TRUE) = TRUE. Thus, a can't be nullable.
+ case a Or (b And c) if !a.nullable && Not(a).semanticEquals(b) => Or(a, c)
+ // (NULL Or (TRUE And NULL)) = NULL, but (NULL Or TRUE) = TRUE. Thus, a can't be nullable.
+ case a Or (b And c) if !a.nullable && Not(a).semanticEquals(c) => Or(a, b)
+ // ((NULL And TRUE) Or NULL) = NULL, but (TRUE Or NULL) = TRUE. Thus, c can't be nullable.
+ case (a And b) Or c if !c.nullable && a.semanticEquals(Not(c)) => Or(b, c)
+ // ((TRUE And NULL) Or NULL) = NULL, but (TRUE Or NULL) = TRUE. Thus, c can't be nullable.
+ case (a And b) Or c if !c.nullable && b.semanticEquals(Not(c)) => Or(a, c)
// Common factor elimination for conjunction
case and @ (left And right) =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
index edbeaf273fd6f..56399e0446b30 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.optimizer
import scala.annotation.tailrec
+import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.planning.ExtractFiltersAndInnerJoins
import org.apache.spark.sql.catalyst.plans._
@@ -85,9 +86,9 @@ object ReorderJoin extends Rule[LogicalPlan] with PredicateHelper {
}
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
- case ExtractFiltersAndInnerJoins(input, conditions)
+ case p @ ExtractFiltersAndInnerJoins(input, conditions)
if input.size > 2 && conditions.nonEmpty =>
- if (SQLConf.get.starSchemaDetection && !SQLConf.get.cboEnabled) {
+ val reordered = if (SQLConf.get.starSchemaDetection && !SQLConf.get.cboEnabled) {
val starJoinPlan = StarSchemaDetection.reorderStarJoins(input, conditions)
if (starJoinPlan.nonEmpty) {
val rest = input.filterNot(starJoinPlan.contains(_))
@@ -98,6 +99,29 @@ object ReorderJoin extends Rule[LogicalPlan] with PredicateHelper {
} else {
createOrderedJoin(input, conditions)
}
+
+ if (sameOutput(p, reordered)) {
+ reordered
+ } else {
+ // Reordering the joins have changed the order of the columns.
+ // Inject a projection to make sure we restore to the expected ordering.
+ Project(p.output, reordered)
+ }
+ }
+
+ /**
+ * Returns true iff output of both plans are semantically the same, ie.:
+ * - they contain the same number of `Attribute`s;
+ * - references are the same;
+ * - the order is equal too.
+ * NOTE: this is copied over from SPARK-25691 from master.
+ */
+ def sameOutput(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
+ val output1 = plan1.output
+ val output2 = plan2.output
+ output1.length == output2.length && output1.zip(output2).forall {
+ case (a1, a2) => a1.semanticEquals(a2)
+ }
}
}
@@ -152,3 +176,47 @@ object EliminateOuterJoin extends Rule[LogicalPlan] with PredicateHelper {
if (j.joinType == newJoinType) f else Filter(condition, j.copy(joinType = newJoinType))
}
}
+
+/**
+ * PythonUDF in join condition can't be evaluated if it refers to attributes from both join sides.
+ * See `ExtractPythonUDFs` for details. This rule will detect un-evaluable PythonUDF and pull them
+ * out from join condition.
+ */
+object PullOutPythonUDFInJoinCondition extends Rule[LogicalPlan] with PredicateHelper {
+
+ private def hasUnevaluablePythonUDF(expr: Expression, j: Join): Boolean = {
+ expr.find { e =>
+ PythonUDF.isScalarPythonUDF(e) && !canEvaluate(e, j.left) && !canEvaluate(e, j.right)
+ }.isDefined
+ }
+
+ override def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
+ case j @ Join(_, _, joinType, Some(cond)) if hasUnevaluablePythonUDF(cond, j) =>
+ if (!joinType.isInstanceOf[InnerLike]) {
+ // The current strategy supports only InnerLike join because for other types,
+ // it breaks SQL semantic if we run the join condition as a filter after join. If we pass
+ // the plan here, it'll still get a an invalid PythonUDF RuntimeException with message
+ // `requires attributes from more than one child`, we throw firstly here for better
+ // readable information.
+ throw new AnalysisException("Using PythonUDF in join condition of join type" +
+ s" $joinType is not supported.")
+ }
+ // If condition expression contains python udf, it will be moved out from
+ // the new join conditions.
+ val (udf, rest) = splitConjunctivePredicates(cond).partition(hasUnevaluablePythonUDF(_, j))
+ val newCondition = if (rest.isEmpty) {
+ logWarning(s"The join condition:$cond of the join plan contains PythonUDF only," +
+ s" it will be moved out and the join plan will be turned to cross join.")
+ None
+ } else {
+ Some(rest.reduceLeft(And))
+ }
+ val newJoin = j.copy(condition = newCondition)
+ joinType match {
+ case _: InnerLike => Filter(udf.reduceLeft(And), newJoin)
+ case _ =>
+ throw new AnalysisException("Using PythonUDF in join condition of join type" +
+ s" $joinType is not supported.")
+ }
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala
index e9b7a8b76e683..5e70d59b04a9a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.optimizer
import scala.collection.mutable.ArrayBuffer
-import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.SubExprUtils._
import org.apache.spark.sql.catalyst.expressions.aggregate._
@@ -43,31 +43,53 @@ import org.apache.spark.sql.types._
* condition.
*/
object RewritePredicateSubquery extends Rule[LogicalPlan] with PredicateHelper {
- private def dedupJoin(joinPlan: LogicalPlan): LogicalPlan = joinPlan match {
+
+ private def buildJoin(
+ outerPlan: LogicalPlan,
+ subplan: LogicalPlan,
+ joinType: JoinType,
+ condition: Option[Expression]): Join = {
+ // Deduplicate conflicting attributes if any.
+ val dedupSubplan = dedupSubqueryOnSelfJoin(outerPlan, subplan, None, condition)
+ Join(outerPlan, dedupSubplan, joinType, condition)
+ }
+
+ private def dedupSubqueryOnSelfJoin(
+ outerPlan: LogicalPlan,
+ subplan: LogicalPlan,
+ valuesOpt: Option[Seq[Expression]],
+ condition: Option[Expression] = None): LogicalPlan = {
// SPARK-21835: It is possibly that the two sides of the join have conflicting attributes,
// the produced join then becomes unresolved and break structural integrity. We should
- // de-duplicate conflicting attributes. We don't use transformation here because we only
- // care about the most top join converted from correlated predicate subquery.
- case j @ Join(left, right, joinType @ (LeftSemi | LeftAnti | ExistenceJoin(_)), joinCond) =>
- val duplicates = right.outputSet.intersect(left.outputSet)
- if (duplicates.nonEmpty) {
- val aliasMap = AttributeMap(duplicates.map { dup =>
- dup -> Alias(dup, dup.toString)()
- }.toSeq)
- val aliasedExpressions = right.output.map { ref =>
- aliasMap.getOrElse(ref, ref)
- }
- val newRight = Project(aliasedExpressions, right)
- val newJoinCond = joinCond.map { condExpr =>
- condExpr transform {
- case a: Attribute => aliasMap.getOrElse(a, a).toAttribute
+ // de-duplicate conflicting attributes.
+ // SPARK-26078: it may also happen that the subquery has conflicting attributes with the outer
+ // values. In this case, the resulting join would contain trivially true conditions (eg.
+ // id#3 = id#3) which cannot be de-duplicated after. In this method, if there are conflicting
+ // attributes in the join condition, the subquery's conflicting attributes are changed using
+ // a projection which aliases them and resolves the problem.
+ val outerReferences = valuesOpt.map(values =>
+ AttributeSet(values.flatMap(_.references))).getOrElse(AttributeSet.empty)
+ val outerRefs = outerPlan.outputSet ++ outerReferences
+ val duplicates = outerRefs.intersect(subplan.outputSet)
+ if (duplicates.nonEmpty) {
+ condition.foreach { e =>
+ val conflictingAttrs = e.references.intersect(duplicates)
+ if (conflictingAttrs.nonEmpty) {
+ throw new AnalysisException("Found conflicting attributes " +
+ s"${conflictingAttrs.mkString(",")} in the condition joining outer plan:\n " +
+ s"$outerPlan\nand subplan:\n $subplan")
}
- }
- Join(left, newRight, joinType, newJoinCond)
- } else {
- j
}
- case _ => joinPlan
+ val rewrites = AttributeMap(duplicates.map { dup =>
+ dup -> Alias(dup, dup.toString)()
+ }.toSeq)
+ val aliasedExpressions = subplan.output.map { ref =>
+ rewrites.getOrElse(ref, ref)
+ }
+ Project(aliasedExpressions, subplan)
+ } else {
+ subplan
+ }
}
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
@@ -85,17 +107,16 @@ object RewritePredicateSubquery extends Rule[LogicalPlan] with PredicateHelper {
withSubquery.foldLeft(newFilter) {
case (p, Exists(sub, conditions, _)) =>
val (joinCond, outerPlan) = rewriteExistentialExpr(conditions, p)
- // Deduplicate conflicting attributes if any.
- dedupJoin(Join(outerPlan, sub, LeftSemi, joinCond))
+ buildJoin(outerPlan, sub, LeftSemi, joinCond)
case (p, Not(Exists(sub, conditions, _))) =>
val (joinCond, outerPlan) = rewriteExistentialExpr(conditions, p)
- // Deduplicate conflicting attributes if any.
- dedupJoin(Join(outerPlan, sub, LeftAnti, joinCond))
+ buildJoin(outerPlan, sub, LeftAnti, joinCond)
case (p, InSubquery(values, ListQuery(sub, conditions, _, _))) =>
- val inConditions = values.zip(sub.output).map(EqualTo.tupled)
- val (joinCond, outerPlan) = rewriteExistentialExpr(inConditions ++ conditions, p)
// Deduplicate conflicting attributes if any.
- dedupJoin(Join(outerPlan, sub, LeftSemi, joinCond))
+ val newSub = dedupSubqueryOnSelfJoin(p, sub, Some(values))
+ val inConditions = values.zip(newSub.output).map(EqualTo.tupled)
+ val (joinCond, outerPlan) = rewriteExistentialExpr(inConditions ++ conditions, p)
+ Join(outerPlan, newSub, LeftSemi, joinCond)
case (p, Not(InSubquery(values, ListQuery(sub, conditions, _, _)))) =>
// This is a NULL-aware (left) anti join (NAAJ) e.g. col NOT IN expr
// Construct the condition. A NULL in one of the conditions is regarded as a positive
@@ -103,7 +124,10 @@ object RewritePredicateSubquery extends Rule[LogicalPlan] with PredicateHelper {
// Note that will almost certainly be planned as a Broadcast Nested Loop join.
// Use EXISTS if performance matters to you.
- val inConditions = values.zip(sub.output).map(EqualTo.tupled)
+
+ // Deduplicate conflicting attributes if any.
+ val newSub = dedupSubqueryOnSelfJoin(p, sub, Some(values))
+ val inConditions = values.zip(newSub.output).map(EqualTo.tupled)
val (joinCond, outerPlan) = rewriteExistentialExpr(inConditions, p)
// Expand the NOT IN expression with the NULL-aware semantic
// to its full form. That is from:
@@ -118,8 +142,7 @@ object RewritePredicateSubquery extends Rule[LogicalPlan] with PredicateHelper {
// will have the final conditions in the LEFT ANTI as
// (A.A1 = B.B1 OR ISNULL(A.A1 = B.B1)) AND (B.B2 = A.A2) AND B.B3 > 1
val finalJoinCond = (nullAwareJoinConds ++ conditions).reduceLeft(And)
- // Deduplicate conflicting attributes if any.
- dedupJoin(Join(outerPlan, sub, LeftAnti, Option(finalJoinCond)))
+ Join(outerPlan, newSub, LeftAnti, Option(finalJoinCond))
case (p, predicate) =>
val (newCond, inputPlan) = rewriteExistentialExpr(Seq(predicate), p)
Project(p.output, Filter(newCond.get, inputPlan))
@@ -140,16 +163,16 @@ object RewritePredicateSubquery extends Rule[LogicalPlan] with PredicateHelper {
e transformUp {
case Exists(sub, conditions, _) =>
val exists = AttributeReference("exists", BooleanType, nullable = false)()
- // Deduplicate conflicting attributes if any.
- newPlan = dedupJoin(
- Join(newPlan, sub, ExistenceJoin(exists), conditions.reduceLeftOption(And)))
+ newPlan =
+ buildJoin(newPlan, sub, ExistenceJoin(exists), conditions.reduceLeftOption(And))
exists
case InSubquery(values, ListQuery(sub, conditions, _, _)) =>
val exists = AttributeReference("exists", BooleanType, nullable = false)()
- val inConditions = values.zip(sub.output).map(EqualTo.tupled)
- val newConditions = (inConditions ++ conditions).reduceLeftOption(And)
// Deduplicate conflicting attributes if any.
- newPlan = dedupJoin(Join(newPlan, sub, ExistenceJoin(exists), newConditions))
+ val newSub = dedupSubqueryOnSelfJoin(newPlan, sub, Some(values))
+ val inConditions = values.zip(newSub.output).map(EqualTo.tupled)
+ val newConditions = (inConditions ++ conditions).reduceLeftOption(And)
+ newPlan = Join(newPlan, newSub, ExistenceJoin(exists), newConditions)
exists
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 7bc1f63e30540..80a4d18adfdac 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -394,6 +394,17 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
Filter(expression(ctx), plan)
}
+ def withHaving(ctx: BooleanExpressionContext, plan: LogicalPlan): LogicalPlan = {
+ // Note that we add a cast to non-predicate expressions. If the expression itself is
+ // already boolean, the optimizer will get rid of the unnecessary cast.
+ val predicate = expression(ctx) match {
+ case p: Predicate => p
+ case e => Cast(e, BooleanType)
+ }
+ Filter(predicate, plan)
+ }
+
+
// Expressions.
val expressions = Option(namedExpressionSeq).toSeq
.flatMap(_.namedExpression.asScala)
@@ -446,30 +457,34 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
case e: NamedExpression => e
case e: Expression => UnresolvedAlias(e)
}
- val withProject = if (aggregation != null) {
- withAggregation(aggregation, namedExpressions, withFilter)
- } else if (namedExpressions.nonEmpty) {
+
+ def createProject() = if (namedExpressions.nonEmpty) {
Project(namedExpressions, withFilter)
} else {
withFilter
}
- // Having
- val withHaving = withProject.optional(having) {
- // Note that we add a cast to non-predicate expressions. If the expression itself is
- // already boolean, the optimizer will get rid of the unnecessary cast.
- val predicate = expression(having) match {
- case p: Predicate => p
- case e => Cast(e, BooleanType)
+ val withProject = if (aggregation == null && having != null) {
+ if (conf.getConf(SQLConf.LEGACY_HAVING_WITHOUT_GROUP_BY_AS_WHERE)) {
+ // If the legacy conf is set, treat HAVING without GROUP BY as WHERE.
+ withHaving(having, createProject())
+ } else {
+ // According to SQL standard, HAVING without GROUP BY means global aggregate.
+ withHaving(having, Aggregate(Nil, namedExpressions, withFilter))
}
- Filter(predicate, withProject)
+ } else if (aggregation != null) {
+ val aggregate = withAggregation(aggregation, namedExpressions, withFilter)
+ aggregate.optionalMap(having)(withHaving)
+ } else {
+ // When hitting this branch, `having` must be null.
+ createProject()
}
// Distinct
val withDistinct = if (setQuantifier() != null && setQuantifier().DISTINCT() != null) {
- Distinct(withHaving)
+ Distinct(withProject)
} else {
- withHaving
+ withProject
}
// Window
@@ -1321,9 +1336,12 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
*/
override def visitLambda(ctx: LambdaContext): Expression = withOrigin(ctx) {
val arguments = ctx.IDENTIFIER().asScala.map { name =>
- UnresolvedAttribute.quoted(name.getText)
+ UnresolvedNamedLambdaVariable(UnresolvedAttribute.quoted(name.getText).nameParts)
+ }
+ val function = expression(ctx.expression).transformUp {
+ case a: UnresolvedAttribute => UnresolvedNamedLambdaVariable(a.nameParts)
}
- LambdaFunction(expression(ctx.expression), arguments)
+ LambdaFunction(function, arguments)
}
/**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
index 7d8cb1f18b4b5..454861a9b7e9d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.types.{DataType, StructType}
/**
* Base SQL parsing infrastructure.
*/
-abstract class AbstractSqlParser extends ParserInterface with Logging {
+abstract class AbstractSqlParser(conf: SQLConf) extends ParserInterface with Logging {
/** Creates/Resolves DataType for a given SQL string. */
override def parseDataType(sqlText: String): DataType = parse(sqlText) { parser =>
@@ -84,14 +84,14 @@ abstract class AbstractSqlParser extends ParserInterface with Logging {
val lexer = new SqlBaseLexer(new UpperCaseCharStream(CharStreams.fromString(command)))
lexer.removeErrorListeners()
lexer.addErrorListener(ParseErrorListener)
- lexer.legacy_setops_precedence_enbled = SQLConf.get.setOpsPrecedenceEnforced
+ lexer.legacy_setops_precedence_enbled = conf.setOpsPrecedenceEnforced
val tokenStream = new CommonTokenStream(lexer)
val parser = new SqlBaseParser(tokenStream)
parser.addParseListener(PostProcessor)
parser.removeErrorListeners()
parser.addErrorListener(ParseErrorListener)
- parser.legacy_setops_precedence_enbled = SQLConf.get.setOpsPrecedenceEnforced
+ parser.legacy_setops_precedence_enbled = conf.setOpsPrecedenceEnforced
try {
try {
@@ -125,13 +125,13 @@ abstract class AbstractSqlParser extends ParserInterface with Logging {
/**
* Concrete SQL parser for Catalyst-only SQL statements.
*/
-class CatalystSqlParser(conf: SQLConf) extends AbstractSqlParser {
+class CatalystSqlParser(conf: SQLConf) extends AbstractSqlParser(conf) {
val astBuilder = new AstBuilder(conf)
}
/** For test-only. */
-object CatalystSqlParser extends AbstractSqlParser {
- val astBuilder = new AstBuilder(new SQLConf())
+object CatalystSqlParser extends AbstractSqlParser(SQLConf.get) {
+ val astBuilder = new AstBuilder(SQLConf.get)
}
/**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/QueryPlanner.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/QueryPlanner.scala
index bc41dd0465e34..6fa5203a06f7c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/QueryPlanner.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/QueryPlanner.scala
@@ -81,7 +81,7 @@ abstract class QueryPlanner[PhysicalPlan <: TreeNode[PhysicalPlan]] {
childPlans.map { childPlan =>
// Replace the placeholder by the child plan
candidateWithPlaceholders.transformUp {
- case p if p == placeholder => childPlan
+ case p if p.eq(placeholder) => childPlan
}
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
index 84be677e438a6..d91b89057b979 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
@@ -66,9 +66,6 @@ object PhysicalOperation extends PredicateHelper {
val substitutedCondition = substitute(aliases)(condition)
(fields, filters ++ splitConjunctivePredicates(substitutedCondition), other, aliases)
- case h: ResolvedHint =>
- collectProjectsAndFilters(h.child)
-
case other =>
(None, Nil, other, Map.empty)
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
index 0e4456ac0e6a9..5f136629eb15b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
@@ -159,7 +159,7 @@ abstract class UnaryNode extends LogicalPlan {
var allConstraints = child.constraints.asInstanceOf[Set[Expression]]
projectList.foreach {
case a @ Alias(l: Literal, _) =>
- allConstraints += EqualTo(a.toAttribute, l)
+ allConstraints += EqualNullSafe(a.toAttribute, l)
case a @ Alias(e, _) =>
// For every alias in `projectList`, replace the reference in constraints by its attribute.
allConstraints ++= allConstraints.map(_ transform {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Statistics.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Statistics.scala
index b3a48860aa63b..d0ca9ebdb6b90 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Statistics.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Statistics.scala
@@ -93,6 +93,7 @@ case class Statistics(
* @param avgLen average length of the values. For fixed-length types, this should be a constant.
* @param maxLen maximum length of the values. For fixed-length types, this should be a constant.
* @param histogram histogram of the values
+ * @param version version of statistics saved to or retrieved from the catalog
*/
case class ColumnStat(
distinctCount: Option[BigInt] = None,
@@ -101,7 +102,8 @@ case class ColumnStat(
nullCount: Option[BigInt] = None,
avgLen: Option[Long] = None,
maxLen: Option[Long] = None,
- histogram: Option[Histogram] = None) {
+ histogram: Option[Histogram] = None,
+ version: Int = CatalogColumnStat.VERSION) {
// Are distinctCount and nullCount statistics defined?
val hasCountStats = distinctCount.isDefined && nullCount.isDefined
@@ -120,7 +122,8 @@ case class ColumnStat(
nullCount = nullCount,
avgLen = avgLen,
maxLen = maxLen,
- histogram = histogram)
+ histogram = histogram,
+ version = version)
}
/**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 7ff83a9be3622..fd6ebff5691b5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -230,9 +230,19 @@ case class Union(children: Seq[LogicalPlan]) extends LogicalPlan {
}
// updating nullability to make all the children consistent
- override def output: Seq[Attribute] =
- children.map(_.output).transpose.map(attrs =>
- attrs.head.withNullability(attrs.exists(_.nullable)))
+ override def output: Seq[Attribute] = {
+ children.map(_.output).transpose.map { attrs =>
+ val firstAttr = attrs.head
+ val nullable = attrs.exists(_.nullable)
+ val newDt = attrs.map(_.dataType).reduce(StructType.merge)
+ if (firstAttr.dataType == newDt) {
+ firstAttr.withNullability(nullable)
+ } else {
+ AttributeReference(firstAttr.name, newDt, nullable, firstAttr.metadata)(
+ firstAttr.exprId, firstAttr.qualifier)
+ }
+ }
+ }
override lazy val resolved: Boolean = {
// allChildrenCompatible needs to be evaluated after childrenResolved
@@ -462,6 +472,9 @@ case class View(
output: Seq[Attribute],
child: LogicalPlan) extends LogicalPlan with MultiInstanceRelation {
+ @transient
+ override lazy val references: AttributeSet = AttributeSet.empty
+
override lazy val resolved: Boolean = child.resolved
override def children: Seq[LogicalPlan] = child :: Nil
@@ -695,6 +708,8 @@ case class Expand(
override def references: AttributeSet =
AttributeSet(projections.flatten.flatMap(_.references))
+ override def producedAttributes: AttributeSet = AttributeSet(output diff child.output)
+
// This operator can reuse attributes (for example making them null when doing a roll up) so
// the constraints of the child may no longer be valid.
override protected def validConstraints: Set[Expression] = Set.empty[Expression]
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala
index 111c594a53e52..b9e72c2cc86c1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala
@@ -39,8 +39,16 @@ object AggregateEstimation {
// Multiply distinct counts of group-by columns. This is an upper bound, which assumes
// the data contains all combinations of distinct values of group-by columns.
var outputRows: BigInt = agg.groupingExpressions.foldLeft(BigInt(1))(
- (res, expr) => res *
- childStats.attributeStats(expr.asInstanceOf[Attribute]).distinctCount.get)
+ (res, expr) => {
+ val columnStat = childStats.attributeStats(expr.asInstanceOf[Attribute])
+ val distinctCount = columnStat.distinctCount.get
+ val distinctValue: BigInt = if (columnStat.nullCount.get > 0) {
+ distinctCount + 1
+ } else {
+ distinctCount
+ }
+ res * distinctValue
+ })
outputRows = if (agg.groupingExpressions.isEmpty) {
// If there's no group-by columns, the output is a single row containing values of aggregate
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/FilterEstimation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/FilterEstimation.scala
index 5a3eeefaedb18..2c5beef43f52a 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/FilterEstimation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/FilterEstimation.scala
@@ -670,6 +670,14 @@ case class FilterEstimation(plan: Filter) extends Logging {
logDebug("[CBO] No range comparison statistics for String/Binary type " + attrLeft)
return None
case _ =>
+ if (!colStatsMap.hasMinMaxStats(attrLeft)) {
+ logDebug("[CBO] No min/max statistics for " + attrLeft)
+ return None
+ }
+ if (!colStatsMap.hasMinMaxStats(attrRight)) {
+ logDebug("[CBO] No min/max statistics for " + attrRight)
+ return None
+ }
}
val colStatLeft = colStatsMap(attrLeft)
@@ -879,13 +887,13 @@ case class ColumnStatsMap(originalMap: AttributeMap[ColumnStat]) {
}
def hasCountStats(a: Attribute): Boolean =
- get(a).map(_.hasCountStats).getOrElse(false)
+ get(a).exists(_.hasCountStats)
def hasDistinctCount(a: Attribute): Boolean =
- get(a).map(_.distinctCount.isDefined).getOrElse(false)
+ get(a).exists(_.distinctCount.isDefined)
def hasMinMaxStats(a: Attribute): Boolean =
- get(a).map(_.hasCountStats).getOrElse(false)
+ get(a).exists(_.hasMinMaxStats)
/**
* Gets column stat for the given attribute. Prefer the column stat in updatedMap than that in
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
index cd28c733f3613..cc1a5e835d9cd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql.catalyst.plans.physical
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.types.{DataType, IntegerType}
@@ -208,18 +206,6 @@ case object SinglePartition extends Partitioning {
}
}
-/**
- * Represents a partitioning where rows are only serialized/deserialized locally. The number
- * of partitions are not changed and also the distribution of rows. This is mainly used to
- * obtain some statistics of map tasks such as number of outputs.
- */
-case class LocalPartitioning(childRDD: RDD[InternalRow]) extends Partitioning {
- val numPartitions = childRDD.getNumPartitions
-
- // We will perform this partitioning no matter what the data distribution is.
- override def satisfies0(required: Distribution): Boolean = false
-}
-
/**
* Represents a partitioning where rows are split up across partitions based on the hash
* of `expressions`. All rows where `expressions` evaluate to the same values are guaranteed to be
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index becfa8d982213..a924f10fb366e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -27,9 +27,8 @@ import org.json4s.JsonAST._
import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods._
-import org.apache.spark.sql.catalyst.FunctionIdentifier
+import org.apache.spark.sql.catalyst.IdentifierWithDatabase
import org.apache.spark.sql.catalyst.ScalaReflection._
-import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType, FunctionResource}
import org.apache.spark.sql.catalyst.errors._
import org.apache.spark.sql.catalyst.expressions._
@@ -211,6 +210,8 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
}
def mapChild(child: Any): Any = child match {
case arg: TreeNode[_] if containsChild(arg) => mapTreeNode(arg)
+ // CaseWhen Case or any tuple type
+ case (left, right) => (mapChild(left), mapChild(right))
case nonChild: AnyRef => nonChild
case null => null
}
@@ -226,6 +227,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
// `mapValues` is lazy and we need to force it to materialize
m.mapValues(mapChild).view.force
case arg: TreeNode[_] if containsChild(arg) => mapTreeNode(arg)
+ case Some(child) => Some(mapChild(child))
case nonChild: AnyRef => nonChild
case null => null
}
@@ -675,9 +677,8 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
private def shouldConvertToJson(product: Product): Boolean = product match {
case exprId: ExprId => true
case field: StructField => true
- case id: TableIdentifier => true
+ case id: IdentifierWithDatabase => true
case join: JoinType => true
- case id: FunctionIdentifier => true
case spec: BucketSpec => true
case catalog: CatalogTable => true
case partition: Partitioning => true
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
new file mode 100644
index 0000000000000..9535a369cb2ec
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.time.{Instant, ZoneId}
+import java.util.Locale
+
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.instantToDays
+
+sealed trait DateFormatter extends Serializable {
+ def parse(s: String): Int // returns days since epoch
+ def format(days: Int): String
+}
+
+class Iso8601DateFormatter(
+ pattern: String,
+ locale: Locale) extends DateFormatter with DateTimeFormatterHelper {
+
+ @transient
+ private lazy val formatter = getOrCreateFormatter(pattern, locale)
+ private val UTC = ZoneId.of("UTC")
+
+ private def toInstant(s: String): Instant = {
+ val temporalAccessor = formatter.parse(s)
+ toInstantWithZoneId(temporalAccessor, UTC)
+ }
+
+ override def parse(s: String): Int = instantToDays(toInstant(s))
+
+ override def format(days: Int): String = {
+ val instant = Instant.ofEpochSecond(days * DateTimeUtils.SECONDS_PER_DAY)
+ formatter.withZone(UTC).format(instant)
+ }
+}
+
+object DateFormatter {
+ val defaultPattern: String = "yyyy-MM-dd"
+ val defaultLocale: Locale = Locale.US
+
+ def apply(format: String, locale: Locale): DateFormatter = {
+ new Iso8601DateFormatter(format, locale)
+ }
+
+ def apply(format: String): DateFormatter = apply(format, defaultLocale)
+
+ def apply(): DateFormatter = apply(defaultPattern)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
new file mode 100644
index 0000000000000..81ad6ad8ca40e
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.time._
+import java.time.chrono.IsoChronology
+import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, ResolverStyle}
+import java.time.temporal.{ChronoField, TemporalAccessor, TemporalQueries}
+import java.util.Locale
+
+import com.google.common.cache.CacheBuilder
+
+import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper._
+
+trait DateTimeFormatterHelper {
+ protected def toInstantWithZoneId(temporalAccessor: TemporalAccessor, zoneId: ZoneId): Instant = {
+ val localTime = if (temporalAccessor.query(TemporalQueries.localTime) == null) {
+ LocalTime.ofNanoOfDay(0)
+ } else {
+ LocalTime.from(temporalAccessor)
+ }
+ val localDate = LocalDate.from(temporalAccessor)
+ val localDateTime = LocalDateTime.of(localDate, localTime)
+ val zonedDateTime = ZonedDateTime.of(localDateTime, zoneId)
+ Instant.from(zonedDateTime)
+ }
+
+ // Gets a formatter from the cache or creates new one. The buildFormatter method can be called
+ // a few times with the same parameters in parallel if the cache does not contain values
+ // associated to those parameters. Since the formatter is immutable, it does not matter.
+ // In this way, synchronised is intentionally omitted in this method to make parallel calls
+ // less synchronised.
+ // The Cache.get method is not used here to avoid creation of additional instances of Callable.
+ protected def getOrCreateFormatter(pattern: String, locale: Locale): DateTimeFormatter = {
+ val key = (pattern, locale)
+ var formatter = cache.getIfPresent(key)
+ if (formatter == null) {
+ formatter = buildFormatter(pattern, locale)
+ cache.put(key, formatter)
+ }
+ formatter
+ }
+}
+
+private object DateTimeFormatterHelper {
+ val cache = CacheBuilder.newBuilder()
+ .maximumSize(128)
+ .build[(String, Locale), DateTimeFormatter]()
+
+ def buildFormatter(pattern: String, locale: Locale): DateTimeFormatter = {
+ new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .appendPattern(pattern)
+ .parseDefaulting(ChronoField.ERA, 1)
+ .parseDefaulting(ChronoField.MONTH_OF_YEAR, 1)
+ .parseDefaulting(ChronoField.DAY_OF_MONTH, 1)
+ .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
+ .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
+ .toFormatter(locale)
+ .withChronology(IsoChronology.INSTANCE)
+ .withResolverStyle(ResolverStyle.STRICT)
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 02813d3939796..f6993ae236447 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -18,14 +18,18 @@
package org.apache.spark.sql.catalyst.util
import java.sql.{Date, Timestamp}
-import java.text.{DateFormat, SimpleDateFormat}
-import java.util.{Calendar, Locale, TimeZone}
+import java.text.{DateFormat, ParsePosition, SimpleDateFormat}
+import java.time.Instant
+import java.util.{Calendar, GregorianCalendar, Locale, TimeZone}
import java.util.concurrent.ConcurrentHashMap
import java.util.function.{Function => JFunction}
import javax.xml.bind.DatatypeConverter
import scala.annotation.tailrec
+import org.apache.commons.lang3.time.FastDateFormat
+
+import org.apache.spark.sql.types.Decimal
import org.apache.spark.unsafe.types.UTF8String
/**
@@ -50,7 +54,7 @@ object DateTimeUtils {
final val MILLIS_PER_SECOND = 1000L
final val NANOS_PER_SECOND = MICROS_PER_SECOND * 1000L
final val MICROS_PER_DAY = MICROS_PER_SECOND * SECONDS_PER_DAY
-
+ final val NANOS_PER_MICROS = 1000L
final val MILLIS_PER_DAY = SECONDS_PER_DAY * 1000L
// number of days in 400 years
@@ -300,28 +304,10 @@ object DateTimeUtils {
* `T[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]+[h]h:[m]m`
*/
def stringToTimestamp(s: UTF8String): Option[SQLTimestamp] = {
- stringToTimestamp(s, defaultTimeZone(), rejectTzInString = false)
+ stringToTimestamp(s, defaultTimeZone())
}
def stringToTimestamp(s: UTF8String, timeZone: TimeZone): Option[SQLTimestamp] = {
- stringToTimestamp(s, timeZone, rejectTzInString = false)
- }
-
- /**
- * Converts a timestamp string to microseconds from the unix epoch, w.r.t. the given timezone.
- * Returns None if the input string is not a valid timestamp format.
- *
- * @param s the input timestamp string.
- * @param timeZone the timezone of the timestamp string, will be ignored if the timestamp string
- * already contains timezone information and `forceTimezone` is false.
- * @param rejectTzInString if true, rejects timezone in the input string, i.e., if the
- * timestamp string contains timezone, like `2000-10-10 00:00:00+00:00`,
- * return None.
- */
- def stringToTimestamp(
- s: UTF8String,
- timeZone: TimeZone,
- rejectTzInString: Boolean): Option[SQLTimestamp] = {
if (s == null) {
return None
}
@@ -395,7 +381,7 @@ object DateTimeUtils {
i += 1
}
} else {
- if (b == ':' || b == ' ') {
+ if (i < segments.length && (b == ':' || b == ' ')) {
segments(i) = currentSegmentValue
currentSegmentValue = 0
i += 1
@@ -439,8 +425,6 @@ object DateTimeUtils {
return None
}
- if (tz.isDefined && rejectTzInString) return None
-
val c = if (tz.isEmpty) {
Calendar.getInstance(timeZone)
} else {
@@ -460,6 +444,18 @@ object DateTimeUtils {
Some(c.getTimeInMillis * 1000 + segments(6))
}
+ def instantToMicros(instant: Instant): Long = {
+ val sec = Math.multiplyExact(instant.getEpochSecond, MICROS_PER_SECOND)
+ val result = Math.addExact(sec, instant.getNano / NANOS_PER_MICROS)
+ result
+ }
+
+ def instantToDays(instant: Instant): Int = {
+ val seconds = instant.getEpochSecond
+ val days = Math.floorDiv(seconds, SECONDS_PER_DAY)
+ days.toInt
+ }
+
/**
* Parses a given UTF8 date string to a corresponding [[Int]] value.
* The return type is [[Option]] in order to distinguish between 0 and null. The following
@@ -505,6 +501,10 @@ object DateTimeUtils {
// year should have exact four digits
return None
}
+ if (i < 2 && j < bytes.length) {
+ // For the `yyyy` and `yyyy-[m]m` formats, entire input must be consumed.
+ return None
+ }
segments(i) = currentSegmentValue
if (isInvalidDate(segments(0), segments(1), segments(2))) {
return None
@@ -1167,4 +1167,62 @@ object DateTimeUtils {
threadLocalTimestampFormat.remove()
threadLocalDateFormat.remove()
}
+
+ /**
+ * The custom sub-class of `GregorianCalendar` is needed to get access to
+ * protected `fields` immediately after parsing. We cannot use
+ * the `get()` method because it performs normalization of the fraction
+ * part. Accordingly, the `MILLISECOND` field doesn't contain original value.
+ *
+ * Also this class allows to set raw value to the `MILLISECOND` field
+ * directly before formatting.
+ */
+ private class MicrosCalendar(tz: TimeZone, digitsInFraction: Int)
+ extends GregorianCalendar(tz, Locale.US) {
+ // Converts parsed `MILLISECOND` field to seconds fraction in microsecond precision.
+ // For example if the fraction pattern is `SSSS` then `digitsInFraction` = 4, and
+ // if the `MILLISECOND` field was parsed to `1234`.
+ def getMicros(): SQLTimestamp = {
+ // Append 6 zeros to the field: 1234 -> 1234000000
+ val d = fields(Calendar.MILLISECOND) * MICROS_PER_SECOND
+ // Take the first 6 digits from `d`: 1234000000 -> 123400
+ // The rest contains exactly `digitsInFraction`: `0000` = 10 ^ digitsInFraction
+ // So, the result is `(1234 * 1000000) / (10 ^ digitsInFraction)
+ d / Decimal.POW_10(digitsInFraction)
+ }
+
+ // Converts the seconds fraction in microsecond precision to a value
+ // that can be correctly formatted according to the specified fraction pattern.
+ // The method performs operations opposite to `getMicros()`.
+ def setMicros(micros: Long): Unit = {
+ val d = micros * Decimal.POW_10(digitsInFraction)
+ fields(Calendar.MILLISECOND) = (d / MICROS_PER_SECOND).toInt
+ }
+ }
+
+ /**
+ * An instance of the class is aimed to re-use many times. It contains helper objects
+ * `cal` which is reused between `parse()` and `format` invokes.
+ */
+ class TimestampParser(fastDateFormat: FastDateFormat) {
+ private val cal = new MicrosCalendar(
+ fastDateFormat.getTimeZone,
+ fastDateFormat.getPattern.count(_ == 'S'))
+
+ def parse(s: String): SQLTimestamp = {
+ cal.clear() // Clear the calendar because it can be re-used many times
+ if (!fastDateFormat.parse(s, new ParsePosition(0), cal)) {
+ throw new IllegalArgumentException(s"'$s' is an invalid timestamp")
+ }
+ val micros = cal.getMicros()
+ cal.set(Calendar.MILLISECOND, 0)
+ cal.getTimeInMillis * MICROS_PER_MILLIS + micros
+ }
+
+ def format(timestamp: SQLTimestamp): String = {
+ cal.setTimeInMillis(Math.floorDiv(timestamp, MICROS_PER_SECOND) * MILLIS_PER_SECOND)
+ cal.setMicros(Math.floorMod(timestamp, MICROS_PER_SECOND))
+ fastDateFormat.format(cal)
+ }
+ }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
new file mode 100644
index 0000000000000..4ec61e1ca4a57
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.text.ParseException
+import java.time._
+import java.time.format.DateTimeParseException
+import java.time.temporal.TemporalQueries
+import java.util.{Locale, TimeZone}
+
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.instantToMicros
+
+sealed trait TimestampFormatter extends Serializable {
+ /**
+ * Parses a timestamp in a string and converts it to microseconds.
+ *
+ * @param s - string with timestamp to parse
+ * @return microseconds since epoch.
+ * @throws ParseException can be thrown by legacy parser
+ * @throws DateTimeParseException can be thrown by new parser
+ * @throws DateTimeException unable to obtain local date or time
+ */
+ @throws(classOf[ParseException])
+ @throws(classOf[DateTimeParseException])
+ @throws(classOf[DateTimeException])
+ def parse(s: String): Long
+ def format(us: Long): String
+}
+
+class Iso8601TimestampFormatter(
+ pattern: String,
+ timeZone: TimeZone,
+ locale: Locale) extends TimestampFormatter with DateTimeFormatterHelper {
+ @transient
+ private lazy val formatter = getOrCreateFormatter(pattern, locale)
+
+ private def toInstant(s: String): Instant = {
+ val temporalAccessor = formatter.parse(s)
+ if (temporalAccessor.query(TemporalQueries.offset()) == null) {
+ toInstantWithZoneId(temporalAccessor, timeZone.toZoneId)
+ } else {
+ Instant.from(temporalAccessor)
+ }
+ }
+
+ override def parse(s: String): Long = instantToMicros(toInstant(s))
+
+ override def format(us: Long): String = {
+ val secs = Math.floorDiv(us, DateTimeUtils.MICROS_PER_SECOND)
+ val mos = Math.floorMod(us, DateTimeUtils.MICROS_PER_SECOND)
+ val instant = Instant.ofEpochSecond(secs, mos * DateTimeUtils.NANOS_PER_MICROS)
+
+ formatter.withZone(timeZone.toZoneId).format(instant)
+ }
+}
+
+object TimestampFormatter {
+ val defaultPattern: String = "yyyy-MM-dd HH:mm:ss"
+ val defaultLocale: Locale = Locale.US
+
+ def apply(format: String, timeZone: TimeZone, locale: Locale): TimestampFormatter = {
+ new Iso8601TimestampFormatter(format, timeZone, locale)
+ }
+
+ def apply(format: String, timeZone: TimeZone): TimestampFormatter = {
+ apply(format, timeZone, defaultLocale)
+ }
+
+ def apply(timeZone: TimeZone): TimestampFormatter = {
+ apply(defaultPattern, timeZone, defaultLocale)
+ }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/RowIterator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/execution/RowIterator.scala
similarity index 100%
rename from sql/core/src/main/scala/org/apache/spark/sql/execution/RowIterator.scala
rename to sql/catalyst/src/main/scala/org/apache/spark/sql/execution/RowIterator.scala
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 738d8fee891d1..c7167f4538098 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -27,7 +27,6 @@ import scala.collection.immutable
import scala.util.matching.Regex
import org.apache.hadoop.fs.Path
-import org.tukaani.xz.LZMA2Options
import org.apache.spark.{SparkContext, TaskContext}
import org.apache.spark.internal.Logging
@@ -36,6 +35,7 @@ import org.apache.spark.network.util.ByteUnit
import org.apache.spark.sql.catalyst.analysis.Resolver
import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator
+import org.apache.spark.unsafe.array.ByteArrayMethods
import org.apache.spark.util.Utils
////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -113,7 +113,9 @@ object SQLConf {
* Returns the active config object within the current scope. If there is an active SparkSession,
* the proper SQLConf associated with the thread's active session is used. If it's called from
* tasks in the executor side, a SQLConf will be created from job local properties, which are set
- * and propagated from the driver side.
+ * and propagated from the driver side, unless a `SQLConf` has been set in the scope by
+ * `withExistingConf` as done for propagating SQLConf for operations performed on RDDs created
+ * from DataFrames.
*
* The way this works is a little bit convoluted, due to the fact that config was added initially
* only for physical plans (and as a result not in sql/catalyst module).
@@ -127,10 +129,16 @@ object SQLConf {
*/
def get: SQLConf = {
if (TaskContext.get != null) {
- new ReadOnlySQLConf(TaskContext.get())
+ val conf = existingConf.get()
+ if (conf != null) {
+ conf
+ } else {
+ new ReadOnlySQLConf(TaskContext.get())
+ }
} else {
val isSchedulerEventLoopThread = SparkContext.getActive
- .map(_.dagScheduler.eventProcessLoop.eventThread)
+ .flatMap { sc => Option(sc.dagScheduler) }
+ .map(_.eventProcessLoop.eventThread)
.exists(_.getId == Thread.currentThread().getId)
if (isSchedulerEventLoopThread) {
// DAGScheduler event loop thread does not have an active SparkSession, the `confGetter`
@@ -235,13 +243,6 @@ object SQLConf {
.intConf
.createWithDefault(4)
- val LIMIT_FLAT_GLOBAL_LIMIT = buildConf("spark.sql.limit.flatGlobalLimit")
- .internal()
- .doc("During global limit, try to evenly distribute limited rows across data " +
- "partitions. If disabled, scanning data partitions sequentially until reaching limit number.")
- .booleanConf
- .createWithDefault(true)
-
val ADVANCED_PARTITION_PREDICATE_PUSHDOWN =
buildConf("spark.sql.hive.advancedPartitionPredicatePushdown.enabled")
.internal()
@@ -249,22 +250,6 @@ object SQLConf {
.booleanConf
.createWithDefault(true)
- val ENABLE_FALL_BACK_TO_HDFS_FOR_STATS =
- buildConf("spark.sql.statistics.fallBackToHdfs")
- .doc("If the table statistics are not available from table metadata enable fall back to hdfs." +
- " This is useful in determining if a table is small enough to use auto broadcast joins.")
- .booleanConf
- .createWithDefault(false)
-
- val DEFAULT_SIZE_IN_BYTES = buildConf("spark.sql.defaultSizeInBytes")
- .internal()
- .doc("The default table size used in query planning. By default, it is set to Long.MaxValue " +
- "which is larger than `spark.sql.autoBroadcastJoinThreshold` to be more conservative. " +
- "That is to say by default the optimizer will not choose to broadcast a table unless it " +
- "knows for sure its size is small enough.")
- .longConf
- .createWithDefault(Long.MaxValue)
-
val SHUFFLE_PARTITIONS = buildConf("spark.sql.shuffle.partitions")
.doc("The default number of partitions to use when shuffling data for joins or aggregations.")
.intConf
@@ -323,13 +308,13 @@ object SQLConf {
.booleanConf
.createWithDefault(false)
- val FILE_COMRESSION_FACTOR = buildConf("spark.sql.sources.fileCompressionFactor")
+ val FILE_COMPRESSION_FACTOR = buildConf("spark.sql.sources.fileCompressionFactor")
.internal()
.doc("When estimating the output data size of a table scan, multiply the file size with this " +
"factor as the estimated data size, in case the data is compressed in the file and lead to" +
" a heavily underestimated result.")
.doubleConf
- .checkValue(_ > 0, "the value of fileDataSizeFactor must be larger than 0")
+ .checkValue(_ > 0, "the value of fileDataSizeFactor must be greater than 0")
.createWithDefault(1.0)
val PARQUET_SCHEMA_MERGING_ENABLED = buildConf("spark.sql.parquet.mergeSchema")
@@ -454,15 +439,19 @@ object SQLConf {
.createWithDefault(10)
val PARQUET_WRITE_LEGACY_FORMAT = buildConf("spark.sql.parquet.writeLegacyFormat")
- .doc("Whether to be compatible with the legacy Parquet format adopted by Spark 1.4 and prior " +
- "versions, when converting Parquet schema to Spark SQL schema and vice versa.")
+ .doc("If true, data will be written in a way of Spark 1.4 and earlier. For example, decimal " +
+ "values will be written in Apache Parquet's fixed-length byte array format, which other " +
+ "systems such as Apache Hive and Apache Impala use. If false, the newer format in Parquet " +
+ "will be used. For example, decimals will be written in int-based format. If Parquet " +
+ "output is intended for use with systems that do not support this newer format, set to true.")
.booleanConf
.createWithDefault(false)
val PARQUET_RECORD_FILTER_ENABLED = buildConf("spark.sql.parquet.recordLevelFilter.enabled")
.doc("If true, enables Parquet's native record-level filtering using the pushed down " +
"filters. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' " +
- "is enabled.")
+ "is enabled and the vectorized reader is not used. You can ensure the vectorized reader " +
+ "is not used by setting 'spark.sql.parquet.enableVectorizedReader' to false.")
.booleanConf
.createWithDefault(false)
@@ -580,23 +569,15 @@ object SQLConf {
.checkValues(HiveCaseSensitiveInferenceMode.values.map(_.toString))
.createWithDefault(HiveCaseSensitiveInferenceMode.INFER_AND_SAVE.toString)
- val TYPECOERCION_COMPARE_DATE_TIMESTAMP_IN_TIMESTAMP =
- buildConf("spark.sql.typeCoercion.compareDateTimestampInTimestamp")
- .internal()
- .doc("When true (default), compare Date with Timestamp after converting both sides to " +
- "Timestamp. This behavior is compatible with Hive 2.2 or later. See HIVE-15236. " +
- "When false, restore the behavior prior to Spark 2.4. Compare Date with Timestamp after " +
- "converting both sides to string. This config will be removed in spark 3.0")
- .booleanConf
- .createWithDefault(true)
-
val OPTIMIZER_METADATA_ONLY = buildConf("spark.sql.optimizer.metadataOnly")
+ .internal()
.doc("When true, enable the metadata-only query optimization that use the table's metadata " +
"to produce the partition columns instead of table scans. It applies when all the columns " +
"scanned are partition columns and the query has an aggregate operator that satisfies " +
- "distinct semantics.")
+ "distinct semantics. By default the optimization is disabled, since it may return " +
+ "incorrect results when the files are empty.")
.booleanConf
- .createWithDefault(true)
+ .createWithDefault(false)
val COLUMN_NAME_OF_CORRUPT_RECORD = buildConf("spark.sql.columnNameOfCorruptRecord")
.doc("The name of internal column for storing raw/un-parsed JSON and CSV records that fail " +
@@ -608,7 +589,7 @@ object SQLConf {
.internal()
.doc("When true, force the output schema of the from_json() function to be nullable " +
"(including all the fields). Otherwise, the schema might not be compatible with" +
- "actual data, which leads to curruptions.")
+ "actual data, which leads to corruptions. This config will be removed in Spark 3.0.")
.booleanConf
.createWithDefault(true)
@@ -677,7 +658,7 @@ object SQLConf {
val BUCKETING_MAX_BUCKETS = buildConf("spark.sql.sources.bucketing.maxBuckets")
.doc("The maximum number of buckets allowed. Defaults to 100000")
.intConf
- .checkValue(_ > 0, "the value of spark.sql.sources.bucketing.maxBuckets must be larger than 0")
+ .checkValue(_ > 0, "the value of spark.sql.sources.bucketing.maxBuckets must be greater than 0")
.createWithDefault(100000)
val CROSS_JOINS_ENABLED = buildConf("spark.sql.crossJoin.enabled")
@@ -1072,7 +1053,7 @@ object SQLConf {
.createWithDefault(10000L)
val STREAMING_NO_DATA_MICRO_BATCHES_ENABLED =
- buildConf("spark.sql.streaming.noDataMicroBatchesEnabled")
+ buildConf("spark.sql.streaming.noDataMicroBatches.enabled")
.doc(
"Whether streaming micro-batch engine will execute batches without data " +
"for eager state management for stateful streaming queries.")
@@ -1098,6 +1079,30 @@ object SQLConf {
.internal()
.stringConf
+ val PARALLEL_FILE_LISTING_IN_STATS_COMPUTATION =
+ buildConf("spark.sql.statistics.parallelFileListingInStatsComputation.enabled")
+ .internal()
+ .doc("When true, SQL commands use parallel file listing, " +
+ "as opposed to single thread listing." +
+ "This usually speeds up commands that need to list many directories.")
+ .booleanConf
+ .createWithDefault(true)
+
+ val ENABLE_FALL_BACK_TO_HDFS_FOR_STATS = buildConf("spark.sql.statistics.fallBackToHdfs")
+ .doc("If the table statistics are not available from table metadata enable fall back to hdfs." +
+ " This is useful in determining if a table is small enough to use auto broadcast joins.")
+ .booleanConf
+ .createWithDefault(false)
+
+ val DEFAULT_SIZE_IN_BYTES = buildConf("spark.sql.defaultSizeInBytes")
+ .internal()
+ .doc("The default table size used in query planning. By default, it is set to Long.MaxValue " +
+ "which is larger than `spark.sql.autoBroadcastJoinThreshold` to be more conservative. " +
+ "That is to say by default the optimizer will not choose to broadcast a table unless it " +
+ "knows for sure its size is small enough.")
+ .longConf
+ .createWithDefault(Long.MaxValue)
+
val NDV_MAX_ERROR =
buildConf("spark.sql.statistics.ndv.maxError")
.internal()
@@ -1121,7 +1126,7 @@ object SQLConf {
.internal()
.doc("The number of bins when generating histograms.")
.intConf
- .checkValue(num => num > 1, "The number of bins must be larger than 1.")
+ .checkValue(num => num > 1, "The number of bins must be greater than 1.")
.createWithDefault(254)
val PERCENTILE_ACCURACY =
@@ -1213,7 +1218,7 @@ object SQLConf {
.doc("Threshold for number of rows guaranteed to be held in memory by the sort merge " +
"join operator")
.intConf
- .createWithDefault(Int.MaxValue)
+ .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
val SORT_MERGE_JOIN_EXEC_BUFFER_SPILL_THRESHOLD =
buildConf("spark.sql.sortMergeJoinExec.buffer.spill.threshold")
@@ -1284,15 +1289,24 @@ object SQLConf {
.booleanConf
.createWithDefault(true)
- val PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_POSITION =
- buildConf("spark.sql.execution.pandas.groupedMap.assignColumnsByPosition")
+ val PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME =
+ buildConf("spark.sql.legacy.execution.pandas.groupedMap.assignColumnsByName")
.internal()
- .doc("When true, a grouped map Pandas UDF will assign columns from the returned " +
- "Pandas DataFrame based on position, regardless of column label type. When false, " +
- "columns will be looked up by name if labeled with a string and fallback to use " +
- "position if not. This configuration will be deprecated in future releases.")
+ .doc("When true, columns will be looked up by name if labeled with a string and fallback " +
+ "to use position if not. When false, a grouped map Pandas UDF will assign columns from " +
+ "the returned Pandas DataFrame based on position, regardless of column label type. " +
+ "This configuration will be deprecated in future releases.")
.booleanConf
- .createWithDefault(false)
+ .createWithDefault(true)
+
+ val USE_CONF_ON_RDD_OPERATION =
+ buildConf("spark.sql.legacy.rdd.applyConf")
+ .internal()
+ .doc("When false, SQL configurations are disregarded when operations on a RDD derived from" +
+ " a dataframe are executed. This is the (buggy) behavior up to 2.4.4. This config is " +
+ "deprecated and it will be removed in 3.0.0.")
+ .booleanConf
+ .createWithDefault(true)
val REPLACE_EXCEPT_WITH_FILTER = buildConf("spark.sql.optimizer.replaceExceptWithFilter")
.internal()
@@ -1317,6 +1331,15 @@ object SQLConf {
.booleanConf
.createWithDefault(true)
+ val LITERAL_PICK_MINIMUM_PRECISION =
+ buildConf("spark.sql.legacy.literal.pickMinimumPrecision")
+ .internal()
+ .doc("When integral literal is used in decimal operations, pick a minimum precision " +
+ "required by the literal if this config is true, to make the resulting precision and/or " +
+ "scale smaller. This can reduce the possibility of precision lose and/or overflow.")
+ .booleanConf
+ .createWithDefault(true)
+
val SQL_OPTIONS_REDACTION_PATTERN =
buildConf("spark.sql.redaction.options.regex")
.doc("Regex to decide which keys in a Spark SQL command's options map contain sensitive " +
@@ -1347,7 +1370,7 @@ object SQLConf {
.createWithDefault(false)
val ALLOW_CREATING_MANAGED_TABLE_USING_NONEMPTY_LOCATION =
- buildConf("spark.sql.allowCreatingManagedTableUsingNonemptyLocation")
+ buildConf("spark.sql.legacy.allowCreatingManagedTableUsingNonemptyLocation")
.internal()
.doc("When this option is set to true, creating managed tables with nonempty location " +
"is allowed. Otherwise, an analysis exception is thrown. ")
@@ -1387,13 +1410,6 @@ object SQLConf {
.stringConf
.createWithDefault("")
- val REJECT_TIMEZONE_IN_STRING = buildConf("spark.sql.function.rejectTimezoneInString")
- .internal()
- .doc("If true, `to_utc_timestamp` and `from_utc_timestamp` return null if the input string " +
- "contains a timezone part, e.g. `2000-10-10 00:00:00+00:00`.")
- .booleanConf
- .createWithDefault(true)
-
object PartitionOverwriteMode extends Enumeration {
val STATIC, DYNAMIC = Value
}
@@ -1429,7 +1445,7 @@ object SQLConf {
.createWithDefault(true)
val NESTED_SCHEMA_PRUNING_ENABLED =
- buildConf("spark.sql.nestedSchemaPruning.enabled")
+ buildConf("spark.sql.optimizer.nestedSchemaPruning.enabled")
.internal()
.doc("Prune nested fields from a logical relation's output which are unnecessary in " +
"satisfying a query. This optimization allows columnar file format readers to avoid " +
@@ -1445,7 +1461,7 @@ object SQLConf {
"'SELECT x FROM t ORDER BY y LIMIT m', if m is under this threshold, do a top-K sort" +
" in memory, otherwise do a global sort which spills to disk if necessary.")
.intConf
- .createWithDefault(Int.MaxValue)
+ .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
object Deprecated {
val MAPRED_REDUCE_TASKS = "mapred.reduce.tasks"
@@ -1462,12 +1478,6 @@ object SQLConf {
.booleanConf
.createWithDefault(true)
- val LEGACY_SIZE_OF_NULL = buildConf("spark.sql.legacy.sizeOfNull")
- .doc("If it is set to true, size of null returns -1. This behavior was inherited from Hive. " +
- "The size function returns null for null input if the flag is disabled.")
- .booleanConf
- .createWithDefault(true)
-
val REPL_EAGER_EVAL_ENABLED = buildConf("spark.sql.repl.eagerEval.enabled")
.doc("Enables eager evaluation or not. When true, the top K rows of Dataset will be " +
"displayed if and only if the REPL supports the eager evaluation. Currently, the " +
@@ -1517,6 +1527,22 @@ object SQLConf {
.checkValues((1 to 9).toSet + Deflater.DEFAULT_COMPRESSION)
.createWithDefault(Deflater.DEFAULT_COMPRESSION)
+ val COMPARE_DATE_TIMESTAMP_IN_TIMESTAMP =
+ buildConf("spark.sql.legacy.compareDateTimestampInTimestamp")
+ .internal()
+ .doc("When true (default), compare Date with Timestamp after converting both sides to " +
+ "Timestamp. This behavior is compatible with Hive 2.2 or later. See HIVE-15236. " +
+ "When false, restore the behavior prior to Spark 2.4. Compare Date with Timestamp after " +
+ "converting both sides to string. This config will be removed in Spark 3.0.")
+ .booleanConf
+ .createWithDefault(true)
+
+ val LEGACY_SIZE_OF_NULL = buildConf("spark.sql.legacy.sizeOfNull")
+ .doc("If it is set to true, size of null returns -1. This behavior was inherited from Hive. " +
+ "The size function returns null for null input if the flag is disabled.")
+ .booleanConf
+ .createWithDefault(true)
+
val LEGACY_REPLACE_DATABRICKS_SPARK_AVRO_ENABLED =
buildConf("spark.sql.legacy.replaceDatabricksSparkAvro.enabled")
.doc("If it is set to true, the data source provider com.databricks.spark.avro is mapped " +
@@ -1534,14 +1560,22 @@ object SQLConf {
.booleanConf
.createWithDefault(false)
- val PARALLEL_FILE_LISTING_IN_STATS_COMPUTATION =
- buildConf("spark.sql.parallelFileListingInStatsComputation.enabled")
+ val LEGACY_HAVING_WITHOUT_GROUP_BY_AS_WHERE =
+ buildConf("spark.sql.legacy.parser.havingWithoutGroupByAsWhere")
.internal()
- .doc("When true, SQL commands use parallel file listing, " +
- "as opposed to single thread listing." +
- "This usually speeds up commands that need to list many directories.")
+ .doc("If it is set to true, the parser will treat HAVING without GROUP BY as a normal " +
+ "WHERE, which does not follow SQL standard.")
.booleanConf
- .createWithDefault(true)
+ .createWithDefault(false)
+
+ val LEGACY_PASS_PARTITION_BY_AS_OPTIONS =
+ buildConf("spark.sql.legacy.sources.write.passPartitionByAsOptions")
+ .internal()
+ .doc("Whether to pass the partitionBy columns as options in DataFrameWriter. " +
+ "Data source V1 now silently drops partitionBy columns for non-file-format sources; " +
+ "turning the flag on provides a way for these sources to see these partitionBy columns.")
+ .booleanConf
+ .createWithDefault(false)
}
/**
@@ -1675,8 +1709,7 @@ class SQLConf extends Serializable with Logging {
def caseSensitiveInferenceMode: HiveCaseSensitiveInferenceMode.Value =
HiveCaseSensitiveInferenceMode.withName(getConf(HIVE_CASE_SENSITIVE_INFERENCE))
- def compareDateTimestampInTimestamp : Boolean =
- getConf(TYPECOERCION_COMPARE_DATE_TIMESTAMP_IN_TIMESTAMP)
+ def compareDateTimestampInTimestamp : Boolean = getConf(COMPARE_DATE_TIMESTAMP_IN_TIMESTAMP)
def gatherFastStats: Boolean = getConf(GATHER_FASTSTAT)
@@ -1712,7 +1745,7 @@ class SQLConf extends Serializable with Logging {
def escapedStringLiterals: Boolean = getConf(ESCAPED_STRING_LITERALS)
- def fileCompressionFactor: Double = getConf(FILE_COMRESSION_FACTOR)
+ def fileCompressionFactor: Double = getConf(FILE_COMPRESSION_FACTOR)
def stringRedactionPattern: Option[Regex] = getConf(SQL_STRING_REDACTION_PATTERN)
@@ -1741,19 +1774,13 @@ class SQLConf extends Serializable with Logging {
def limitScaleUpFactor: Int = getConf(LIMIT_SCALE_UP_FACTOR)
- def limitFlatGlobalLimit: Boolean = getConf(LIMIT_FLAT_GLOBAL_LIMIT)
-
def advancedPartitionPredicatePushdownEnabled: Boolean =
getConf(ADVANCED_PARTITION_PREDICATE_PUSHDOWN)
- def fallBackToHdfsForStatsEnabled: Boolean = getConf(ENABLE_FALL_BACK_TO_HDFS_FOR_STATS)
-
def preferSortMergeJoin: Boolean = getConf(PREFER_SORTMERGEJOIN)
def enableRadixSort: Boolean = getConf(RADIX_SORT_ENABLED)
- def defaultSizeInBytes: Long = getConf(DEFAULT_SIZE_IN_BYTES)
-
def isParquetSchemaMergingEnabled: Boolean = getConf(PARQUET_SCHEMA_MERGING_ENABLED)
def isParquetSchemaRespectSummaries: Boolean = getConf(PARQUET_SCHEMA_RESPECT_SUMMARIES)
@@ -1845,6 +1872,13 @@ class SQLConf extends Serializable with Logging {
def sessionLocalTimeZone: String = getConf(SQLConf.SESSION_LOCAL_TIMEZONE)
+ def parallelFileListingInStatsComputation: Boolean =
+ getConf(SQLConf.PARALLEL_FILE_LISTING_IN_STATS_COMPUTATION)
+
+ def fallBackToHdfsForStatsEnabled: Boolean = getConf(ENABLE_FALL_BACK_TO_HDFS_FOR_STATS)
+
+ def defaultSizeInBytes: Long = getConf(DEFAULT_SIZE_IN_BYTES)
+
def ndvMaxError: Double = getConf(NDV_MAX_ERROR)
def histogramEnabled: Boolean = getConf(HISTOGRAM_ENABLED)
@@ -1899,13 +1933,15 @@ class SQLConf extends Serializable with Logging {
def pandasRespectSessionTimeZone: Boolean = getConf(PANDAS_RESPECT_SESSION_LOCAL_TIMEZONE)
- def pandasGroupedMapAssignColumnssByPosition: Boolean =
- getConf(SQLConf.PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_POSITION)
+ def pandasGroupedMapAssignColumnsByName: Boolean =
+ getConf(SQLConf.PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME)
def replaceExceptWithFilter: Boolean = getConf(REPLACE_EXCEPT_WITH_FILTER)
def decimalOperationsAllowPrecisionLoss: Boolean = getConf(DECIMAL_OPERATIONS_ALLOW_PREC_LOSS)
+ def literalPickMinimumPrecision: Boolean = getConf(LITERAL_PICK_MINIMUM_PRECISION)
+
def continuousStreamingExecutorQueueSize: Int = getConf(CONTINUOUS_STREAMING_EXECUTOR_QUEUE_SIZE)
def continuousStreamingExecutorPollIntervalMs: Long =
@@ -1947,9 +1983,6 @@ class SQLConf extends Serializable with Logging {
def setOpsPrecedenceEnforced: Boolean = getConf(SQLConf.LEGACY_SETOPS_PRECEDENCE_ENABLED)
- def parallelFileListingInStatsComputation: Boolean =
- getConf(SQLConf.PARALLEL_FILE_LISTING_IN_STATS_COMPUTATION)
-
/** ********************** SQLConf functionality methods ************ */
/** Set Spark SQL configuration properties. */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index 9eed2eb202045..0da28c4038161 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -185,9 +185,21 @@ final class Decimal extends Ordered[Decimal] with Serializable {
}
}
- def toScalaBigInt: BigInt = BigInt(toLong)
+ def toScalaBigInt: BigInt = {
+ if (decimalVal.ne(null)) {
+ decimalVal.toBigInt()
+ } else {
+ BigInt(toLong)
+ }
+ }
- def toJavaBigInteger: java.math.BigInteger = java.math.BigInteger.valueOf(toLong)
+ def toJavaBigInteger: java.math.BigInteger = {
+ if (decimalVal.ne(null)) {
+ decimalVal.underlying().toBigInteger()
+ } else {
+ java.math.BigInteger.valueOf(toLong)
+ }
+ }
def toUnscaledLong: Long = {
if (decimalVal.ne(null)) {
@@ -420,7 +432,7 @@ object Decimal {
/** Maximum number of decimal digits a Long can represent */
val MAX_LONG_DIGITS = 18
- private val POW_10 = Array.tabulate[Long](MAX_LONG_DIGITS + 1)(i => math.pow(10, i).toLong)
+ val POW_10 = Array.tabulate[Long](MAX_LONG_DIGITS + 1)(i => math.pow(10, i).toLong)
private val BIG_DEC_ZERO = BigDecimal(0)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala
index 7c15dc0de4b6b..e79ab7a6bdbb5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala
@@ -190,6 +190,8 @@ object Metadata {
JBool(x)
case x: String =>
JString(x)
+ case null =>
+ JNull
case x: Metadata =>
toJsonValue(x.map)
case other =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala
index 902cae9150ede..35f9970a0aaec 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructField.scala
@@ -79,6 +79,8 @@ case class StructField(
/**
* Returns a string containing a schema in DDL format. For example, the following value:
* `StructField("eventId", IntegerType)` will be converted to `eventId` INT.
+ *
+ * @since 2.4.0
*/
def toDDL: String = {
val comment = getComment()
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
index c5ca169c955dc..06289b1483203 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
@@ -365,6 +365,8 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
* `StructType(Seq(StructField("eventId", IntegerType), StructField("s", StringType)))`
* will be converted to `eventId` INT, `s` STRING.
* The returned DDL schema can be used in a table creation.
+ *
+ * @since 2.4.0
*/
def toDDL: String = fields.map(_.toDDL).mkString(",")
@@ -441,6 +443,8 @@ object StructType extends AbstractDataType {
/**
* Creates StructType for a given DDL-formatted string, which is a comma separated list of field
* definitions, e.g., a INT, b STRING.
+ *
+ * @since 2.2.0
*/
def fromDDL(ddl: String): StructType = CatalystSqlParser.parseTableSchema(ddl)
diff --git a/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/HiveHasherSuite.java b/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/HiveHasherSuite.java
index 76930f9368514..b67c6f3e6e85e 100644
--- a/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/HiveHasherSuite.java
+++ b/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/HiveHasherSuite.java
@@ -17,8 +17,7 @@
package org.apache.spark.sql.catalyst.expressions;
-import org.apache.spark.unsafe.memory.ByteArrayMemoryBlock;
-import org.apache.spark.unsafe.memory.MemoryBlock;
+import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.types.UTF8String;
import org.junit.Assert;
import org.junit.Test;
@@ -54,7 +53,7 @@ public void testKnownStringAndIntInputs() {
for (int i = 0; i < inputs.length; i++) {
UTF8String s = UTF8String.fromString("val_" + inputs[i]);
- int hash = HiveHasher.hashUnsafeBytesBlock(s.getMemoryBlock());
+ int hash = HiveHasher.hashUnsafeBytes(s.getBaseObject(), s.getBaseOffset(), s.numBytes());
Assert.assertEquals(expected[i], ((31 * inputs[i]) + hash));
}
}
@@ -90,13 +89,13 @@ public void randomizedStressTestBytes() {
int byteArrSize = rand.nextInt(100) * 8;
byte[] bytes = new byte[byteArrSize];
rand.nextBytes(bytes);
- MemoryBlock mb = ByteArrayMemoryBlock.fromArray(bytes);
Assert.assertEquals(
- HiveHasher.hashUnsafeBytesBlock(mb),
- HiveHasher.hashUnsafeBytesBlock(mb));
+ HiveHasher.hashUnsafeBytes(bytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize),
+ HiveHasher.hashUnsafeBytes(bytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize));
- hashcodes.add(HiveHasher.hashUnsafeBytesBlock(mb));
+ hashcodes.add(HiveHasher.hashUnsafeBytes(
+ bytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize));
}
// A very loose bound.
@@ -113,13 +112,13 @@ public void randomizedStressTestPaddedStrings() {
byte[] strBytes = String.valueOf(i).getBytes(StandardCharsets.UTF_8);
byte[] paddedBytes = new byte[byteArrSize];
System.arraycopy(strBytes, 0, paddedBytes, 0, strBytes.length);
- MemoryBlock mb = ByteArrayMemoryBlock.fromArray(paddedBytes);
Assert.assertEquals(
- HiveHasher.hashUnsafeBytesBlock(mb),
- HiveHasher.hashUnsafeBytesBlock(mb));
+ HiveHasher.hashUnsafeBytes(paddedBytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize),
+ HiveHasher.hashUnsafeBytes(paddedBytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize));
- hashcodes.add(HiveHasher.hashUnsafeBytesBlock(mb));
+ hashcodes.add(HiveHasher.hashUnsafeBytes(
+ paddedBytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize));
}
// A very loose bound.
diff --git a/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/XXH64Suite.java b/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/XXH64Suite.java
index cd8bce623c5df..1baee91b3439c 100644
--- a/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/XXH64Suite.java
+++ b/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/XXH64Suite.java
@@ -24,8 +24,6 @@
import java.util.Set;
import org.apache.spark.unsafe.Platform;
-import org.apache.spark.unsafe.memory.ByteArrayMemoryBlock;
-import org.apache.spark.unsafe.memory.MemoryBlock;
import org.junit.Assert;
import org.junit.Test;
@@ -144,13 +142,13 @@ public void randomizedStressTestBytes() {
int byteArrSize = rand.nextInt(100) * 8;
byte[] bytes = new byte[byteArrSize];
rand.nextBytes(bytes);
- MemoryBlock mb = ByteArrayMemoryBlock.fromArray(bytes);
Assert.assertEquals(
- hasher.hashUnsafeWordsBlock(mb),
- hasher.hashUnsafeWordsBlock(mb));
+ hasher.hashUnsafeWords(bytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize),
+ hasher.hashUnsafeWords(bytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize));
- hashcodes.add(hasher.hashUnsafeWordsBlock(mb));
+ hashcodes.add(hasher.hashUnsafeWords(
+ bytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize));
}
// A very loose bound.
@@ -167,13 +165,13 @@ public void randomizedStressTestPaddedStrings() {
byte[] strBytes = String.valueOf(i).getBytes(StandardCharsets.UTF_8);
byte[] paddedBytes = new byte[byteArrSize];
System.arraycopy(strBytes, 0, paddedBytes, 0, strBytes.length);
- MemoryBlock mb = ByteArrayMemoryBlock.fromArray(paddedBytes);
Assert.assertEquals(
- hasher.hashUnsafeWordsBlock(mb),
- hasher.hashUnsafeWordsBlock(mb));
+ hasher.hashUnsafeWords(paddedBytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize),
+ hasher.hashUnsafeWords(paddedBytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize));
- hashcodes.add(hasher.hashUnsafeWordsBlock(mb));
+ hashcodes.add(hasher.hashUnsafeWords(
+ paddedBytes, Platform.BYTE_ARRAY_OFFSET, byteArrSize));
}
// A very loose bound.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
index 8ae3ff5043e68..d361e6248e2f5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql
-import java.lang.Double.longBitsToDouble
-import java.lang.Float.intBitsToFloat
import java.math.MathContext
import scala.collection.mutable
@@ -69,6 +67,28 @@ object RandomDataGenerator {
Some(f)
}
+ /**
+ * A wrapper of Float.intBitsToFloat to use a unique NaN value for all NaN values.
+ * This prevents `checkEvaluationWithUnsafeProjection` from failing due to
+ * the difference between `UnsafeRow` binary presentation for NaN.
+ * This is visible for testing.
+ */
+ def intBitsToFloat(bits: Int): Float = {
+ val value = java.lang.Float.intBitsToFloat(bits)
+ if (value.isNaN) Float.NaN else value
+ }
+
+ /**
+ * A wrapper of Double.longBitsToDouble to use a unique NaN value for all NaN values.
+ * This prevents `checkEvaluationWithUnsafeProjection` from failing due to
+ * the difference between `UnsafeRow` binary presentation for NaN.
+ * This is visible for testing.
+ */
+ def longBitsToDouble(bits: Long): Double = {
+ val value = java.lang.Double.longBitsToDouble(bits)
+ if (value.isNaN) Double.NaN else value
+ }
+
/**
* Returns a randomly generated schema, based on the given accepted types.
*
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala
index 3c2f8a28875f7..3e62ca069e9ea 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala
@@ -17,6 +17,9 @@
package org.apache.spark.sql
+import java.nio.ByteBuffer
+import java.util.Arrays
+
import scala.util.Random
import org.apache.spark.SparkFunSuite
@@ -106,4 +109,32 @@ class RandomDataGeneratorSuite extends SparkFunSuite {
assert(deviation.toDouble / expectedTotalElements < 2e-1)
}
}
+
+ test("Use Float.NaN for all NaN values") {
+ val bits = -6966608
+ val nan1 = java.lang.Float.intBitsToFloat(bits)
+ val nan2 = RandomDataGenerator.intBitsToFloat(bits)
+ assert(nan1.isNaN)
+ assert(nan2.isNaN)
+
+ val arrayExpected = ByteBuffer.allocate(4).putFloat(Float.NaN).array
+ val array1 = ByteBuffer.allocate(4).putFloat(nan1).array
+ val array2 = ByteBuffer.allocate(4).putFloat(nan2).array
+ assert(!Arrays.equals(array1, arrayExpected))
+ assert(Arrays.equals(array2, arrayExpected))
+ }
+
+ test("Use Double.NaN for all NaN values") {
+ val bits = -6966608
+ val nan1 = java.lang.Double.longBitsToDouble(bits)
+ val nan2 = RandomDataGenerator.longBitsToDouble(bits)
+ assert(nan1.isNaN)
+ assert(nan2.isNaN)
+
+ val arrayExpected = ByteBuffer.allocate(8).putDouble(Double.NaN).array
+ val array1 = ByteBuffer.allocate(8).putDouble(nan1).array
+ val array2 = ByteBuffer.allocate(8).putDouble(nan2).array
+ assert(!Arrays.equals(array1, arrayExpected))
+ assert(Arrays.equals(array2, arrayExpected))
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
index f9ee948b97e0a..38b8e7beab6e6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
@@ -112,6 +112,12 @@ object TestingUDT {
class ScalaReflectionSuite extends SparkFunSuite {
import org.apache.spark.sql.catalyst.ScalaReflection._
+ test("isSubtype") {
+ assert(isSubtype(localTypeOf[Option[Int]], localTypeOf[Option[_]]))
+ assert(isSubtype(localTypeOf[Option[Int]], localTypeOf[Option[Int]]))
+ assert(!isSubtype(localTypeOf[Option[_]], localTypeOf[Option[Int]]))
+ }
+
test("SQLUserDefinedType annotation on Scala structure") {
val schema = schemaFor[TestingUDT.NestedStruct]
assert(schema === Schema(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 3b3edac0a314e..d8cb6f7caa99e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -32,6 +32,8 @@ import org.apache.spark.sql.catalyst.plans.{Cross, Inner}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning,
RangePartitioning, RoundRobinPartitioning}
+import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
@@ -312,24 +314,24 @@ class AnalysisSuite extends AnalysisTest with Matchers {
}
// non-primitive parameters do not need special null handling
- val udf1 = ScalaUDF((s: String) => "x", StringType, string :: Nil)
+ val udf1 = ScalaUDF((s: String) => "x", StringType, string :: Nil, true :: Nil)
val expected1 = udf1
checkUDF(udf1, expected1)
// only primitive parameter needs special null handling
val udf2 = ScalaUDF((s: String, d: Double) => "x", StringType, string :: double :: Nil,
- nullableTypes = true :: false :: Nil)
+ true :: false :: Nil)
val expected2 =
- If(IsNull(double), nullResult, udf2.copy(children = string :: KnownNotNull(double) :: Nil))
+ If(IsNull(double), nullResult, udf2.copy(inputsNullSafe = true :: true :: Nil))
checkUDF(udf2, expected2)
// special null handling should apply to all primitive parameters
val udf3 = ScalaUDF((s: Short, d: Double) => "x", StringType, short :: double :: Nil,
- nullableTypes = false :: false :: Nil)
+ false :: false :: Nil)
val expected3 = If(
IsNull(short) || IsNull(double),
nullResult,
- udf3.copy(children = KnownNotNull(short) :: KnownNotNull(double) :: Nil))
+ udf3.copy(inputsNullSafe = true :: true :: Nil))
checkUDF(udf3, expected3)
// we can skip special null handling for primitive parameters that are not nullable
@@ -338,19 +340,19 @@ class AnalysisSuite extends AnalysisTest with Matchers {
(s: Short, d: Double) => "x",
StringType,
short :: double.withNullability(false) :: Nil,
- nullableTypes = false :: false :: Nil)
+ false :: false :: Nil)
val expected4 = If(
IsNull(short),
nullResult,
- udf4.copy(children = KnownNotNull(short) :: double.withNullability(false) :: Nil))
+ udf4.copy(inputsNullSafe = true :: true :: Nil))
// checkUDF(udf4, expected4)
}
test("SPARK-24891 Fix HandleNullInputsForUDF rule") {
val a = testRelation.output(0)
val func = (x: Int, y: Int) => x + y
- val udf1 = ScalaUDF(func, IntegerType, a :: a :: Nil)
- val udf2 = ScalaUDF(func, IntegerType, a :: udf1 :: Nil)
+ val udf1 = ScalaUDF(func, IntegerType, a :: a :: Nil, false :: false :: Nil)
+ val udf2 = ScalaUDF(func, IntegerType, a :: udf1 :: Nil, false :: false :: Nil)
val plan = Project(Alias(udf2, "")() :: Nil, testRelation)
comparePlans(plan.analyze, plan.analyze.analyze)
}
@@ -586,4 +588,20 @@ class AnalysisSuite extends AnalysisTest with Matchers {
listRelation.select(MultiAlias(MultiAlias(
PosExplode('list), Seq("first_pos", "first_val")), Seq("second_pos", "second_val"))))
}
+
+ test("SPARK-24151: CURRENT_DATE, CURRENT_TIMESTAMP should be case insensitive") {
+ withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+ val input = Project(Seq(
+ UnresolvedAttribute("current_date"),
+ UnresolvedAttribute("CURRENT_DATE"),
+ UnresolvedAttribute("CURRENT_TIMESTAMP"),
+ UnresolvedAttribute("current_timestamp")), testRelation)
+ val expected = Project(Seq(
+ Alias(CurrentDate(), toPrettySQL(CurrentDate()))(),
+ Alias(CurrentDate(), toPrettySQL(CurrentDate()))(),
+ Alias(CurrentTimestamp(), toPrettySQL(CurrentTimestamp()))(),
+ Alias(CurrentTimestamp(), toPrettySQL(CurrentTimestamp()))()), testRelation).analyze
+ checkAnalysis(input, expected)
+ }
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DSLHintSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DSLHintSuite.scala
index 48a3ca2ccfb0b..388eb238db0c0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DSLHintSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DSLHintSuite.scala
@@ -15,12 +15,10 @@
* limitations under the License.
*/
-package org.apache.spark.sql
+package org.apache.spark.sql.catalyst.analysis
-import org.apache.spark.sql.catalyst.analysis.AnalysisTest
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.logical._
class DSLHintSuite extends AnalysisTest {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveLambdaVariablesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveLambdaVariablesSuite.scala
index c4171c75ecd03..a5847ba7c522d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveLambdaVariablesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveLambdaVariablesSuite.scala
@@ -49,19 +49,21 @@ class ResolveLambdaVariablesSuite extends PlanTest {
comparePlans(Analyzer.execute(plan(e1)), plan(e2))
}
+ private def lv(s: Symbol) = UnresolvedNamedLambdaVariable(Seq(s.name))
+
test("resolution - no op") {
checkExpression(key, key)
}
test("resolution - simple") {
- val in = ArrayTransform(values1, LambdaFunction('x.attr + 1, 'x.attr :: Nil))
+ val in = ArrayTransform(values1, LambdaFunction(lv('x) + 1, lv('x) :: Nil))
val out = ArrayTransform(values1, LambdaFunction(lvInt + 1, lvInt :: Nil))
checkExpression(in, out)
}
test("resolution - nested") {
val in = ArrayTransform(values2, LambdaFunction(
- ArrayTransform('x.attr, LambdaFunction('x.attr + 1, 'x.attr :: Nil)), 'x.attr :: Nil))
+ ArrayTransform(lv('x), LambdaFunction(lv('x) + 1, lv('x) :: Nil)), lv('x) :: Nil))
val out = ArrayTransform(values2, LambdaFunction(
ArrayTransform(lvArray, LambdaFunction(lvInt + 1, lvInt :: Nil)), lvArray :: Nil))
checkExpression(in, out)
@@ -75,14 +77,14 @@ class ResolveLambdaVariablesSuite extends PlanTest {
test("fail - name collisions") {
val p = plan(ArrayTransform(values1,
- LambdaFunction('x.attr + 'X.attr, 'x.attr :: 'X.attr :: Nil)))
+ LambdaFunction(lv('x) + lv('X), lv('x) :: lv('X) :: Nil)))
val msg = intercept[AnalysisException](Analyzer.execute(p)).getMessage
assert(msg.contains("arguments should not have names that are semantically the same"))
}
test("fail - lambda arguments") {
val p = plan(ArrayTransform(values1,
- LambdaFunction('x.attr + 'y.attr + 'z.attr, 'x.attr :: 'y.attr :: 'z.attr :: Nil)))
+ LambdaFunction(lv('x) + lv('y) + lv('z), lv('x) :: lv('y) :: lv('z) :: Nil)))
val msg = intercept[AnalysisException](Analyzer.execute(p)).getMessage
assert(msg.contains("does not match the number of arguments expected"))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
index 461eda4334bb9..0eba1c537d67d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
@@ -257,12 +257,43 @@ class TypeCoercionSuite extends AnalysisTest {
shouldNotCast(checkedType, IntegralType)
}
- test("implicit type cast - MapType(StringType, StringType)") {
- val checkedType = MapType(StringType, StringType)
- checkTypeCasting(checkedType, castableTypes = Seq(checkedType))
- shouldNotCast(checkedType, DecimalType)
- shouldNotCast(checkedType, NumericType)
- shouldNotCast(checkedType, IntegralType)
+ test("implicit type cast between two Map types") {
+ val sourceType = MapType(IntegerType, IntegerType, true)
+ val castableTypes = numericTypes ++ Seq(StringType).filter(!Cast.forceNullable(IntegerType, _))
+ val targetTypes = numericTypes.filter(!Cast.forceNullable(IntegerType, _)).map { t =>
+ MapType(t, sourceType.valueType, valueContainsNull = true)
+ }
+ val nonCastableTargetTypes = allTypes.filterNot(castableTypes.contains(_)).map {t =>
+ MapType(t, sourceType.valueType, valueContainsNull = true)
+ }
+
+ // Tests that its possible to setup implicit casts between two map types when
+ // source map's key type is integer and the target map's key type are either Byte, Short,
+ // Long, Double, Float, Decimal(38, 18) or String.
+ targetTypes.foreach { targetType =>
+ shouldCast(sourceType, targetType, targetType)
+ }
+
+ // Tests that its not possible to setup implicit casts between two map types when
+ // source map's key type is integer and the target map's key type are either Binary,
+ // Boolean, Date, Timestamp, Array, Struct, CaleandarIntervalType or NullType
+ nonCastableTargetTypes.foreach { targetType =>
+ shouldNotCast(sourceType, targetType)
+ }
+
+ // Tests that its not possible to cast from nullable map type to not nullable map type.
+ val targetNotNullableTypes = allTypes.filterNot(_ == IntegerType).map { t =>
+ MapType(t, sourceType.valueType, valueContainsNull = false)
+ }
+ val sourceMapExprWithValueNull =
+ CreateMap(Seq(Literal.default(sourceType.keyType),
+ Literal.create(null, sourceType.valueType)))
+ targetNotNullableTypes.foreach { targetType =>
+ val castDefault =
+ TypeCoercion.ImplicitTypeCasts.implicitCast(sourceMapExprWithValueNull, targetType)
+ assert(castDefault.isEmpty,
+ s"Should not be able to cast $sourceType to $targetType, but got $castDefault")
+ }
}
test("implicit type cast - StructType().add(\"a1\", StringType)") {
@@ -680,11 +711,11 @@ class TypeCoercionSuite extends AnalysisTest {
test("cast NullType for expressions that implement ExpectsInputTypes") {
import TypeCoercionSuite._
- ruleTest(new TypeCoercion.ImplicitTypeCasts(conf),
+ ruleTest(TypeCoercion.ImplicitTypeCasts,
AnyTypeUnaryExpression(Literal.create(null, NullType)),
AnyTypeUnaryExpression(Literal.create(null, NullType)))
- ruleTest(new TypeCoercion.ImplicitTypeCasts(conf),
+ ruleTest(TypeCoercion.ImplicitTypeCasts,
NumericTypeUnaryExpression(Literal.create(null, NullType)),
NumericTypeUnaryExpression(Literal.create(null, DoubleType)))
}
@@ -692,11 +723,11 @@ class TypeCoercionSuite extends AnalysisTest {
test("cast NullType for binary operators") {
import TypeCoercionSuite._
- ruleTest(new TypeCoercion.ImplicitTypeCasts(conf),
+ ruleTest(TypeCoercion.ImplicitTypeCasts,
AnyTypeBinaryOperator(Literal.create(null, NullType), Literal.create(null, NullType)),
AnyTypeBinaryOperator(Literal.create(null, NullType), Literal.create(null, NullType)))
- ruleTest(new TypeCoercion.ImplicitTypeCasts(conf),
+ ruleTest(TypeCoercion.ImplicitTypeCasts,
NumericTypeBinaryOperator(Literal.create(null, NullType), Literal.create(null, NullType)),
NumericTypeBinaryOperator(Literal.create(null, DoubleType), Literal.create(null, DoubleType)))
}
@@ -976,7 +1007,7 @@ class TypeCoercionSuite extends AnalysisTest {
}
test("type coercion for CaseKeyWhen") {
- ruleTest(new TypeCoercion.ImplicitTypeCasts(conf),
+ ruleTest(TypeCoercion.ImplicitTypeCasts,
CaseKeyWhen(Literal(1.toShort), Seq(Literal(1), Literal("a"))),
CaseKeyWhen(Cast(Literal(1.toShort), IntegerType), Seq(Literal(1), Literal("a")))
)
@@ -1436,7 +1467,7 @@ class TypeCoercionSuite extends AnalysisTest {
}
test("SPARK-17117 null type coercion in divide") {
- val rules = Seq(FunctionArgumentConversion, Division, new ImplicitTypeCasts(conf))
+ val rules = Seq(FunctionArgumentConversion, Division, ImplicitTypeCasts)
val nullLit = Literal.create(null, NullType)
ruleTest(rules, Divide(1L, nullLit), Divide(Cast(1L, DoubleType), Cast(nullLit, DoubleType)))
ruleTest(rules, Divide(nullLit, 1L), Divide(Cast(nullLit, DoubleType), Cast(1L, DoubleType)))
@@ -1459,7 +1490,7 @@ class TypeCoercionSuite extends AnalysisTest {
DoubleType)))
Seq(true, false).foreach { convertToTS =>
withSQLConf(
- "spark.sql.typeCoercion.compareDateTimestampInTimestamp" -> convertToTS.toString) {
+ "spark.sql.legacy.compareDateTimestampInTimestamp" -> convertToTS.toString) {
val date0301 = Literal(java.sql.Date.valueOf("2017-03-01"))
val timestamp0301000000 = Literal(Timestamp.valueOf("2017-03-01 00:00:00"))
val timestamp0301000001 = Literal(Timestamp.valueOf("2017-03-01 00:00:01"))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
index 89fabd4774065..19e8c0334689c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
@@ -1427,6 +1427,7 @@ abstract class SessionCatalogSuite extends AnalysisTest {
Seq(true, false) foreach { caseSensitive =>
val conf = new SQLConf().copy(SQLConf.CASE_SENSITIVE -> caseSensitive)
val catalog = new SessionCatalog(newBasicCatalog(), new SimpleFunctionRegistry, conf)
+ catalog.setCurrentDatabase("db1")
try {
val analyzer = new Analyzer(catalog, conf)
@@ -1440,6 +1441,8 @@ abstract class SessionCatalogSuite extends AnalysisTest {
}
assert(cause.getMessage.contains("Undefined function: 'undefined_fn'"))
+ // SPARK-21318: the error message should contains the current database name
+ assert(cause.getMessage.contains("db1"))
} finally {
catalog.reset()
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
index 28e6940f3cca3..9802a6e5891b8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.plans.logical.Range
+import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
class CanonicalizeSuite extends SparkFunSuite {
@@ -50,4 +51,32 @@ class CanonicalizeSuite extends SparkFunSuite {
assert(range.where(arrays1).sameResult(range.where(arrays2)))
assert(!range.where(arrays1).sameResult(range.where(arrays3)))
}
+
+ test("SPARK-26402: accessing nested fields with different cases in case insensitive mode") {
+ val expId = NamedExpression.newExprId
+ val qualifier = Seq.empty[String]
+ val structType = StructType(
+ StructField("a", StructType(StructField("b", IntegerType, false) :: Nil), false) :: Nil)
+
+ // GetStructField with different names are semantically equal
+ val fieldA1 = GetStructField(
+ AttributeReference("data1", structType, false)(expId, qualifier),
+ 0, Some("a1"))
+ val fieldA2 = GetStructField(
+ AttributeReference("data2", structType, false)(expId, qualifier),
+ 0, Some("a2"))
+ assert(fieldA1.semanticEquals(fieldA2))
+
+ val fieldB1 = GetStructField(
+ GetStructField(
+ AttributeReference("data1", structType, false)(expId, qualifier),
+ 0, Some("a1")),
+ 0, Some("b1"))
+ val fieldB2 = GetStructField(
+ GetStructField(
+ AttributeReference("data2", structType, false)(expId, qualifier),
+ 0, Some("a2")),
+ 0, Some("b2"))
+ assert(fieldB1.semanticEquals(fieldB2))
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index d9f32c000a885..c9a8c294dab0c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -23,6 +23,7 @@ import java.util.{Calendar, Locale, TimeZone}
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.TypeCoercion.numericPrecedence
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
import org.apache.spark.sql.catalyst.util.DateTimeUtils
@@ -953,4 +954,54 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper {
val ret6 = cast(Literal.create((1, Map(1 -> "a", 2 -> "b", 3 -> "c"))), StringType)
checkEvaluation(ret6, "[1, [1 -> a, 2 -> b, 3 -> c]]")
}
+
+ test("SPARK-26706: Fix Cast.mayTruncate for bytes") {
+ assert(!Cast.mayTruncate(ByteType, ByteType))
+ assert(!Cast.mayTruncate(DecimalType.ByteDecimal, ByteType))
+ assert(Cast.mayTruncate(ShortType, ByteType))
+ assert(Cast.mayTruncate(IntegerType, ByteType))
+ assert(Cast.mayTruncate(LongType, ByteType))
+ assert(Cast.mayTruncate(FloatType, ByteType))
+ assert(Cast.mayTruncate(DoubleType, ByteType))
+ assert(Cast.mayTruncate(DecimalType.IntDecimal, ByteType))
+ }
+
+ test("canSafeCast and mayTruncate must be consistent for numeric types") {
+ import DataTypeTestUtils._
+
+ def isCastSafe(from: NumericType, to: NumericType): Boolean = (from, to) match {
+ case (_, dt: DecimalType) => dt.isWiderThan(from)
+ case (dt: DecimalType, _) => dt.isTighterThan(to)
+ case _ => numericPrecedence.indexOf(from) <= numericPrecedence.indexOf(to)
+ }
+
+ numericTypes.foreach { from =>
+ val (safeTargetTypes, unsafeTargetTypes) = numericTypes.partition(to => isCastSafe(from, to))
+
+ safeTargetTypes.foreach { to =>
+ assert(Cast.canSafeCast(from, to), s"It should be possible to safely cast $from to $to")
+ assert(!Cast.mayTruncate(from, to), s"No truncation is expected when casting $from to $to")
+ }
+
+ unsafeTargetTypes.foreach { to =>
+ assert(!Cast.canSafeCast(from, to), s"It shouldn't be possible to safely cast $from to $to")
+ assert(Cast.mayTruncate(from, to), s"Truncation is expected when casting $from to $to")
+ }
+ }
+ }
+
+ test("SPARK-27671: cast from nested null type in struct") {
+ import DataTypeTestUtils._
+
+ atomicTypes.foreach { atomicType =>
+ val struct = Literal.create(
+ InternalRow(null),
+ StructType(Seq(StructField("a", NullType, nullable = true))))
+
+ val ret = cast(struct, StructType(Seq(
+ StructField("a", atomicType, nullable = true))))
+ assert(ret.resolved)
+ checkEvaluation(ret, InternalRow(null))
+ }
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index c7db4ec9e16b1..1984fa1d4b9d1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -83,30 +83,6 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(MapValues(m2), null)
}
- test("MapEntries") {
- def r(values: Any*): InternalRow = create_row(values: _*)
-
- // Primitive-type keys/values
- val mi0 = Literal.create(Map(1 -> 1, 2 -> null, 3 -> 2), MapType(IntegerType, IntegerType))
- val mi1 = Literal.create(Map[Int, Int](), MapType(IntegerType, IntegerType))
- val mi2 = Literal.create(null, MapType(IntegerType, IntegerType))
- val mid0 = Literal.create(Map(1 -> 1.1, 2 -> 2.2), MapType(IntegerType, DoubleType))
-
- checkEvaluation(MapEntries(mi0), Seq(r(1, 1), r(2, null), r(3, 2)))
- checkEvaluation(MapEntries(mi1), Seq.empty)
- checkEvaluation(MapEntries(mi2), null)
- checkEvaluation(MapEntries(mid0), Seq(r(1, 1.1), r(2, 2.2)))
-
- // Non-primitive-type keys/values
- val ms0 = Literal.create(Map("a" -> "c", "b" -> null), MapType(StringType, StringType))
- val ms1 = Literal.create(Map[Int, Int](), MapType(StringType, StringType))
- val ms2 = Literal.create(null, MapType(StringType, StringType))
-
- checkEvaluation(MapEntries(ms0), Seq(r("a", "c"), r("b", null)))
- checkEvaluation(MapEntries(ms1), Seq.empty)
- checkEvaluation(MapEntries(ms2), null)
- }
-
test("Map Concat") {
val m0 = Literal.create(Map("a" -> "1", "b" -> "2"), MapType(StringType, StringType,
valueContainsNull = false))
@@ -1339,6 +1315,8 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
ArrayType(DoubleType))
val a7 = Literal.create(Seq(1.123f, 0.1234f, 1.121f, 1.123f, 1.1230f, 1.121f, 0.1234f),
ArrayType(FloatType))
+ val a8 =
+ Literal.create(Seq(2, 1, 2, 3, 4, 4, 5).map(_.toString.getBytes), ArrayType(BinaryType))
checkEvaluation(new ArrayDistinct(a0), Seq(2, 1, 3, 4, 5))
checkEvaluation(new ArrayDistinct(a1), Seq.empty[Integer])
@@ -1348,6 +1326,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(new ArrayDistinct(a5), Seq(true, false))
checkEvaluation(new ArrayDistinct(a6), Seq(1.123, 0.1234, 1.121))
checkEvaluation(new ArrayDistinct(a7), Seq(1.123f, 0.1234f, 1.121f))
+ checkEvaluation(new ArrayDistinct(a8), Seq(2, 1, 3, 4, 5).map(_.toString.getBytes))
// complex data types
val b0 = Literal.create(Seq[Array[Byte]](Array[Byte](5, 6), Array[Byte](1, 2),
@@ -1368,9 +1347,17 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
ArrayType(ArrayType(IntegerType)))
val c2 = Literal.create(Seq[Seq[Int]](null, Seq[Int](2, 1), null, null, Seq[Int](2, 1), null),
ArrayType(ArrayType(IntegerType)))
+ val c3 = Literal.create(Seq[Seq[Int]](Seq[Int](1, 2), Seq[Int](1, 2), Seq[Int](1, 2),
+ Seq[Int](3, 4), Seq[Int](4, 5)), ArrayType(ArrayType(IntegerType)))
+ val c4 = Literal.create(Seq[Seq[Int]](null, Seq[Int](1, 2), Seq[Int](1, 2),
+ Seq[Int](3, 4), Seq[Int](4, 5), null), ArrayType(ArrayType(IntegerType)))
checkEvaluation(ArrayDistinct(c0), Seq[Seq[Int]](Seq[Int](1, 2), Seq[Int](3, 4)))
checkEvaluation(ArrayDistinct(c1), Seq[Seq[Int]](Seq[Int](5, 6), Seq[Int](2, 1)))
checkEvaluation(ArrayDistinct(c2), Seq[Seq[Int]](null, Seq[Int](2, 1)))
+ checkEvaluation(ArrayDistinct(c3), Seq[Seq[Int]](Seq[Int](1, 2), Seq[Int](3, 4),
+ Seq[Int](4, 5)))
+ checkEvaluation(ArrayDistinct(c4), Seq[Seq[Int]](null, Seq[Int](1, 2), Seq[Int](3, 4),
+ Seq[Int](4, 5)))
}
test("Array Union") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
index f489d330cf453..2e1858c188bb1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
@@ -222,4 +222,28 @@ class ConditionalExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
CaseWhen(Seq((Literal.create(false, BooleanType), Literal(1))), Literal(-1)).genCode(ctx)
assert(ctx.inlinedMutableStates.size == 1)
}
+
+ test("SPARK-27917 test semantic equals of CaseWhen") {
+ val attrRef = AttributeReference("ACCESS_CHECK", StringType)()
+ val aliasAttrRef = attrRef.withName("access_check")
+ // Test for Equality
+ var caseWhenObj1 = CaseWhen(Seq((attrRef, Literal("A"))))
+ var caseWhenObj2 = CaseWhen(Seq((aliasAttrRef, Literal("A"))))
+ assert(caseWhenObj1.semanticEquals(caseWhenObj2))
+ assert(caseWhenObj2.semanticEquals(caseWhenObj1))
+ // Test for inEquality
+ caseWhenObj2 = CaseWhen(Seq((attrRef, Literal("a"))))
+ assert(!caseWhenObj1.semanticEquals(caseWhenObj2))
+ assert(!caseWhenObj2.semanticEquals(caseWhenObj1))
+ // Test with elseValue with Equality
+ caseWhenObj1 = CaseWhen(Seq((attrRef, Literal("A"))), attrRef.withName("ELSEVALUE"))
+ caseWhenObj2 = CaseWhen(Seq((aliasAttrRef, Literal("A"))), aliasAttrRef.withName("elsevalue"))
+ assert(caseWhenObj1.semanticEquals(caseWhenObj2))
+ assert(caseWhenObj2.semanticEquals(caseWhenObj1))
+ caseWhenObj1 = CaseWhen(Seq((attrRef, Literal("A"))), Literal("ELSEVALUE"))
+ caseWhenObj2 = CaseWhen(Seq((aliasAttrRef, Literal("A"))), Literal("elsevalue"))
+ // Test with elseValue with inEquality
+ assert(!caseWhenObj1.semanticEquals(caseWhenObj2))
+ assert(!caseWhenObj2.semanticEquals(caseWhenObj1))
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
index e13f4d98295be..a0155ca98b8ec 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
@@ -88,11 +88,6 @@ class HigherOrderFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper
ArrayFilter(expr, createLambda(et, cn, f)).bind(validateBinding)
}
- def transformKeys(expr: Expression, f: (Expression, Expression) => Expression): Expression = {
- val MapType(kt, vt, vcn) = expr.dataType
- TransformKeys(expr, createLambda(kt, false, vt, vcn, f)).bind(validateBinding)
- }
-
def aggregate(
expr: Expression,
zero: Expression,
@@ -115,11 +110,6 @@ class HigherOrderFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper
aggregate(expr, zero, merge, identity)
}
- def transformValues(expr: Expression, f: (Expression, Expression) => Expression): Expression = {
- val MapType(kt, vt, vcn) = expr.dataType
- TransformValues(expr, createLambda(kt, false, vt, vcn, f)).bind(validateBinding)
- }
-
test("ArrayTransform") {
val ai0 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType, containsNull = false))
val ai1 = Literal.create(Seq[Integer](1, null, 3), ArrayType(IntegerType, containsNull = true))
@@ -161,55 +151,6 @@ class HigherOrderFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper
Seq("[1, 3, 5]", null, "[4, 6]"))
}
- test("MapFilter") {
- def mapFilter(expr: Expression, f: (Expression, Expression) => Expression): Expression = {
- val MapType(kt, vt, vcn) = expr.dataType
- MapFilter(expr, createLambda(kt, false, vt, vcn, f)).bind(validateBinding)
- }
- val mii0 = Literal.create(Map(1 -> 0, 2 -> 10, 3 -> -1),
- MapType(IntegerType, IntegerType, valueContainsNull = false))
- val mii1 = Literal.create(Map(1 -> null, 2 -> 10, 3 -> null),
- MapType(IntegerType, IntegerType, valueContainsNull = true))
- val miin = Literal.create(null, MapType(IntegerType, IntegerType, valueContainsNull = false))
-
- val kGreaterThanV: (Expression, Expression) => Expression = (k, v) => k > v
-
- checkEvaluation(mapFilter(mii0, kGreaterThanV), Map(1 -> 0, 3 -> -1))
- checkEvaluation(mapFilter(mii1, kGreaterThanV), Map())
- checkEvaluation(mapFilter(miin, kGreaterThanV), null)
-
- val valueIsNull: (Expression, Expression) => Expression = (_, v) => v.isNull
-
- checkEvaluation(mapFilter(mii0, valueIsNull), Map())
- checkEvaluation(mapFilter(mii1, valueIsNull), Map(1 -> null, 3 -> null))
- checkEvaluation(mapFilter(miin, valueIsNull), null)
-
- val msi0 = Literal.create(Map("abcdf" -> 5, "abc" -> 10, "" -> 0),
- MapType(StringType, IntegerType, valueContainsNull = false))
- val msi1 = Literal.create(Map("abcdf" -> 5, "abc" -> 10, "" -> null),
- MapType(StringType, IntegerType, valueContainsNull = true))
- val msin = Literal.create(null, MapType(StringType, IntegerType, valueContainsNull = false))
-
- val isLengthOfKey: (Expression, Expression) => Expression = (k, v) => Length(k) === v
-
- checkEvaluation(mapFilter(msi0, isLengthOfKey), Map("abcdf" -> 5, "" -> 0))
- checkEvaluation(mapFilter(msi1, isLengthOfKey), Map("abcdf" -> 5))
- checkEvaluation(mapFilter(msin, isLengthOfKey), null)
-
- val mia0 = Literal.create(Map(1 -> Seq(0, 1, 2), 2 -> Seq(10), -3 -> Seq(-1, 0, -2, 3)),
- MapType(IntegerType, ArrayType(IntegerType), valueContainsNull = false))
- val mia1 = Literal.create(Map(1 -> Seq(0, 1, 2), 2 -> null, -3 -> Seq(-1, 0, -2, 3)),
- MapType(IntegerType, ArrayType(IntegerType), valueContainsNull = true))
- val mian = Literal.create(
- null, MapType(IntegerType, ArrayType(IntegerType), valueContainsNull = false))
-
- val customFunc: (Expression, Expression) => Expression = (k, v) => Size(v) + k > 3
-
- checkEvaluation(mapFilter(mia0, customFunc), Map(1 -> Seq(0, 1, 2)))
- checkEvaluation(mapFilter(mia1, customFunc), Map(1 -> Seq(0, 1, 2)))
- checkEvaluation(mapFilter(mian, customFunc), null)
- }
-
test("ArrayFilter") {
val ai0 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType, containsNull = false))
val ai1 = Literal.create(Seq[Integer](1, null, 3), ArrayType(IntegerType, containsNull = true))
@@ -308,262 +249,6 @@ class HigherOrderFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper
15)
}
- test("TransformKeys") {
- val ai0 = Literal.create(
- Map(1 -> 1, 2 -> 2, 3 -> 3, 4 -> 4),
- MapType(IntegerType, IntegerType, valueContainsNull = false))
- val ai1 = Literal.create(
- Map.empty[Int, Int],
- MapType(IntegerType, IntegerType, valueContainsNull = true))
- val ai2 = Literal.create(
- Map(1 -> 1, 2 -> null, 3 -> 3),
- MapType(IntegerType, IntegerType, valueContainsNull = true))
- val ai3 = Literal.create(null, MapType(IntegerType, IntegerType, valueContainsNull = false))
-
- val plusOne: (Expression, Expression) => Expression = (k, v) => k + 1
- val plusValue: (Expression, Expression) => Expression = (k, v) => k + v
- val modKey: (Expression, Expression) => Expression = (k, v) => k % 3
-
- checkEvaluation(transformKeys(ai0, plusOne), Map(2 -> 1, 3 -> 2, 4 -> 3, 5 -> 4))
- checkEvaluation(transformKeys(ai0, plusValue), Map(2 -> 1, 4 -> 2, 6 -> 3, 8 -> 4))
- checkEvaluation(
- transformKeys(transformKeys(ai0, plusOne), plusValue), Map(3 -> 1, 5 -> 2, 7 -> 3, 9 -> 4))
- checkEvaluation(transformKeys(ai0, modKey),
- ArrayBasedMapData(Array(1, 2, 0, 1), Array(1, 2, 3, 4)))
- checkEvaluation(transformKeys(ai1, plusOne), Map.empty[Int, Int])
- checkEvaluation(transformKeys(ai1, plusOne), Map.empty[Int, Int])
- checkEvaluation(
- transformKeys(transformKeys(ai1, plusOne), plusValue), Map.empty[Int, Int])
- checkEvaluation(transformKeys(ai2, plusOne), Map(2 -> 1, 3 -> null, 4 -> 3))
- checkEvaluation(
- transformKeys(transformKeys(ai2, plusOne), plusOne), Map(3 -> 1, 4 -> null, 5 -> 3))
- checkEvaluation(transformKeys(ai3, plusOne), null)
-
- val as0 = Literal.create(
- Map("a" -> "xy", "bb" -> "yz", "ccc" -> "zx"),
- MapType(StringType, StringType, valueContainsNull = false))
- val as1 = Literal.create(
- Map("a" -> "xy", "bb" -> "yz", "ccc" -> null),
- MapType(StringType, StringType, valueContainsNull = true))
- val as2 = Literal.create(null,
- MapType(StringType, StringType, valueContainsNull = false))
- val as3 = Literal.create(Map.empty[StringType, StringType],
- MapType(StringType, StringType, valueContainsNull = true))
-
- val concatValue: (Expression, Expression) => Expression = (k, v) => Concat(Seq(k, v))
- val convertKeyToKeyLength: (Expression, Expression) => Expression =
- (k, v) => Length(k) + 1
-
- checkEvaluation(
- transformKeys(as0, concatValue), Map("axy" -> "xy", "bbyz" -> "yz", "ccczx" -> "zx"))
- checkEvaluation(
- transformKeys(transformKeys(as0, concatValue), concatValue),
- Map("axyxy" -> "xy", "bbyzyz" -> "yz", "ccczxzx" -> "zx"))
- checkEvaluation(transformKeys(as3, concatValue), Map.empty[String, String])
- checkEvaluation(
- transformKeys(transformKeys(as3, concatValue), convertKeyToKeyLength),
- Map.empty[Int, String])
- checkEvaluation(transformKeys(as0, convertKeyToKeyLength),
- Map(2 -> "xy", 3 -> "yz", 4 -> "zx"))
- checkEvaluation(transformKeys(as1, convertKeyToKeyLength),
- Map(2 -> "xy", 3 -> "yz", 4 -> null))
- checkEvaluation(transformKeys(as2, convertKeyToKeyLength), null)
- checkEvaluation(transformKeys(as3, convertKeyToKeyLength), Map.empty[Int, String])
-
- val ax0 = Literal.create(
- Map(1 -> "x", 2 -> "y", 3 -> "z"),
- MapType(IntegerType, StringType, valueContainsNull = false))
-
- checkEvaluation(transformKeys(ax0, plusOne), Map(2 -> "x", 3 -> "y", 4 -> "z"))
- }
-
- test("TransformValues") {
- val ai0 = Literal.create(
- Map(1 -> 1, 2 -> 2, 3 -> 3),
- MapType(IntegerType, IntegerType, valueContainsNull = false))
- val ai1 = Literal.create(
- Map(1 -> 1, 2 -> null, 3 -> 3),
- MapType(IntegerType, IntegerType, valueContainsNull = true))
- val ai2 = Literal.create(
- Map.empty[Int, Int],
- MapType(IntegerType, IntegerType, valueContainsNull = true))
- val ai3 = Literal.create(null, MapType(IntegerType, IntegerType, valueContainsNull = false))
-
- val plusOne: (Expression, Expression) => Expression = (k, v) => v + 1
- val valueUpdate: (Expression, Expression) => Expression = (k, v) => k * k
-
- checkEvaluation(transformValues(ai0, plusOne), Map(1 -> 2, 2 -> 3, 3 -> 4))
- checkEvaluation(transformValues(ai0, valueUpdate), Map(1 -> 1, 2 -> 4, 3 -> 9))
- checkEvaluation(
- transformValues(transformValues(ai0, plusOne), valueUpdate), Map(1 -> 1, 2 -> 4, 3 -> 9))
- checkEvaluation(transformValues(ai1, plusOne), Map(1 -> 2, 2 -> null, 3 -> 4))
- checkEvaluation(transformValues(ai1, valueUpdate), Map(1 -> 1, 2 -> 4, 3 -> 9))
- checkEvaluation(
- transformValues(transformValues(ai1, plusOne), valueUpdate), Map(1 -> 1, 2 -> 4, 3 -> 9))
- checkEvaluation(transformValues(ai2, plusOne), Map.empty[Int, Int])
- checkEvaluation(transformValues(ai3, plusOne), null)
-
- val as0 = Literal.create(
- Map("a" -> "xy", "bb" -> "yz", "ccc" -> "zx"),
- MapType(StringType, StringType, valueContainsNull = false))
- val as1 = Literal.create(
- Map("a" -> "xy", "bb" -> null, "ccc" -> "zx"),
- MapType(StringType, StringType, valueContainsNull = true))
- val as2 = Literal.create(Map.empty[StringType, StringType],
- MapType(StringType, StringType, valueContainsNull = true))
- val as3 = Literal.create(null, MapType(StringType, StringType, valueContainsNull = true))
-
- val concatValue: (Expression, Expression) => Expression = (k, v) => Concat(Seq(k, v))
- val valueTypeUpdate: (Expression, Expression) => Expression =
- (k, v) => Length(v) + 1
-
- checkEvaluation(
- transformValues(as0, concatValue), Map("a" -> "axy", "bb" -> "bbyz", "ccc" -> "ccczx"))
- checkEvaluation(transformValues(as0, valueTypeUpdate),
- Map("a" -> 3, "bb" -> 3, "ccc" -> 3))
- checkEvaluation(
- transformValues(transformValues(as0, concatValue), concatValue),
- Map("a" -> "aaxy", "bb" -> "bbbbyz", "ccc" -> "cccccczx"))
- checkEvaluation(transformValues(as1, concatValue),
- Map("a" -> "axy", "bb" -> null, "ccc" -> "ccczx"))
- checkEvaluation(transformValues(as1, valueTypeUpdate),
- Map("a" -> 3, "bb" -> null, "ccc" -> 3))
- checkEvaluation(
- transformValues(transformValues(as1, concatValue), concatValue),
- Map("a" -> "aaxy", "bb" -> null, "ccc" -> "cccccczx"))
- checkEvaluation(transformValues(as2, concatValue), Map.empty[String, String])
- checkEvaluation(transformValues(as2, valueTypeUpdate), Map.empty[String, Int])
- checkEvaluation(
- transformValues(transformValues(as2, concatValue), valueTypeUpdate),
- Map.empty[String, Int])
- checkEvaluation(transformValues(as3, concatValue), null)
-
- val ax0 = Literal.create(
- Map(1 -> "x", 2 -> "y", 3 -> "z"),
- MapType(IntegerType, StringType, valueContainsNull = false))
-
- checkEvaluation(transformValues(ax0, valueUpdate), Map(1 -> 1, 2 -> 4, 3 -> 9))
- }
-
- test("MapZipWith") {
- def map_zip_with(
- left: Expression,
- right: Expression,
- f: (Expression, Expression, Expression) => Expression): Expression = {
- val MapType(kt, vt1, _) = left.dataType
- val MapType(_, vt2, _) = right.dataType
- MapZipWith(left, right, createLambda(kt, false, vt1, true, vt2, true, f))
- .bind(validateBinding)
- }
-
- val mii0 = Literal.create(Map(1 -> 10, 2 -> 20, 3 -> 30),
- MapType(IntegerType, IntegerType, valueContainsNull = false))
- val mii1 = Literal.create(Map(1 -> -1, 2 -> -2, 4 -> -4),
- MapType(IntegerType, IntegerType, valueContainsNull = false))
- val mii2 = Literal.create(Map(1 -> null, 2 -> -2, 3 -> null),
- MapType(IntegerType, IntegerType, valueContainsNull = true))
- val mii3 = Literal.create(Map(), MapType(IntegerType, IntegerType, valueContainsNull = false))
- val mii4 = MapFromArrays(
- Literal.create(Seq(2, 2), ArrayType(IntegerType, false)),
- Literal.create(Seq(20, 200), ArrayType(IntegerType, false)))
- val miin = Literal.create(null, MapType(IntegerType, IntegerType, valueContainsNull = false))
-
- val multiplyKeyWithValues: (Expression, Expression, Expression) => Expression = {
- (k, v1, v2) => k * v1 * v2
- }
-
- checkEvaluation(
- map_zip_with(mii0, mii1, multiplyKeyWithValues),
- Map(1 -> -10, 2 -> -80, 3 -> null, 4 -> null))
- checkEvaluation(
- map_zip_with(mii0, mii2, multiplyKeyWithValues),
- Map(1 -> null, 2 -> -80, 3 -> null))
- checkEvaluation(
- map_zip_with(mii0, mii3, multiplyKeyWithValues),
- Map(1 -> null, 2 -> null, 3 -> null))
- checkEvaluation(
- map_zip_with(mii0, mii4, multiplyKeyWithValues),
- Map(1 -> null, 2 -> 800, 3 -> null))
- checkEvaluation(
- map_zip_with(mii4, mii0, multiplyKeyWithValues),
- Map(2 -> 800, 1 -> null, 3 -> null))
- checkEvaluation(
- map_zip_with(mii0, miin, multiplyKeyWithValues),
- null)
- assert(map_zip_with(mii0, mii1, multiplyKeyWithValues).dataType ===
- MapType(IntegerType, IntegerType, valueContainsNull = true))
-
- val mss0 = Literal.create(Map("a" -> "x", "b" -> "y", "d" -> "z"),
- MapType(StringType, StringType, valueContainsNull = false))
- val mss1 = Literal.create(Map("d" -> "b", "b" -> "d"),
- MapType(StringType, StringType, valueContainsNull = false))
- val mss2 = Literal.create(Map("c" -> null, "b" -> "t", "a" -> null),
- MapType(StringType, StringType, valueContainsNull = true))
- val mss3 = Literal.create(Map(), MapType(StringType, StringType, valueContainsNull = false))
- val mss4 = MapFromArrays(
- Literal.create(Seq("a", "a"), ArrayType(StringType, false)),
- Literal.create(Seq("a", "n"), ArrayType(StringType, false)))
- val mssn = Literal.create(null, MapType(StringType, StringType, valueContainsNull = false))
-
- val concat: (Expression, Expression, Expression) => Expression = {
- (k, v1, v2) => Concat(Seq(k, v1, v2))
- }
-
- checkEvaluation(
- map_zip_with(mss0, mss1, concat),
- Map("a" -> null, "b" -> "byd", "d" -> "dzb"))
- checkEvaluation(
- map_zip_with(mss1, mss2, concat),
- Map("d" -> null, "b" -> "bdt", "c" -> null, "a" -> null))
- checkEvaluation(
- map_zip_with(mss0, mss3, concat),
- Map("a" -> null, "b" -> null, "d" -> null))
- checkEvaluation(
- map_zip_with(mss0, mss4, concat),
- Map("a" -> "axa", "b" -> null, "d" -> null))
- checkEvaluation(
- map_zip_with(mss4, mss0, concat),
- Map("a" -> "aax", "b" -> null, "d" -> null))
- checkEvaluation(
- map_zip_with(mss0, mssn, concat),
- null)
- assert(map_zip_with(mss0, mss1, concat).dataType ===
- MapType(StringType, StringType, valueContainsNull = true))
-
- def b(data: Byte*): Array[Byte] = Array[Byte](data: _*)
-
- val mbb0 = Literal.create(Map(b(1, 2) -> b(4), b(2, 1) -> b(5), b(1, 3) -> b(8)),
- MapType(BinaryType, BinaryType, valueContainsNull = false))
- val mbb1 = Literal.create(Map(b(2, 1) -> b(7), b(1, 2) -> b(3), b(1, 1) -> b(6)),
- MapType(BinaryType, BinaryType, valueContainsNull = false))
- val mbb2 = Literal.create(Map(b(1, 3) -> null, b(1, 2) -> b(2), b(2, 1) -> null),
- MapType(BinaryType, BinaryType, valueContainsNull = true))
- val mbb3 = Literal.create(Map(), MapType(BinaryType, BinaryType, valueContainsNull = false))
- val mbb4 = MapFromArrays(
- Literal.create(Seq(b(2, 1), b(2, 1)), ArrayType(BinaryType, false)),
- Literal.create(Seq(b(1), b(9)), ArrayType(BinaryType, false)))
- val mbbn = Literal.create(null, MapType(BinaryType, BinaryType, valueContainsNull = false))
-
- checkEvaluation(
- map_zip_with(mbb0, mbb1, concat),
- Map(b(1, 2) -> b(1, 2, 4, 3), b(2, 1) -> b(2, 1, 5, 7), b(1, 3) -> null, b(1, 1) -> null))
- checkEvaluation(
- map_zip_with(mbb1, mbb2, concat),
- Map(b(2, 1) -> null, b(1, 2) -> b(1, 2, 3, 2), b(1, 1) -> null, b(1, 3) -> null))
- checkEvaluation(
- map_zip_with(mbb0, mbb3, concat),
- Map(b(1, 2) -> null, b(2, 1) -> null, b(1, 3) -> null))
- checkEvaluation(
- map_zip_with(mbb0, mbb4, concat),
- Map(b(1, 2) -> null, b(2, 1) -> b(2, 1, 5, 1), b(1, 3) -> null))
- checkEvaluation(
- map_zip_with(mbb4, mbb0, concat),
- Map(b(2, 1) -> b(2, 1, 1, 5), b(1, 2) -> null, b(1, 3) -> null))
- checkEvaluation(
- map_zip_with(mbb0, mbbn, concat),
- null)
- }
-
test("ZipWith") {
def zip_with(
left: Expression,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
index ac76b17ef4761..3b60d1d88b3c6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
@@ -268,7 +268,7 @@ class PredicateSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(InSet(nl, nS), null)
val primitiveTypes = Seq(IntegerType, FloatType, DoubleType, StringType, ByteType, ShortType,
- LongType, BinaryType, BooleanType, DecimalType.USER_DEFAULT, TimestampType)
+ LongType, BooleanType, DecimalType.USER_DEFAULT, TimestampType)
primitiveTypes.foreach { t =>
val dataGen = RandomDataGenerator.forType(t, nullable = true).get
val inputData = Seq.fill(10) {
@@ -293,6 +293,54 @@ class PredicateSuite extends SparkFunSuite with ExpressionEvalHelper {
}
}
+ test("INSET: binary") {
+ val hS = HashSet[Any]() + Array(1.toByte, 2.toByte) + Array(3.toByte)
+ val nS = HashSet[Any]() + Array(1.toByte, 2.toByte) + Array(3.toByte) + null
+ val onetwo = Literal(Array(1.toByte, 2.toByte))
+ val three = Literal(Array(3.toByte))
+ val threefour = Literal(Array(3.toByte, 4.toByte))
+ val nl = Literal(null, onetwo.dataType)
+ checkEvaluation(InSet(onetwo, hS), true)
+ checkEvaluation(InSet(three, hS), true)
+ checkEvaluation(InSet(three, nS), true)
+ checkEvaluation(InSet(threefour, hS), false)
+ checkEvaluation(InSet(threefour, nS), null)
+ checkEvaluation(InSet(nl, hS), null)
+ checkEvaluation(InSet(nl, nS), null)
+ }
+
+ test("INSET: struct") {
+ val hS = HashSet[Any]() + Literal.create((1, "a")).value + Literal.create((2, "b")).value
+ val nS = HashSet[Any]() + Literal.create((1, "a")).value + Literal.create((2, "b")).value + null
+ val oneA = Literal.create((1, "a"))
+ val twoB = Literal.create((2, "b"))
+ val twoC = Literal.create((2, "c"))
+ val nl = Literal(null, oneA.dataType)
+ checkEvaluation(InSet(oneA, hS), true)
+ checkEvaluation(InSet(twoB, hS), true)
+ checkEvaluation(InSet(twoB, nS), true)
+ checkEvaluation(InSet(twoC, hS), false)
+ checkEvaluation(InSet(twoC, nS), null)
+ checkEvaluation(InSet(nl, hS), null)
+ checkEvaluation(InSet(nl, nS), null)
+ }
+
+ test("INSET: array") {
+ val hS = HashSet[Any]() + Literal.create(Seq(1, 2)).value + Literal.create(Seq(3)).value
+ val nS = HashSet[Any]() + Literal.create(Seq(1, 2)).value + Literal.create(Seq(3)).value + null
+ val onetwo = Literal.create(Seq(1, 2))
+ val three = Literal.create(Seq(3))
+ val threefour = Literal.create(Seq(3, 4))
+ val nl = Literal(null, onetwo.dataType)
+ checkEvaluation(InSet(onetwo, hS), true)
+ checkEvaluation(InSet(three, hS), true)
+ checkEvaluation(InSet(three, nS), true)
+ checkEvaluation(InSet(threefour, hS), false)
+ checkEvaluation(InSet(threefour, nS), null)
+ checkEvaluation(InSet(nl, hS), null)
+ checkEvaluation(InSet(nl, nS), null)
+ }
+
private case class MyStruct(a: Long, b: String)
private case class MyStruct2(a: MyStruct, b: Array[Int])
private val udt = new ExamplePointUDT
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
index e083ae0089244..467cfd5598ff1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
@@ -26,10 +26,10 @@ import org.apache.spark.sql.types.{IntegerType, StringType}
class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
test("basic") {
- val intUdf = ScalaUDF((i: Int) => i + 1, IntegerType, Literal(1) :: Nil)
+ val intUdf = ScalaUDF((i: Int) => i + 1, IntegerType, Literal(1) :: Nil, true :: Nil)
checkEvaluation(intUdf, 2)
- val stringUdf = ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil)
+ val stringUdf = ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil, true :: Nil)
checkEvaluation(stringUdf, "ax")
}
@@ -37,7 +37,8 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
val udf = ScalaUDF(
(s: String) => s.toLowerCase(Locale.ROOT),
StringType,
- Literal.create(null, StringType) :: Nil)
+ Literal.create(null, StringType) :: Nil,
+ true :: Nil)
val e1 = intercept[SparkException](udf.eval())
assert(e1.getMessage.contains("Failed to execute user defined function"))
@@ -50,7 +51,7 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
test("SPARK-22695: ScalaUDF should not use global variables") {
val ctx = new CodegenContext
- ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil).genCode(ctx)
+ ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil, true :: Nil).genCode(ctx)
assert(ctx.inlinedMutableStates.isEmpty)
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AggregateExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AggregateExpressionSuite.scala
new file mode 100644
index 0000000000000..8e9c9972071ad
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AggregateExpressionSuite.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions.aggregate
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
+import org.apache.spark.sql.catalyst.expressions.{Add, AttributeSet}
+
+class AggregateExpressionSuite extends SparkFunSuite {
+
+ test("test references from unresolved aggregate functions") {
+ val x = UnresolvedAttribute("x")
+ val y = UnresolvedAttribute("y")
+ val actual = AggregateExpression(Sum(Add(x, y)), mode = Complete, isDistinct = false).references
+ val expected = AttributeSet(x :: y :: Nil)
+ assert(expected == actual, s"Expected: $expected. Actual: $actual")
+ }
+
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/UTF8StringBuilderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/UTF8StringBuilderSuite.scala
deleted file mode 100644
index 1b25a4b191f86..0000000000000
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/UTF8StringBuilderSuite.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.expressions.codegen
-
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.unsafe.types.UTF8String
-
-class UTF8StringBuilderSuite extends SparkFunSuite {
-
- test("basic test") {
- val sb = new UTF8StringBuilder()
- assert(sb.build() === UTF8String.EMPTY_UTF8)
-
- sb.append("")
- assert(sb.build() === UTF8String.EMPTY_UTF8)
-
- sb.append("abcd")
- assert(sb.build() === UTF8String.fromString("abcd"))
-
- sb.append(UTF8String.fromString("1234"))
- assert(sb.build() === UTF8String.fromString("abcd1234"))
-
- // expect to grow an internal buffer
- sb.append(UTF8String.fromString("efgijk567890"))
- assert(sb.build() === UTF8String.fromString("abcd1234efgijk567890"))
- }
-}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriterSuite.scala
new file mode 100644
index 0000000000000..fb651b76fc16d
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeRowWriterSuite.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions.codegen
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.types.Decimal
+
+class UnsafeRowWriterSuite extends SparkFunSuite {
+
+ def checkDecimalSizeInBytes(decimal: Decimal, numBytes: Int): Unit = {
+ assert(decimal.toJavaBigDecimal.unscaledValue().toByteArray.length == numBytes)
+ }
+
+ test("SPARK-25538: zero-out all bits for decimals") {
+ val decimal1 = Decimal(0.431)
+ decimal1.changePrecision(38, 18)
+ checkDecimalSizeInBytes(decimal1, 8)
+
+ val decimal2 = Decimal(123456789.1232456789)
+ decimal2.changePrecision(38, 18)
+ checkDecimalSizeInBytes(decimal2, 11)
+ // On an UnsafeRowWriter we write decimal2 first and then decimal1
+ val unsafeRowWriter1 = new UnsafeRowWriter(1)
+ unsafeRowWriter1.resetRowWriter()
+ unsafeRowWriter1.write(0, decimal2, decimal2.precision, decimal2.scale)
+ unsafeRowWriter1.reset()
+ unsafeRowWriter1.write(0, decimal1, decimal1.precision, decimal1.scale)
+ val res1 = unsafeRowWriter1.getRow
+ // On a second UnsafeRowWriter we write directly decimal1
+ val unsafeRowWriter2 = new UnsafeRowWriter(1)
+ unsafeRowWriter2.resetRowWriter()
+ unsafeRowWriter2.write(0, decimal1, decimal1.precision, decimal1.scale)
+ val res2 = unsafeRowWriter2.getRow
+ // The two rows should be the equal
+ assert(res1 == res2)
+ }
+
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
index a313681eeb8f0..5794691a365a9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLite
import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
class BinaryComparisonSimplificationSuite extends PlanTest with PredicateHelper {
@@ -92,4 +93,33 @@ class BinaryComparisonSimplificationSuite extends PlanTest with PredicateHelper
val correctAnswer = nonNullableRelation.analyze
comparePlans(actual, correctAnswer)
}
+
+ test("SPARK-26402: accessing nested fields with different cases in case insensitive mode") {
+ val expId = NamedExpression.newExprId
+ val qualifier = Seq.empty[String]
+ val structType = StructType(
+ StructField("a", StructType(StructField("b", IntegerType, false) :: Nil), false) :: Nil)
+
+ val fieldA1 = GetStructField(
+ GetStructField(
+ AttributeReference("data1", structType, false)(expId, qualifier),
+ 0, Some("a1")),
+ 0, Some("b1"))
+ val fieldA2 = GetStructField(
+ GetStructField(
+ AttributeReference("data2", structType, false)(expId, qualifier),
+ 0, Some("a2")),
+ 0, Some("b2"))
+
+ // GetStructField with different names are semantically equal; thus, `EqualTo(fieldA1, fieldA2)`
+ // will be optimized to `TrueLiteral` by `SimplifyBinaryComparison`.
+ val originalQuery = nonNullableRelation
+ .where(EqualTo(fieldA1, fieldA2))
+ .analyze
+
+ val optimized = Optimize.execute(originalQuery)
+ val correctAnswer = nonNullableRelation.analyze
+
+ comparePlans(optimized, correctAnswer)
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
index 653c07f1835ca..a0de5f6930958 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
@@ -27,8 +27,9 @@ import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.BooleanType
-class BooleanSimplificationSuite extends PlanTest with PredicateHelper {
+class BooleanSimplificationSuite extends PlanTest with ExpressionEvalHelper with PredicateHelper {
object Optimize extends RuleExecutor[LogicalPlan] {
val batches =
@@ -37,6 +38,7 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper {
Batch("Constant Folding", FixedPoint(50),
NullPropagation,
ConstantFolding,
+ SimplifyConditionals,
BooleanSimplification,
PruneFilters) :: Nil
}
@@ -48,6 +50,14 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper {
testRelation.output, Seq(Row(1, 2, 3, "abc"))
)
+ val testNotNullableRelation = LocalRelation('a.int.notNull, 'b.int.notNull, 'c.int.notNull,
+ 'd.string.notNull, 'e.boolean.notNull, 'f.boolean.notNull, 'g.boolean.notNull,
+ 'h.boolean.notNull)
+
+ val testNotNullableRelationWithData = LocalRelation.fromExternalRows(
+ testNotNullableRelation.output, Seq(Row(1, 2, 3, "abc"))
+ )
+
private def checkCondition(input: Expression, expected: LogicalPlan): Unit = {
val plan = testRelationWithData.where(input).analyze
val actual = Optimize.execute(plan)
@@ -61,6 +71,21 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper {
comparePlans(actual, correctAnswer)
}
+ private def checkConditionInNotNullableRelation(
+ input: Expression, expected: Expression): Unit = {
+ val plan = testNotNullableRelationWithData.where(input).analyze
+ val actual = Optimize.execute(plan)
+ val correctAnswer = testNotNullableRelationWithData.where(expected).analyze
+ comparePlans(actual, correctAnswer)
+ }
+
+ private def checkConditionInNotNullableRelation(
+ input: Expression, expected: LogicalPlan): Unit = {
+ val plan = testNotNullableRelationWithData.where(input).analyze
+ val actual = Optimize.execute(plan)
+ comparePlans(actual, expected)
+ }
+
test("a && a => a") {
checkCondition(Literal(1) < 'a && Literal(1) < 'a, Literal(1) < 'a)
checkCondition(Literal(1) < 'a && Literal(1) < 'a && Literal(1) < 'a, Literal(1) < 'a)
@@ -102,42 +127,55 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper {
'a === 'b || 'b > 3 && 'a > 3 && 'a < 5)
}
- test("e && (!e || f)") {
- checkCondition('e && (!'e || 'f ), 'e && 'f)
+ test("e && (!e || f) - not nullable") {
+ checkConditionInNotNullableRelation('e && (!'e || 'f ), 'e && 'f)
+
+ checkConditionInNotNullableRelation('e && ('f || !'e ), 'e && 'f)
- checkCondition('e && ('f || !'e ), 'e && 'f)
+ checkConditionInNotNullableRelation((!'e || 'f ) && 'e, 'f && 'e)
- checkCondition((!'e || 'f ) && 'e, 'f && 'e)
+ checkConditionInNotNullableRelation(('f || !'e ) && 'e, 'f && 'e)
+ }
- checkCondition(('f || !'e ) && 'e, 'f && 'e)
+ test("e && (!e || f) - nullable") {
+ Seq ('e && (!'e || 'f ),
+ 'e && ('f || !'e ),
+ (!'e || 'f ) && 'e,
+ ('f || !'e ) && 'e,
+ 'e || (!'e && 'f),
+ 'e || ('f && !'e),
+ ('e && 'f) || !'e,
+ ('f && 'e) || !'e).foreach { expr =>
+ checkCondition(expr, expr)
+ }
}
- test("a < 1 && (!(a < 1) || f)") {
- checkCondition('a < 1 && (!('a < 1) || 'f), ('a < 1) && 'f)
- checkCondition('a < 1 && ('f || !('a < 1)), ('a < 1) && 'f)
+ test("a < 1 && (!(a < 1) || f) - not nullable") {
+ checkConditionInNotNullableRelation('a < 1 && (!('a < 1) || 'f), ('a < 1) && 'f)
+ checkConditionInNotNullableRelation('a < 1 && ('f || !('a < 1)), ('a < 1) && 'f)
- checkCondition('a <= 1 && (!('a <= 1) || 'f), ('a <= 1) && 'f)
- checkCondition('a <= 1 && ('f || !('a <= 1)), ('a <= 1) && 'f)
+ checkConditionInNotNullableRelation('a <= 1 && (!('a <= 1) || 'f), ('a <= 1) && 'f)
+ checkConditionInNotNullableRelation('a <= 1 && ('f || !('a <= 1)), ('a <= 1) && 'f)
- checkCondition('a > 1 && (!('a > 1) || 'f), ('a > 1) && 'f)
- checkCondition('a > 1 && ('f || !('a > 1)), ('a > 1) && 'f)
+ checkConditionInNotNullableRelation('a > 1 && (!('a > 1) || 'f), ('a > 1) && 'f)
+ checkConditionInNotNullableRelation('a > 1 && ('f || !('a > 1)), ('a > 1) && 'f)
- checkCondition('a >= 1 && (!('a >= 1) || 'f), ('a >= 1) && 'f)
- checkCondition('a >= 1 && ('f || !('a >= 1)), ('a >= 1) && 'f)
+ checkConditionInNotNullableRelation('a >= 1 && (!('a >= 1) || 'f), ('a >= 1) && 'f)
+ checkConditionInNotNullableRelation('a >= 1 && ('f || !('a >= 1)), ('a >= 1) && 'f)
}
- test("a < 1 && ((a >= 1) || f)") {
- checkCondition('a < 1 && ('a >= 1 || 'f ), ('a < 1) && 'f)
- checkCondition('a < 1 && ('f || 'a >= 1), ('a < 1) && 'f)
+ test("a < 1 && ((a >= 1) || f) - not nullable") {
+ checkConditionInNotNullableRelation('a < 1 && ('a >= 1 || 'f ), ('a < 1) && 'f)
+ checkConditionInNotNullableRelation('a < 1 && ('f || 'a >= 1), ('a < 1) && 'f)
- checkCondition('a <= 1 && ('a > 1 || 'f ), ('a <= 1) && 'f)
- checkCondition('a <= 1 && ('f || 'a > 1), ('a <= 1) && 'f)
+ checkConditionInNotNullableRelation('a <= 1 && ('a > 1 || 'f ), ('a <= 1) && 'f)
+ checkConditionInNotNullableRelation('a <= 1 && ('f || 'a > 1), ('a <= 1) && 'f)
- checkCondition('a > 1 && (('a <= 1) || 'f), ('a > 1) && 'f)
- checkCondition('a > 1 && ('f || ('a <= 1)), ('a > 1) && 'f)
+ checkConditionInNotNullableRelation('a > 1 && (('a <= 1) || 'f), ('a > 1) && 'f)
+ checkConditionInNotNullableRelation('a > 1 && ('f || ('a <= 1)), ('a > 1) && 'f)
- checkCondition('a >= 1 && (('a < 1) || 'f), ('a >= 1) && 'f)
- checkCondition('a >= 1 && ('f || ('a < 1)), ('a >= 1) && 'f)
+ checkConditionInNotNullableRelation('a >= 1 && (('a < 1) || 'f), ('a >= 1) && 'f)
+ checkConditionInNotNullableRelation('a >= 1 && ('f || ('a < 1)), ('a >= 1) && 'f)
}
test("DeMorgan's law") {
@@ -174,10 +212,72 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper {
}
test("Complementation Laws") {
- checkCondition('a && !'a, testRelation)
- checkCondition(!'a && 'a, testRelation)
+ checkConditionInNotNullableRelation('e && !'e, testNotNullableRelation)
+ checkConditionInNotNullableRelation(!'e && 'e, testNotNullableRelation)
+
+ checkConditionInNotNullableRelation('e || !'e, testNotNullableRelationWithData)
+ checkConditionInNotNullableRelation(!'e || 'e, testNotNullableRelationWithData)
+ }
+
+ test("Complementation Laws - null handling") {
+ checkCondition('e && !'e,
+ testRelationWithData.where(If('e.isNull, Literal.create(null, BooleanType), false)).analyze)
+ checkCondition(!'e && 'e,
+ testRelationWithData.where(If('e.isNull, Literal.create(null, BooleanType), false)).analyze)
+
+ checkCondition('e || !'e,
+ testRelationWithData.where(If('e.isNull, Literal.create(null, BooleanType), true)).analyze)
+ checkCondition(!'e || 'e,
+ testRelationWithData.where(If('e.isNull, Literal.create(null, BooleanType), true)).analyze)
+ }
+
+ test("Complementation Laws - negative case") {
+ checkCondition('e && !'f, testRelationWithData.where('e && !'f).analyze)
+ checkCondition(!'f && 'e, testRelationWithData.where(!'f && 'e).analyze)
+
+ checkCondition('e || !'f, testRelationWithData.where('e || !'f).analyze)
+ checkCondition(!'f || 'e, testRelationWithData.where(!'f || 'e).analyze)
+ }
+
+ protected def assertEquivalent(e1: Expression, e2: Expression): Unit = {
+ val correctAnswer = Project(Alias(e2, "out")() :: Nil, OneRowRelation()).analyze
+ val actual = Optimize.execute(Project(Alias(e1, "out")() :: Nil, OneRowRelation()).analyze)
+ comparePlans(actual, correctAnswer)
+ }
- checkCondition('a || !'a, testRelationWithData)
- checkCondition(!'a || 'a, testRelationWithData)
+ test("filter reduction - positive cases") {
+ val fields = Seq(
+ 'col1NotNULL.boolean.notNull,
+ 'col2NotNULL.boolean.notNull
+ )
+ val Seq(col1NotNULL, col2NotNULL) = fields.zipWithIndex.map { case (f, i) => f.at(i) }
+
+ val exprs = Seq(
+ // actual expressions of the transformations: original -> transformed
+ (col1NotNULL && (!col1NotNULL || col2NotNULL)) -> (col1NotNULL && col2NotNULL),
+ (col1NotNULL && (col2NotNULL || !col1NotNULL)) -> (col1NotNULL && col2NotNULL),
+ ((!col1NotNULL || col2NotNULL) && col1NotNULL) -> (col2NotNULL && col1NotNULL),
+ ((col2NotNULL || !col1NotNULL) && col1NotNULL) -> (col2NotNULL && col1NotNULL),
+
+ (col1NotNULL || (!col1NotNULL && col2NotNULL)) -> (col1NotNULL || col2NotNULL),
+ (col1NotNULL || (col2NotNULL && !col1NotNULL)) -> (col1NotNULL || col2NotNULL),
+ ((!col1NotNULL && col2NotNULL) || col1NotNULL) -> (col2NotNULL || col1NotNULL),
+ ((col2NotNULL && !col1NotNULL) || col1NotNULL) -> (col2NotNULL || col1NotNULL)
+ )
+
+ // check plans
+ for ((originalExpr, expectedExpr) <- exprs) {
+ assertEquivalent(originalExpr, expectedExpr)
+ }
+
+ // check evaluation
+ val binaryBooleanValues = Seq(true, false)
+ for (col1NotNULLVal <- binaryBooleanValues;
+ col2NotNULLVal <- binaryBooleanValues;
+ (originalExpr, expectedExpr) <- exprs) {
+ val inputRow = create_row(col1NotNULLVal, col2NotNULLVal)
+ val optimizedVal = evaluateWithoutCodegen(expectedExpr, inputRow)
+ checkEvaluation(originalExpr, optimizedVal, inputRow)
+ }
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConvertToLocalRelationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConvertToLocalRelationSuite.scala
index 0c015f88e1e84..b0bea019d2f52 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConvertToLocalRelationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConvertToLocalRelationSuite.scala
@@ -21,10 +21,12 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.expressions.{LessThan, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Expression, GenericInternalRow, LessThan, Literal, UnaryExpression}
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types.{DataType, StructType}
class ConvertToLocalRelationSuite extends PlanTest {
@@ -70,4 +72,36 @@ class ConvertToLocalRelationSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}
+
+ test("SPARK-27798: Expression reusing output shouldn't override values in local relation") {
+ val testRelation = LocalRelation(
+ LocalRelation('a.int).output,
+ InternalRow(1) :: InternalRow(2) :: Nil)
+
+ val correctAnswer = LocalRelation(
+ LocalRelation('a.struct('a1.int)).output,
+ InternalRow(InternalRow(1)) :: InternalRow(InternalRow(2)) :: Nil)
+
+ val projected = testRelation.select(ExprReuseOutput(UnresolvedAttribute("a")).as("a"))
+ val optimized = Optimize.execute(projected.analyze)
+
+ comparePlans(optimized, correctAnswer)
+ }
+}
+
+// Dummy expression used for testing. It reuses output row. Assumes child expr outputs an integer.
+case class ExprReuseOutput(child: Expression) extends UnaryExpression {
+ override def dataType: DataType = StructType.fromDDL("a1 int")
+
+ override def nullable: Boolean = true
+
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
+ throw new UnsupportedOperationException("Should not trigger codegen")
+
+ private val row: InternalRow = new GenericInternalRow(1)
+
+ override def eval(input: InternalRow): Any = {
+ row.update(0, child.eval(input))
+ row
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/InferFiltersFromConstraintsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/InferFiltersFromConstraintsSuite.scala
index e4671f0d1cce6..a40ba2dc38b70 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/InferFiltersFromConstraintsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/InferFiltersFromConstraintsSuite.scala
@@ -196,7 +196,7 @@ class InferFiltersFromConstraintsSuite extends PlanTest {
test("constraints should be inferred from aliased literals") {
val originalLeft = testRelation.subquery('left).as("left")
- val optimizedLeft = testRelation.subquery('left).where(IsNotNull('a) && 'a === 2).as("left")
+ val optimizedLeft = testRelation.subquery('left).where(IsNotNull('a) && 'a <=> 2).as("left")
val right = Project(Seq(Literal(2).as("two")), testRelation.subquery('right)).as("right")
val condition = Some("left.a".attr === "right.two".attr)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinOptimizationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinOptimizationSuite.scala
index ccd9d8dd4d213..e9438b2eee550 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinOptimizationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinOptimizationSuite.scala
@@ -102,16 +102,19 @@ class JoinOptimizationSuite extends PlanTest {
x.join(y).join(z).where(("x.b".attr === "z.b".attr) && ("y.d".attr === "z.a".attr)),
x.join(z, condition = Some("x.b".attr === "z.b".attr))
.join(y, condition = Some("y.d".attr === "z.a".attr))
+ .select(Seq("x.a", "x.b", "x.c", "y.d", "z.a", "z.b", "z.c").map(_.attr): _*)
),
(
x.join(y, Cross).join(z, Cross)
.where(("x.b".attr === "z.b".attr) && ("y.d".attr === "z.a".attr)),
x.join(z, Cross, Some("x.b".attr === "z.b".attr))
.join(y, Cross, Some("y.d".attr === "z.a".attr))
+ .select(Seq("x.a", "x.b", "x.c", "y.d", "z.a", "z.b", "z.c").map(_.attr): _*)
),
(
x.join(y, Inner).join(z, Cross).where("x.b".attr === "z.a".attr),
x.join(z, Cross, Some("x.b".attr === "z.a".attr)).join(y, Inner)
+ .select(Seq("x.a", "x.b", "x.c", "y.d", "z.a", "z.b", "z.c").map(_.attr): _*)
)
)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
index 565b0a10154a8..38a70f0691dd4 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
@@ -20,8 +20,8 @@ package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap}
-import org.apache.spark.sql.catalyst.plans.{Inner, PlanTest}
-import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, LogicalPlan}
+import org.apache.spark.sql.catalyst.plans.{Cross, Inner, PlanTest}
+import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, LocalRelation, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.catalyst.statsEstimation.{StatsEstimationTestBase, StatsTestPlan}
import org.apache.spark.sql.internal.SQLConf.{CBO_ENABLED, JOIN_REORDER_ENABLED}
@@ -124,7 +124,8 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
// the original order (t1 J t2) J t3.
val bestPlan =
t1.join(t3, Inner, Some(nameToAttr("t1.v-1-10") === nameToAttr("t3.v-1-100")))
- .join(t2, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
+ .join(t2, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
+ .select(outputsOf(t1, t2, t3): _*)
assertEqualPlans(originalPlan, bestPlan)
}
@@ -139,7 +140,9 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
val bestPlan =
t1.join(t3, Inner, Some(nameToAttr("t1.v-1-10") === nameToAttr("t3.v-1-100")))
.join(t2, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
+ .select(outputsOf(t1, t2, t3): _*) // this is redundant but we'll take it for now
.join(t4)
+ .select(outputsOf(t1, t2, t4, t3): _*)
assertEqualPlans(originalPlan, bestPlan)
}
@@ -202,6 +205,7 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
t1.join(t2, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
.join(t4.join(t3, Inner, Some(nameToAttr("t4.v-1-10") === nameToAttr("t3.v-1-100"))),
Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t4.k-1-2")))
+ .select(outputsOf(t1, t4, t2, t3): _*)
assertEqualPlans(originalPlan, bestPlan)
}
@@ -219,6 +223,23 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
}
}
+ test("SPARK-26352: join reordering should not change the order of attributes") {
+ // This test case does not rely on CBO.
+ // It's similar to the test case above, but catches a reordering bug that the one above doesn't
+ val tab1 = LocalRelation('x.int, 'y.int)
+ val tab2 = LocalRelation('i.int, 'j.int)
+ val tab3 = LocalRelation('a.int, 'b.int)
+ val original =
+ tab1.join(tab2, Cross)
+ .join(tab3, Inner, Some('a === 'x && 'b === 'i))
+ val expected =
+ tab1.join(tab3, Inner, Some('a === 'x))
+ .join(tab2, Cross, Some('b === 'i))
+ .select(outputsOf(tab1, tab2, tab3): _*)
+
+ assertEqualPlans(original, expected)
+ }
+
test("reorder recursively") {
// Original order:
// Join
@@ -266,8 +287,32 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
private def assertEqualPlans(
originalPlan: LogicalPlan,
groundTruthBestPlan: LogicalPlan): Unit = {
- val optimized = Optimize.execute(originalPlan.analyze)
+ val analyzed = originalPlan.analyze
+ val optimized = Optimize.execute(analyzed)
val expected = groundTruthBestPlan.analyze
+
+ assert(sameOutput(analyzed, expected)) // if this fails, the expected plan itself is incorrect
+ assert(sameOutput(analyzed, optimized))
+
compareJoinOrder(optimized, expected)
}
+
+ private def outputsOf(plans: LogicalPlan*): Seq[Attribute] = {
+ plans.map(_.output).reduce(_ ++ _)
+ }
+
+ /**
+ * Returns true iff output of both plans are semantically the same, ie.:
+ * - they contain the same number of `Attribute`s;
+ * - references are the same;
+ * - the order is equal too.
+ * NOTE: this is copied over from SPARK-25691 from master.
+ */
+ def sameOutput(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
+ val output1 = plan1.output
+ val output2 = plan2.output
+ output1.length == output2.length && output1.zip(output2).forall {
+ case (a1, a2) => a1.semanticEquals(a2)
+ }
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
index f1ce7543ffdc1..d395bba105a7b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
@@ -147,7 +147,7 @@ class PropagateEmptyRelationSuite extends PlanTest {
.where(false)
.select('a)
.where('a > 1)
- .where('a != 200)
+ .where('a =!= 200)
.orderBy('a.asc)
val optimized = Optimize.execute(query.analyze)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullOutPythonUDFInJoinConditionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullOutPythonUDFInJoinConditionSuite.scala
new file mode 100644
index 0000000000000..4a25ddf3ed9e9
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullOutPythonUDFInJoinConditionSuite.scala
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.api.python.PythonEvalType
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.PythonUDF
+import org.apache.spark.sql.catalyst.plans._
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.internal.SQLConf._
+import org.apache.spark.sql.types.{BooleanType, IntegerType}
+
+class PullOutPythonUDFInJoinConditionSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+ val batches =
+ Batch("Extract PythonUDF From JoinCondition", Once,
+ PullOutPythonUDFInJoinCondition) ::
+ Batch("Check Cartesian Products", Once,
+ CheckCartesianProducts) :: Nil
+ }
+
+ val attrA = 'a.int
+ val attrB = 'b.int
+ val attrC = 'c.int
+ val attrD = 'd.int
+
+ val testRelationLeft = LocalRelation(attrA, attrB)
+ val testRelationRight = LocalRelation(attrC, attrD)
+
+ // This join condition refers to attributes from 2 tables, but the PythonUDF inside it only
+ // refer to attributes from one side.
+ val evaluableJoinCond = {
+ val pythonUDF = PythonUDF("evaluable", null,
+ IntegerType,
+ Seq(attrA),
+ PythonEvalType.SQL_BATCHED_UDF,
+ udfDeterministic = true)
+ pythonUDF === attrC
+ }
+
+ // This join condition is a PythonUDF which refers to attributes from 2 tables.
+ val unevaluableJoinCond = PythonUDF("unevaluable", null,
+ BooleanType,
+ Seq(attrA, attrC),
+ PythonEvalType.SQL_BATCHED_UDF,
+ udfDeterministic = true)
+
+ val unsupportedJoinTypes = Seq(LeftOuter, RightOuter, FullOuter, LeftAnti, LeftSemi)
+
+ private def comparePlanWithCrossJoinEnable(query: LogicalPlan, expected: LogicalPlan): Unit = {
+ // AnalysisException thrown by CheckCartesianProducts while spark.sql.crossJoin.enabled=false
+ val exception = intercept[AnalysisException] {
+ Optimize.execute(query.analyze)
+ }
+ assert(exception.message.startsWith("Detected implicit cartesian product"))
+
+ // pull out the python udf while set spark.sql.crossJoin.enabled=true
+ withSQLConf(CROSS_JOINS_ENABLED.key -> "true") {
+ val optimized = Optimize.execute(query.analyze)
+ comparePlans(optimized, expected)
+ }
+ }
+
+ test("inner join condition with python udf") {
+ val query1 = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = Some(unevaluableJoinCond))
+ val expected1 = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = None).where(unevaluableJoinCond).analyze
+ comparePlanWithCrossJoinEnable(query1, expected1)
+
+ // evaluable PythonUDF will not be touched
+ val query2 = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = Some(evaluableJoinCond))
+ comparePlans(Optimize.execute(query2), query2)
+ }
+
+ test("unevaluable python udf and common condition") {
+ val query = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = Some(unevaluableJoinCond && 'a.attr === 'c.attr))
+ val expected = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = Some('a.attr === 'c.attr)).where(unevaluableJoinCond).analyze
+ val optimized = Optimize.execute(query.analyze)
+ comparePlans(optimized, expected)
+ }
+
+ test("unevaluable python udf or common condition") {
+ val query = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = Some(unevaluableJoinCond || 'a.attr === 'c.attr))
+ val expected = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = None).where(unevaluableJoinCond || 'a.attr === 'c.attr).analyze
+ comparePlanWithCrossJoinEnable(query, expected)
+ }
+
+ test("pull out whole complex condition with multiple unevaluable python udf") {
+ val pythonUDF1 = PythonUDF("pythonUDF1", null,
+ BooleanType,
+ Seq(attrA, attrC),
+ PythonEvalType.SQL_BATCHED_UDF,
+ udfDeterministic = true)
+ val condition = (unevaluableJoinCond || 'a.attr === 'c.attr) && pythonUDF1
+
+ val query = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = Some(condition))
+ val expected = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = None).where(condition).analyze
+ comparePlanWithCrossJoinEnable(query, expected)
+ }
+
+ test("partial pull out complex condition with multiple unevaluable python udf") {
+ val pythonUDF1 = PythonUDF("pythonUDF1", null,
+ BooleanType,
+ Seq(attrA, attrC),
+ PythonEvalType.SQL_BATCHED_UDF,
+ udfDeterministic = true)
+ val condition = (unevaluableJoinCond || pythonUDF1) && 'a.attr === 'c.attr
+
+ val query = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = Some(condition))
+ val expected = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = Some('a.attr === 'c.attr)).where(unevaluableJoinCond || pythonUDF1).analyze
+ val optimized = Optimize.execute(query.analyze)
+ comparePlans(optimized, expected)
+ }
+
+ test("pull out unevaluable python udf when it's mixed with evaluable one") {
+ val query = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = Some(evaluableJoinCond && unevaluableJoinCond))
+ val expected = testRelationLeft.join(
+ testRelationRight,
+ joinType = Inner,
+ condition = Some(evaluableJoinCond)).where(unevaluableJoinCond).analyze
+ val optimized = Optimize.execute(query.analyze)
+ comparePlans(optimized, expected)
+ }
+
+ test("throw an exception for not supported join types") {
+ for (joinType <- unsupportedJoinTypes) {
+ val e = intercept[AnalysisException] {
+ val query = testRelationLeft.join(
+ testRelationRight,
+ joinType,
+ condition = Some(unevaluableJoinCond))
+ Optimize.execute(query.analyze)
+ }
+ assert(e.message.contentEquals(
+ s"Using PythonUDF in join condition of join type $joinType is not supported."))
+
+ val query2 = testRelationLeft.join(
+ testRelationRight,
+ joinType,
+ condition = Some(evaluableJoinCond))
+ comparePlans(Optimize.execute(query2), query2)
+ }
+ }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PushProjectThroughUnionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PushProjectThroughUnionSuite.scala
new file mode 100644
index 0000000000000..294d29842b045
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PushProjectThroughUnionSuite.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
+import org.apache.spark.sql.catalyst.rules.RuleExecutor
+
+class PushProjectThroughUnionSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+ val batches = Batch("Optimizer Batch", FixedPoint(100),
+ PushProjectionThroughUnion,
+ FoldablePropagation) :: Nil
+ }
+
+ test("SPARK-25450 PushProjectThroughUnion rule uses the same exprId for project expressions " +
+ "in each Union child, causing mistakes in constant propagation") {
+ val testRelation1 = LocalRelation('a.string, 'b.int, 'c.string)
+ val testRelation2 = LocalRelation('d.string, 'e.int, 'f.string)
+ val query = testRelation1
+ .union(testRelation2.select("bar".as("d"), 'e, 'f))
+ .select('a.as("n"))
+ .select('n, "dummy").analyze
+ val optimized = Optimize.execute(query)
+
+ val expected = testRelation1
+ .select('a.as("n"))
+ .select('n, "dummy")
+ .union(testRelation2
+ .select("bar".as("d"), 'e, 'f)
+ .select("bar".as("n"))
+ .select("bar".as("n"), "dummy")).analyze
+
+ comparePlans(optimized, expected)
+ }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala
index 3b1b2d588ef67..c8e15c7da763e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceOperatorSuite.scala
@@ -20,11 +20,12 @@ package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.expressions.{Alias, Literal, Not}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Coalesce, If, Literal, Not}
import org.apache.spark.sql.catalyst.expressions.aggregate.First
import org.apache.spark.sql.catalyst.plans.{LeftAnti, LeftSemi, PlanTest}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.types.BooleanType
class ReplaceOperatorSuite extends PlanTest {
@@ -65,8 +66,7 @@ class ReplaceOperatorSuite extends PlanTest {
val correctAnswer =
Aggregate(table1.output, table1.output,
- Filter(Not((attributeA.isNotNull && attributeB.isNotNull) &&
- (attributeA >= 2 && attributeB < 1)),
+ Filter(Not(Coalesce(Seq(attributeA >= 2 && attributeB < 1, Literal.FalseLiteral))),
Filter(attributeB === 2, Filter(attributeA === 1, table1)))).analyze
comparePlans(optimized, correctAnswer)
@@ -84,8 +84,8 @@ class ReplaceOperatorSuite extends PlanTest {
val correctAnswer =
Aggregate(table1.output, table1.output,
- Filter(Not((attributeA.isNotNull && attributeB.isNotNull) &&
- (attributeA >= 2 && attributeB < 1)), table1)).analyze
+ Filter(Not(Coalesce(Seq(attributeA >= 2 && attributeB < 1, Literal.FalseLiteral))),
+ table1)).analyze
comparePlans(optimized, correctAnswer)
}
@@ -104,8 +104,7 @@ class ReplaceOperatorSuite extends PlanTest {
val correctAnswer =
Aggregate(table1.output, table1.output,
- Filter(Not((attributeA.isNotNull && attributeB.isNotNull) &&
- (attributeA >= 2 && attributeB < 1)),
+ Filter(Not(Coalesce(Seq(attributeA >= 2 && attributeB < 1, Literal.FalseLiteral))),
Project(Seq(attributeA, attributeB), table1))).analyze
comparePlans(optimized, correctAnswer)
@@ -125,8 +124,7 @@ class ReplaceOperatorSuite extends PlanTest {
val correctAnswer =
Aggregate(table1.output, table1.output,
- Filter(Not((attributeA.isNotNull && attributeB.isNotNull) &&
- (attributeA >= 2 && attributeB < 1)),
+ Filter(Not(Coalesce(Seq(attributeA >= 2 && attributeB < 1, Literal.FalseLiteral))),
Filter(attributeB === 2, Filter(attributeA === 1, table1)))).analyze
comparePlans(optimized, correctAnswer)
@@ -146,8 +144,7 @@ class ReplaceOperatorSuite extends PlanTest {
val correctAnswer =
Aggregate(table1.output, table1.output,
- Filter(Not((attributeA.isNotNull && attributeB.isNotNull) &&
- (attributeA === 1 && attributeB === 2)),
+ Filter(Not(Coalesce(Seq(attributeA === 1 && attributeB === 2, Literal.FalseLiteral))),
Project(Seq(attributeA, attributeB),
Filter(attributeB < 1, Filter(attributeA >= 2, table1))))).analyze
@@ -229,4 +226,29 @@ class ReplaceOperatorSuite extends PlanTest {
comparePlans(optimized, query)
}
+
+ test("SPARK-26366: ReplaceExceptWithFilter should handle properly NULL") {
+ val basePlan = LocalRelation(Seq('a.int, 'b.int))
+ val otherPlan = basePlan.where('a.in(1, 2) || 'b.in())
+ val except = Except(basePlan, otherPlan, false)
+ val result = OptimizeIn(Optimize.execute(except.analyze))
+ val correctAnswer = Aggregate(basePlan.output, basePlan.output,
+ Filter(!Coalesce(Seq(
+ 'a.in(1, 2) || If('b.isNotNull, Literal.FalseLiteral, Literal(null, BooleanType)),
+ Literal.FalseLiteral)),
+ basePlan)).analyze
+ comparePlans(result, correctAnswer)
+ }
+
+ test("SPARK-26366: ReplaceExceptWithFilter should not transform non-detrministic") {
+ val basePlan = LocalRelation(Seq('a.int, 'b.int))
+ val otherPlan = basePlan.where('a > rand(1L))
+ val except = Except(basePlan, otherPlan, false)
+ val result = Optimize.execute(except.analyze)
+ val condition = basePlan.output.zip(otherPlan.output).map { case (a1, a2) =>
+ a1 <=> a2 }.reduce( _ && _)
+ val correctAnswer = Aggregate(basePlan.output, otherPlan.output,
+ Join(basePlan, otherPlan, LeftAnti, Option(condition))).analyze
+ comparePlans(result, correctAnswer)
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
index d4d23ad69b2c2..baae934e1e4fe 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
@@ -218,6 +218,7 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
.join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk")))
.join(t2, Inner, Some(nameToAttr("f1_c2") === nameToAttr("t2_c1")))
.join(t1, Inner, Some(nameToAttr("f1_c1") === nameToAttr("t1_c1")))
+ .select(outputsOf(f1, t1, t2, d1, d2): _*)
assertEqualPlans(query, expected)
}
@@ -256,6 +257,7 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
.join(t3.join(t2, Inner, Some(nameToAttr("t2_c2") === nameToAttr("t3_c1"))), Inner,
Some(nameToAttr("d1_c2") === nameToAttr("t2_c1")))
.join(t1, Inner, Some(nameToAttr("t1_c1") === nameToAttr("f1_c1")))
+ .select(outputsOf(d1, t1, t2, f1, d2, t3): _*)
assertEqualPlans(query, expected)
}
@@ -297,6 +299,7 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
Some(nameToAttr("t3_c1") === nameToAttr("t4_c1")))
.join(t1.join(t2, Inner, Some(nameToAttr("t1_c1") === nameToAttr("t2_c1"))), Inner,
Some(nameToAttr("t1_c2") === nameToAttr("t4_c2")))
+ .select(outputsOf(d1, t1, t2, t3, t4, f1, d2): _*)
assertEqualPlans(query, expected)
}
@@ -347,6 +350,7 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
Some(nameToAttr("d3_c2") === nameToAttr("t1_c1")))
.join(t5.join(t6, Inner, Some(nameToAttr("t5_c2") === nameToAttr("t6_c2"))), Inner,
Some(nameToAttr("d2_c2") === nameToAttr("t5_c1")))
+ .select(outputsOf(d1, t3, t4, f1, d2, t5, t6, d3, t1, t2): _*)
assertEqualPlans(query, expected)
}
@@ -375,6 +379,7 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
f1.join(d3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("d3_pk")))
.join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk")))
.join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk")))
+ .select(outputsOf(d1, d2, f1, d3): _*)
assertEqualPlans(query, expected)
}
@@ -400,13 +405,27 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
f1.join(t3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("t3_c1")))
.join(t2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("t2_c1")))
.join(t1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("t1_c1")))
+ .select(outputsOf(t1, f1, t2, t3): _*)
assertEqualPlans(query, expected)
}
private def assertEqualPlans( plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
- val optimized = Optimize.execute(plan1.analyze)
+ val analyzed = plan1.analyze
+ val optimized = Optimize.execute(analyzed)
val expected = plan2.analyze
+
+ assert(equivalentOutput(analyzed, expected)) // if this fails, the expected itself is incorrect
+ assert(equivalentOutput(analyzed, optimized))
+
compareJoinOrder(optimized, expected)
}
+
+ private def outputsOf(plans: LogicalPlan*): Seq[Attribute] = {
+ plans.map(_.output).reduce(_ ++ _)
+ }
+
+ private def equivalentOutput(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
+ normalizeExprIds(plan1).output == normalizeExprIds(plan2).output
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
index 4e0883e91e84a..9dc653b9d6c44 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
@@ -182,6 +182,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
.join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk1")))
.join(d3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("d3_pk1")))
.join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
+ .select(outputsOf(d1, d2, f1, d3, s3): _*)
assertEqualPlans(query, expected)
}
@@ -220,6 +221,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
.join(d3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("d3_pk1")))
.join(d2, Inner, Some(nameToAttr("f1_fk2") < nameToAttr("d2_pk1")))
.join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
+ .select(outputsOf(d1, f1, d2, s3, d3): _*)
assertEqualPlans(query, expected)
}
@@ -255,7 +257,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
.join(d3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
.join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk1")))
.join(s3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("s3_c2")))
-
+ .select(outputsOf(d1, f1, d2, s3, d3): _*)
assertEqualPlans(query, expected)
}
@@ -292,6 +294,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
.join(d3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("d3_pk1")))
.join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_c2")))
.join(s3, Inner, Some(nameToAttr("d3_fk1") < nameToAttr("s3_pk1")))
+ .select(outputsOf(d1, f1, d2, s3, d3): _*)
assertEqualPlans(query, expected)
}
@@ -395,6 +398,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
.join(d2.where(nameToAttr("d2_c2") === 2), Inner,
Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk1")))
.join(s3, Inner, Some(nameToAttr("f11_fk1") === nameToAttr("s3_pk1")))
+ .select(outputsOf(d1, f11, f1, d2, s3): _*)
assertEqualPlans(query, equivQuery)
}
@@ -430,6 +434,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
.join(d2.where(nameToAttr("d2_c2") === 2), Inner,
Some(nameToAttr("f1_fk2") === nameToAttr("d2_c4")))
.join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
+ .select(outputsOf(d1, d3, f1, d2, s3): _*)
assertEqualPlans(query, expected)
}
@@ -465,6 +470,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
.join(d2.where(nameToAttr("d2_c2") === 2), Inner,
Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk1")))
.join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
+ .select(outputsOf(d1, d3, f1, d2, s3): _*)
assertEqualPlans(query, expected)
}
@@ -499,6 +505,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
.join(d2.where(nameToAttr("d2_c2") === 2),
Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk1")))
.join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
+ .select(outputsOf(d1, d3, f1, d2, s3): _*)
assertEqualPlans(query, expected)
}
@@ -532,6 +539,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
.join(d3, Inner, Some(nameToAttr("f1_fk3") < nameToAttr("d3_pk1")))
.join(d2, Inner, Some(nameToAttr("f1_fk2") < nameToAttr("d2_pk1")))
.join(s3, Inner, Some(nameToAttr("d3_fk1") < nameToAttr("s3_pk1")))
+ .select(outputsOf(d1, d3, f1, d2, s3): _*)
assertEqualPlans(query, expected)
}
@@ -565,13 +573,27 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
.join(d3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("d3_pk1")))
.join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk1")))
.join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
+ .select(outputsOf(d1, d3, f1, d2, s3): _*)
assertEqualPlans(query, expected)
}
- private def assertEqualPlans( plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
- val optimized = Optimize.execute(plan1.analyze)
+ private def assertEqualPlans(plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
+ val analyzed = plan1.analyze
+ val optimized = Optimize.execute(analyzed)
val expected = plan2.analyze
+
+ assert(equivalentOutput(analyzed, expected)) // if this fails, the expected itself is incorrect
+ assert(equivalentOutput(analyzed, optimized))
+
compareJoinOrder(optimized, expected)
}
+
+ private def outputsOf(plans: LogicalPlan*): Seq[Attribute] = {
+ plans.map(_.output).reduce(_ ++ _)
+ }
+
+ private def equivalentOutput(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
+ normalizeExprIds(plan1).output == normalizeExprIds(plan2).output
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 781fc1e957ae0..2bc41dad9d46d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, _}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate.{First, Last}
import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.CalendarInterval
@@ -246,9 +247,11 @@ class ExpressionParserSuite extends PlanTest {
intercept("foo(a x)", "extraneous input 'x'")
}
+ private def lv(s: Symbol) = UnresolvedNamedLambdaVariable(Seq(s.name))
+
test("lambda functions") {
- assertEqual("x -> x + 1", LambdaFunction('x + 1, Seq('x.attr)))
- assertEqual("(x, y) -> x + y", LambdaFunction('x + 'y, Seq('x.attr, 'y.attr)))
+ assertEqual("x -> x + 1", LambdaFunction(lv('x) + 1, Seq(lv('x))))
+ assertEqual("(x, y) -> x + y", LambdaFunction(lv('x) + lv('y), Seq(lv('x), lv('y))))
}
test("window function expressions") {
@@ -606,7 +609,19 @@ class ExpressionParserSuite extends PlanTest {
// Hive nanosecond notation.
assertEqual("interval 13.123456789 seconds", intervalLiteral("second", "13.123456789"))
- assertEqual("interval -13.123456789 second", intervalLiteral("second", "-13.123456789"))
+ assertEqual(
+ "interval -13.123456789 second",
+ Literal(new CalendarInterval(
+ 0,
+ -13 * DateTimeUtils.MICROS_PER_SECOND - 123 * DateTimeUtils.MICROS_PER_MILLIS - 456)))
+ assertEqual(
+ "interval 13.123456 second",
+ Literal(new CalendarInterval(
+ 0,
+ 13 * DateTimeUtils.MICROS_PER_SECOND + 123 * DateTimeUtils.MICROS_PER_MILLIS + 456)))
+ assertEqual(
+ "interval 1.001 second",
+ Literal(CalendarInterval.fromString("interval 1 second 1 millisecond")))
// Non Existing unit
intercept("interval 10 nanoseconds", "No interval can be constructed")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
index 422bf97e30e7e..f5da90f7cf0c6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
@@ -108,7 +108,7 @@ class PlanParserSuite extends AnalysisTest {
assertEqual("select a, b from db.c where x < 1", table("db", "c").where('x < 1).select('a, 'b))
assertEqual(
"select a, b from db.c having x < 1",
- table("db", "c").select('a, 'b).where('x < 1))
+ table("db", "c").groupBy()('a, 'b).where('x < 1))
assertEqual("select distinct a, b from db.c", Distinct(table("db", "c").select('a, 'b)))
assertEqual("select all a, b from db.c", table("db", "c").select('a, 'b))
assertEqual("select from tbl", OneRowRelation().select('from.as("tbl")))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala
new file mode 100644
index 0000000000000..4d869d79ad594
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.catalyst.plans
+
+import java.io.File
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.util.Utils
+
+trait SQLHelper {
+
+ /**
+ * Sets all SQL configurations specified in `pairs`, calls `f`, and then restores all SQL
+ * configurations.
+ */
+ protected def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = {
+ val conf = SQLConf.get
+ val (keys, values) = pairs.unzip
+ val currentValues = keys.map { key =>
+ if (conf.contains(key)) {
+ Some(conf.getConfString(key))
+ } else {
+ None
+ }
+ }
+ (keys, values).zipped.foreach { (k, v) =>
+ if (SQLConf.staticConfKeys.contains(k)) {
+ throw new AnalysisException(s"Cannot modify the value of a static config: $k")
+ }
+ conf.setConfString(k, v)
+ }
+ try f finally {
+ keys.zip(currentValues).foreach {
+ case (key, Some(value)) => conf.setConfString(key, value)
+ case (key, None) => conf.unsetConf(key)
+ }
+ }
+ }
+
+ /**
+ * Generates a temporary path without creating the actual file/directory, then pass it to `f`. If
+ * a file/directory is created there by `f`, it will be delete after `f` returns.
+ */
+ protected def withTempPath(f: File => Unit): Unit = {
+ val path = Utils.createTempDir()
+ path.delete()
+ try f(path) finally Utils.deleteRecursively(path)
+ }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/AggregateEstimationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/AggregateEstimationSuite.scala
index 8213d568fe85e..d89b9dfe57970 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/AggregateEstimationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/AggregateEstimationSuite.scala
@@ -38,7 +38,11 @@ class AggregateEstimationSuite extends StatsEstimationTestBase with PlanTest {
attr("key22") -> ColumnStat(distinctCount = Some(2), min = Some(10), max = Some(20),
nullCount = Some(0), avgLen = Some(4), maxLen = Some(4)),
attr("key31") -> ColumnStat(distinctCount = Some(0), min = None, max = None,
- nullCount = Some(0), avgLen = Some(4), maxLen = Some(4))
+ nullCount = Some(0), avgLen = Some(4), maxLen = Some(4)),
+ attr("key32") -> ColumnStat(distinctCount = Some(0), min = None, max = None,
+ nullCount = Some(4), avgLen = Some(4), maxLen = Some(4)),
+ attr("key33") -> ColumnStat(distinctCount = Some(2), min = None, max = None,
+ nullCount = Some(2), avgLen = Some(4), maxLen = Some(4))
))
private val nameToAttr: Map[String, Attribute] = columnInfo.map(kv => kv._1.name -> kv._1)
@@ -92,6 +96,23 @@ class AggregateEstimationSuite extends StatsEstimationTestBase with PlanTest {
expectedOutputRowCount = 0)
}
+ test("group-by column with only null value") {
+ checkAggStats(
+ tableColumns = Seq("key22", "key32"),
+ tableRowCount = 6,
+ groupByColumns = Seq("key22", "key32"),
+ expectedOutputRowCount = nameToColInfo("key22")._2.distinctCount.get)
+ }
+
+ test("group-by column with null value") {
+ checkAggStats(
+ tableColumns = Seq("key21", "key33"),
+ tableRowCount = 6,
+ groupByColumns = Seq("key21", "key33"),
+ expectedOutputRowCount = nameToColInfo("key21")._2.distinctCount.get *
+ (nameToColInfo("key33")._2.distinctCount.get + 1))
+ }
+
test("non-cbo estimation") {
val attributes = Seq("key12").map(nameToAttr)
val child = StatsTestPlan(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala
index 47bfa62569583..b0a47e7835129 100755
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
import org.apache.spark.sql.catalyst.plans.LeftOuter
import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.{ColumnStatsMap, FilterEstimation}
import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.EstimationUtils._
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.types._
@@ -821,6 +822,32 @@ class FilterEstimationSuite extends StatsEstimationTestBase {
expectedRowCount = 3)
}
+ test("ColumnStatsMap tests") {
+ val attrNoDistinct = AttributeReference("att_without_distinct", IntegerType)()
+ val attrNoCount = AttributeReference("att_without_count", BooleanType)()
+ val attrNoMinMax = AttributeReference("att_without_min_max", DateType)()
+ val colStatNoDistinct = ColumnStat(distinctCount = None, min = Some(1), max = Some(10),
+ nullCount = Some(0), avgLen = Some(4), maxLen = Some(4))
+ val colStatNoCount = ColumnStat(distinctCount = Some(2), min = Some(false), max = Some(true),
+ nullCount = None, avgLen = Some(1), maxLen = Some(1))
+ val colStatNoMinMax = ColumnStat(distinctCount = Some(1), min = None, max = None,
+ nullCount = Some(1), avgLen = None, maxLen = None)
+ val columnStatsMap = ColumnStatsMap(AttributeMap(Seq(
+ attrNoDistinct -> colStatNoDistinct,
+ attrNoCount -> colStatNoCount,
+ attrNoMinMax -> colStatNoMinMax
+ )))
+ assert(!columnStatsMap.hasDistinctCount(attrNoDistinct))
+ assert(columnStatsMap.hasDistinctCount(attrNoCount))
+ assert(columnStatsMap.hasDistinctCount(attrNoMinMax))
+ assert(!columnStatsMap.hasCountStats(attrNoDistinct))
+ assert(!columnStatsMap.hasCountStats(attrNoCount))
+ assert(columnStatsMap.hasCountStats(attrNoMinMax))
+ assert(columnStatsMap.hasMinMaxStats(attrNoDistinct))
+ assert(columnStatsMap.hasMinMaxStats(attrNoCount))
+ assert(!columnStatsMap.hasMinMaxStats(attrNoMinMax))
+ }
+
private def childStatsTestPlan(outList: Seq[Attribute], tableRowCount: BigInt): StatsTestPlan = {
StatsTestPlan(
outputList = outList,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
index b7092f4c42d4c..e37cf8a8e2170 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
@@ -28,13 +28,13 @@ import org.json4s.jackson.JsonMethods
import org.json4s.jackson.JsonMethods._
import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow, TableIdentifier}
+import org.apache.spark.sql.catalyst.{AliasIdentifier, FunctionIdentifier, InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.catalyst.dsl.expressions.DslString
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.plans.{LeftOuter, NaturalJoin}
-import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, Union}
+import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, SubqueryAlias, Union}
import org.apache.spark.sql.catalyst.plans.physical.{IdentityBroadcastMode, RoundRobinPartitioning, SinglePartition}
import org.apache.spark.sql.types._
import org.apache.spark.storage.StorageLevel
@@ -425,6 +425,28 @@ class TreeNodeSuite extends SparkFunSuite {
"product-class" -> JString(classOf[FunctionIdentifier].getName),
"funcName" -> "function"))
+ // Converts AliasIdentifier to JSON
+ assertJSON(
+ AliasIdentifier("alias"),
+ JObject(
+ "product-class" -> JString(classOf[AliasIdentifier].getName),
+ "identifier" -> "alias"))
+
+ // Converts SubqueryAlias to JSON
+ assertJSON(
+ SubqueryAlias("t1", JsonTestTreeNode("0")),
+ List(
+ JObject(
+ "class" -> classOf[SubqueryAlias].getName,
+ "num-children" -> 1,
+ "name" -> JObject("product-class" -> JString(classOf[AliasIdentifier].getName),
+ "identifier" -> "t1"),
+ "child" -> 0),
+ JObject(
+ "class" -> classOf[JsonTestTreeNode].getName,
+ "num-children" -> 0,
+ "arg" -> "0")))
+
// Converts BucketSpec to JSON
assertJSON(
BucketSpec(1, Seq("bucket"), Seq("sort")),
@@ -564,7 +586,7 @@ class TreeNodeSuite extends SparkFunSuite {
}
test("toJSON should not throws java.lang.StackOverflowError") {
- val udf = ScalaUDF(SelfReferenceUDF(), BooleanType, Seq("col1".attr))
+ val udf = ScalaUDF(SelfReferenceUDF(), BooleanType, Seq("col1".attr), true :: Nil)
// Should not throw java.lang.StackOverflowError
udf.toJSON
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
index 0c1feb3aa0882..66d8d28988f89 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
@@ -26,6 +26,17 @@ object DateTimeTestUtils {
val ALL_TIMEZONES: Seq[TimeZone] = TimeZone.getAvailableIDs.toSeq.map(TimeZone.getTimeZone)
+ val outstandingTimezonesIds: Seq[String] = Seq(
+ "UTC",
+ "PST",
+ "CET",
+ "Africa/Dakar",
+ "America/Los_Angeles",
+ "Antarctica/Vostok",
+ "Asia/Hong_Kong",
+ "Europe/Amsterdam")
+ val outstandingTimezones: Seq[TimeZone] = outstandingTimezonesIds.map(TimeZone.getTimeZone)
+
def withDefaultTimeZone[T](newDefaultTimeZone: TimeZone)(block: => T): T = {
val originalDefaultTimeZone = TimeZone.getDefault
try {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 2423668392231..7eb3d2b3a8854 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -21,6 +21,8 @@ import java.sql.{Date, Timestamp}
import java.text.SimpleDateFormat
import java.util.{Calendar, Locale, TimeZone}
+import org.apache.commons.lang3.time.FastDateFormat
+
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.catalyst.util.DateTimeUtils._
import org.apache.spark.unsafe.types.UTF8String
@@ -160,6 +162,9 @@ class DateTimeUtilsSuite extends SparkFunSuite {
assert(stringToDate(UTF8String.fromString("015-03-18")).isEmpty)
assert(stringToDate(UTF8String.fromString("015")).isEmpty)
assert(stringToDate(UTF8String.fromString("02015")).isEmpty)
+ assert(stringToDate(UTF8String.fromString("1999 08 01")).isEmpty)
+ assert(stringToDate(UTF8String.fromString("1999-08 01")).isEmpty)
+ assert(stringToDate(UTF8String.fromString("1999 08")).isEmpty)
}
test("string to time") {
@@ -336,6 +341,9 @@ class DateTimeUtilsSuite extends SparkFunSuite {
checkStringToTimestamp("2015-03-18T12:03.17-20:0", None)
checkStringToTimestamp("2015-03-18T12:03.17-0:70", None)
checkStringToTimestamp("2015-03-18T12:03.17-1:0:0", None)
+ checkStringToTimestamp("1999 08 01", None)
+ checkStringToTimestamp("1999-08 01", None)
+ checkStringToTimestamp("1999 08", None)
// Truncating the fractional seconds
c = Calendar.getInstance(TimeZone.getTimeZone("GMT+00:00"))
@@ -575,6 +583,12 @@ class DateTimeUtilsSuite extends SparkFunSuite {
}
}
+ test("trailing characters while converting string to timestamp") {
+ val s = UTF8String.fromString("2019-10-31T10:59:23Z:::")
+ val time = DateTimeUtils.stringToTimestamp(s, DateTimeUtils.defaultTimeZone())
+ assert(time == None)
+ }
+
test("truncTimestamp") {
def testTrunc(
level: Int,
@@ -663,12 +677,12 @@ class DateTimeUtilsSuite extends SparkFunSuite {
// There are some days are skipped entirely in some timezone, skip them here.
val skipped_days = Map[String, Set[Int]](
- "Kwajalein" -> Set(8632),
+ "Kwajalein" -> Set(8632, 8633),
"Pacific/Apia" -> Set(15338),
"Pacific/Enderbury" -> Set(9130, 9131),
"Pacific/Fakaofo" -> Set(15338),
"Pacific/Kiritimati" -> Set(9130, 9131),
- "Pacific/Kwajalein" -> Set(8632),
+ "Pacific/Kwajalein" -> Set(8632, 8633),
"MIT" -> Set(15338))
for (tz <- DateTimeTestUtils.ALL_TIMEZONES) {
val skipped = skipped_days.getOrElse(tz.getID, Set.empty)
@@ -680,4 +694,81 @@ class DateTimeUtilsSuite extends SparkFunSuite {
}
}
}
+
+ test("parsing timestamp strings up to microsecond precision") {
+ DateTimeTestUtils.outstandingTimezones.foreach { timeZone =>
+ def check(pattern: String, input: String, reference: String): Unit = {
+ val parser = new TimestampParser(FastDateFormat.getInstance(pattern, timeZone, Locale.US))
+ val expected = DateTimeUtils.stringToTimestamp(
+ UTF8String.fromString(reference), timeZone).get
+ val actual = parser.parse(input)
+ assert(actual === expected)
+ }
+
+ check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSSXXX",
+ "2019-10-14T09:39:07.3220000Z", "2019-10-14T09:39:07.322Z")
+ check("yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
+ "2019-10-14T09:39:07.322000", "2019-10-14T09:39:07.322")
+ check("yyyy-MM-dd'T'HH:mm:ss.SSSX",
+ "2019-10-14T09:39:07.322Z", "2019-10-14T09:39:07.322Z")
+ check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSX",
+ "2019-10-14T09:39:07.123456Z", "2019-10-14T09:39:07.123456Z")
+ check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSX",
+ "2019-10-14T09:39:07.000010Z", "2019-10-14T09:39:07.00001Z")
+ check("yyyy-MM-dd'T'HH:mm:ss.S",
+ "2019-10-14T09:39:07.1", "2019-10-14T09:39:07.1")
+ check("yyyy-MM-dd'T'HH:mm:ss.SS",
+ "2019-10-14T09:39:07.10", "2019-10-14T09:39:07.1")
+
+ try {
+ new TimestampParser(
+ FastDateFormat.getInstance("yyyy/MM/dd HH_mm_ss.SSSSSS", timeZone, Locale.US))
+ .parse("2019/11/14 20#25#30.123456")
+ fail("Expected to throw an exception for the invalid input")
+ } catch {
+ case e: IllegalArgumentException =>
+ assert(e.getMessage.contains("is an invalid timestamp"))
+ }
+ }
+ }
+
+ test("formatting timestamp strings up to microsecond precision") {
+ DateTimeTestUtils.outstandingTimezones.foreach { timeZone =>
+ def check(pattern: String, input: String, expected: String): Unit = {
+ val parser = new TimestampParser(FastDateFormat.getInstance(pattern, timeZone, Locale.US))
+ val timestamp = DateTimeUtils.stringToTimestamp(
+ UTF8String.fromString(input), timeZone).get
+ val actual = parser.format(timestamp)
+ assert(actual === expected)
+ }
+
+ check(
+ "yyyy-MM-dd HH:mm:ss.SSSSSSS", "2019-10-14T09:39:07.123456",
+ "2019-10-14 09:39:07.1234560")
+ check(
+ "yyyy-MM-dd HH:mm:ss.SSSSSS", "1960-01-01T09:39:07.123456",
+ "1960-01-01 09:39:07.123456")
+ check(
+ "yyyy-MM-dd HH:mm:ss.SSSSS", "0001-10-14T09:39:07.1",
+ "0001-10-14 09:39:07.10000")
+ check(
+ "yyyy-MM-dd HH:mm:ss.SSSS", "9999-12-31T23:59:59.999",
+ "9999-12-31 23:59:59.9990")
+ check(
+ "yyyy-MM-dd HH:mm:ss.SSS", "1970-01-01T00:00:00.0101",
+ "1970-01-01 00:00:00.010")
+ check(
+ "yyyy-MM-dd HH:mm:ss.SS", "2019-10-14T09:39:07.09",
+ "2019-10-14 09:39:07.09")
+ check(
+ "yyyy-MM-dd HH:mm:ss.S", "2019-10-14T09:39:07.2",
+ "2019-10-14 09:39:07.2")
+ check(
+ "yyyy-MM-dd HH:mm:ss.S", "2019-10-14T09:39:07",
+ "2019-10-14 09:39:07.0")
+ check(
+ "yyyy-MM-dd HH:mm:ss", "2019-10-14T09:39:07.123456",
+ "2019-10-14 09:39:07")
+ }
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala
index 8f75c14192c9b..f485320b395cd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala
@@ -17,11 +17,13 @@
package org.apache.spark.sql.catalyst.util
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.serializer.{JavaSerializer, KryoSerializer}
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
import org.apache.spark.sql.catalyst.expressions.UnsafeArrayData
import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.Platform
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
class UnsafeArraySuite extends SparkFunSuite {
@@ -52,6 +54,16 @@ class UnsafeArraySuite extends SparkFunSuite {
val doubleMultiDimArray = Array(
Array(1.1, 11.1), Array(2.2, 22.2, 222.2), Array(3.3, 33.3, 333.3, 3333.3))
+ val serialArray = {
+ val offset = 32
+ val data = new Array[Byte](1024)
+ Platform.putLong(data, offset, 1)
+ val arrayData = new UnsafeArrayData()
+ arrayData.pointTo(data, offset, data.length)
+ arrayData.setLong(0, 19285)
+ arrayData
+ }
+
test("read array") {
val unsafeBoolean = ExpressionEncoder[Array[Boolean]].resolveAndBind().
toRow(booleanArray).getArray(0)
@@ -114,7 +126,7 @@ class UnsafeArraySuite extends SparkFunSuite {
assert(unsafeDate.isInstanceOf[UnsafeArrayData])
assert(unsafeDate.numElements == dateArray.length)
dateArray.zipWithIndex.map { case (e, i) =>
- assert(unsafeDate.get(i, DateType) == e)
+ assert(unsafeDate.get(i, DateType).asInstanceOf[Int] == e)
}
val unsafeTimestamp = ExpressionEncoder[Array[Long]].resolveAndBind().
@@ -122,7 +134,7 @@ class UnsafeArraySuite extends SparkFunSuite {
assert(unsafeTimestamp.isInstanceOf[UnsafeArrayData])
assert(unsafeTimestamp.numElements == timestampArray.length)
timestampArray.zipWithIndex.map { case (e, i) =>
- assert(unsafeTimestamp.get(i, TimestampType) == e)
+ assert(unsafeTimestamp.get(i, TimestampType).asInstanceOf[Long] == e)
}
Seq(decimalArray4_1, decimalArray20_20).map { decimalArray =>
@@ -204,4 +216,18 @@ class UnsafeArraySuite extends SparkFunSuite {
val doubleEncoder = ExpressionEncoder[Array[Double]].resolveAndBind()
assert(doubleEncoder.toRow(doubleArray).getArray(0).toDoubleArray.sameElements(doubleArray))
}
+
+ test("unsafe java serialization") {
+ val ser = new JavaSerializer(new SparkConf).newInstance()
+ val arrayDataSer = ser.deserialize[UnsafeArrayData](ser.serialize(serialArray))
+ assert(arrayDataSer.getLong(0) == 19285)
+ assert(arrayDataSer.getBaseObject.asInstanceOf[Array[Byte]].length == 1024)
+ }
+
+ test("unsafe Kryo serialization") {
+ val ser = new KryoSerializer(new SparkConf).newInstance()
+ val arrayDataSer = ser.deserialize[UnsafeArrayData](ser.serialize(serialArray))
+ assert(arrayDataSer.getLong(0) == 19285)
+ assert(arrayDataSer.getBaseObject.asInstanceOf[Array[Byte]].length == 1024)
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala
new file mode 100644
index 0000000000000..ebc88612be22a
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.serializer.{JavaSerializer, KryoSerializer}
+import org.apache.spark.sql.catalyst.expressions.{UnsafeArrayData, UnsafeMapData}
+import org.apache.spark.unsafe.Platform
+
+class UnsafeMapSuite extends SparkFunSuite {
+
+ val unsafeMapData = {
+ val offset = 32
+ val keyArraySize = 256
+ val baseObject = new Array[Byte](1024)
+ Platform.putLong(baseObject, offset, keyArraySize)
+
+ val unsafeMap = new UnsafeMapData
+ Platform.putLong(baseObject, offset + 8, 1)
+ val keyArray = new UnsafeArrayData()
+ keyArray.pointTo(baseObject, offset + 8, keyArraySize)
+ keyArray.setLong(0, 19285)
+
+ val valueArray = new UnsafeArrayData()
+ Platform.putLong(baseObject, offset + 8 + keyArray.getSizeInBytes, 1)
+ valueArray.pointTo(baseObject, offset + 8 + keyArray.getSizeInBytes, keyArraySize)
+ valueArray.setLong(0, 19286)
+ unsafeMap.pointTo(baseObject, offset, baseObject.length)
+ unsafeMap
+ }
+
+ test("unsafe java serialization") {
+ val ser = new JavaSerializer(new SparkConf).newInstance()
+ val mapDataSer = ser.deserialize[UnsafeMapData](ser.serialize(unsafeMapData))
+ assert(mapDataSer.numElements() == 1)
+ assert(mapDataSer.keyArray().getInt(0) == 19285)
+ assert(mapDataSer.valueArray().getInt(0) == 19286)
+ assert(mapDataSer.getBaseObject.asInstanceOf[Array[Byte]].length == 1024)
+ }
+
+ test("unsafe Kryo serialization") {
+ val ser = new KryoSerializer(new SparkConf).newInstance()
+ val mapDataSer = ser.deserialize[UnsafeMapData](ser.serialize(unsafeMapData))
+ assert(mapDataSer.numElements() == 1)
+ assert(mapDataSer.keyArray().getInt(0) == 19285)
+ assert(mapDataSer.valueArray().getInt(0) == 19286)
+ assert(mapDataSer.getBaseObject.asInstanceOf[Array[Byte]].length == 1024)
+ }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
index 10de90c6a44ca..8abd7625c21aa 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
@@ -228,4 +228,15 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester {
val decimal = Decimal.apply(bigInt)
assert(decimal.toJavaBigDecimal.unscaledValue.toString === "9223372036854775808")
}
+
+ test("SPARK-26038: toScalaBigInt/toJavaBigInteger") {
+ // not fitting long
+ val decimal = Decimal("1234568790123456789012348790.1234879012345678901234568790")
+ assert(decimal.toScalaBigInt == scala.math.BigInt("1234568790123456789012348790"))
+ assert(decimal.toJavaBigInteger == new java.math.BigInteger("1234568790123456789012348790"))
+ // fitting long
+ val decimalLong = Decimal(123456789123456789L, 18, 9)
+ assert(decimalLong.toScalaBigInt == scala.math.BigInt("123456789"))
+ assert(decimalLong.toJavaBigInteger == new java.math.BigInteger("123456789"))
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/MetadataSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/MetadataSuite.scala
index 210e65708170f..b4aeac562d2b8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/MetadataSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/MetadataSuite.scala
@@ -26,6 +26,7 @@ class MetadataSuite extends SparkFunSuite {
assert(meta.## !== 0)
assert(meta.getString("key") === "value")
assert(meta.contains("key"))
+ assert(meta === Metadata.fromJson(meta.json))
intercept[NoSuchElementException](meta.getString("no_such_key"))
intercept[ClassCastException](meta.getBoolean("key"))
}
@@ -36,6 +37,7 @@ class MetadataSuite extends SparkFunSuite {
assert(meta.## !== 0)
assert(meta.getLong("key") === 12)
assert(meta.contains("key"))
+ assert(meta === Metadata.fromJson(meta.json))
intercept[NoSuchElementException](meta.getLong("no_such_key"))
intercept[ClassCastException](meta.getBoolean("key"))
}
@@ -46,6 +48,7 @@ class MetadataSuite extends SparkFunSuite {
assert(meta.## !== 0)
assert(meta.getDouble("key") === 12)
assert(meta.contains("key"))
+ assert(meta === Metadata.fromJson(meta.json))
intercept[NoSuchElementException](meta.getDouble("no_such_key"))
intercept[ClassCastException](meta.getBoolean("key"))
}
@@ -56,6 +59,7 @@ class MetadataSuite extends SparkFunSuite {
assert(meta.## !== 0)
assert(meta.getBoolean("key") === true)
assert(meta.contains("key"))
+ assert(meta === Metadata.fromJson(meta.json))
intercept[NoSuchElementException](meta.getBoolean("no_such_key"))
intercept[ClassCastException](meta.getString("key"))
}
@@ -69,6 +73,7 @@ class MetadataSuite extends SparkFunSuite {
assert(meta.getLong("key") === 0)
assert(meta.getBoolean("key") === false)
assert(meta.contains("key"))
+ assert(meta === Metadata.fromJson(meta.json))
intercept[NoSuchElementException](meta.getLong("no_such_key"))
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
new file mode 100644
index 0000000000000..602542fb33db0
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.util
+
+import java.time.LocalDate
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.internal.SQLConf
+
+class DateFormatterSuite extends SparkFunSuite with SQLHelper {
+ test("parsing dates") {
+ DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+ withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+ val formatter = DateFormatter()
+ val daysSinceEpoch = formatter.parse("2018-12-02")
+ assert(daysSinceEpoch === 17867)
+ }
+ }
+ }
+
+ test("format dates") {
+ DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+ withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+ val formatter = DateFormatter()
+ val date = formatter.format(17867)
+ assert(date === "2018-12-02")
+ }
+ }
+ }
+
+ test("roundtrip date -> days -> date") {
+ Seq(
+ "0050-01-01",
+ "0953-02-02",
+ "1423-03-08",
+ "1969-12-31",
+ "1972-08-25",
+ "1975-09-26",
+ "2018-12-12",
+ "2038-01-01",
+ "5010-11-17").foreach { date =>
+ DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+ withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+ val formatter = DateFormatter()
+ val days = formatter.parse(date)
+ val formatted = formatter.format(days)
+ assert(date === formatted)
+ }
+ }
+ }
+ }
+
+ test("roundtrip days -> date -> days") {
+ Seq(
+ -701265,
+ -371419,
+ -199722,
+ -1,
+ 0,
+ 967,
+ 2094,
+ 17877,
+ 24837,
+ 1110657).foreach { days =>
+ DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+ withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+ val formatter = DateFormatter()
+ val date = formatter.format(days)
+ val parsed = formatter.parse(date)
+ assert(days === parsed)
+ }
+ }
+ }
+ }
+
+ test("parsing date without explicit day") {
+ val formatter = DateFormatter("yyyy MMM")
+ val daysSinceEpoch = formatter.parse("2018 Dec")
+ assert(daysSinceEpoch === LocalDate.of(2018, 12, 1).toEpochDay)
+ }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
new file mode 100644
index 0000000000000..192ca13ddaf6e
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.util
+
+import java.time.{LocalDateTime, ZoneOffset}
+import java.util.TimeZone
+import java.util.concurrent.TimeUnit
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, TimestampFormatter}
+
+class TimestampFormatterSuite extends SparkFunSuite with SQLHelper {
+
+ test("parsing timestamps using time zones") {
+ val localDate = "2018-12-02T10:11:12.001234"
+ val expectedMicros = Map(
+ "UTC" -> 1543745472001234L,
+ "PST" -> 1543774272001234L,
+ "CET" -> 1543741872001234L,
+ "Africa/Dakar" -> 1543745472001234L,
+ "America/Los_Angeles" -> 1543774272001234L,
+ "Antarctica/Vostok" -> 1543723872001234L,
+ "Asia/Hong_Kong" -> 1543716672001234L,
+ "Europe/Amsterdam" -> 1543741872001234L)
+ DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+ val formatter = TimestampFormatter(
+ "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
+ TimeZone.getTimeZone(timeZone))
+ val microsSinceEpoch = formatter.parse(localDate)
+ assert(microsSinceEpoch === expectedMicros(timeZone))
+ }
+ }
+
+ test("format timestamps using time zones") {
+ val microsSinceEpoch = 1543745472001234L
+ val expectedTimestamp = Map(
+ "UTC" -> "2018-12-02T10:11:12.001234",
+ "PST" -> "2018-12-02T02:11:12.001234",
+ "CET" -> "2018-12-02T11:11:12.001234",
+ "Africa/Dakar" -> "2018-12-02T10:11:12.001234",
+ "America/Los_Angeles" -> "2018-12-02T02:11:12.001234",
+ "Antarctica/Vostok" -> "2018-12-02T16:11:12.001234",
+ "Asia/Hong_Kong" -> "2018-12-02T18:11:12.001234",
+ "Europe/Amsterdam" -> "2018-12-02T11:11:12.001234")
+ DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+ val formatter = TimestampFormatter(
+ "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
+ TimeZone.getTimeZone(timeZone))
+ val timestamp = formatter.format(microsSinceEpoch)
+ assert(timestamp === expectedTimestamp(timeZone))
+ }
+ }
+
+ test("roundtrip micros -> timestamp -> micros using timezones") {
+ Seq("yyyy-MM-dd'T'HH:mm:ss.SSSSSS", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXXXX").foreach { pattern =>
+ Seq(
+ -58710115316212000L,
+ -18926315945345679L,
+ -9463427405253013L,
+ -244000001L,
+ 0L,
+ 99628200102030L,
+ 1543749753123456L,
+ 2177456523456789L,
+ 11858049903010203L).foreach { micros =>
+ DateTimeTestUtils.outstandingTimezones.foreach { timeZone =>
+ val formatter = TimestampFormatter(pattern, timeZone)
+ val timestamp = formatter.format(micros)
+ val parsed = formatter.parse(timestamp)
+ assert(micros === parsed)
+ }
+ }
+ }
+ }
+
+ test("roundtrip timestamp -> micros -> timestamp using timezones") {
+ Seq(
+ "0109-07-20T18:38:03.788000",
+ "1370-04-01T10:00:54.654321",
+ "1670-02-11T14:09:54.746987",
+ "1969-12-31T23:55:55.999999",
+ "1970-01-01T00:00:00.000000",
+ "1973-02-27T02:30:00.102030",
+ "2018-12-02T11:22:33.123456",
+ "2039-01-01T01:02:03.456789",
+ "2345-10-07T22:45:03.010203").foreach { timestamp =>
+ DateTimeTestUtils.outstandingTimezones.foreach { timeZone =>
+ val formatter = TimestampFormatter("yyyy-MM-dd'T'HH:mm:ss.SSSSSS", timeZone)
+ val micros = formatter.parse(timestamp)
+ val formatted = formatter.format(micros)
+ assert(timestamp === formatted)
+ }
+ }
+ }
+
+ test(" case insensitive parsing of am and pm") {
+ val formatter = TimestampFormatter(
+ "yyyy MMM dd hh:mm:ss a",
+ TimeZone.getTimeZone("UTC"))
+ val micros = formatter.parse("2009 Mar 20 11:30:01 am")
+ assert(micros === TimeUnit.SECONDS.toMicros(
+ LocalDateTime.of(2009, 3, 20, 11, 30, 1).toEpochSecond(ZoneOffset.UTC)))
+ }
+}
diff --git a/sql/core/benchmarks/FilterPushdownBenchmark-results.txt b/sql/core/benchmarks/FilterPushdownBenchmark-results.txt
index a75a15c99328a..e680ddff53dd1 100644
--- a/sql/core/benchmarks/FilterPushdownBenchmark-results.txt
+++ b/sql/core/benchmarks/FilterPushdownBenchmark-results.txt
@@ -2,737 +2,669 @@
Pushdown for many distinct value case
================================================================================================
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
-
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 0 string row (value IS NULL): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8970 / 9122 1.8 570.3 1.0X
-Parquet Vectorized (Pushdown) 471 / 491 33.4 30.0 19.0X
-Native ORC Vectorized 7661 / 7853 2.1 487.0 1.2X
-Native ORC Vectorized (Pushdown) 1134 / 1161 13.9 72.1 7.9X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 11405 / 11485 1.4 725.1 1.0X
+Parquet Vectorized (Pushdown) 675 / 690 23.3 42.9 16.9X
+Native ORC Vectorized 7127 / 7170 2.2 453.1 1.6X
+Native ORC Vectorized (Pushdown) 519 / 541 30.3 33.0 22.0X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 0 string row ('7864320' < value < '7864320'): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 9246 / 9297 1.7 587.8 1.0X
-Parquet Vectorized (Pushdown) 480 / 488 32.8 30.5 19.3X
-Native ORC Vectorized 7838 / 7850 2.0 498.3 1.2X
-Native ORC Vectorized (Pushdown) 1054 / 1118 14.9 67.0 8.8X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 11457 / 11473 1.4 728.4 1.0X
+Parquet Vectorized (Pushdown) 656 / 686 24.0 41.7 17.5X
+Native ORC Vectorized 7328 / 7342 2.1 465.9 1.6X
+Native ORC Vectorized (Pushdown) 539 / 565 29.2 34.2 21.3X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 string row (value = '7864320'): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8989 / 9100 1.7 571.5 1.0X
-Parquet Vectorized (Pushdown) 448 / 467 35.1 28.5 20.1X
-Native ORC Vectorized 7680 / 7768 2.0 488.3 1.2X
-Native ORC Vectorized (Pushdown) 1067 / 1118 14.7 67.8 8.4X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 11878 / 11888 1.3 755.2 1.0X
+Parquet Vectorized (Pushdown) 630 / 654 25.0 40.1 18.9X
+Native ORC Vectorized 7342 / 7362 2.1 466.8 1.6X
+Native ORC Vectorized (Pushdown) 519 / 537 30.3 33.0 22.9X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 string row (value <=> '7864320'): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 9115 / 9266 1.7 579.5 1.0X
-Parquet Vectorized (Pushdown) 466 / 492 33.7 29.7 19.5X
-Native ORC Vectorized 7800 / 7914 2.0 495.9 1.2X
-Native ORC Vectorized (Pushdown) 1075 / 1102 14.6 68.4 8.5X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 11423 / 11440 1.4 726.2 1.0X
+Parquet Vectorized (Pushdown) 625 / 643 25.2 39.7 18.3X
+Native ORC Vectorized 7315 / 7335 2.2 465.1 1.6X
+Native ORC Vectorized (Pushdown) 507 / 520 31.0 32.2 22.5X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 string row ('7864320' <= value <= '7864320'): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 9099 / 9237 1.7 578.5 1.0X
-Parquet Vectorized (Pushdown) 462 / 475 34.1 29.3 19.7X
-Native ORC Vectorized 7847 / 7925 2.0 498.9 1.2X
-Native ORC Vectorized (Pushdown) 1078 / 1114 14.6 68.5 8.4X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 11440 / 11478 1.4 727.3 1.0X
+Parquet Vectorized (Pushdown) 634 / 652 24.8 40.3 18.0X
+Native ORC Vectorized 7311 / 7324 2.2 464.8 1.6X
+Native ORC Vectorized (Pushdown) 517 / 548 30.4 32.8 22.1X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select all string rows (value IS NOT NULL): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 19303 / 19547 0.8 1227.3 1.0X
-Parquet Vectorized (Pushdown) 19924 / 20089 0.8 1266.7 1.0X
-Native ORC Vectorized 18725 / 19079 0.8 1190.5 1.0X
-Native ORC Vectorized (Pushdown) 19310 / 19492 0.8 1227.7 1.0X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 20750 / 20872 0.8 1319.3 1.0X
+Parquet Vectorized (Pushdown) 21002 / 21032 0.7 1335.3 1.0X
+Native ORC Vectorized 16714 / 16742 0.9 1062.6 1.2X
+Native ORC Vectorized (Pushdown) 16926 / 16965 0.9 1076.1 1.2X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 0 int row (value IS NULL): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8117 / 8323 1.9 516.1 1.0X
-Parquet Vectorized (Pushdown) 484 / 494 32.5 30.8 16.8X
-Native ORC Vectorized 6811 / 7036 2.3 433.0 1.2X
-Native ORC Vectorized (Pushdown) 1061 / 1082 14.8 67.5 7.6X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10510 / 10532 1.5 668.2 1.0X
+Parquet Vectorized (Pushdown) 642 / 665 24.5 40.8 16.4X
+Native ORC Vectorized 6609 / 6618 2.4 420.2 1.6X
+Native ORC Vectorized (Pushdown) 502 / 512 31.4 31.9 21.0X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 0 int row (7864320 < value < 7864320): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8105 / 8140 1.9 515.3 1.0X
-Parquet Vectorized (Pushdown) 478 / 505 32.9 30.4 17.0X
-Native ORC Vectorized 6914 / 7211 2.3 439.6 1.2X
-Native ORC Vectorized (Pushdown) 1044 / 1064 15.1 66.4 7.8X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10505 / 10514 1.5 667.9 1.0X
+Parquet Vectorized (Pushdown) 659 / 673 23.9 41.9 15.9X
+Native ORC Vectorized 6634 / 6641 2.4 421.8 1.6X
+Native ORC Vectorized (Pushdown) 513 / 526 30.7 32.6 20.5X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 int row (value = 7864320): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7983 / 8116 2.0 507.6 1.0X
-Parquet Vectorized (Pushdown) 464 / 487 33.9 29.5 17.2X
-Native ORC Vectorized 6703 / 6774 2.3 426.1 1.2X
-Native ORC Vectorized (Pushdown) 1017 / 1058 15.5 64.6 7.9X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10555 / 10570 1.5 671.1 1.0X
+Parquet Vectorized (Pushdown) 651 / 668 24.2 41.4 16.2X
+Native ORC Vectorized 6721 / 6728 2.3 427.3 1.6X
+Native ORC Vectorized (Pushdown) 508 / 519 31.0 32.3 20.8X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 int row (value <=> 7864320): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7942 / 7983 2.0 504.9 1.0X
-Parquet Vectorized (Pushdown) 468 / 479 33.6 29.7 17.0X
-Native ORC Vectorized 6677 / 6779 2.4 424.5 1.2X
-Native ORC Vectorized (Pushdown) 1021 / 1068 15.4 64.9 7.8X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10556 / 10566 1.5 671.1 1.0X
+Parquet Vectorized (Pushdown) 647 / 654 24.3 41.1 16.3X
+Native ORC Vectorized 6716 / 6728 2.3 427.0 1.6X
+Native ORC Vectorized (Pushdown) 510 / 521 30.9 32.4 20.7X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 int row (7864320 <= value <= 7864320): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7909 / 7958 2.0 502.8 1.0X
-Parquet Vectorized (Pushdown) 485 / 494 32.4 30.8 16.3X
-Native ORC Vectorized 6751 / 6846 2.3 429.2 1.2X
-Native ORC Vectorized (Pushdown) 1043 / 1077 15.1 66.3 7.6X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10556 / 10565 1.5 671.1 1.0X
+Parquet Vectorized (Pushdown) 649 / 654 24.2 41.3 16.3X
+Native ORC Vectorized 6700 / 6712 2.3 426.0 1.6X
+Native ORC Vectorized (Pushdown) 509 / 520 30.9 32.3 20.8X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 int row (7864319 < value < 7864321): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8010 / 8033 2.0 509.2 1.0X
-Parquet Vectorized (Pushdown) 472 / 489 33.3 30.0 17.0X
-Native ORC Vectorized 6655 / 6808 2.4 423.1 1.2X
-Native ORC Vectorized (Pushdown) 1015 / 1067 15.5 64.5 7.9X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10547 / 10566 1.5 670.5 1.0X
+Parquet Vectorized (Pushdown) 649 / 653 24.2 41.3 16.3X
+Native ORC Vectorized 6703 / 6713 2.3 426.2 1.6X
+Native ORC Vectorized (Pushdown) 510 / 520 30.8 32.5 20.7X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 10% int rows (value < 1572864): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8983 / 9035 1.8 571.1 1.0X
-Parquet Vectorized (Pushdown) 2204 / 2231 7.1 140.1 4.1X
-Native ORC Vectorized 7864 / 8011 2.0 500.0 1.1X
-Native ORC Vectorized (Pushdown) 2674 / 2789 5.9 170.0 3.4X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 11478 / 11525 1.4 729.7 1.0X
+Parquet Vectorized (Pushdown) 2576 / 2587 6.1 163.8 4.5X
+Native ORC Vectorized 7633 / 7657 2.1 485.3 1.5X
+Native ORC Vectorized (Pushdown) 2076 / 2096 7.6 132.0 5.5X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 50% int rows (value < 7864320): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 12723 / 12903 1.2 808.9 1.0X
-Parquet Vectorized (Pushdown) 9112 / 9282 1.7 579.3 1.4X
-Native ORC Vectorized 12090 / 12230 1.3 768.7 1.1X
-Native ORC Vectorized (Pushdown) 9242 / 9372 1.7 587.6 1.4X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 14785 / 14802 1.1 940.0 1.0X
+Parquet Vectorized (Pushdown) 9971 / 9977 1.6 633.9 1.5X
+Native ORC Vectorized 11082 / 11107 1.4 704.6 1.3X
+Native ORC Vectorized (Pushdown) 8061 / 8073 2.0 512.5 1.8X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 90% int rows (value < 14155776): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 16453 / 16678 1.0 1046.1 1.0X
-Parquet Vectorized (Pushdown) 15997 / 16262 1.0 1017.0 1.0X
-Native ORC Vectorized 16652 / 17070 0.9 1058.7 1.0X
-Native ORC Vectorized (Pushdown) 15843 / 16112 1.0 1007.2 1.0X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 18174 / 18214 0.9 1155.5 1.0X
+Parquet Vectorized (Pushdown) 17387 / 17403 0.9 1105.5 1.0X
+Native ORC Vectorized 14465 / 14492 1.1 919.7 1.3X
+Native ORC Vectorized (Pushdown) 14024 / 14041 1.1 891.6 1.3X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select all int rows (value IS NOT NULL): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 17098 / 17254 0.9 1087.1 1.0X
-Parquet Vectorized (Pushdown) 17302 / 17529 0.9 1100.1 1.0X
-Native ORC Vectorized 16790 / 17098 0.9 1067.5 1.0X
-Native ORC Vectorized (Pushdown) 17329 / 17914 0.9 1101.7 1.0X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 19004 / 19014 0.8 1208.2 1.0X
+Parquet Vectorized (Pushdown) 19219 / 19232 0.8 1221.9 1.0X
+Native ORC Vectorized 15266 / 15290 1.0 970.6 1.2X
+Native ORC Vectorized (Pushdown) 15469 / 15482 1.0 983.5 1.2X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select all int rows (value > -1): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 17088 / 17392 0.9 1086.4 1.0X
-Parquet Vectorized (Pushdown) 17609 / 17863 0.9 1119.5 1.0X
-Native ORC Vectorized 18334 / 69831 0.9 1165.7 0.9X
-Native ORC Vectorized (Pushdown) 17465 / 17629 0.9 1110.4 1.0X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 19036 / 19052 0.8 1210.3 1.0X
+Parquet Vectorized (Pushdown) 19287 / 19306 0.8 1226.2 1.0X
+Native ORC Vectorized 15311 / 15371 1.0 973.5 1.2X
+Native ORC Vectorized (Pushdown) 15517 / 15590 1.0 986.5 1.2X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select all int rows (value != -1): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 16903 / 17233 0.9 1074.6 1.0X
-Parquet Vectorized (Pushdown) 16945 / 17032 0.9 1077.3 1.0X
-Native ORC Vectorized 16377 / 16762 1.0 1041.2 1.0X
-Native ORC Vectorized (Pushdown) 16950 / 17212 0.9 1077.7 1.0X
+Parquet Vectorized 19072 / 19102 0.8 1212.6 1.0X
+Parquet Vectorized (Pushdown) 19288 / 19318 0.8 1226.3 1.0X
+Native ORC Vectorized 15277 / 15293 1.0 971.3 1.2X
+Native ORC Vectorized (Pushdown) 15479 / 15499 1.0 984.1 1.2X
================================================================================================
Pushdown for few distinct value case (use dictionary encoding)
================================================================================================
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
-
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 0 distinct string row (value IS NULL): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7245 / 7322 2.2 460.7 1.0X
-Parquet Vectorized (Pushdown) 378 / 389 41.6 24.0 19.2X
-Native ORC Vectorized 6720 / 6778 2.3 427.2 1.1X
-Native ORC Vectorized (Pushdown) 1009 / 1032 15.6 64.2 7.2X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10250 / 10274 1.5 651.7 1.0X
+Parquet Vectorized (Pushdown) 571 / 576 27.5 36.3 17.9X
+Native ORC Vectorized 8651 / 8660 1.8 550.0 1.2X
+Native ORC Vectorized (Pushdown) 909 / 933 17.3 57.8 11.3X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 0 distinct string row ('100' < value < '100'): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7627 / 7795 2.1 484.9 1.0X
-Parquet Vectorized (Pushdown) 384 / 406 41.0 24.4 19.9X
-Native ORC Vectorized 6724 / 7824 2.3 427.5 1.1X
-Native ORC Vectorized (Pushdown) 968 / 986 16.3 61.5 7.9X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10420 / 10426 1.5 662.5 1.0X
+Parquet Vectorized (Pushdown) 574 / 579 27.4 36.5 18.2X
+Native ORC Vectorized 8973 / 8982 1.8 570.5 1.2X
+Native ORC Vectorized (Pushdown) 916 / 955 17.2 58.2 11.4X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 distinct string row (value = '100'): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7157 / 7534 2.2 455.0 1.0X
-Parquet Vectorized (Pushdown) 542 / 565 29.0 34.5 13.2X
-Native ORC Vectorized 6716 / 7214 2.3 427.0 1.1X
-Native ORC Vectorized (Pushdown) 1212 / 1288 13.0 77.0 5.9X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10428 / 10441 1.5 663.0 1.0X
+Parquet Vectorized (Pushdown) 789 / 809 19.9 50.2 13.2X
+Native ORC Vectorized 9042 / 9055 1.7 574.9 1.2X
+Native ORC Vectorized (Pushdown) 1130 / 1145 13.9 71.8 9.2X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 distinct string row (value <=> '100'): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7368 / 7552 2.1 468.4 1.0X
-Parquet Vectorized (Pushdown) 544 / 556 28.9 34.6 13.5X
-Native ORC Vectorized 6740 / 6867 2.3 428.5 1.1X
-Native ORC Vectorized (Pushdown) 1230 / 1426 12.8 78.2 6.0X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10402 / 10416 1.5 661.3 1.0X
+Parquet Vectorized (Pushdown) 791 / 806 19.9 50.3 13.2X
+Native ORC Vectorized 9042 / 9055 1.7 574.9 1.2X
+Native ORC Vectorized (Pushdown) 1112 / 1145 14.1 70.7 9.4X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 distinct string row ('100' <= value <= '100'): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7427 / 7734 2.1 472.2 1.0X
-Parquet Vectorized (Pushdown) 556 / 568 28.3 35.4 13.3X
-Native ORC Vectorized 6847 / 7059 2.3 435.3 1.1X
-Native ORC Vectorized (Pushdown) 1226 / 1230 12.8 77.9 6.1X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10548 / 10563 1.5 670.6 1.0X
+Parquet Vectorized (Pushdown) 790 / 796 19.9 50.2 13.4X
+Native ORC Vectorized 9144 / 9153 1.7 581.3 1.2X
+Native ORC Vectorized (Pushdown) 1117 / 1148 14.1 71.0 9.4X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select all distinct string rows (value IS NOT NULL): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 16998 / 17311 0.9 1080.7 1.0X
-Parquet Vectorized (Pushdown) 16977 / 17250 0.9 1079.4 1.0X
-Native ORC Vectorized 18447 / 19852 0.9 1172.8 0.9X
-Native ORC Vectorized (Pushdown) 16614 / 17102 0.9 1056.3 1.0X
+Parquet Vectorized 20445 / 20469 0.8 1299.8 1.0X
+Parquet Vectorized (Pushdown) 20686 / 20699 0.8 1315.2 1.0X
+Native ORC Vectorized 18851 / 18953 0.8 1198.5 1.1X
+Native ORC Vectorized (Pushdown) 19255 / 19268 0.8 1224.2 1.1X
================================================================================================
Pushdown benchmark for StringStartsWith
================================================================================================
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
-
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
StringStartsWith filter: (value like '10%'): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 9705 / 10814 1.6 617.0 1.0X
-Parquet Vectorized (Pushdown) 3086 / 3574 5.1 196.2 3.1X
-Native ORC Vectorized 10094 / 10695 1.6 641.8 1.0X
-Native ORC Vectorized (Pushdown) 9611 / 9999 1.6 611.0 1.0X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 14265 / 15213 1.1 907.0 1.0X
+Parquet Vectorized (Pushdown) 4228 / 4870 3.7 268.8 3.4X
+Native ORC Vectorized 10116 / 10977 1.6 643.2 1.4X
+Native ORC Vectorized (Pushdown) 10653 / 11376 1.5 677.3 1.3X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
StringStartsWith filter: (value like '1000%'): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8016 / 8183 2.0 509.7 1.0X
-Parquet Vectorized (Pushdown) 444 / 457 35.4 28.2 18.0X
-Native ORC Vectorized 6970 / 7169 2.3 443.2 1.2X
-Native ORC Vectorized (Pushdown) 7447 / 7503 2.1 473.5 1.1X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 11499 / 11539 1.4 731.1 1.0X
+Parquet Vectorized (Pushdown) 669 / 672 23.5 42.5 17.2X
+Native ORC Vectorized 7343 / 7363 2.1 466.8 1.6X
+Native ORC Vectorized (Pushdown) 7559 / 7568 2.1 480.6 1.5X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
StringStartsWith filter: (value like '786432%'): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7908 / 8046 2.0 502.8 1.0X
-Parquet Vectorized (Pushdown) 408 / 429 38.6 25.9 19.4X
-Native ORC Vectorized 7021 / 7100 2.2 446.4 1.1X
-Native ORC Vectorized (Pushdown) 7310 / 7490 2.2 464.8 1.1X
+Parquet Vectorized 11463 / 11468 1.4 728.8 1.0X
+Parquet Vectorized (Pushdown) 647 / 651 24.3 41.1 17.7X
+Native ORC Vectorized 7322 / 7338 2.1 465.5 1.6X
+Native ORC Vectorized (Pushdown) 7533 / 7544 2.1 478.9 1.5X
================================================================================================
Pushdown benchmark for decimal
================================================================================================
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
-
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 decimal(9, 2) row (value = 7864320): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 4546 / 4743 3.5 289.0 1.0X
-Parquet Vectorized (Pushdown) 161 / 175 98.0 10.2 28.3X
-Native ORC Vectorized 5721 / 5842 2.7 363.7 0.8X
-Native ORC Vectorized (Pushdown) 1019 / 1070 15.4 64.8 4.5X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 5543 / 5564 2.8 352.4 1.0X
+Parquet Vectorized (Pushdown) 168 / 174 93.7 10.7 33.0X
+Native ORC Vectorized 4992 / 5052 3.2 317.4 1.1X
+Native ORC Vectorized (Pushdown) 840 / 850 18.7 53.4 6.6X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 10% decimal(9, 2) rows (value < 1572864): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 6340 / 7236 2.5 403.1 1.0X
-Parquet Vectorized (Pushdown) 3052 / 3164 5.2 194.1 2.1X
-Native ORC Vectorized 8370 / 9214 1.9 532.1 0.8X
-Native ORC Vectorized (Pushdown) 4137 / 4242 3.8 263.0 1.5X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 7312 / 7358 2.2 464.9 1.0X
+Parquet Vectorized (Pushdown) 3008 / 3078 5.2 191.2 2.4X
+Native ORC Vectorized 6775 / 6798 2.3 430.7 1.1X
+Native ORC Vectorized (Pushdown) 6819 / 6832 2.3 433.5 1.1X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 50% decimal(9, 2) rows (value < 7864320): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 12976 / 13249 1.2 825.0 1.0X
-Parquet Vectorized (Pushdown) 12655 / 13570 1.2 804.6 1.0X
-Native ORC Vectorized 15562 / 15950 1.0 989.4 0.8X
-Native ORC Vectorized (Pushdown) 15042 / 15668 1.0 956.3 0.9X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 13232 / 13241 1.2 841.3 1.0X
+Parquet Vectorized (Pushdown) 12555 / 12569 1.3 798.2 1.1X
+Native ORC Vectorized 12597 / 12627 1.2 800.9 1.1X
+Native ORC Vectorized (Pushdown) 12677 / 12711 1.2 806.0 1.0X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 90% decimal(9, 2) rows (value < 14155776): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 14303 / 14616 1.1 909.3 1.0X
-Parquet Vectorized (Pushdown) 14380 / 14649 1.1 914.3 1.0X
-Native ORC Vectorized 16964 / 17358 0.9 1078.5 0.8X
-Native ORC Vectorized (Pushdown) 17255 / 17874 0.9 1097.0 0.8X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 14725 / 14729 1.1 936.2 1.0X
+Parquet Vectorized (Pushdown) 14781 / 14800 1.1 939.7 1.0X
+Native ORC Vectorized 15360 / 15453 1.0 976.5 1.0X
+Native ORC Vectorized (Pushdown) 15444 / 15466 1.0 981.9 1.0X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 decimal(18, 2) row (value = 7864320): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 4701 / 6416 3.3 298.9 1.0X
-Parquet Vectorized (Pushdown) 128 / 164 122.8 8.1 36.7X
-Native ORC Vectorized 5698 / 7904 2.8 362.3 0.8X
-Native ORC Vectorized (Pushdown) 913 / 942 17.2 58.0 5.2X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 5746 / 5763 2.7 365.3 1.0X
+Parquet Vectorized (Pushdown) 166 / 169 94.8 10.6 34.6X
+Native ORC Vectorized 5007 / 5023 3.1 318.3 1.1X
+Native ORC Vectorized (Pushdown) 2629 / 2640 6.0 167.1 2.2X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 10% decimal(18, 2) rows (value < 1572864): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 5376 / 5461 2.9 341.8 1.0X
-Parquet Vectorized (Pushdown) 1479 / 1543 10.6 94.0 3.6X
-Native ORC Vectorized 6640 / 6748 2.4 422.2 0.8X
-Native ORC Vectorized (Pushdown) 2438 / 2479 6.5 155.0 2.2X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 6827 / 6864 2.3 434.0 1.0X
+Parquet Vectorized (Pushdown) 1809 / 1827 8.7 115.0 3.8X
+Native ORC Vectorized 6287 / 6296 2.5 399.7 1.1X
+Native ORC Vectorized (Pushdown) 6364 / 6377 2.5 404.6 1.1X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 50% decimal(18, 2) rows (value < 7864320): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 9224 / 9356 1.7 586.5 1.0X
-Parquet Vectorized (Pushdown) 7172 / 7415 2.2 456.0 1.3X
-Native ORC Vectorized 11017 / 11408 1.4 700.4 0.8X
-Native ORC Vectorized (Pushdown) 8771 / 10218 1.8 557.7 1.1X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 11315 / 11342 1.4 719.4 1.0X
+Parquet Vectorized (Pushdown) 8431 / 8450 1.9 536.0 1.3X
+Native ORC Vectorized 11591 / 11611 1.4 736.9 1.0X
+Native ORC Vectorized (Pushdown) 11424 / 11475 1.4 726.3 1.0X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 90% decimal(18, 2) rows (value < 14155776): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 13933 / 15990 1.1 885.8 1.0X
-Parquet Vectorized (Pushdown) 12683 / 12942 1.2 806.4 1.1X
-Native ORC Vectorized 16344 / 20196 1.0 1039.1 0.9X
-Native ORC Vectorized (Pushdown) 15162 / 16627 1.0 964.0 0.9X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 15703 / 15712 1.0 998.4 1.0X
+Parquet Vectorized (Pushdown) 14982 / 15009 1.0 952.5 1.0X
+Native ORC Vectorized 16887 / 16955 0.9 1073.7 0.9X
+Native ORC Vectorized (Pushdown) 16518 / 16530 1.0 1050.2 1.0X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 decimal(38, 2) row (value = 7864320): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7102 / 8282 2.2 451.5 1.0X
-Parquet Vectorized (Pushdown) 124 / 150 126.4 7.9 57.1X
-Native ORC Vectorized 5811 / 6883 2.7 369.5 1.2X
-Native ORC Vectorized (Pushdown) 1121 / 1502 14.0 71.3 6.3X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 8101 / 8130 1.9 515.1 1.0X
+Parquet Vectorized (Pushdown) 184 / 187 85.6 11.7 44.1X
+Native ORC Vectorized 4998 / 5027 3.1 317.8 1.6X
+Native ORC Vectorized (Pushdown) 165 / 168 95.6 10.5 49.2X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 10% decimal(38, 2) rows (value < 1572864): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 6894 / 7562 2.3 438.3 1.0X
-Parquet Vectorized (Pushdown) 1863 / 1980 8.4 118.4 3.7X
-Native ORC Vectorized 6812 / 6848 2.3 433.1 1.0X
-Native ORC Vectorized (Pushdown) 2511 / 2598 6.3 159.7 2.7X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 9405 / 9447 1.7 597.9 1.0X
+Parquet Vectorized (Pushdown) 2269 / 2275 6.9 144.2 4.1X
+Native ORC Vectorized 6167 / 6203 2.6 392.1 1.5X
+Native ORC Vectorized (Pushdown) 1783 / 1787 8.8 113.3 5.3X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 50% decimal(38, 2) rows (value < 7864320): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 11732 / 12183 1.3 745.9 1.0X
-Parquet Vectorized (Pushdown) 8912 / 9945 1.8 566.6 1.3X
-Native ORC Vectorized 11499 / 12387 1.4 731.1 1.0X
-Native ORC Vectorized (Pushdown) 9328 / 9382 1.7 593.1 1.3X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 14700 / 14707 1.1 934.6 1.0X
+Parquet Vectorized (Pushdown) 10699 / 10712 1.5 680.2 1.4X
+Native ORC Vectorized 10687 / 10703 1.5 679.5 1.4X
+Native ORC Vectorized (Pushdown) 8364 / 8415 1.9 531.8 1.8X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 90% decimal(38, 2) rows (value < 14155776): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 16272 / 16328 1.0 1034.6 1.0X
-Parquet Vectorized (Pushdown) 15714 / 18100 1.0 999.1 1.0X
-Native ORC Vectorized 16539 / 18897 1.0 1051.5 1.0X
-Native ORC Vectorized (Pushdown) 16328 / 17306 1.0 1038.1 1.0X
+Parquet Vectorized 19780 / 19894 0.8 1257.6 1.0X
+Parquet Vectorized (Pushdown) 19003 / 19025 0.8 1208.1 1.0X
+Native ORC Vectorized 15385 / 15404 1.0 978.2 1.3X
+Native ORC Vectorized (Pushdown) 15032 / 15060 1.0 955.7 1.3X
================================================================================================
Pushdown benchmark for InSet -> InFilters
================================================================================================
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
-
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
InSet -> InFilters (values count: 5, distribution: 10): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7993 / 8104 2.0 508.2 1.0X
-Parquet Vectorized (Pushdown) 507 / 532 31.0 32.2 15.8X
-Native ORC Vectorized 6922 / 7163 2.3 440.1 1.2X
-Native ORC Vectorized (Pushdown) 1017 / 1058 15.5 64.6 7.9X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10521 / 10534 1.5 668.9 1.0X
+Parquet Vectorized (Pushdown) 677 / 691 23.2 43.1 15.5X
+Native ORC Vectorized 6768 / 6776 2.3 430.3 1.6X
+Native ORC Vectorized (Pushdown) 501 / 512 31.4 31.8 21.0X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
InSet -> InFilters (values count: 5, distribution: 50): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7855 / 7963 2.0 499.4 1.0X
-Parquet Vectorized (Pushdown) 503 / 516 31.3 32.0 15.6X
-Native ORC Vectorized 6825 / 6954 2.3 433.9 1.2X
-Native ORC Vectorized (Pushdown) 1019 / 1044 15.4 64.8 7.7X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10531 / 10538 1.5 669.5 1.0X
+Parquet Vectorized (Pushdown) 677 / 718 23.2 43.0 15.6X
+Native ORC Vectorized 6765 / 6773 2.3 430.1 1.6X
+Native ORC Vectorized (Pushdown) 499 / 507 31.5 31.7 21.1X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
InSet -> InFilters (values count: 5, distribution: 90): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7858 / 7928 2.0 499.6 1.0X
-Parquet Vectorized (Pushdown) 490 / 519 32.1 31.1 16.0X
-Native ORC Vectorized 7079 / 7966 2.2 450.1 1.1X
-Native ORC Vectorized (Pushdown) 1276 / 1673 12.3 81.1 6.2X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10540 / 10553 1.5 670.1 1.0X
+Parquet Vectorized (Pushdown) 678 / 710 23.2 43.1 15.5X
+Native ORC Vectorized 6787 / 6794 2.3 431.5 1.6X
+Native ORC Vectorized (Pushdown) 501 / 509 31.4 31.9 21.0X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
InSet -> InFilters (values count: 10, distribution: 10): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8007 / 11155 2.0 509.0 1.0X
-Parquet Vectorized (Pushdown) 519 / 540 30.3 33.0 15.4X
-Native ORC Vectorized 6848 / 7072 2.3 435.4 1.2X
-Native ORC Vectorized (Pushdown) 1026 / 1050 15.3 65.2 7.8X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10551 / 10559 1.5 670.8 1.0X
+Parquet Vectorized (Pushdown) 703 / 708 22.4 44.7 15.0X
+Native ORC Vectorized 6791 / 6802 2.3 431.7 1.6X
+Native ORC Vectorized (Pushdown) 519 / 526 30.3 33.0 20.3X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
InSet -> InFilters (values count: 10, distribution: 50): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7876 / 7956 2.0 500.7 1.0X
-Parquet Vectorized (Pushdown) 521 / 535 30.2 33.1 15.1X
-Native ORC Vectorized 7051 / 7368 2.2 448.3 1.1X
-Native ORC Vectorized (Pushdown) 1014 / 1035 15.5 64.5 7.8X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10561 / 10565 1.5 671.4 1.0X
+Parquet Vectorized (Pushdown) 711 / 716 22.1 45.2 14.9X
+Native ORC Vectorized 6791 / 6806 2.3 431.8 1.6X
+Native ORC Vectorized (Pushdown) 529 / 537 29.8 33.6 20.0X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
InSet -> InFilters (values count: 10, distribution: 90): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7897 / 8229 2.0 502.1 1.0X
-Parquet Vectorized (Pushdown) 513 / 530 30.7 32.6 15.4X
-Native ORC Vectorized 6730 / 6990 2.3 427.9 1.2X
-Native ORC Vectorized (Pushdown) 1003 / 1036 15.7 63.8 7.9X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10572 / 10590 1.5 672.1 1.0X
+Parquet Vectorized (Pushdown) 713 / 716 22.1 45.3 14.8X
+Native ORC Vectorized 6808 / 6815 2.3 432.9 1.6X
+Native ORC Vectorized (Pushdown) 530 / 541 29.7 33.7 19.9X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
InSet -> InFilters (values count: 50, distribution: 10): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7967 / 8175 2.0 506.5 1.0X
-Parquet Vectorized (Pushdown) 8155 / 8434 1.9 518.5 1.0X
-Native ORC Vectorized 7002 / 7107 2.2 445.2 1.1X
-Native ORC Vectorized (Pushdown) 1092 / 1139 14.4 69.4 7.3X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10871 / 10882 1.4 691.2 1.0X
+Parquet Vectorized (Pushdown) 11104 / 11110 1.4 706.0 1.0X
+Native ORC Vectorized 7088 / 7104 2.2 450.7 1.5X
+Native ORC Vectorized (Pushdown) 665 / 677 23.6 42.3 16.3X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
InSet -> InFilters (values count: 50, distribution: 50): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8032 / 8122 2.0 510.7 1.0X
-Parquet Vectorized (Pushdown) 8141 / 8908 1.9 517.6 1.0X
-Native ORC Vectorized 7140 / 7387 2.2 454.0 1.1X
-Native ORC Vectorized (Pushdown) 1156 / 1220 13.6 73.5 6.9X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10861 / 10867 1.4 690.5 1.0X
+Parquet Vectorized (Pushdown) 11094 / 11099 1.4 705.3 1.0X
+Native ORC Vectorized 7075 / 7092 2.2 449.8 1.5X
+Native ORC Vectorized (Pushdown) 718 / 733 21.9 45.6 15.1X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
InSet -> InFilters (values count: 50, distribution: 90): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8088 / 8350 1.9 514.2 1.0X
-Parquet Vectorized (Pushdown) 8629 / 8702 1.8 548.6 0.9X
-Native ORC Vectorized 7480 / 7886 2.1 475.6 1.1X
-Native ORC Vectorized (Pushdown) 1106 / 1145 14.2 70.3 7.3X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10868 / 10887 1.4 691.0 1.0X
+Parquet Vectorized (Pushdown) 11100 / 11106 1.4 705.7 1.0X
+Native ORC Vectorized 7087 / 7093 2.2 450.6 1.5X
+Native ORC Vectorized (Pushdown) 712 / 731 22.1 45.3 15.3X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
InSet -> InFilters (values count: 100, distribution: 10): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8028 / 8165 2.0 510.4 1.0X
-Parquet Vectorized (Pushdown) 8349 / 8674 1.9 530.8 1.0X
-Native ORC Vectorized 7107 / 7354 2.2 451.8 1.1X
-Native ORC Vectorized (Pushdown) 1175 / 1207 13.4 74.7 6.8X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10850 / 10888 1.4 689.8 1.0X
+Parquet Vectorized (Pushdown) 11086 / 11105 1.4 704.9 1.0X
+Native ORC Vectorized 7090 / 7101 2.2 450.8 1.5X
+Native ORC Vectorized (Pushdown) 867 / 882 18.1 55.1 12.5X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
InSet -> InFilters (values count: 100, distribution: 50): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8041 / 8195 2.0 511.2 1.0X
-Parquet Vectorized (Pushdown) 8466 / 8604 1.9 538.2 0.9X
-Native ORC Vectorized 7116 / 7286 2.2 452.4 1.1X
-Native ORC Vectorized (Pushdown) 1197 / 1214 13.1 76.1 6.7X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10816 / 10819 1.5 687.7 1.0X
+Parquet Vectorized (Pushdown) 11052 / 11059 1.4 702.7 1.0X
+Native ORC Vectorized 7037 / 7044 2.2 447.4 1.5X
+Native ORC Vectorized (Pushdown) 919 / 931 17.1 58.4 11.8X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
InSet -> InFilters (values count: 100, distribution: 90): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7998 / 8311 2.0 508.5 1.0X
-Parquet Vectorized (Pushdown) 9366 / 11257 1.7 595.5 0.9X
-Native ORC Vectorized 7856 / 9273 2.0 499.5 1.0X
-Native ORC Vectorized (Pushdown) 1350 / 1747 11.7 85.8 5.9X
+Parquet Vectorized 10807 / 10815 1.5 687.1 1.0X
+Parquet Vectorized (Pushdown) 11047 / 11054 1.4 702.4 1.0X
+Native ORC Vectorized 7042 / 7047 2.2 447.7 1.5X
+Native ORC Vectorized (Pushdown) 950 / 961 16.6 60.4 11.4X
================================================================================================
Pushdown benchmark for tinyint
================================================================================================
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
-
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 tinyint row (value = CAST(63 AS tinyint)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 3461 / 3997 4.5 220.1 1.0X
-Parquet Vectorized (Pushdown) 270 / 315 58.4 17.1 12.8X
-Native ORC Vectorized 4107 / 5372 3.8 261.1 0.8X
-Native ORC Vectorized (Pushdown) 778 / 1553 20.2 49.5 4.4X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 6034 / 6048 2.6 383.6 1.0X
+Parquet Vectorized (Pushdown) 333 / 344 47.2 21.2 18.1X
+Native ORC Vectorized 3240 / 3307 4.9 206.0 1.9X
+Native ORC Vectorized (Pushdown) 330 / 341 47.6 21.0 18.3X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 10% tinyint rows (value < CAST(12 AS tinyint)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 4771 / 6655 3.3 303.3 1.0X
-Parquet Vectorized (Pushdown) 1322 / 1606 11.9 84.0 3.6X
-Native ORC Vectorized 4437 / 4572 3.5 282.1 1.1X
-Native ORC Vectorized (Pushdown) 1781 / 1976 8.8 113.2 2.7X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 6759 / 6800 2.3 429.7 1.0X
+Parquet Vectorized (Pushdown) 1533 / 1537 10.3 97.5 4.4X
+Native ORC Vectorized 3863 / 3874 4.1 245.6 1.7X
+Native ORC Vectorized (Pushdown) 1235 / 1248 12.7 78.5 5.5X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 50% tinyint rows (value < CAST(63 AS tinyint)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 7433 / 7752 2.1 472.6 1.0X
-Parquet Vectorized (Pushdown) 5863 / 5913 2.7 372.8 1.3X
-Native ORC Vectorized 7986 / 8084 2.0 507.7 0.9X
-Native ORC Vectorized (Pushdown) 6522 / 6608 2.4 414.6 1.1X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10247 / 10289 1.5 651.5 1.0X
+Parquet Vectorized (Pushdown) 7430 / 7453 2.1 472.4 1.4X
+Native ORC Vectorized 6995 / 7009 2.2 444.7 1.5X
+Native ORC Vectorized (Pushdown) 5561 / 5571 2.8 353.6 1.8X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 90% tinyint rows (value < CAST(114 AS tinyint)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 11190 / 11519 1.4 711.4 1.0X
-Parquet Vectorized (Pushdown) 10861 / 11206 1.4 690.5 1.0X
-Native ORC Vectorized 11622 / 12196 1.4 738.9 1.0X
-Native ORC Vectorized (Pushdown) 11377 / 11654 1.4 723.3 1.0X
+Parquet Vectorized 13949 / 13991 1.1 886.9 1.0X
+Parquet Vectorized (Pushdown) 13486 / 13511 1.2 857.4 1.0X
+Native ORC Vectorized 10149 / 10186 1.5 645.3 1.4X
+Native ORC Vectorized (Pushdown) 9889 / 9905 1.6 628.7 1.4X
================================================================================================
Pushdown benchmark for Timestamp
================================================================================================
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
-
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 timestamp stored as INT96 row (value = CAST(7864320 AS timestamp)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 4784 / 4956 3.3 304.2 1.0X
-Parquet Vectorized (Pushdown) 4838 / 4917 3.3 307.6 1.0X
-Native ORC Vectorized 3923 / 4173 4.0 249.4 1.2X
-Native ORC Vectorized (Pushdown) 894 / 943 17.6 56.8 5.4X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 6307 / 6310 2.5 401.0 1.0X
+Parquet Vectorized (Pushdown) 6360 / 6397 2.5 404.3 1.0X
+Native ORC Vectorized 2912 / 2917 5.4 185.1 2.2X
+Native ORC Vectorized (Pushdown) 138 / 141 114.4 8.7 45.9X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 10% timestamp stored as INT96 rows (value < CAST(1572864 AS timestamp)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 5686 / 5901 2.8 361.5 1.0X
-Parquet Vectorized (Pushdown) 5555 / 5895 2.8 353.2 1.0X
-Native ORC Vectorized 4844 / 4957 3.2 308.0 1.2X
-Native ORC Vectorized (Pushdown) 2141 / 2230 7.3 136.1 2.7X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 7225 / 7233 2.2 459.4 1.0X
+Parquet Vectorized (Pushdown) 7250 / 7255 2.2 461.0 1.0X
+Native ORC Vectorized 3772 / 3783 4.2 239.8 1.9X
+Native ORC Vectorized (Pushdown) 1277 / 1282 12.3 81.2 5.7X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 50% timestamp stored as INT96 rows (value < CAST(7864320 AS timestamp)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 9100 / 9421 1.7 578.6 1.0X
-Parquet Vectorized (Pushdown) 9122 / 9496 1.7 580.0 1.0X
-Native ORC Vectorized 8365 / 8874 1.9 531.9 1.1X
-Native ORC Vectorized (Pushdown) 7128 / 7376 2.2 453.2 1.3X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10952 / 10965 1.4 696.3 1.0X
+Parquet Vectorized (Pushdown) 10985 / 10998 1.4 698.4 1.0X
+Native ORC Vectorized 7178 / 7227 2.2 456.3 1.5X
+Native ORC Vectorized (Pushdown) 5825 / 5830 2.7 370.3 1.9X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 90% timestamp stored as INT96 rows (value < CAST(14155776 AS timestamp)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 12764 / 13120 1.2 811.5 1.0X
-Parquet Vectorized (Pushdown) 12656 / 13003 1.2 804.7 1.0X
-Native ORC Vectorized 13096 / 13233 1.2 832.6 1.0X
-Native ORC Vectorized (Pushdown) 12710 / 15611 1.2 808.1 1.0X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 14560 / 14583 1.1 925.7 1.0X
+Parquet Vectorized (Pushdown) 14608 / 14620 1.1 928.7 1.0X
+Native ORC Vectorized 10601 / 10640 1.5 674.0 1.4X
+Native ORC Vectorized (Pushdown) 10392 / 10406 1.5 660.7 1.4X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 timestamp stored as TIMESTAMP_MICROS row (value = CAST(7864320 AS timestamp)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 4381 / 4796 3.6 278.5 1.0X
-Parquet Vectorized (Pushdown) 122 / 137 129.3 7.7 36.0X
-Native ORC Vectorized 3913 / 3988 4.0 248.8 1.1X
-Native ORC Vectorized (Pushdown) 905 / 945 17.4 57.6 4.8X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 5653 / 5658 2.8 359.4 1.0X
+Parquet Vectorized (Pushdown) 165 / 169 95.1 10.5 34.2X
+Native ORC Vectorized 2918 / 2921 5.4 185.5 1.9X
+Native ORC Vectorized (Pushdown) 137 / 145 114.9 8.7 41.3X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 10% timestamp stored as TIMESTAMP_MICROS rows (value < CAST(1572864 AS timestamp)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 5145 / 5184 3.1 327.1 1.0X
-Parquet Vectorized (Pushdown) 1426 / 1519 11.0 90.7 3.6X
-Native ORC Vectorized 4827 / 4901 3.3 306.9 1.1X
-Native ORC Vectorized (Pushdown) 2133 / 2210 7.4 135.6 2.4X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 6540 / 6552 2.4 415.8 1.0X
+Parquet Vectorized (Pushdown) 1610 / 1614 9.8 102.3 4.1X
+Native ORC Vectorized 3775 / 3788 4.2 240.0 1.7X
+Native ORC Vectorized (Pushdown) 1274 / 1277 12.3 81.0 5.1X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 50% timestamp stored as TIMESTAMP_MICROS rows (value < CAST(7864320 AS timestamp)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 9234 / 9516 1.7 587.1 1.0X
-Parquet Vectorized (Pushdown) 6752 / 7046 2.3 429.3 1.4X
-Native ORC Vectorized 8418 / 8998 1.9 535.2 1.1X
-Native ORC Vectorized (Pushdown) 7199 / 7314 2.2 457.7 1.3X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10259 / 10278 1.5 652.3 1.0X
+Parquet Vectorized (Pushdown) 7591 / 7601 2.1 482.6 1.4X
+Native ORC Vectorized 7185 / 7194 2.2 456.8 1.4X
+Native ORC Vectorized (Pushdown) 5828 / 5843 2.7 370.6 1.8X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 90% timestamp stored as TIMESTAMP_MICROS rows (value < CAST(14155776 AS timestamp)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 12414 / 12458 1.3 789.2 1.0X
-Parquet Vectorized (Pushdown) 12094 / 12249 1.3 768.9 1.0X
-Native ORC Vectorized 12198 / 13755 1.3 775.5 1.0X
-Native ORC Vectorized (Pushdown) 12205 / 12431 1.3 776.0 1.0X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 13850 / 13868 1.1 880.5 1.0X
+Parquet Vectorized (Pushdown) 13433 / 13450 1.2 854.0 1.0X
+Native ORC Vectorized 10635 / 10669 1.5 676.1 1.3X
+Native ORC Vectorized (Pushdown) 10437 / 10448 1.5 663.6 1.3X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 timestamp stored as TIMESTAMP_MILLIS row (value = CAST(7864320 AS timestamp)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 4369 / 4515 3.6 277.8 1.0X
-Parquet Vectorized (Pushdown) 116 / 125 136.2 7.3 37.8X
-Native ORC Vectorized 3965 / 4703 4.0 252.1 1.1X
-Native ORC Vectorized (Pushdown) 892 / 1162 17.6 56.7 4.9X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 5884 / 5888 2.7 374.1 1.0X
+Parquet Vectorized (Pushdown) 166 / 170 94.7 10.6 35.4X
+Native ORC Vectorized 2913 / 2916 5.4 185.2 2.0X
+Native ORC Vectorized (Pushdown) 136 / 144 115.4 8.7 43.2X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 10% timestamp stored as TIMESTAMP_MILLIS rows (value < CAST(1572864 AS timestamp)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 5211 / 5409 3.0 331.3 1.0X
-Parquet Vectorized (Pushdown) 1427 / 1438 11.0 90.7 3.7X
-Native ORC Vectorized 4719 / 4883 3.3 300.1 1.1X
-Native ORC Vectorized (Pushdown) 2191 / 2228 7.2 139.3 2.4X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 6763 / 6776 2.3 430.0 1.0X
+Parquet Vectorized (Pushdown) 1634 / 1638 9.6 103.9 4.1X
+Native ORC Vectorized 3777 / 3785 4.2 240.1 1.8X
+Native ORC Vectorized (Pushdown) 1276 / 1279 12.3 81.2 5.3X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 50% timestamp stored as TIMESTAMP_MILLIS rows (value < CAST(7864320 AS timestamp)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 8716 / 8953 1.8 554.2 1.0X
-Parquet Vectorized (Pushdown) 6632 / 6968 2.4 421.7 1.3X
-Native ORC Vectorized 8376 / 9118 1.9 532.5 1.0X
-Native ORC Vectorized (Pushdown) 7218 / 7609 2.2 458.9 1.2X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_151-b12 on Mac OS X 10.12.6
-Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+Parquet Vectorized 10460 / 10469 1.5 665.0 1.0X
+Parquet Vectorized (Pushdown) 7689 / 7698 2.0 488.9 1.4X
+Native ORC Vectorized 7190 / 7197 2.2 457.1 1.5X
+Native ORC Vectorized (Pushdown) 5820 / 5834 2.7 370.0 1.8X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 90% timestamp stored as TIMESTAMP_MILLIS rows (value < CAST(14155776 AS timestamp)): Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 12264 / 12452 1.3 779.7 1.0X
-Parquet Vectorized (Pushdown) 11766 / 11927 1.3 748.0 1.0X
-Native ORC Vectorized 12101 / 12301 1.3 769.3 1.0X
-Native ORC Vectorized (Pushdown) 11983 / 12651 1.3 761.9 1.0X
+Parquet Vectorized 14033 / 14039 1.1 892.2 1.0X
+Parquet Vectorized (Pushdown) 13608 / 13636 1.2 865.2 1.0X
+Native ORC Vectorized 10635 / 10686 1.5 676.2 1.3X
+Native ORC Vectorized (Pushdown) 10420 / 10442 1.5 662.5 1.3X
================================================================================================
Pushdown benchmark with many filters
================================================================================================
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_181-b13 on Mac OS X 10.13.6
-Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz
-
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 row with 1 filters: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 158 / 182 0.0 158442969.0 1.0X
-Parquet Vectorized (Pushdown) 150 / 158 0.0 149718289.0 1.1X
-Native ORC Vectorized 141 / 148 0.0 141259852.0 1.1X
-Native ORC Vectorized (Pushdown) 142 / 147 0.0 142016472.0 1.1X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_181-b13 on Mac OS X 10.13.6
-Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz
+Parquet Vectorized 319 / 323 0.0 318789986.0 1.0X
+Parquet Vectorized (Pushdown) 323 / 347 0.0 322755287.0 1.0X
+Native ORC Vectorized 316 / 336 0.0 315670745.0 1.0X
+Native ORC Vectorized (Pushdown) 317 / 320 0.0 317392594.0 1.0X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 row with 250 filters: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 1013 / 1026 0.0 1013194322.0 1.0X
-Parquet Vectorized (Pushdown) 1326 / 1332 0.0 1326301956.0 0.8X
-Native ORC Vectorized 1005 / 1010 0.0 1005266379.0 1.0X
-Native ORC Vectorized (Pushdown) 1068 / 1071 0.0 1067964993.0 0.9X
-
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_181-b13 on Mac OS X 10.13.6
-Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz
+Parquet Vectorized 2192 / 2218 0.0 2191883823.0 1.0X
+Parquet Vectorized (Pushdown) 2675 / 2687 0.0 2675439029.0 0.8X
+Native ORC Vectorized 2158 / 2162 0.0 2157646071.0 1.0X
+Native ORC Vectorized (Pushdown) 2309 / 2326 0.0 2309096612.0 0.9X
+OpenJDK 64-Bit Server VM 1.8.0_181-b13 on Linux 3.10.0-862.3.2.el7.x86_64
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
Select 1 row with 500 filters: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------
-Parquet Vectorized 3598 / 3614 0.0 3598001202.0 1.0X
-Parquet Vectorized (Pushdown) 4282 / 4333 0.0 4281849770.0 0.8X
-Native ORC Vectorized 3594 / 3619 0.0 3593551548.0 1.0X
-Native ORC Vectorized (Pushdown) 3834 / 3840 0.0 3834240570.0 0.9X
+Parquet Vectorized 6219 / 6248 0.0 6218727737.0 1.0X
+Parquet Vectorized (Pushdown) 7376 / 7436 0.0 7375977710.0 0.8X
+Native ORC Vectorized 6252 / 6279 0.0 6252473320.0 1.0X
+Native ORC Vectorized (Pushdown) 6858 / 6876 0.0 6857854486.0 0.9X
+
+
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index ba17f5f33f2b6..f9883a740fab1 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.4.0-SNAPSHOT
+ 2.4.5-SNAPSHOT
../../pom.xml
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
index 6fdadde628551..3b919c7d471f4 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
@@ -23,7 +23,6 @@
import org.apache.spark.sql.types.*;
import org.apache.spark.unsafe.Platform;
-import org.apache.spark.unsafe.memory.OffHeapMemoryBlock;
import org.apache.spark.unsafe.types.UTF8String;
/**
@@ -207,7 +206,7 @@ public byte[] getBytes(int rowId, int count) {
@Override
protected UTF8String getBytesAsUTF8String(int rowId, int count) {
- return new UTF8String(new OffHeapMemoryBlock(data + rowId, count));
+ return UTF8String.fromAddress(null, data + rowId, count);
}
//
@@ -418,7 +417,7 @@ public void putFloats(int rowId, int count, byte[] src, int srcIndex) {
Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex,
null, data + rowId * 4L, count * 4L);
} else {
- ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+ ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.BIG_ENDIAN);
long offset = data + 4L * rowId;
for (int i = 0; i < count; ++i, offset += 4) {
Platform.putFloat(null, offset, bb.getFloat(srcIndex + (4 * i)));
@@ -473,7 +472,7 @@ public void putDoubles(int rowId, int count, byte[] src, int srcIndex) {
Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex,
null, data + rowId * 8L, count * 8L);
} else {
- ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+ ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.BIG_ENDIAN);
long offset = data + 8L * rowId;
for (int i = 0; i < count; ++i, offset += 8) {
Platform.putDouble(null, offset, bb.getDouble(srcIndex + (8 * i)));
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
index 577eab6ed14c8..1bf3126664177 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
@@ -396,7 +396,7 @@ public void putFloats(int rowId, int count, byte[] src, int srcIndex) {
Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex, floatData,
Platform.DOUBLE_ARRAY_OFFSET + rowId * 4L, count * 4L);
} else {
- ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+ ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.BIG_ENDIAN);
for (int i = 0; i < count; ++i) {
floatData[i + rowId] = bb.getFloat(srcIndex + (4 * i));
}
@@ -445,7 +445,7 @@ public void putDoubles(int rowId, int count, byte[] src, int srcIndex) {
Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex, doubleData,
Platform.DOUBLE_ARRAY_OFFSET + rowId * 8L, count * 8L);
} else {
- ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+ ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.BIG_ENDIAN);
for (int i = 0; i < count; ++i) {
doubleData[i + rowId] = bb.getDouble(srcIndex + (8 * i));
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/internal/NonClosableMutableURLClassLoader.java b/sql/core/src/main/java/org/apache/spark/sql/internal/NonClosableMutableURLClassLoader.java
new file mode 100644
index 0000000000000..db77d9136aaba
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/internal/NonClosableMutableURLClassLoader.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.internal;
+
+import java.net.URL;
+
+import org.apache.spark.util.MutableURLClassLoader;
+
+/**
+ * This class loader cannot be closed (its `close` method is a no-op).
+ */
+public class NonClosableMutableURLClassLoader extends MutableURLClassLoader {
+
+ static {
+ ClassLoader.registerAsParallelCapable();
+ }
+
+ public NonClosableMutableURLClassLoader(ClassLoader parent) {
+ super(new URL[]{}, parent);
+ }
+
+ @Override
+ public void close() {}
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/ContinuousReadSupport.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/ContinuousReadSupport.java
new file mode 100644
index 0000000000000..7df5a451ae5f3
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/ContinuousReadSupport.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.sources.v2;
+
+import java.util.Optional;
+
+import org.apache.spark.annotation.InterfaceStability;
+import org.apache.spark.sql.sources.v2.reader.streaming.ContinuousReader;
+import org.apache.spark.sql.types.StructType;
+
+/**
+ * A mix-in interface for {@link DataSourceV2}. Data sources can implement this interface to
+ * provide data reading ability for continuous stream processing.
+ */
+@InterfaceStability.Evolving
+public interface ContinuousReadSupport extends DataSourceV2 {
+ /**
+ * Creates a {@link ContinuousReader} to scan the data from this data source.
+ *
+ * @param schema the user provided schema, or empty() if none was provided
+ * @param checkpointLocation a path to Hadoop FS scratch space that can be used for failure
+ * recovery. Readers for the same logical source in the same query
+ * will be given the same checkpointLocation.
+ * @param options the options for the returned data source reader, which is an immutable
+ * case-insensitive string-to-string map.
+ */
+ ContinuousReader createContinuousReader(
+ Optional schema,
+ String checkpointLocation,
+ DataSourceOptions options);
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/ContinuousReadSupportProvider.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/ContinuousReadSupportProvider.java
deleted file mode 100644
index 824c290518acf..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/ContinuousReadSupportProvider.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2;
-
-import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Utils;
-import org.apache.spark.sql.sources.v2.reader.streaming.ContinuousReadSupport;
-import org.apache.spark.sql.types.StructType;
-
-/**
- * A mix-in interface for {@link DataSourceV2}. Data sources can implement this interface to
- * provide data reading ability for continuous stream processing.
- *
- * This interface is used to create {@link ContinuousReadSupport} instances when end users run
- * {@code SparkSession.readStream.format(...).option(...).load()} with a continuous trigger.
- */
-@InterfaceStability.Evolving
-public interface ContinuousReadSupportProvider extends DataSourceV2 {
-
- /**
- * Creates a {@link ContinuousReadSupport} instance to scan the data from this streaming data
- * source with a user specified schema, which is called by Spark at the beginning of each
- * continuous streaming query.
- *
- * By default this method throws {@link UnsupportedOperationException}, implementations should
- * override this method to handle user specified schema.
- *
- * @param schema the user provided schema.
- * @param checkpointLocation a path to Hadoop FS scratch space that can be used for failure
- * recovery. Readers for the same logical source in the same query
- * will be given the same checkpointLocation.
- * @param options the options for the returned data source reader, which is an immutable
- * case-insensitive string-to-string map.
- */
- default ContinuousReadSupport createContinuousReadSupport(
- StructType schema,
- String checkpointLocation,
- DataSourceOptions options) {
- return DataSourceV2Utils.failForUserSpecifiedSchema(this);
- }
-
- /**
- * Creates a {@link ContinuousReadSupport} instance to scan the data from this streaming data
- * source, which is called by Spark at the beginning of each continuous streaming query.
- *
- * @param checkpointLocation a path to Hadoop FS scratch space that can be used for failure
- * recovery. Readers for the same logical source in the same query
- * will be given the same checkpointLocation.
- * @param options the options for the returned data source reader, which is an immutable
- * case-insensitive string-to-string map.
- */
- ContinuousReadSupport createContinuousReadSupport(
- String checkpointLocation,
- DataSourceOptions options);
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/DataSourceV2.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/DataSourceV2.java
index 6e31e84bf6c72..6234071320dc9 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/DataSourceV2.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/DataSourceV2.java
@@ -22,13 +22,9 @@
/**
* The base interface for data source v2. Implementations must have a public, 0-arg constructor.
*
- * Note that this is an empty interface. Data source implementations must mix in interfaces such as
- * {@link BatchReadSupportProvider} or {@link BatchWriteSupportProvider}, which can provide
- * batch or streaming read/write support instances. Otherwise it's just a dummy data source which
- * is un-readable/writable.
- *
- * If Spark fails to execute any methods in the implementations of this interface (by throwing an
- * exception), the read action will fail and no Spark job will be submitted.
+ * Note that this is an empty interface. Data source implementations should mix-in at least one of
+ * the plug-in interfaces like {@link ReadSupport} and {@link WriteSupport}. Otherwise it's just
+ * a dummy data source which is un-readable/writable.
*/
@InterfaceStability.Evolving
public interface DataSourceV2 {}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/MicroBatchReadSupport.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/MicroBatchReadSupport.java
new file mode 100644
index 0000000000000..7f4a2c9593c76
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/MicroBatchReadSupport.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.sources.v2;
+
+import java.util.Optional;
+
+import org.apache.spark.annotation.InterfaceStability;
+import org.apache.spark.sql.sources.v2.reader.streaming.MicroBatchReader;
+import org.apache.spark.sql.types.StructType;
+
+/**
+ * A mix-in interface for {@link DataSourceV2}. Data sources can implement this interface to
+ * provide streaming micro-batch data reading ability.
+ */
+@InterfaceStability.Evolving
+public interface MicroBatchReadSupport extends DataSourceV2 {
+ /**
+ * Creates a {@link MicroBatchReader} to read batches of data from this data source in a
+ * streaming query.
+ *
+ * The execution engine will create a micro-batch reader at the start of a streaming query,
+ * alternate calls to setOffsetRange and planInputPartitions for each batch to process, and
+ * then call stop() when the execution is complete. Note that a single query may have multiple
+ * executions due to restart or failure recovery.
+ *
+ * @param schema the user provided schema, or empty() if none was provided
+ * @param checkpointLocation a path to Hadoop FS scratch space that can be used for failure
+ * recovery. Readers for the same logical source in the same query
+ * will be given the same checkpointLocation.
+ * @param options the options for the returned data source reader, which is an immutable
+ * case-insensitive string-to-string map.
+ */
+ MicroBatchReader createMicroBatchReader(
+ Optional schema,
+ String checkpointLocation,
+ DataSourceOptions options);
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/MicroBatchReadSupportProvider.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/MicroBatchReadSupportProvider.java
deleted file mode 100644
index 61c08e7fa89df..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/MicroBatchReadSupportProvider.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2;
-
-import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Utils;
-import org.apache.spark.sql.sources.v2.reader.streaming.MicroBatchReadSupport;
-import org.apache.spark.sql.types.StructType;
-
-/**
- * A mix-in interface for {@link DataSourceV2}. Data sources can implement this interface to
- * provide data reading ability for micro-batch stream processing.
- *
- * This interface is used to create {@link MicroBatchReadSupport} instances when end users run
- * {@code SparkSession.readStream.format(...).option(...).load()} with a micro-batch trigger.
- */
-@InterfaceStability.Evolving
-public interface MicroBatchReadSupportProvider extends DataSourceV2 {
-
- /**
- * Creates a {@link MicroBatchReadSupport} instance to scan the data from this streaming data
- * source with a user specified schema, which is called by Spark at the beginning of each
- * micro-batch streaming query.
- *
- * By default this method throws {@link UnsupportedOperationException}, implementations should
- * override this method to handle user specified schema.
- *
- * @param schema the user provided schema.
- * @param checkpointLocation a path to Hadoop FS scratch space that can be used for failure
- * recovery. Readers for the same logical source in the same query
- * will be given the same checkpointLocation.
- * @param options the options for the returned data source reader, which is an immutable
- * case-insensitive string-to-string map.
- */
- default MicroBatchReadSupport createMicroBatchReadSupport(
- StructType schema,
- String checkpointLocation,
- DataSourceOptions options) {
- return DataSourceV2Utils.failForUserSpecifiedSchema(this);
- }
-
- /**
- * Creates a {@link MicroBatchReadSupport} instance to scan the data from this streaming data
- * source, which is called by Spark at the beginning of each micro-batch streaming query.
- *
- * @param checkpointLocation a path to Hadoop FS scratch space that can be used for failure
- * recovery. Readers for the same logical source in the same query
- * will be given the same checkpointLocation.
- * @param options the options for the returned data source reader, which is an immutable
- * case-insensitive string-to-string map.
- */
- MicroBatchReadSupport createMicroBatchReadSupport(
- String checkpointLocation,
- DataSourceOptions options);
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/BatchReadSupportProvider.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/ReadSupport.java
similarity index 59%
rename from sql/core/src/main/java/org/apache/spark/sql/sources/v2/BatchReadSupportProvider.java
rename to sql/core/src/main/java/org/apache/spark/sql/sources/v2/ReadSupport.java
index f403dc619e86c..80ac08ee5ff52 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/BatchReadSupportProvider.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/ReadSupport.java
@@ -18,44 +18,48 @@
package org.apache.spark.sql.sources.v2;
import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Utils;
-import org.apache.spark.sql.sources.v2.reader.BatchReadSupport;
+import org.apache.spark.sql.sources.DataSourceRegister;
+import org.apache.spark.sql.sources.v2.reader.DataSourceReader;
import org.apache.spark.sql.types.StructType;
/**
* A mix-in interface for {@link DataSourceV2}. Data sources can implement this interface to
- * provide data reading ability for batch processing.
- *
- * This interface is used to create {@link BatchReadSupport} instances when end users run
- * {@code SparkSession.read.format(...).option(...).load()}.
+ * provide data reading ability and scan the data from the data source.
*/
@InterfaceStability.Evolving
-public interface BatchReadSupportProvider extends DataSourceV2 {
+public interface ReadSupport extends DataSourceV2 {
/**
- * Creates a {@link BatchReadSupport} instance to load the data from this data source with a user
- * specified schema, which is called by Spark at the beginning of each batch query.
- *
- * Spark will call this method at the beginning of each batch query to create a
- * {@link BatchReadSupport} instance.
+ * Creates a {@link DataSourceReader} to scan the data from this data source.
*
- * By default this method throws {@link UnsupportedOperationException}, implementations should
- * override this method to handle user specified schema.
+ * If this method fails (by throwing an exception), the action will fail and no Spark job will be
+ * submitted.
*
* @param schema the user specified schema.
* @param options the options for the returned data source reader, which is an immutable
* case-insensitive string-to-string map.
+ *
+ * By default this method throws {@link UnsupportedOperationException}, implementations should
+ * override this method to handle user specified schema.
*/
- default BatchReadSupport createBatchReadSupport(StructType schema, DataSourceOptions options) {
- return DataSourceV2Utils.failForUserSpecifiedSchema(this);
+ default DataSourceReader createReader(StructType schema, DataSourceOptions options) {
+ String name;
+ if (this instanceof DataSourceRegister) {
+ name = ((DataSourceRegister) this).shortName();
+ } else {
+ name = this.getClass().getName();
+ }
+ throw new UnsupportedOperationException(name + " does not support user specified schema");
}
/**
- * Creates a {@link BatchReadSupport} instance to scan the data from this data source, which is
- * called by Spark at the beginning of each batch query.
+ * Creates a {@link DataSourceReader} to scan the data from this data source.
+ *
+ * If this method fails (by throwing an exception), the action will fail and no Spark job will be
+ * submitted.
*
* @param options the options for the returned data source reader, which is an immutable
* case-insensitive string-to-string map.
*/
- BatchReadSupport createBatchReadSupport(DataSourceOptions options);
+ DataSourceReader createReader(DataSourceOptions options);
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/SessionConfigSupport.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/SessionConfigSupport.java
index bbe430e299261..926c6fd8fd228 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/SessionConfigSupport.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/SessionConfigSupport.java
@@ -28,10 +28,9 @@
public interface SessionConfigSupport extends DataSourceV2 {
/**
- * Key prefix of the session configs to propagate, which is usually the data source name. Spark
- * will extract all session configs that starts with `spark.datasource.$keyPrefix`, turn
- * `spark.datasource.$keyPrefix.xxx -> yyy` into `xxx -> yyy`, and propagate them to all
- * data source operations in this session.
+ * Key prefix of the session configs to propagate. Spark will extract all session configs that
+ * starts with `spark.datasource.$keyPrefix`, turn `spark.datasource.$keyPrefix.xxx -> yyy`
+ * into `xxx -> yyy`, and propagate them to all data source operations in this session.
*/
String keyPrefix();
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/StreamWriteSupport.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/StreamWriteSupport.java
new file mode 100644
index 0000000000000..a77b01497269e
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/StreamWriteSupport.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.sources.v2;
+
+import org.apache.spark.annotation.InterfaceStability;
+import org.apache.spark.sql.execution.streaming.BaseStreamingSink;
+import org.apache.spark.sql.sources.v2.writer.DataSourceWriter;
+import org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter;
+import org.apache.spark.sql.streaming.OutputMode;
+import org.apache.spark.sql.types.StructType;
+
+/**
+ * A mix-in interface for {@link DataSourceV2}. Data sources can implement this interface to
+ * provide data writing ability for structured streaming.
+ */
+@InterfaceStability.Evolving
+public interface StreamWriteSupport extends DataSourceV2, BaseStreamingSink {
+
+ /**
+ * Creates an optional {@link StreamWriter} to save the data to this data source. Data
+ * sources can return None if there is no writing needed to be done.
+ *
+ * @param queryId A unique string for the writing query. It's possible that there are many
+ * writing queries running at the same time, and the returned
+ * {@link DataSourceWriter} can use this id to distinguish itself from others.
+ * @param schema the schema of the data to be written.
+ * @param mode the output mode which determines what successive epoch output means to this
+ * sink, please refer to {@link OutputMode} for more details.
+ * @param options the options for the returned data source writer, which is an immutable
+ * case-insensitive string-to-string map.
+ */
+ StreamWriter createStreamWriter(
+ String queryId,
+ StructType schema,
+ OutputMode mode,
+ DataSourceOptions options);
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/StreamingWriteSupportProvider.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/StreamingWriteSupportProvider.java
deleted file mode 100644
index f9ca85d8089b4..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/StreamingWriteSupportProvider.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2;
-
-import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.execution.streaming.BaseStreamingSink;
-import org.apache.spark.sql.sources.v2.writer.streaming.StreamingWriteSupport;
-import org.apache.spark.sql.streaming.OutputMode;
-import org.apache.spark.sql.types.StructType;
-
-/**
- * A mix-in interface for {@link DataSourceV2}. Data sources can implement this interface to
- * provide data writing ability for structured streaming.
- *
- * This interface is used to create {@link StreamingWriteSupport} instances when end users run
- * {@code Dataset.writeStream.format(...).option(...).start()}.
- */
-@InterfaceStability.Evolving
-public interface StreamingWriteSupportProvider extends DataSourceV2, BaseStreamingSink {
-
- /**
- * Creates a {@link StreamingWriteSupport} instance to save the data to this data source, which is
- * called by Spark at the beginning of each streaming query.
- *
- * @param queryId A unique string for the writing query. It's possible that there are many
- * writing queries running at the same time, and the returned
- * {@link StreamingWriteSupport} can use this id to distinguish itself from others.
- * @param schema the schema of the data to be written.
- * @param mode the output mode which determines what successive epoch output means to this
- * sink, please refer to {@link OutputMode} for more details.
- * @param options the options for the returned data source writer, which is an immutable
- * case-insensitive string-to-string map.
- */
- StreamingWriteSupport createStreamingWriteSupport(
- String queryId,
- StructType schema,
- OutputMode mode,
- DataSourceOptions options);
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/BatchWriteSupportProvider.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/WriteSupport.java
similarity index 58%
rename from sql/core/src/main/java/org/apache/spark/sql/sources/v2/BatchWriteSupportProvider.java
rename to sql/core/src/main/java/org/apache/spark/sql/sources/v2/WriteSupport.java
index bd10c3353bf12..048787a7a0a05 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/BatchWriteSupportProvider.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/WriteSupport.java
@@ -21,39 +21,33 @@
import org.apache.spark.annotation.InterfaceStability;
import org.apache.spark.sql.SaveMode;
-import org.apache.spark.sql.sources.v2.writer.BatchWriteSupport;
+import org.apache.spark.sql.sources.v2.writer.DataSourceWriter;
import org.apache.spark.sql.types.StructType;
/**
* A mix-in interface for {@link DataSourceV2}. Data sources can implement this interface to
- * provide data writing ability for batch processing.
- *
- * This interface is used to create {@link BatchWriteSupport} instances when end users run
- * {@code Dataset.write.format(...).option(...).save()}.
+ * provide data writing ability and save the data to the data source.
*/
@InterfaceStability.Evolving
-public interface BatchWriteSupportProvider extends DataSourceV2 {
+public interface WriteSupport extends DataSourceV2 {
/**
- * Creates an optional {@link BatchWriteSupport} instance to save the data to this data source,
- * which is called by Spark at the beginning of each batch query.
+ * Creates an optional {@link DataSourceWriter} to save the data to this data source. Data
+ * sources can return None if there is no writing needed to be done according to the save mode.
*
- * Data sources can return None if there is no writing needed to be done according to the save
- * mode.
+ * If this method fails (by throwing an exception), the action will fail and no Spark job will be
+ * submitted.
*
- * @param queryId A unique string for the writing query. It's possible that there are many
- * writing queries running at the same time, and the returned
- * {@link BatchWriteSupport} can use this id to distinguish itself from others.
+ * @param writeUUID A unique string for the writing job. It's possible that there are many writing
+ * jobs running at the same time, and the returned {@link DataSourceWriter} can
+ * use this job id to distinguish itself from other jobs.
* @param schema the schema of the data to be written.
* @param mode the save mode which determines what to do when the data are already in this data
* source, please refer to {@link SaveMode} for more details.
* @param options the options for the returned data source writer, which is an immutable
* case-insensitive string-to-string map.
- * @return a write support to write data to this data source.
+ * @return a writer to append data to this data source
*/
- Optional createBatchWriteSupport(
- String queryId,
- StructType schema,
- SaveMode mode,
- DataSourceOptions options);
+ Optional createWriter(
+ String writeUUID, StructType schema, SaveMode mode, DataSourceOptions options);
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/BatchReadSupport.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/BatchReadSupport.java
deleted file mode 100644
index 452ee86675b42..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/BatchReadSupport.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2.reader;
-
-import org.apache.spark.annotation.InterfaceStability;
-
-/**
- * An interface that defines how to load the data from data source for batch processing.
- *
- * The execution engine will get an instance of this interface from a data source provider
- * (e.g. {@link org.apache.spark.sql.sources.v2.BatchReadSupportProvider}) at the start of a batch
- * query, then call {@link #newScanConfigBuilder()} and create an instance of {@link ScanConfig}.
- * The {@link ScanConfigBuilder} can apply operator pushdown and keep the pushdown result in
- * {@link ScanConfig}. The {@link ScanConfig} will be used to create input partitions and reader
- * factory to scan data from the data source with a Spark job.
- */
-@InterfaceStability.Evolving
-public interface BatchReadSupport extends ReadSupport {
-
- /**
- * Returns a builder of {@link ScanConfig}. Spark will call this method and create a
- * {@link ScanConfig} for each data scanning job.
- *
- * The builder can take some query specific information to do operators pushdown, and keep these
- * information in the created {@link ScanConfig}.
- *
- * This is the first step of the data scan. All other methods in {@link BatchReadSupport} needs
- * to take {@link ScanConfig} as an input.
- */
- ScanConfigBuilder newScanConfigBuilder();
-
- /**
- * Returns a factory, which produces one {@link PartitionReader} for one {@link InputPartition}.
- */
- PartitionReaderFactory createReaderFactory(ScanConfig config);
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/ContinuousInputPartition.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/ContinuousInputPartition.java
new file mode 100644
index 0000000000000..dcb87715d0b6f
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/ContinuousInputPartition.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.sources.v2.reader;
+
+import org.apache.spark.annotation.InterfaceStability;
+import org.apache.spark.sql.sources.v2.reader.streaming.PartitionOffset;
+
+/**
+ * A mix-in interface for {@link InputPartition}. Continuous input partitions can
+ * implement this interface to provide creating {@link InputPartitionReader} with particular offset.
+ */
+@InterfaceStability.Evolving
+public interface ContinuousInputPartition extends InputPartition {
+ /**
+ * Create an input partition reader with particular offset as its startOffset.
+ *
+ * @param offset offset want to set as the input partition reader's startOffset.
+ */
+ InputPartitionReader createContinuousReader(PartitionOffset offset);
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/DataSourceReader.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/DataSourceReader.java
new file mode 100644
index 0000000000000..da98fab1284ef
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/DataSourceReader.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.sources.v2.reader;
+
+import java.util.List;
+
+import org.apache.spark.annotation.InterfaceStability;
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.sources.v2.DataSourceOptions;
+import org.apache.spark.sql.sources.v2.ReadSupport;
+import org.apache.spark.sql.types.StructType;
+
+/**
+ * A data source reader that is returned by
+ * {@link ReadSupport#createReader(DataSourceOptions)} or
+ * {@link ReadSupport#createReader(StructType, DataSourceOptions)}.
+ * It can mix in various query optimization interfaces to speed up the data scan. The actual scan
+ * logic is delegated to {@link InputPartition}s, which are returned by
+ * {@link #planInputPartitions()}.
+ *
+ * There are mainly 3 kinds of query optimizations:
+ * 1. Operators push-down. E.g., filter push-down, required columns push-down(aka column
+ * pruning), etc. Names of these interfaces start with `SupportsPushDown`.
+ * 2. Information Reporting. E.g., statistics reporting, ordering reporting, etc.
+ * Names of these interfaces start with `SupportsReporting`.
+ * 3. Columnar scan if implements {@link SupportsScanColumnarBatch}.
+ *
+ * If an exception was throw when applying any of these query optimizations, the action will fail
+ * and no Spark job will be submitted.
+ *
+ * Spark first applies all operator push-down optimizations that this data source supports. Then
+ * Spark collects information this data source reported for further optimizations. Finally Spark
+ * issues the scan request and does the actual data reading.
+ */
+@InterfaceStability.Evolving
+public interface DataSourceReader {
+
+ /**
+ * Returns the actual schema of this data source reader, which may be different from the physical
+ * schema of the underlying storage, as column pruning or other optimizations may happen.
+ *
+ * If this method fails (by throwing an exception), the action will fail and no Spark job will be
+ * submitted.
+ */
+ StructType readSchema();
+
+ /**
+ * Returns a list of {@link InputPartition}s. Each {@link InputPartition} is responsible for
+ * creating a data reader to output data of one RDD partition. The number of input partitions
+ * returned here is the same as the number of RDD partitions this scan outputs.
+ *
+ * Note that, this may not be a full scan if the data source reader mixes in other optimization
+ * interfaces like column pruning, filter push-down, etc. These optimizations are applied before
+ * Spark issues the scan request.
+ *
+ * If this method fails (by throwing an exception), the action will fail and no Spark job will be
+ * submitted.
+ */
+ List> planInputPartitions();
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/InputPartition.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/InputPartition.java
index 95c30de907e44..f2038d0de3ffe 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/InputPartition.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/InputPartition.java
@@ -22,18 +22,18 @@
import org.apache.spark.annotation.InterfaceStability;
/**
- * A serializable representation of an input partition returned by
- * {@link ReadSupport#planInputPartitions(ScanConfig)}.
+ * An input partition returned by {@link DataSourceReader#planInputPartitions()} and is
+ * responsible for creating the actual data reader of one RDD partition.
+ * The relationship between {@link InputPartition} and {@link InputPartitionReader}
+ * is similar to the relationship between {@link Iterable} and {@link java.util.Iterator}.
*
- * Note that {@link InputPartition} will be serialized and sent to executors, then
- * {@link PartitionReader} will be created by
- * {@link PartitionReaderFactory#createReader(InputPartition)} or
- * {@link PartitionReaderFactory#createColumnarReader(InputPartition)} on executors to do
- * the actual reading. So {@link InputPartition} must be serializable while {@link PartitionReader}
- * doesn't need to be.
+ * Note that {@link InputPartition}s will be serialized and sent to executors, then
+ * {@link InputPartitionReader}s will be created on executors to do the actual reading. So
+ * {@link InputPartition} must be serializable while {@link InputPartitionReader} doesn't need to
+ * be.
*/
@InterfaceStability.Evolving
-public interface InputPartition extends Serializable {
+public interface InputPartition extends Serializable {
/**
* The preferred locations where the input partition reader returned by this partition can run
@@ -51,4 +51,12 @@ public interface InputPartition extends Serializable {
default String[] preferredLocations() {
return new String[0];
}
+
+ /**
+ * Returns an input partition reader to do the actual reading work.
+ *
+ * If this method fails (by throwing an exception), the corresponding Spark task would fail and
+ * get retried until hitting the maximum retry times.
+ */
+ InputPartitionReader createPartitionReader();
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/PartitionReader.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/InputPartitionReader.java
similarity index 67%
rename from sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/PartitionReader.java
rename to sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/InputPartitionReader.java
index 04ff8d0a19fc3..f3ff7f5cc0f20 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/PartitionReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/InputPartitionReader.java
@@ -23,27 +23,31 @@
import org.apache.spark.annotation.InterfaceStability;
/**
- * A partition reader returned by {@link PartitionReaderFactory#createReader(InputPartition)} or
- * {@link PartitionReaderFactory#createColumnarReader(InputPartition)}. It's responsible for
- * outputting data for a RDD partition.
+ * An input partition reader returned by {@link InputPartition#createPartitionReader()} and is
+ * responsible for outputting data for a RDD partition.
*
* Note that, Currently the type `T` can only be {@link org.apache.spark.sql.catalyst.InternalRow}
- * for normal data sources, or {@link org.apache.spark.sql.vectorized.ColumnarBatch} for columnar
- * data sources(whose {@link PartitionReaderFactory#supportColumnarReads(InputPartition)}
- * returns true).
+ * for normal data source readers, {@link org.apache.spark.sql.vectorized.ColumnarBatch} for data
+ * source readers that mix in {@link SupportsScanColumnarBatch}.
*/
@InterfaceStability.Evolving
-public interface PartitionReader extends Closeable {
+public interface InputPartitionReader extends Closeable {
/**
* Proceed to next record, returns false if there is no more records.
*
+ * If this method fails (by throwing an exception), the corresponding Spark task would fail and
+ * get retried until hitting the maximum retry times.
+ *
* @throws IOException if failure happens during disk/network IO like reading files.
*/
boolean next() throws IOException;
/**
* Return the current record. This method should return same value until `next` is called.
+ *
+ * If this method fails (by throwing an exception), the corresponding Spark task would fail and
+ * get retried until hitting the maximum retry times.
*/
T get();
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/PartitionReaderFactory.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/PartitionReaderFactory.java
deleted file mode 100644
index f35de9310eee3..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/PartitionReaderFactory.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2.reader;
-
-import java.io.Serializable;
-
-import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.vectorized.ColumnarBatch;
-
-/**
- * A factory used to create {@link PartitionReader} instances.
- *
- * If Spark fails to execute any methods in the implementations of this interface or in the returned
- * {@link PartitionReader} (by throwing an exception), corresponding Spark task would fail and
- * get retried until hitting the maximum retry times.
- */
-@InterfaceStability.Evolving
-public interface PartitionReaderFactory extends Serializable {
-
- /**
- * Returns a row-based partition reader to read data from the given {@link InputPartition}.
- *
- * Implementations probably need to cast the input partition to the concrete
- * {@link InputPartition} class defined for the data source.
- */
- PartitionReader createReader(InputPartition partition);
-
- /**
- * Returns a columnar partition reader to read data from the given {@link InputPartition}.
- *
- * Implementations probably need to cast the input partition to the concrete
- * {@link InputPartition} class defined for the data source.
- */
- default PartitionReader createColumnarReader(InputPartition partition) {
- throw new UnsupportedOperationException("Cannot create columnar reader.");
- }
-
- /**
- * Returns true if the given {@link InputPartition} should be read by Spark in a columnar way.
- * This means, implementations must also implement {@link #createColumnarReader(InputPartition)}
- * for the input partitions that this method returns true.
- *
- * As of Spark 2.4, Spark can only read all input partition in a columnar way, or none of them.
- * Data source can't mix columnar and row-based partitions. This may be relaxed in future
- * versions.
- */
- default boolean supportColumnarReads(InputPartition partition) {
- return false;
- }
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/ReadSupport.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/ReadSupport.java
deleted file mode 100644
index a58ddb288f1ed..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/ReadSupport.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2.reader;
-
-import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.types.StructType;
-
-/**
- * The base interface for all the batch and streaming read supports. Data sources should implement
- * concrete read support interfaces like {@link BatchReadSupport}.
- *
- * If Spark fails to execute any methods in the implementations of this interface (by throwing an
- * exception), the read action will fail and no Spark job will be submitted.
- */
-@InterfaceStability.Evolving
-public interface ReadSupport {
-
- /**
- * Returns the full schema of this data source, which is usually the physical schema of the
- * underlying storage. This full schema should not be affected by column pruning or other
- * optimizations.
- */
- StructType fullSchema();
-
- /**
- * Returns a list of {@link InputPartition input partitions}. Each {@link InputPartition}
- * represents a data split that can be processed by one Spark task. The number of input
- * partitions returned here is the same as the number of RDD partitions this scan outputs.
- *
- * Note that, this may not be a full scan if the data source supports optimization like filter
- * push-down. Implementations should check the input {@link ScanConfig} and adjust the resulting
- * {@link InputPartition input partitions}.
- */
- InputPartition[] planInputPartitions(ScanConfig config);
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/ScanConfig.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/ScanConfig.java
deleted file mode 100644
index 7462ce2820585..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/ScanConfig.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2.reader;
-
-import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.types.StructType;
-
-/**
- * An interface that carries query specific information for the data scanning job, like operator
- * pushdown information and streaming query offsets. This is defined as an empty interface, and data
- * sources should define their own {@link ScanConfig} classes.
- *
- * For APIs that take a {@link ScanConfig} as input, like
- * {@link ReadSupport#planInputPartitions(ScanConfig)},
- * {@link BatchReadSupport#createReaderFactory(ScanConfig)} and
- * {@link SupportsReportStatistics#estimateStatistics(ScanConfig)}, implementations mostly need to
- * cast the input {@link ScanConfig} to the concrete {@link ScanConfig} class of the data source.
- */
-@InterfaceStability.Evolving
-public interface ScanConfig {
-
- /**
- * Returns the actual schema of this data source reader, which may be different from the physical
- * schema of the underlying storage, as column pruning or other optimizations may happen.
- *
- * If this method fails (by throwing an exception), the action will fail and no Spark job will be
- * submitted.
- */
- StructType readSchema();
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/Statistics.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/Statistics.java
index 44799c7d49137..031c7a73c367d 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/Statistics.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/Statistics.java
@@ -23,7 +23,7 @@
/**
* An interface to represent statistics for a data source, which is returned by
- * {@link SupportsReportStatistics#estimateStatistics(ScanConfig)}.
+ * {@link SupportsReportStatistics#estimateStatistics()}.
*/
@InterfaceStability.Evolving
public interface Statistics {
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownFilters.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownFilters.java
index 5e7985f645a06..7e0020f38a73e 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownFilters.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownFilters.java
@@ -21,11 +21,11 @@
import org.apache.spark.sql.sources.Filter;
/**
- * A mix-in interface for {@link ScanConfigBuilder}. Data sources can implement this interface to
- * push down filters to the data source and reduce the size of the data to be read.
+ * A mix-in interface for {@link DataSourceReader}. Data source readers can implement this
+ * interface to push down filters to the data source and reduce the size of the data to be read.
*/
@InterfaceStability.Evolving
-public interface SupportsPushDownFilters extends ScanConfigBuilder {
+public interface SupportsPushDownFilters extends DataSourceReader {
/**
* Pushes down filters, and returns filters that need to be evaluated after scanning.
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownRequiredColumns.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownRequiredColumns.java
index edb164937d6ef..427b4d00a1128 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownRequiredColumns.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsPushDownRequiredColumns.java
@@ -21,12 +21,12 @@
import org.apache.spark.sql.types.StructType;
/**
- * A mix-in interface for {@link ScanConfigBuilder}. Data sources can implement this
+ * A mix-in interface for {@link DataSourceReader}. Data source readers can implement this
* interface to push down required columns to the data source and only read these columns during
* scan to reduce the size of the data to be read.
*/
@InterfaceStability.Evolving
-public interface SupportsPushDownRequiredColumns extends ScanConfigBuilder {
+public interface SupportsPushDownRequiredColumns extends DataSourceReader {
/**
* Applies column pruning w.r.t. the given requiredSchema.
@@ -35,8 +35,8 @@ public interface SupportsPushDownRequiredColumns extends ScanConfigBuilder {
* also OK to do the pruning partially, e.g., a data source may not be able to prune nested
* fields, and only prune top-level columns.
*
- * Note that, {@link ScanConfig#readSchema()} implementation should take care of the column
- * pruning applied here.
+ * Note that, data source readers should update {@link DataSourceReader#readSchema()} after
+ * applying column pruning.
*/
void pruneColumns(StructType requiredSchema);
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsReportPartitioning.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsReportPartitioning.java
index db62cd4515362..6b60da7c4dc1d 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsReportPartitioning.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsReportPartitioning.java
@@ -21,17 +21,17 @@
import org.apache.spark.sql.sources.v2.reader.partitioning.Partitioning;
/**
- * A mix in interface for {@link BatchReadSupport}. Data sources can implement this interface to
- * report data partitioning and try to avoid shuffle at Spark side.
+ * A mix in interface for {@link DataSourceReader}. Data source readers can implement this
+ * interface to report data partitioning and try to avoid shuffle at Spark side.
*
- * Note that, when a {@link ReadSupport} implementation creates exactly one {@link InputPartition},
- * Spark may avoid adding a shuffle even if the reader does not implement this interface.
+ * Note that, when the reader creates exactly one {@link InputPartition}, Spark may avoid
+ * adding a shuffle even if the reader does not implement this interface.
*/
@InterfaceStability.Evolving
-public interface SupportsReportPartitioning extends ReadSupport {
+public interface SupportsReportPartitioning extends DataSourceReader {
/**
* Returns the output data partitioning that this reader guarantees.
*/
- Partitioning outputPartitioning(ScanConfig config);
+ Partitioning outputPartitioning();
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsReportStatistics.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsReportStatistics.java
index 1831488ba096f..44d0ce3c6e74f 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsReportStatistics.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsReportStatistics.java
@@ -20,18 +20,18 @@
import org.apache.spark.annotation.InterfaceStability;
/**
- * A mix in interface for {@link BatchReadSupport}. Data sources can implement this interface to
- * report statistics to Spark.
+ * A mix in interface for {@link DataSourceReader}. Data source readers can implement this
+ * interface to report statistics to Spark.
*
* As of Spark 2.4, statistics are reported to the optimizer before any operator is pushed to the
- * data source. Implementations that return more accurate statistics based on pushed operators will
- * not improve query performance until the planner can push operators before getting stats.
+ * DataSourceReader. Implementations that return more accurate statistics based on pushed operators
+ * will not improve query performance until the planner can push operators before getting stats.
*/
@InterfaceStability.Evolving
-public interface SupportsReportStatistics extends ReadSupport {
+public interface SupportsReportStatistics extends DataSourceReader {
/**
- * Returns the estimated statistics of this data source scan.
+ * Returns the estimated statistics of this data source.
*/
- Statistics estimateStatistics(ScanConfig config);
+ Statistics estimateStatistics();
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsScanColumnarBatch.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsScanColumnarBatch.java
new file mode 100644
index 0000000000000..f4da686740d11
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/SupportsScanColumnarBatch.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.sources.v2.reader;
+
+import java.util.List;
+
+import org.apache.spark.annotation.InterfaceStability;
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.vectorized.ColumnarBatch;
+
+/**
+ * A mix-in interface for {@link DataSourceReader}. Data source readers can implement this
+ * interface to output {@link ColumnarBatch} and make the scan faster.
+ */
+@InterfaceStability.Evolving
+public interface SupportsScanColumnarBatch extends DataSourceReader {
+ @Override
+ default List> planInputPartitions() {
+ throw new IllegalStateException(
+ "planInputPartitions not supported by default within SupportsScanColumnarBatch.");
+ }
+
+ /**
+ * Similar to {@link DataSourceReader#planInputPartitions()}, but returns columnar data
+ * in batches.
+ */
+ List> planBatchInputPartitions();
+
+ /**
+ * Returns true if the concrete data source reader can read data in batch according to the scan
+ * properties like required columns, pushes filters, etc. It's possible that the implementation
+ * can only support some certain columns with certain types. Users can overwrite this method and
+ * {@link #planInputPartitions()} to fallback to normal read path under some conditions.
+ */
+ default boolean enableBatchRead() {
+ return true;
+ }
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/partitioning/ClusteredDistribution.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/partitioning/ClusteredDistribution.java
index 6764d4b7665c7..38ca5fc6387b2 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/partitioning/ClusteredDistribution.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/partitioning/ClusteredDistribution.java
@@ -18,12 +18,12 @@
package org.apache.spark.sql.sources.v2.reader.partitioning;
import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.sources.v2.reader.PartitionReader;
+import org.apache.spark.sql.sources.v2.reader.InputPartitionReader;
/**
* A concrete implementation of {@link Distribution}. Represents a distribution where records that
* share the same values for the {@link #clusteredColumns} will be produced by the same
- * {@link PartitionReader}.
+ * {@link InputPartitionReader}.
*/
@InterfaceStability.Evolving
public class ClusteredDistribution implements Distribution {
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/partitioning/Distribution.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/partitioning/Distribution.java
index 364a3f553923c..5e32ba6952e1c 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/partitioning/Distribution.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/partitioning/Distribution.java
@@ -18,14 +18,14 @@
package org.apache.spark.sql.sources.v2.reader.partitioning;
import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.sources.v2.reader.PartitionReader;
+import org.apache.spark.sql.sources.v2.reader.InputPartitionReader;
/**
* An interface to represent data distribution requirement, which specifies how the records should
- * be distributed among the data partitions (one {@link PartitionReader} outputs data for one
+ * be distributed among the data partitions (one {@link InputPartitionReader} outputs data for one
* partition).
* Note that this interface has nothing to do with the data ordering inside one
- * partition(the output records of a single {@link PartitionReader}).
+ * partition(the output records of a single {@link InputPartitionReader}).
*
* The instance of this interface is created and provided by Spark, then consumed by
* {@link Partitioning#satisfy(Distribution)}. This means data source developers don't need to
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/partitioning/Partitioning.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/partitioning/Partitioning.java
index fb0b6f1df43bb..f460f6bfe3bb9 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/partitioning/Partitioning.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/partitioning/Partitioning.java
@@ -19,13 +19,12 @@
import org.apache.spark.annotation.InterfaceStability;
import org.apache.spark.sql.sources.v2.reader.InputPartition;
-import org.apache.spark.sql.sources.v2.reader.ScanConfig;
import org.apache.spark.sql.sources.v2.reader.SupportsReportPartitioning;
/**
* An interface to represent the output data partitioning for a data source, which is returned by
- * {@link SupportsReportPartitioning#outputPartitioning(ScanConfig)}. Note that this should work
- * like a snapshot. Once created, it should be deterministic and always report the same number of
+ * {@link SupportsReportPartitioning#outputPartitioning()}. Note that this should work like a
+ * snapshot. Once created, it should be deterministic and always report the same number of
* partitions and the same "satisfy" result for a certain distribution.
*/
@InterfaceStability.Evolving
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousPartitionReader.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousInputPartitionReader.java
similarity index 60%
rename from sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousPartitionReader.java
rename to sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousInputPartitionReader.java
index 9101c8a44d34e..7b0ba0bbdda90 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousPartitionReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousInputPartitionReader.java
@@ -18,20 +18,19 @@
package org.apache.spark.sql.sources.v2.reader.streaming;
import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.sources.v2.reader.PartitionReader;
+import org.apache.spark.sql.sources.v2.reader.InputPartitionReader;
/**
- * A variation on {@link PartitionReader} for use with continuous streaming processing.
+ * A variation on {@link InputPartitionReader} for use with streaming in continuous processing mode.
*/
@InterfaceStability.Evolving
-public interface ContinuousPartitionReader extends PartitionReader {
-
- /**
- * Get the offset of the current record, or the start offset if no records have been read.
- *
- * The execution engine will call this method along with get() to keep track of the current
- * offset. When an epoch ends, the offset of the previous record in each partition will be saved
- * as a restart checkpoint.
- */
- PartitionOffset getOffset();
+public interface ContinuousInputPartitionReader extends InputPartitionReader {
+ /**
+ * Get the offset of the current record, or the start offset if no records have been read.
+ *
+ * The execution engine will call this method along with get() to keep track of the current
+ * offset. When an epoch ends, the offset of the previous record in each partition will be saved
+ * as a restart checkpoint.
+ */
+ PartitionOffset getOffset();
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousPartitionReaderFactory.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousPartitionReaderFactory.java
deleted file mode 100644
index 2d9f1ca1686a1..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousPartitionReaderFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2.reader.streaming;
-
-import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.sources.v2.reader.InputPartition;
-import org.apache.spark.sql.sources.v2.reader.PartitionReaderFactory;
-import org.apache.spark.sql.vectorized.ColumnarBatch;
-
-/**
- * A variation on {@link PartitionReaderFactory} that returns {@link ContinuousPartitionReader}
- * instead of {@link org.apache.spark.sql.sources.v2.reader.PartitionReader}. It's used for
- * continuous streaming processing.
- */
-@InterfaceStability.Evolving
-public interface ContinuousPartitionReaderFactory extends PartitionReaderFactory {
- @Override
- ContinuousPartitionReader createReader(InputPartition partition);
-
- @Override
- default ContinuousPartitionReader createColumnarReader(InputPartition partition) {
- throw new UnsupportedOperationException("Cannot create columnar reader.");
- }
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousReadSupport.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousReadSupport.java
deleted file mode 100644
index 9a3ad2eb8a801..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousReadSupport.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2.reader.streaming;
-
-import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.execution.streaming.BaseStreamingSource;
-import org.apache.spark.sql.sources.v2.reader.InputPartition;
-import org.apache.spark.sql.sources.v2.reader.ScanConfig;
-import org.apache.spark.sql.sources.v2.reader.ScanConfigBuilder;
-
-/**
- * An interface that defines how to load the data from data source for continuous streaming
- * processing.
- *
- * The execution engine will get an instance of this interface from a data source provider
- * (e.g. {@link org.apache.spark.sql.sources.v2.ContinuousReadSupportProvider}) at the start of a
- * streaming query, then call {@link #newScanConfigBuilder(Offset)} and create an instance of
- * {@link ScanConfig} for the duration of the streaming query or until
- * {@link #needsReconfiguration(ScanConfig)} is true. The {@link ScanConfig} will be used to create
- * input partitions and reader factory to scan data with a Spark job for its duration. At the end
- * {@link #stop()} will be called when the streaming execution is completed. Note that a single
- * query may have multiple executions due to restart or failure recovery.
- */
-@InterfaceStability.Evolving
-public interface ContinuousReadSupport extends StreamingReadSupport, BaseStreamingSource {
-
- /**
- * Returns a builder of {@link ScanConfig}. Spark will call this method and create a
- * {@link ScanConfig} for each data scanning job.
- *
- * The builder can take some query specific information to do operators pushdown, store streaming
- * offsets, etc., and keep these information in the created {@link ScanConfig}.
- *
- * This is the first step of the data scan. All other methods in {@link ContinuousReadSupport}
- * needs to take {@link ScanConfig} as an input.
- */
- ScanConfigBuilder newScanConfigBuilder(Offset start);
-
- /**
- * Returns a factory, which produces one {@link ContinuousPartitionReader} for one
- * {@link InputPartition}.
- */
- ContinuousPartitionReaderFactory createContinuousReaderFactory(ScanConfig config);
-
- /**
- * Merge partitioned offsets coming from {@link ContinuousPartitionReader} instances
- * for each partition to a single global offset.
- */
- Offset mergeOffsets(PartitionOffset[] offsets);
-
- /**
- * The execution engine will call this method in every epoch to determine if new input
- * partitions need to be generated, which may be required if for example the underlying
- * source system has had partitions added or removed.
- *
- * If true, the query will be shut down and restarted with a new {@link ContinuousReadSupport}
- * instance.
- */
- default boolean needsReconfiguration(ScanConfig config) {
- return false;
- }
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousReader.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousReader.java
new file mode 100644
index 0000000000000..6e960bedf8020
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/ContinuousReader.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.sources.v2.reader.streaming;
+
+import org.apache.spark.annotation.InterfaceStability;
+import org.apache.spark.sql.execution.streaming.BaseStreamingSource;
+import org.apache.spark.sql.sources.v2.reader.DataSourceReader;
+
+import java.util.Optional;
+
+/**
+ * A mix-in interface for {@link DataSourceReader}. Data source readers can implement this
+ * interface to allow reading in a continuous processing mode stream.
+ *
+ * Implementations must ensure each partition reader is a {@link ContinuousInputPartitionReader}.
+ *
+ * Note: This class currently extends {@link BaseStreamingSource} to maintain compatibility with
+ * DataSource V1 APIs. This extension will be removed once we get rid of V1 completely.
+ */
+@InterfaceStability.Evolving
+public interface ContinuousReader extends BaseStreamingSource, DataSourceReader {
+ /**
+ * Merge partitioned offsets coming from {@link ContinuousInputPartitionReader} instances
+ * for each partition to a single global offset.
+ */
+ Offset mergeOffsets(PartitionOffset[] offsets);
+
+ /**
+ * Deserialize a JSON string into an Offset of the implementation-defined offset type.
+ * @throws IllegalArgumentException if the JSON does not encode a valid offset for this reader
+ */
+ Offset deserializeOffset(String json);
+
+ /**
+ * Set the desired start offset for partitions created from this reader. The scan will
+ * start from the first record after the provided offset, or from an implementation-defined
+ * inferred starting point if no offset is provided.
+ */
+ void setStartOffset(Optional start);
+
+ /**
+ * Return the specified or inferred start offset for this reader.
+ *
+ * @throws IllegalStateException if setStartOffset has not been called
+ */
+ Offset getStartOffset();
+
+ /**
+ * The execution engine will call this method in every epoch to determine if new input
+ * partitions need to be generated, which may be required if for example the underlying
+ * source system has had partitions added or removed.
+ *
+ * If true, the query will be shut down and restarted with a new reader.
+ */
+ default boolean needsReconfiguration() {
+ return false;
+ }
+
+ /**
+ * Informs the source that Spark has completed processing all data for offsets less than or
+ * equal to `end` and will only request offsets greater than `end` in the future.
+ */
+ void commit(Offset end);
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/MicroBatchReadSupport.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/MicroBatchReadSupport.java
deleted file mode 100644
index edb0db11bff2c..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/MicroBatchReadSupport.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2.reader.streaming;
-
-import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.execution.streaming.BaseStreamingSource;
-import org.apache.spark.sql.sources.v2.reader.*;
-
-/**
- * An interface that defines how to scan the data from data source for micro-batch streaming
- * processing.
- *
- * The execution engine will get an instance of this interface from a data source provider
- * (e.g. {@link org.apache.spark.sql.sources.v2.MicroBatchReadSupportProvider}) at the start of a
- * streaming query, then call {@link #newScanConfigBuilder(Offset, Offset)} and create an instance
- * of {@link ScanConfig} for each micro-batch. The {@link ScanConfig} will be used to create input
- * partitions and reader factory to scan a micro-batch with a Spark job. At the end {@link #stop()}
- * will be called when the streaming execution is completed. Note that a single query may have
- * multiple executions due to restart or failure recovery.
- */
-@InterfaceStability.Evolving
-public interface MicroBatchReadSupport extends StreamingReadSupport, BaseStreamingSource {
-
- /**
- * Returns a builder of {@link ScanConfig}. Spark will call this method and create a
- * {@link ScanConfig} for each data scanning job.
- *
- * The builder can take some query specific information to do operators pushdown, store streaming
- * offsets, etc., and keep these information in the created {@link ScanConfig}.
- *
- * This is the first step of the data scan. All other methods in {@link MicroBatchReadSupport}
- * needs to take {@link ScanConfig} as an input.
- */
- ScanConfigBuilder newScanConfigBuilder(Offset start, Offset end);
-
- /**
- * Returns a factory, which produces one {@link PartitionReader} for one {@link InputPartition}.
- */
- PartitionReaderFactory createReaderFactory(ScanConfig config);
-
- /**
- * Returns the most recent offset available.
- */
- Offset latestOffset();
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/MicroBatchReader.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/MicroBatchReader.java
new file mode 100644
index 0000000000000..0159c731762d9
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/MicroBatchReader.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.sources.v2.reader.streaming;
+
+import org.apache.spark.annotation.InterfaceStability;
+import org.apache.spark.sql.sources.v2.reader.DataSourceReader;
+import org.apache.spark.sql.execution.streaming.BaseStreamingSource;
+
+import java.util.Optional;
+
+/**
+ * A mix-in interface for {@link DataSourceReader}. Data source readers can implement this
+ * interface to indicate they allow micro-batch streaming reads.
+ *
+ * Note: This class currently extends {@link BaseStreamingSource} to maintain compatibility with
+ * DataSource V1 APIs. This extension will be removed once we get rid of V1 completely.
+ */
+@InterfaceStability.Evolving
+public interface MicroBatchReader extends DataSourceReader, BaseStreamingSource {
+ /**
+ * Set the desired offset range for input partitions created from this reader. Partition readers
+ * will generate only data within (`start`, `end`]; that is, from the first record after `start`
+ * to the record with offset `end`.
+ *
+ * @param start The initial offset to scan from. If not specified, scan from an
+ * implementation-specified start point, such as the earliest available record.
+ * @param end The last offset to include in the scan. If not specified, scan up to an
+ * implementation-defined endpoint, such as the last available offset
+ * or the start offset plus a target batch size.
+ */
+ void setOffsetRange(Optional start, Optional end);
+
+ /**
+ * Returns the specified (if explicitly set through setOffsetRange) or inferred start offset
+ * for this reader.
+ *
+ * @throws IllegalStateException if setOffsetRange has not been called
+ */
+ Offset getStartOffset();
+
+ /**
+ * Return the specified (if explicitly set through setOffsetRange) or inferred end offset
+ * for this reader.
+ *
+ * @throws IllegalStateException if setOffsetRange has not been called
+ */
+ Offset getEndOffset();
+
+ /**
+ * Deserialize a JSON string into an Offset of the implementation-defined offset type.
+ * @throws IllegalArgumentException if the JSON does not encode a valid offset for this reader
+ */
+ Offset deserializeOffset(String json);
+
+ /**
+ * Informs the source that Spark has completed processing all data for offsets less than or
+ * equal to `end` and will only request offsets greater than `end` in the future.
+ */
+ void commit(Offset end);
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/Offset.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/Offset.java
index 6cf27734867cb..e41c0351edc82 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/Offset.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/Offset.java
@@ -20,8 +20,8 @@
import org.apache.spark.annotation.InterfaceStability;
/**
- * An abstract representation of progress through a {@link MicroBatchReadSupport} or
- * {@link ContinuousReadSupport}.
+ * An abstract representation of progress through a {@link MicroBatchReader} or
+ * {@link ContinuousReader}.
* During execution, offsets provided by the data source implementation will be logged and used as
* restart checkpoints. Each source should provide an offset implementation which the source can use
* to reconstruct a position in the stream up to which data has been seen/processed.
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/StreamingReadSupport.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/StreamingReadSupport.java
deleted file mode 100644
index 84872d1ebc26e..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/streaming/StreamingReadSupport.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2.reader.streaming;
-
-import org.apache.spark.sql.sources.v2.reader.ReadSupport;
-
-/**
- * A base interface for streaming read support. This is package private and is invisible to data
- * sources. Data sources should implement concrete streaming read support interfaces:
- * {@link MicroBatchReadSupport} or {@link ContinuousReadSupport}.
- */
-interface StreamingReadSupport extends ReadSupport {
-
- /**
- * Returns the initial offset for a streaming query to start reading from. Note that the
- * streaming data source should not assume that it will start reading from its initial offset:
- * if Spark is restarting an existing query, it will restart from the check-pointed offset rather
- * than the initial one.
- */
- Offset initialOffset();
-
- /**
- * Deserialize a JSON string into an Offset of the implementation-defined offset type.
- *
- * @throws IllegalArgumentException if the JSON does not encode a valid offset for this reader
- */
- Offset deserializeOffset(String json);
-
- /**
- * Informs the source that Spark has completed processing all data for offsets less than or
- * equal to `end` and will only request offsets greater than `end` in the future.
- */
- void commit(Offset end);
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/BatchWriteSupport.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/DataSourceWriter.java
similarity index 79%
rename from sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/BatchWriteSupport.java
rename to sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/DataSourceWriter.java
index 0ec9e05d6a02b..385fc294fea82 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/BatchWriteSupport.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/DataSourceWriter.java
@@ -18,13 +18,28 @@
package org.apache.spark.sql.sources.v2.writer;
import org.apache.spark.annotation.InterfaceStability;
+import org.apache.spark.sql.SaveMode;
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.sources.v2.DataSourceOptions;
+import org.apache.spark.sql.sources.v2.StreamWriteSupport;
+import org.apache.spark.sql.sources.v2.WriteSupport;
+import org.apache.spark.sql.streaming.OutputMode;
+import org.apache.spark.sql.types.StructType;
/**
- * An interface that defines how to write the data to data source for batch processing.
+ * A data source writer that is returned by
+ * {@link WriteSupport#createWriter(String, StructType, SaveMode, DataSourceOptions)}/
+ * {@link StreamWriteSupport#createStreamWriter(
+ * String, StructType, OutputMode, DataSourceOptions)}.
+ * It can mix in various writing optimization interfaces to speed up the data saving. The actual
+ * writing logic is delegated to {@link DataWriter}.
+ *
+ * If an exception was throw when applying any of these writing optimizations, the action will fail
+ * and no Spark job will be submitted.
*
* The writing procedure is:
- * 1. Create a writer factory by {@link #createBatchWriterFactory()}, serialize and send it to all
- * the partitions of the input data(RDD).
+ * 1. Create a writer factory by {@link #createWriterFactory()}, serialize and send it to all the
+ * partitions of the input data(RDD).
* 2. For each partition, create the data writer, and write the data of the partition with this
* writer. If all the data are written successfully, call {@link DataWriter#commit()}. If
* exception happens during the writing, call {@link DataWriter#abort()}.
@@ -38,7 +53,7 @@
* Please refer to the documentation of commit/abort methods for detailed specifications.
*/
@InterfaceStability.Evolving
-public interface BatchWriteSupport {
+public interface DataSourceWriter {
/**
* Creates a writer factory which will be serialized and sent to executors.
@@ -46,7 +61,7 @@ public interface BatchWriteSupport {
* If this method fails (by throwing an exception), the action will fail and no Spark job will be
* submitted.
*/
- DataWriterFactory createBatchWriterFactory();
+ DataWriterFactory createWriterFactory();
/**
* Returns whether Spark should use the commit coordinator to ensure that at most one task for
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/DataWriter.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/DataWriter.java
index 5fb067966ee67..27dc5ea224fe2 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/DataWriter.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/DataWriter.java
@@ -22,7 +22,7 @@
import org.apache.spark.annotation.InterfaceStability;
/**
- * A data writer returned by {@link DataWriterFactory#createWriter(int, long)} and is
+ * A data writer returned by {@link DataWriterFactory#createDataWriter(int, long, long)} and is
* responsible for writing data for an input RDD partition.
*
* One Spark task has one exclusive data writer, so there is no thread-safe concern.
@@ -36,11 +36,11 @@
*
* If this data writer succeeds(all records are successfully written and {@link #commit()}
* succeeds), a {@link WriterCommitMessage} will be sent to the driver side and pass to
- * {@link BatchWriteSupport#commit(WriterCommitMessage[])} with commit messages from other data
+ * {@link DataSourceWriter#commit(WriterCommitMessage[])} with commit messages from other data
* writers. If this data writer fails(one record fails to write or {@link #commit()} fails), an
* exception will be sent to the driver side, and Spark may retry this writing task a few times.
- * In each retry, {@link DataWriterFactory#createWriter(int, long)} will receive a
- * different `taskId`. Spark will call {@link BatchWriteSupport#abort(WriterCommitMessage[])}
+ * In each retry, {@link DataWriterFactory#createDataWriter(int, long, long)} will receive a
+ * different `taskId`. Spark will call {@link DataSourceWriter#abort(WriterCommitMessage[])}
* when the configured number of retries is exhausted.
*
* Besides the retry mechanism, Spark may launch speculative tasks if the existing writing task
@@ -71,11 +71,11 @@ public interface DataWriter {
/**
* Commits this writer after all records are written successfully, returns a commit message which
* will be sent back to driver side and passed to
- * {@link BatchWriteSupport#commit(WriterCommitMessage[])}.
+ * {@link DataSourceWriter#commit(WriterCommitMessage[])}.
*
* The written data should only be visible to data source readers after
- * {@link BatchWriteSupport#commit(WriterCommitMessage[])} succeeds, which means this method
- * should still "hide" the written data and ask the {@link BatchWriteSupport} at driver side to
+ * {@link DataSourceWriter#commit(WriterCommitMessage[])} succeeds, which means this method
+ * should still "hide" the written data and ask the {@link DataSourceWriter} at driver side to
* do the final commit via {@link WriterCommitMessage}.
*
* If this method fails (by throwing an exception), {@link #abort()} will be called and this
@@ -93,7 +93,7 @@ public interface DataWriter {
* failed.
*
* If this method fails(by throwing an exception), the underlying data source may have garbage
- * that need to be cleaned by {@link BatchWriteSupport#abort(WriterCommitMessage[])} or manually,
+ * that need to be cleaned by {@link DataSourceWriter#abort(WriterCommitMessage[])} or manually,
* but these garbage should not be visible to data source readers.
*
* @throws IOException if failure happens during disk/network IO like writing files.
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/DataWriterFactory.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/DataWriterFactory.java
index 19a36dd232456..3d337b6e0bdfd 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/DataWriterFactory.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/DataWriterFactory.java
@@ -19,20 +19,18 @@
import java.io.Serializable;
-import org.apache.spark.TaskContext;
import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.catalyst.InternalRow;
/**
- * A factory of {@link DataWriter} returned by {@link BatchWriteSupport#createBatchWriterFactory()},
+ * A factory of {@link DataWriter} returned by {@link DataSourceWriter#createWriterFactory()},
* which is responsible for creating and initializing the actual data writer at executor side.
*
* Note that, the writer factory will be serialized and sent to executors, then the data writer
- * will be created on executors and do the actual writing. So this interface must be
+ * will be created on executors and do the actual writing. So {@link DataWriterFactory} must be
* serializable and {@link DataWriter} doesn't need to be.
*/
@InterfaceStability.Evolving
-public interface DataWriterFactory extends Serializable {
+public interface DataWriterFactory extends Serializable {
/**
* Returns a data writer to do the actual writing work. Note that, Spark will reuse the same data
@@ -40,16 +38,19 @@ public interface DataWriterFactory extends Serializable {
* are responsible for defensive copies if necessary, e.g. copy the data before buffer it in a
* list.
*
- * If this method fails (by throwing an exception), the corresponding Spark write task would fail
- * and get retried until hitting the maximum retry times.
+ * If this method fails (by throwing an exception), the action will fail and no Spark job will be
+ * submitted.
*
* @param partitionId A unique id of the RDD partition that the returned writer will process.
* Usually Spark processes many RDD partitions at the same time,
* implementations should use the partition id to distinguish writers for
* different partitions.
- * @param taskId The task id returned by {@link TaskContext#taskAttemptId()}. Spark may run
- * multiple tasks for the same partition (due to speculation or task failures,
- * for example).
+ * @param taskId A unique identifier for a task that is performing the write of the partition
+ * data. Spark may run multiple tasks for the same partition (due to speculation
+ * or task failures, for example).
+ * @param epochId A monotonically increasing id for streaming queries that are split in to
+ * discrete periods of execution. For non-streaming queries,
+ * this ID will always be 0.
*/
- DataWriter createWriter(int partitionId, long taskId);
+ DataWriter createDataWriter(int partitionId, long taskId, long epochId);
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/WriterCommitMessage.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/WriterCommitMessage.java
index 123335c414e9f..9e38836c0edf9 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/WriterCommitMessage.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/WriterCommitMessage.java
@@ -19,16 +19,15 @@
import java.io.Serializable;
-import org.apache.spark.sql.sources.v2.writer.streaming.StreamingWriteSupport;
import org.apache.spark.annotation.InterfaceStability;
/**
* A commit message returned by {@link DataWriter#commit()} and will be sent back to the driver side
- * as the input parameter of {@link BatchWriteSupport#commit(WriterCommitMessage[])} or
- * {@link StreamingWriteSupport#commit(long, WriterCommitMessage[])}.
+ * as the input parameter of {@link DataSourceWriter#commit(WriterCommitMessage[])}.
*
- * This is an empty interface, data sources should define their own message class and use it when
- * generating messages at executor side and handling the messages at driver side.
+ * This is an empty interface, data sources should define their own message class and use it in
+ * their {@link DataWriter#commit()} and {@link DataSourceWriter#commit(WriterCommitMessage[])}
+ * implementations.
*/
@InterfaceStability.Evolving
public interface WriterCommitMessage extends Serializable {}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/streaming/StreamingWriteSupport.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/streaming/StreamWriter.java
similarity index 78%
rename from sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/streaming/StreamingWriteSupport.java
rename to sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/streaming/StreamWriter.java
index 3fdfac5e1c84a..a316b2a4c1d82 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/streaming/StreamingWriteSupport.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/streaming/StreamWriter.java
@@ -18,36 +18,27 @@
package org.apache.spark.sql.sources.v2.writer.streaming;
import org.apache.spark.annotation.InterfaceStability;
+import org.apache.spark.sql.sources.v2.writer.DataSourceWriter;
import org.apache.spark.sql.sources.v2.writer.DataWriter;
import org.apache.spark.sql.sources.v2.writer.WriterCommitMessage;
/**
- * An interface that defines how to write the data to data source for streaming processing.
+ * A {@link DataSourceWriter} for use with structured streaming.
*
* Streaming queries are divided into intervals of data called epochs, with a monotonically
* increasing numeric ID. This writer handles commits and aborts for each successive epoch.
*/
@InterfaceStability.Evolving
-public interface StreamingWriteSupport {
-
- /**
- * Creates a writer factory which will be serialized and sent to executors.
- *
- * If this method fails (by throwing an exception), the action will fail and no Spark job will be
- * submitted.
- */
- StreamingDataWriterFactory createStreamingWriterFactory();
-
+public interface StreamWriter extends DataSourceWriter {
/**
* Commits this writing job for the specified epoch with a list of commit messages. The commit
* messages are collected from successful data writers and are produced by
* {@link DataWriter#commit()}.
*
* If this method fails (by throwing an exception), this writing job is considered to have been
- * failed, and the execution engine will attempt to call
- * {@link #abort(long, WriterCommitMessage[])}.
+ * failed, and the execution engine will attempt to call {@link #abort(WriterCommitMessage[])}.
*
- * The execution engine may call `commit` multiple times for the same epoch in some circumstances.
+ * The execution engine may call commit() multiple times for the same epoch in some circumstances.
* To support exactly-once data semantics, implementations must ensure that multiple commits for
* the same epoch are idempotent.
*/
@@ -55,8 +46,7 @@ public interface StreamingWriteSupport {
/**
* Aborts this writing job because some data writers are failed and keep failing when retried, or
- * the Spark job fails with some unknown reasons, or {@link #commit(long, WriterCommitMessage[])}
- * fails.
+ * the Spark job fails with some unknown reasons, or {@link #commit(WriterCommitMessage[])} fails.
*
* If this method fails (by throwing an exception), the underlying data source may require manual
* cleanup.
@@ -68,4 +58,14 @@ public interface StreamingWriteSupport {
* clean up the data left by data writers.
*/
void abort(long epochId, WriterCommitMessage[] messages);
+
+ default void commit(WriterCommitMessage[] messages) {
+ throw new UnsupportedOperationException(
+ "Commit without epoch should not be called with StreamWriter");
+ }
+
+ default void abort(WriterCommitMessage[] messages) {
+ throw new UnsupportedOperationException(
+ "Abort without epoch should not be called with StreamWriter");
+ }
}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/streaming/StreamingDataWriterFactory.java b/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/streaming/StreamingDataWriterFactory.java
deleted file mode 100644
index a4da24fc5ae68..0000000000000
--- a/sql/core/src/main/java/org/apache/spark/sql/sources/v2/writer/streaming/StreamingDataWriterFactory.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.sources.v2.writer.streaming;
-
-import java.io.Serializable;
-
-import org.apache.spark.TaskContext;
-import org.apache.spark.annotation.InterfaceStability;
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.sources.v2.writer.DataWriter;
-
-/**
- * A factory of {@link DataWriter} returned by
- * {@link StreamingWriteSupport#createStreamingWriterFactory()}, which is responsible for creating
- * and initializing the actual data writer at executor side.
- *
- * Note that, the writer factory will be serialized and sent to executors, then the data writer
- * will be created on executors and do the actual writing. So this interface must be
- * serializable and {@link DataWriter} doesn't need to be.
- */
-@InterfaceStability.Evolving
-public interface StreamingDataWriterFactory extends Serializable {
-
- /**
- * Returns a data writer to do the actual writing work. Note that, Spark will reuse the same data
- * object instance when sending data to the data writer, for better performance. Data writers
- * are responsible for defensive copies if necessary, e.g. copy the data before buffer it in a
- * list.
- *
- * If this method fails (by throwing an exception), the corresponding Spark write task would fail
- * and get retried until hitting the maximum retry times.
- *
- * @param partitionId A unique id of the RDD partition that the returned writer will process.
- * Usually Spark processes many RDD partitions at the same time,
- * implementations should use the partition id to distinguish writers for
- * different partitions.
- * @param taskId The task id returned by {@link TaskContext#taskAttemptId()}. Spark may run
- * multiple tasks for the same partition (due to speculation or task failures,
- * for example).
- * @param epochId A monotonically increasing id for streaming queries that are split in to
- * discrete periods of execution.
- */
- DataWriter createWriter(int partitionId, long taskId, long epochId);
-}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
index 1c9beda404356..5f58b031f6aef 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
@@ -25,7 +25,6 @@
import org.apache.spark.annotation.InterfaceStability;
import org.apache.spark.sql.execution.arrow.ArrowUtils;
import org.apache.spark.sql.types.*;
-import org.apache.spark.unsafe.memory.OffHeapMemoryBlock;
import org.apache.spark.unsafe.types.UTF8String;
/**
@@ -378,10 +377,9 @@ final UTF8String getUTF8String(int rowId) {
if (stringResult.isSet == 0) {
return null;
} else {
- return new UTF8String(new OffHeapMemoryBlock(
+ return UTF8String.fromAddress(null,
stringResult.buffer.memoryAddress() + stringResult.start,
- stringResult.end - stringResult.start
- ));
+ stringResult.end - stringResult.start);
}
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index ae27690f2e5ba..a046127c3edb4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -199,13 +199,13 @@ class Column(val expr: Expression) extends Logging {
/**
* Extracts a value or values from a complex type.
* The following types of extraction are supported:
- *
- * - Given an Array, an integer ordinal can be used to retrieve a single value.
- * - Given a Map, a key of the correct type can be used to retrieve an individual value.
- * - Given a Struct, a string fieldName can be used to extract that field.
- * - Given an Array of Structs, a string fieldName can be used to extract filed
- * of every struct in that array, and return an Array of fields
- *
+ *
+ * Given an Array, an integer ordinal can be used to retrieve a single value.
+ * Given a Map, a key of the correct type can be used to retrieve an individual value.
+ * Given a Struct, a string fieldName can be used to extract that field.
+ * Given an Array of Structs, a string fieldName can be used to extract filed
+ * of every struct in that array, and return an Array of fields.
+ *
* @group expr_ops
* @since 1.4.0
*/
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
index 5288907b7d7ff..78df89d491ac1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
@@ -455,7 +455,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
val keyExpr = df.col(col.name).expr
def buildExpr(v: Any) = Cast(Literal(v), keyExpr.dataType)
val branches = replacementMap.flatMap { case (source, target) =>
- Seq(buildExpr(source), buildExpr(target))
+ Seq(Literal(source), buildExpr(target))
}.toSeq
new Column(CaseKeyWhen(keyExpr, branches :+ keyExpr)).as(col.name)
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index 0cfcc45fb3d31..c71f871c1035d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -37,7 +37,7 @@ import org.apache.spark.sql.execution.datasources.jdbc._
import org.apache.spark.sql.execution.datasources.json.TextInputJsonDataSource
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Utils
-import org.apache.spark.sql.sources.v2.{BatchReadSupportProvider, DataSourceOptions, DataSourceV2}
+import org.apache.spark.sql.sources.v2.{DataSourceOptions, DataSourceV2, ReadSupport}
import org.apache.spark.sql.types.{StringType, StructType}
import org.apache.spark.unsafe.types.UTF8String
@@ -194,7 +194,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
val cls = DataSource.lookupDataSource(source, sparkSession.sessionState.conf)
if (classOf[DataSourceV2].isAssignableFrom(cls)) {
val ds = cls.newInstance().asInstanceOf[DataSourceV2]
- if (ds.isInstanceOf[BatchReadSupportProvider]) {
+ if (ds.isInstanceOf[ReadSupport]) {
val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
ds = ds, conf = sparkSession.sessionState.conf)
val pathsOption = {
@@ -202,7 +202,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
DataSourceOptions.PATHS_KEY -> objectMapper.writeValueAsString(paths.toArray)
}
Dataset.ofRows(sparkSession, DataSourceV2Relation.create(
- ds, extraOptions.toMap ++ sessionOptions + pathsOption,
+ ds, sessionOptions ++ extraOptions.toMap + pathsOption,
userSpecifiedSchema = userSpecifiedSchema))
} else {
loadV1Source(paths: _*)
@@ -248,7 +248,8 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
*
* @param url JDBC database url of the form `jdbc:subprotocol:subname`.
* @param table Name of the table in the external database.
- * @param columnName the name of a column of integral type that will be used for partitioning.
+ * @param columnName the name of a column of numeric, date, or timestamp type
+ * that will be used for partitioning.
* @param lowerBound the minimum value of `columnName` used to decide partition stride.
* @param upperBound the maximum value of `columnName` used to decide partition stride.
* @param numPartitions the number of partitions. This, along with `lowerBound` (inclusive),
@@ -450,8 +451,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
input => rawParser.parse(input, createParser, UTF8String.fromString),
parsedOptions.parseMode,
schema,
- parsedOptions.columnNameOfCorruptRecord,
- parsedOptions.multiLine)
+ parsedOptions.columnNameOfCorruptRecord)
iter.flatMap(parser.parse)
}
sparkSession.internalCreateDataFrame(parsed, schema, isStreaming = jsonDataset.isStreaming)
@@ -505,7 +505,8 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
val actualSchema =
StructType(schema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord))
- val linesWithoutHeader: RDD[String] = maybeFirstLine.map { firstLine =>
+ val linesWithoutHeader = if (parsedOptions.headerFlag && maybeFirstLine.isDefined) {
+ val firstLine = maybeFirstLine.get
val parser = new CsvParser(parsedOptions.asParserSettings)
val columnNames = parser.parseLine(firstLine)
CSVDataSource.checkHeaderColumnNames(
@@ -515,7 +516,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
parsedOptions.enforceSchema,
sparkSession.sessionState.conf.caseSensitiveAnalysis)
filteredLines.rdd.mapPartitions(CSVUtils.filterHeaderLine(_, firstLine, parsedOptions))
- }.getOrElse(filteredLines.rdd)
+ } else {
+ filteredLines.rdd
+ }
val parsed = linesWithoutHeader.mapPartitions { iter =>
val rawParser = new UnivocityParser(actualSchema, parsedOptions)
@@ -523,8 +526,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
input => Seq(rawParser.parse(input)),
parsedOptions.parseMode,
schema,
- parsedOptions.columnNameOfCorruptRecord,
- parsedOptions.multiLine)
+ parsedOptions.columnNameOfCorruptRecord)
iter.flatMap(parser.parse)
}
sparkSession.internalCreateDataFrame(parsed, schema, isStreaming = csvDataset.isStreaming)
@@ -571,6 +573,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
* whitespaces from values being read should be skipped.
* `nullValue` (default empty string): sets the string representation of a null value. Since
* 2.0.1, this applies to all supported types including the string type.
+ * `emptyValue` (default empty string): sets the string representation of an empty value.
* `nanValue` (default `NaN`): sets the string representation of a non-number" value.
* `positiveInf` (default `Inf`): sets the string representation of a positive infinity
* value.
@@ -587,7 +590,10 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
* `maxCharsPerColumn` (default `-1`): defines the maximum number of characters allowed
* for any given value being read. By default, it is -1 meaning unlimited length
* `mode` (default `PERMISSIVE`): allows a mode for dealing with corrupt records
- * during parsing. It supports the following case-insensitive modes.
+ * during parsing. It supports the following case-insensitive modes. Note that Spark tries
+ * to parse only required columns in CSV under column pruning. Therefore, corrupt records
+ * can be different based on required set of fields. This behavior can be controlled by
+ * `spark.sql.csv.parser.columnPruning.enabled` (enabled by default).
*
* `PERMISSIVE` : when it meets a corrupted record, puts the malformed string into a
* field configured by `columnNameOfCorruptRecord`, and sets other fields to `null`. To keep
@@ -644,7 +650,6 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
*
* @param path input path
* @since 1.5.0
- * @note Currently, this method can only be used after enabling Hive support.
*/
def orc(path: String): DataFrame = {
// This method ensures that calls that explicit need single argument works, see SPARK-16009
@@ -656,7 +661,6 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
*
* @param paths input paths
* @since 2.0.0
- * @note Currently, this method can only be used after enabling Hive support.
*/
@scala.annotation.varargs
def orc(paths: String*): DataFrame = format("orc").load(paths: _*)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index eca2d5b971905..f90d3532995f3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -28,8 +28,9 @@ import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.catalyst.plans.logical.{AppendData, InsertIntoTable, LogicalPlan}
import org.apache.spark.sql.execution.SQLExecution
import org.apache.spark.sql.execution.command.DDLUtils
-import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource, LogicalRelation}
+import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource, DataSourceUtils, LogicalRelation}
import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, DataSourceV2Utils, WriteToDataSourceV2}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.BaseRelation
import org.apache.spark.sql.sources.v2._
import org.apache.spark.sql.types.StructType
@@ -47,10 +48,12 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
/**
* Specifies the behavior when data or table already exists. Options include:
- * - `SaveMode.Overwrite`: overwrite the existing data.
- * - `SaveMode.Append`: append the data.
- * - `SaveMode.Ignore`: ignore the operation (i.e. no-op).
- * - `SaveMode.ErrorIfExists`: default option, throw an exception at runtime.
+ *
+ * `SaveMode.Overwrite`: overwrite the existing data.
+ * `SaveMode.Append`: append the data.
+ * `SaveMode.Ignore`: ignore the operation (i.e. no-op).
+ * `SaveMode.ErrorIfExists`: default option, throw an exception at runtime.
+ *
*
* @since 1.4.0
*/
@@ -61,10 +64,12 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
/**
* Specifies the behavior when data or table already exists. Options include:
- * - `overwrite`: overwrite the existing data.
- * - `append`: append the data.
- * - `ignore`: ignore the operation (i.e. no-op).
- * - `error` or `errorifexists`: default option, throw an exception at runtime.
+ *
+ * `overwrite`: overwrite the existing data.
+ * `append`: append the data.
+ * `ignore`: ignore the operation (i.e. no-op).
+ * `error` or `errorifexists`: default option, throw an exception at runtime.
+ *
*
* @since 1.4.0
*/
@@ -163,9 +168,10 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
* Partitions the output by the given columns on the file system. If specified, the output is
* laid out on the file system similar to Hive's partitioning scheme. As an example, when we
* partition a dataset by year and then month, the directory layout would look like:
- *
- * - year=2016/month=01/
- * - year=2016/month=02/
+ *
+ * year=2016/month=01/
+ * year=2016/month=02/
+ *
*
* Partitioning is one of the most widely used techniques to optimize physical data layout.
* It provides a coarse-grained index for skipping unnecessary data reads when queries have
@@ -240,27 +246,19 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
if (classOf[DataSourceV2].isAssignableFrom(cls)) {
val source = cls.newInstance().asInstanceOf[DataSourceV2]
source match {
- case provider: BatchWriteSupportProvider =>
- val options = extraOptions ++
- DataSourceV2Utils.extractSessionConfigs(source, df.sparkSession.sessionState.conf)
+ case ws: WriteSupport =>
+ val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
+ source,
+ df.sparkSession.sessionState.conf)
+ val options = sessionOptions ++ extraOptions
- val relation = DataSourceV2Relation.create(source, options.toMap)
- if (mode == SaveMode.Append) {
- runCommand(df.sparkSession, "save") {
- AppendData.byName(relation, df.logicalPlan)
- }
+ val writer = ws.createWriter(
+ UUID.randomUUID.toString, df.logicalPlan.output.toStructType, mode,
+ new DataSourceOptions(options.asJava))
- } else {
- val writer = provider.createBatchWriteSupport(
- UUID.randomUUID().toString,
- df.logicalPlan.output.toStructType,
- mode,
- new DataSourceOptions(options.asJava))
-
- if (writer.isPresent) {
- runCommand(df.sparkSession, "save") {
- WriteToDataSourceV2(writer.get, df.logicalPlan)
- }
+ if (writer.isPresent) {
+ runCommand(df.sparkSession, "save") {
+ WriteToDataSourceV2(writer.get, df.logicalPlan)
}
}
@@ -275,6 +273,14 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
}
private def saveToV1Source(): Unit = {
+ if (SparkSession.active.sessionState.conf.getConf(
+ SQLConf.LEGACY_PASS_PARTITION_BY_AS_OPTIONS)) {
+ partitioningColumns.foreach { columns =>
+ extraOptions += (DataSourceUtils.PARTITIONING_COLUMNS_KEY ->
+ DataSourceUtils.encodePartitioningColumns(columns))
+ }
+ }
+
// Code path for data source v1.
runCommand(df.sparkSession, "save") {
DataSource(
@@ -577,7 +583,6 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
*
*
* @since 1.5.0
- * @note Currently, this method can only be used after enabling Hive support
*/
def orc(path: String): Unit = {
format("orc").save(path)
@@ -635,6 +640,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
* enclosed in quotes. Default is to only escape values containing a quote character.
* `header` (default `false`): writes the names of columns as the first line.
* `nullValue` (default empty string): sets the string representation of a null value.
+ * `emptyValue` (default `""`): sets the string representation of an empty value.
* `encoding` (by default it is not set): specifies encoding (charset) of saved csv
* files. If it is not set, the UTF-8 charset will be used.
* `compression` (default `null`): compression codec to use when saving to file. This can be
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index db439b1ee76f1..c90b2e857e664 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql
import java.io.CharArrayWriter
-import java.sql.{Date, Timestamp}
import scala.collection.JavaConverters._
import scala.language.implicitConversions
@@ -46,7 +45,6 @@ import org.apache.spark.sql.catalyst.parser.{ParseException, ParserUtils}
import org.apache.spark.sql.catalyst.plans._
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, PartitioningCollection}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.arrow.{ArrowBatchStreamWriter, ArrowConverters}
import org.apache.spark.sql.execution.command._
@@ -57,6 +55,7 @@ import org.apache.spark.sql.streaming.DataStreamWriter
import org.apache.spark.sql.types._
import org.apache.spark.sql.util.SchemaUtils
import org.apache.spark.storage.StorageLevel
+import org.apache.spark.unsafe.array.ByteArrayMethods
import org.apache.spark.unsafe.types.CalendarInterval
import org.apache.spark.util.Utils
@@ -287,7 +286,7 @@ class Dataset[T] private[sql](
_numRows: Int,
truncate: Int = 20,
vertical: Boolean = false): String = {
- val numRows = _numRows.max(0).min(Int.MaxValue - 1)
+ val numRows = _numRows.max(0).min(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH - 1)
// Get rows represented by Seq[Seq[String]], we may get one more line if it has more data.
val tmpRows = getRows(numRows, truncate)
@@ -306,16 +305,16 @@ class Dataset[T] private[sql](
// Compute the width of each column
for (row <- rows) {
for ((cell, i) <- row.zipWithIndex) {
- colWidths(i) = math.max(colWidths(i), cell.length)
+ colWidths(i) = math.max(colWidths(i), Utils.stringHalfWidth(cell))
}
}
val paddedRows = rows.map { row =>
row.zipWithIndex.map { case (cell, i) =>
if (truncate > 0) {
- StringUtils.leftPad(cell, colWidths(i))
+ StringUtils.leftPad(cell, colWidths(i) - Utils.stringHalfWidth(cell) + cell.length)
} else {
- StringUtils.rightPad(cell, colWidths(i))
+ StringUtils.rightPad(cell, colWidths(i) - Utils.stringHalfWidth(cell) + cell.length)
}
}
}
@@ -337,12 +336,10 @@ class Dataset[T] private[sql](
// Compute the width of field name and data columns
val fieldNameColWidth = fieldNames.foldLeft(minimumColWidth) { case (curMax, fieldName) =>
- math.max(curMax, fieldName.length)
+ math.max(curMax, Utils.stringHalfWidth(fieldName))
}
val dataColWidth = dataRows.foldLeft(minimumColWidth) { case (curMax, row) =>
- math.max(curMax, row.map(_.length).reduceLeftOption[Int] { case (cellMax, cell) =>
- math.max(cellMax, cell)
- }.getOrElse(0))
+ math.max(curMax, row.map(cell => Utils.stringHalfWidth(cell)).max)
}
dataRows.zipWithIndex.foreach { case (row, i) =>
@@ -351,8 +348,10 @@ class Dataset[T] private[sql](
s"-RECORD $i", fieldNameColWidth + dataColWidth + 5, "-")
sb.append(rowHeader).append("\n")
row.zipWithIndex.map { case (cell, j) =>
- val fieldName = StringUtils.rightPad(fieldNames(j), fieldNameColWidth)
- val data = StringUtils.rightPad(cell, dataColWidth)
+ val fieldName = StringUtils.rightPad(fieldNames(j),
+ fieldNameColWidth - Utils.stringHalfWidth(fieldNames(j)) + fieldNames(j).length)
+ val data = StringUtils.rightPad(cell,
+ dataColWidth - Utils.stringHalfWidth(cell) + cell.length)
s" $fieldName | $data "
}.addString(sb, "", "\n", "\n")
}
@@ -677,8 +676,14 @@ class Dataset[T] private[sql](
// defined on a derived column cannot referenced elsewhere in the plan.
def withWatermark(eventTime: String, delayThreshold: String): Dataset[T] = withTypedPlan {
val parsedDelay =
- Option(CalendarInterval.fromString("interval " + delayThreshold))
- .getOrElse(throw new AnalysisException(s"Unable to parse time delay '$delayThreshold'"))
+ try {
+ CalendarInterval.fromCaseInsensitiveString(delayThreshold)
+ } catch {
+ case e: IllegalArgumentException =>
+ throw new AnalysisException(
+ s"Unable to parse time delay '$delayThreshold'",
+ cause = Some(e))
+ }
require(parsedDelay.milliseconds >= 0 && parsedDelay.months >= 0,
s"delay threshold ($delayThreshold) should not be negative.")
EliminateEventTimeWatermark(
@@ -3264,7 +3269,7 @@ class Dataset[T] private[sql](
_numRows: Int,
truncate: Int): Array[Any] = {
EvaluatePython.registerPicklers()
- val numRows = _numRows.max(0).min(Int.MaxValue - 1)
+ val numRows = _numRows.max(0).min(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH - 1)
val rows = getRows(numRows, truncate).map(_.toArray).toArray
val toJava: (Any) => Any = EvaluatePython.toJava(_, ArrayType(ArrayType(StringType)))
val iter: Iterator[Array[Byte]] = new SerDeUtil.AutoBatchedPickler(
@@ -3279,7 +3284,7 @@ class Dataset[T] private[sql](
val timeZoneId = sparkSession.sessionState.conf.sessionLocalTimeZone
withAction("collectAsArrowToPython", queryExecution) { plan =>
- PythonRDD.serveToStream("serve-Arrow") { out =>
+ PythonRDD.serveToStreamWithSync("serve-Arrow") { out =>
val batchWriter = new ArrowBatchStreamWriter(schema, out, timeZoneId)
val arrowBatchRdd = toArrowBatchRdd(plan)
val numPartitions = arrowBatchRdd.partitions.length
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala
index b21c50af18433..5cf294e3b2d7c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/ForeachWriter.scala
@@ -50,14 +50,11 @@ import org.apache.spark.annotation.InterfaceStability
*
* Important points to note:
*
- * The `partitionId` and `epochId` can be used to deduplicate generated data when failures
- * cause reprocessing of some input data. This depends on the execution mode of the query. If
- * the streaming query is being executed in the micro-batch mode, then every partition
- * represented by a unique tuple (partitionId, epochId) is guaranteed to have the same data.
- * Hence, (partitionId, epochId) can be used to deduplicate and/or transactionally commit data
- * and achieve exactly-once guarantees. However, if the streaming query is being executed in the
- * continuous mode, then this guarantee does not hold and therefore should not be used for
- * deduplication.
+ * Spark doesn't guarantee same output for (partitionId, epochId), so deduplication
+ * cannot be achieved with (partitionId, epochId). e.g. source provides different number of
+ * partitions for some reason, Spark optimization changes number of partitions, etc.
+ * Refer SPARK-28650 for more details. If you need deduplication on output, try out
+ * `foreachBatch` instead.
*
* The `close()` method will be called if `open()` method returns successfully (irrespective
* of the return value), except if the JVM crashes in the middle.
@@ -130,8 +127,11 @@ abstract class ForeachWriter[T] extends Serializable {
* Called when stopping to process one partition of new data in the executor side. This is
* guaranteed to be called either `open` returns `true` or `false`. However,
* `close` won't be called in the following cases:
- * - JVM crashes without throwing a `Throwable`
- * - `open` throws a `Throwable`.
+ *
+ *
+ * JVM crashes without throwing a `Throwable`
+ * `open` throws a `Throwable`.
+ *
*
* @param errorOrNull the error thrown during processing data or null if there was no error.
*/
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
index f99c108161f94..6b02ac2ded8d7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
@@ -30,12 +30,15 @@ import org.apache.spark.sql.catalyst.rules.Rule
* regarding binary compatibility and source compatibility of methods here.
*
* This current provides the following extension points:
- * - Analyzer Rules.
- * - Check Analysis Rules
- * - Optimizer Rules.
- * - Planning Strategies.
- * - Customized Parser.
- * - (External) Catalog listeners.
+ *
+ *
+ * Analyzer Rules.
+ * Check Analysis Rules.
+ * Optimizer Rules.
+ * Planning Strategies.
+ * Customized Parser.
+ * (External) Catalog listeners.
+ *
*
* The extensions can be used by calling withExtension on the [[SparkSession.Builder]], for
* example:
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
index c37ba0c60c3d4..aa3a6c3bf122f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
@@ -113,7 +113,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
(0 to 22).foreach { x =>
val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"})
val typeTags = (1 to x).map(i => s"A$i: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _)
- val inputSchemas = (1 to x).foldRight("Nil")((i, s) => {s"ScalaReflection.schemaFor[A$i] :: $s"})
+ val inputSchemas = (1 to x).foldRight("Nil")((i, s) => {s"Try(ScalaReflection.schemaFor[A$i]).toOption :: $s"})
println(s"""
|/**
| * Registers a deterministic Scala closure of $x arguments as user-defined function (UDF).
@@ -122,10 +122,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
| */
|def register[$typeTags](name: String, func: Function$x[$types]): UserDefinedFunction = {
| val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- | val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try($inputSchemas).toOption
+ | val inputSchemas: Seq[Option[ScalaReflection.Schema]] = $inputSchemas
| def builder(e: Seq[Expression]) = if (e.length == $x) {
- | ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- | udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ | ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ | if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ | Some(name), nullable, udfDeterministic = true)
| } else {
| throw new AnalysisException("Invalid number of arguments for function " + name +
| ". Expected: $x; Found: " + e.length)
@@ -151,7 +152,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
|def register(name: String, f: UDF$i[$extTypeArgs], returnType: DataType): Unit = {
| val func = f$anyCast.call($anyParams)
| def builder(e: Seq[Expression]) = if (e.length == $i) {
- | ScalaUDF($funcCall, returnType, e, udfName = Some(name))
+ | ScalaUDF($funcCall, returnType, e, e.map(_ => true), udfName = Some(name))
| } else {
| throw new AnalysisException("Invalid number of arguments for function " + name +
| ". Expected: $i; Found: " + e.length)
@@ -168,10 +169,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag](name: String, func: Function0[RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Nil
def builder(e: Seq[Expression]) = if (e.length == 0) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 0; Found: " + e.length)
@@ -188,10 +190,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag](name: String, func: Function1[A1, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 1) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 1; Found: " + e.length)
@@ -208,10 +211,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag](name: String, func: Function2[A1, A2, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 2) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 2; Found: " + e.length)
@@ -228,10 +232,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](name: String, func: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 3) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 3; Found: " + e.length)
@@ -248,10 +253,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](name: String, func: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 4) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 4; Found: " + e.length)
@@ -268,10 +274,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](name: String, func: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 5) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 5; Found: " + e.length)
@@ -288,10 +295,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](name: String, func: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 6) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 6; Found: " + e.length)
@@ -308,10 +316,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](name: String, func: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 7) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 7; Found: " + e.length)
@@ -328,10 +337,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](name: String, func: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 8) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 8; Found: " + e.length)
@@ -348,10 +358,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](name: String, func: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 9) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 9; Found: " + e.length)
@@ -368,10 +379,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](name: String, func: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 10) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 10; Found: " + e.length)
@@ -388,10 +400,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](name: String, func: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: ScalaReflection.schemaFor[A11] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 11) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 11; Found: " + e.length)
@@ -408,10 +421,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](name: String, func: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: ScalaReflection.schemaFor[A11] :: ScalaReflection.schemaFor[A12] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 12) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 12; Found: " + e.length)
@@ -428,10 +442,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](name: String, func: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: ScalaReflection.schemaFor[A11] :: ScalaReflection.schemaFor[A12] :: ScalaReflection.schemaFor[A13] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 13) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 13; Found: " + e.length)
@@ -448,10 +463,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](name: String, func: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: ScalaReflection.schemaFor[A11] :: ScalaReflection.schemaFor[A12] :: ScalaReflection.schemaFor[A13] :: ScalaReflection.schemaFor[A14] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 14) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 14; Found: " + e.length)
@@ -468,10 +484,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](name: String, func: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: ScalaReflection.schemaFor[A11] :: ScalaReflection.schemaFor[A12] :: ScalaReflection.schemaFor[A13] :: ScalaReflection.schemaFor[A14] :: ScalaReflection.schemaFor[A15] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 15) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 15; Found: " + e.length)
@@ -488,10 +505,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](name: String, func: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: ScalaReflection.schemaFor[A11] :: ScalaReflection.schemaFor[A12] :: ScalaReflection.schemaFor[A13] :: ScalaReflection.schemaFor[A14] :: ScalaReflection.schemaFor[A15] :: ScalaReflection.schemaFor[A16] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 16) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 16; Found: " + e.length)
@@ -508,10 +526,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](name: String, func: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: ScalaReflection.schemaFor[A11] :: ScalaReflection.schemaFor[A12] :: ScalaReflection.schemaFor[A13] :: ScalaReflection.schemaFor[A14] :: ScalaReflection.schemaFor[A15] :: ScalaReflection.schemaFor[A16] :: ScalaReflection.schemaFor[A17] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 17) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 17; Found: " + e.length)
@@ -528,10 +547,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](name: String, func: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: ScalaReflection.schemaFor[A11] :: ScalaReflection.schemaFor[A12] :: ScalaReflection.schemaFor[A13] :: ScalaReflection.schemaFor[A14] :: ScalaReflection.schemaFor[A15] :: ScalaReflection.schemaFor[A16] :: ScalaReflection.schemaFor[A17] :: ScalaReflection.schemaFor[A18] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 18) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 18; Found: " + e.length)
@@ -548,10 +568,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](name: String, func: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: ScalaReflection.schemaFor[A11] :: ScalaReflection.schemaFor[A12] :: ScalaReflection.schemaFor[A13] :: ScalaReflection.schemaFor[A14] :: ScalaReflection.schemaFor[A15] :: ScalaReflection.schemaFor[A16] :: ScalaReflection.schemaFor[A17] :: ScalaReflection.schemaFor[A18] :: ScalaReflection.schemaFor[A19] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Try(ScalaReflection.schemaFor[A19]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 19) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 19; Found: " + e.length)
@@ -568,10 +589,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](name: String, func: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: ScalaReflection.schemaFor[A11] :: ScalaReflection.schemaFor[A12] :: ScalaReflection.schemaFor[A13] :: ScalaReflection.schemaFor[A14] :: ScalaReflection.schemaFor[A15] :: ScalaReflection.schemaFor[A16] :: ScalaReflection.schemaFor[A17] :: ScalaReflection.schemaFor[A18] :: ScalaReflection.schemaFor[A19] :: ScalaReflection.schemaFor[A20] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Try(ScalaReflection.schemaFor[A19]).toOption :: Try(ScalaReflection.schemaFor[A20]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 20) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 20; Found: " + e.length)
@@ -588,10 +610,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](name: String, func: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: ScalaReflection.schemaFor[A11] :: ScalaReflection.schemaFor[A12] :: ScalaReflection.schemaFor[A13] :: ScalaReflection.schemaFor[A14] :: ScalaReflection.schemaFor[A15] :: ScalaReflection.schemaFor[A16] :: ScalaReflection.schemaFor[A17] :: ScalaReflection.schemaFor[A18] :: ScalaReflection.schemaFor[A19] :: ScalaReflection.schemaFor[A20] :: ScalaReflection.schemaFor[A21] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Try(ScalaReflection.schemaFor[A19]).toOption :: Try(ScalaReflection.schemaFor[A20]).toOption :: Try(ScalaReflection.schemaFor[A21]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 21) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 21; Found: " + e.length)
@@ -608,10 +631,11 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](name: String, func: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas: Option[Seq[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1] :: ScalaReflection.schemaFor[A2] :: ScalaReflection.schemaFor[A3] :: ScalaReflection.schemaFor[A4] :: ScalaReflection.schemaFor[A5] :: ScalaReflection.schemaFor[A6] :: ScalaReflection.schemaFor[A7] :: ScalaReflection.schemaFor[A8] :: ScalaReflection.schemaFor[A9] :: ScalaReflection.schemaFor[A10] :: ScalaReflection.schemaFor[A11] :: ScalaReflection.schemaFor[A12] :: ScalaReflection.schemaFor[A13] :: ScalaReflection.schemaFor[A14] :: ScalaReflection.schemaFor[A15] :: ScalaReflection.schemaFor[A16] :: ScalaReflection.schemaFor[A17] :: ScalaReflection.schemaFor[A18] :: ScalaReflection.schemaFor[A19] :: ScalaReflection.schemaFor[A20] :: ScalaReflection.schemaFor[A21] :: ScalaReflection.schemaFor[A22] :: Nil).toOption
+ val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Try(ScalaReflection.schemaFor[A19]).toOption :: Try(ScalaReflection.schemaFor[A20]).toOption :: Try(ScalaReflection.schemaFor[A21]).toOption :: Try(ScalaReflection.schemaFor[A22]).toOption :: Nil
def builder(e: Seq[Expression]) = if (e.length == 22) {
- ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.dataType)).getOrElse(Nil), Some(name), nullable,
- udfDeterministic = true, nullableTypes = inputSchemas.map(_.map(_.nullable)).getOrElse(Nil))
+ ScalaUDF(func, dataType, e, inputSchemas.map(_.map(_.nullable).getOrElse(true)),
+ if (inputSchemas.contains(None)) Nil else inputSchemas.map(_.get.dataType),
+ Some(name), nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 22; Found: " + e.length)
@@ -719,7 +743,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF0[_], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF0[Any]].call()
def builder(e: Seq[Expression]) = if (e.length == 0) {
- ScalaUDF(() => func, returnType, e, udfName = Some(name))
+ ScalaUDF(() => func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 0; Found: " + e.length)
@@ -734,7 +758,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF1[_, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF1[Any, Any]].call(_: Any)
def builder(e: Seq[Expression]) = if (e.length == 1) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 1; Found: " + e.length)
@@ -749,7 +773,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF2[_, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF2[Any, Any, Any]].call(_: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 2) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 2; Found: " + e.length)
@@ -764,7 +788,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF3[_, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF3[Any, Any, Any, Any]].call(_: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 3) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 3; Found: " + e.length)
@@ -779,7 +803,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF4[_, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF4[Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 4) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 4; Found: " + e.length)
@@ -794,7 +818,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF5[_, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 5) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 5; Found: " + e.length)
@@ -809,7 +833,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF6[_, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 6) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 6; Found: " + e.length)
@@ -824,7 +848,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 7) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 7; Found: " + e.length)
@@ -839,7 +863,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 8) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 8; Found: " + e.length)
@@ -854,7 +878,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 9) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 9; Found: " + e.length)
@@ -869,7 +893,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 10) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 10; Found: " + e.length)
@@ -884,7 +908,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF11[_, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF11[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 11) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 11; Found: " + e.length)
@@ -899,7 +923,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF12[_, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF12[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 12) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 12; Found: " + e.length)
@@ -914,7 +938,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF13[_, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF13[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 13) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 13; Found: " + e.length)
@@ -929,7 +953,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF14[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF14[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 14) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 14; Found: " + e.length)
@@ -944,7 +968,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF15[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 15) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 15; Found: " + e.length)
@@ -959,7 +983,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF16[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 16) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 16; Found: " + e.length)
@@ -974,7 +998,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF17[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 17) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 17; Found: " + e.length)
@@ -989,7 +1013,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF18[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 18) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 18; Found: " + e.length)
@@ -1004,7 +1028,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF19[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 19) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 19; Found: " + e.length)
@@ -1019,7 +1043,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF20[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 20) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 20; Found: " + e.length)
@@ -1034,7 +1058,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF21[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 21) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 21; Found: " + e.length)
@@ -1049,7 +1073,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
def register(name: String, f: UDF22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
val func = f.asInstanceOf[UDF22[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
def builder(e: Seq[Expression]) = if (e.length == 22) {
- ScalaUDF(func, returnType, e, udfName = Some(name))
+ ScalaUDF(func, returnType, e, e.map(_ => true), udfName = Some(name))
} else {
throw new AnalysisException("Invalid number of arguments for function " + name +
". Expected: 22; Found: " + e.length)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
index c0830e77b5a87..482e2bfeb7098 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
@@ -17,6 +17,12 @@
package org.apache.spark.sql.api.python
+import java.io.InputStream
+import java.nio.channels.Channels
+
+import org.apache.spark.api.java.JavaRDD
+import org.apache.spark.api.python.PythonRDDServer
+import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, SQLContext}
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
@@ -33,19 +39,36 @@ private[sql] object PythonSQLUtils {
}
/**
- * Python callable function to read a file in Arrow stream format and create a [[DataFrame]]
+ * Python callable function to read a file in Arrow stream format and create a [[RDD]]
* using each serialized ArrowRecordBatch as a partition.
- *
- * @param sqlContext The active [[SQLContext]].
- * @param filename File to read the Arrow stream from.
- * @param schemaString JSON Formatted Spark schema for Arrow batches.
- * @return A new [[DataFrame]].
*/
- def arrowReadStreamFromFile(
- sqlContext: SQLContext,
- filename: String,
- schemaString: String): DataFrame = {
- val jrdd = ArrowConverters.readArrowStreamFromFile(sqlContext, filename)
- ArrowConverters.toDataFrame(jrdd, schemaString, sqlContext)
+ def readArrowStreamFromFile(sqlContext: SQLContext, filename: String): JavaRDD[Array[Byte]] = {
+ ArrowConverters.readArrowStreamFromFile(sqlContext, filename)
+ }
+
+ /**
+ * Python callable function to read a file in Arrow stream format and create a [[DataFrame]]
+ * from an RDD.
+ */
+ def toDataFrame(
+ arrowBatchRDD: JavaRDD[Array[Byte]],
+ schemaString: String,
+ sqlContext: SQLContext): DataFrame = {
+ ArrowConverters.toDataFrame(arrowBatchRDD, schemaString, sqlContext)
}
}
+
+/**
+ * Helper for making a dataframe from arrow data from data sent from python over a socket. This is
+ * used when encryption is enabled, and we don't want to write data to a file.
+ */
+private[sql] class ArrowRDDServer(sqlContext: SQLContext) extends PythonRDDServer {
+
+ override protected def streamToRDD(input: InputStream): RDD[Array[Byte]] = {
+ // Create array to consume iterator so that we can safely close the inputStream
+ val batches = ArrowConverters.getBatchesFromStream(Channels.newChannel(input)).toArray
+ // Parallelize the record batches to create an RDD
+ JavaRDD.fromRDD(sqlContext.sparkContext.parallelize(batches, batches.length))
+ }
+
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
index af20764f9a968..4c717957c5526 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
@@ -49,9 +49,17 @@ private[sql] object SQLUtils extends Logging {
sparkConfigMap: JMap[Object, Object],
enableHiveSupport: Boolean): SparkSession = {
val spark =
- if (SparkSession.hiveClassesArePresent && enableHiveSupport &&
+ if (enableHiveSupport &&
jsc.sc.conf.get(CATALOG_IMPLEMENTATION.key, "hive").toLowerCase(Locale.ROOT) ==
- "hive") {
+ "hive" &&
+ // Note that the order of conditions here are on purpose.
+ // `SparkSession.hiveClassesArePresent` checks if Hive's `HiveConf` is loadable or not;
+ // however, `HiveConf` itself has some static logic to check if Hadoop version is
+ // supported or not, which throws an `IllegalArgumentException` if unsupported.
+ // If this is checked first, there's no way to disable Hive support in the case above.
+ // So, we intentionally check if Hive classes are loadable or not only when
+ // Hive support is explicitly enabled by short-circuiting. See also SPARK-26422.
+ SparkSession.hiveClassesArePresent) {
SparkSession.builder().sparkContext(withHiveExternalCatalog(jsc.sc)).getOrCreate()
} else {
if (enableHiveSupport) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
index c9929935fb8ac..5b30596529227 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
@@ -166,16 +166,34 @@ class CacheManager extends Logging {
val needToRecache = scala.collection.mutable.ArrayBuffer.empty[CachedData]
while (it.hasNext) {
val cd = it.next()
- if (condition(cd.plan)) {
- if (clearCache) {
- cd.cachedRepresentation.cacheBuilder.clearCache()
- }
+ // If `clearCache` is false (which means the recache request comes from a non-cascading
+ // cache invalidation) and the cache buffer has already been loaded, we do not need to
+ // re-compile a physical plan because the old plan will not be used any more by the
+ // CacheManager although it still lives in compiled `Dataset`s and it could still work.
+ // Otherwise, it means either `clearCache` is true, then we have to clear the cache buffer
+ // and re-compile the physical plan; or it is a non-cascading cache invalidation and cache
+ // buffer is still empty, then we could have a more efficient new plan by removing
+ // dependency on the previously removed cache entries.
+ // Note that the `CachedRDDBuilder`.`isCachedColumnBuffersLoaded` call is a non-locking
+ // status test and may not return the most accurate cache buffer state. So the worse case
+ // scenario can be:
+ // 1) The buffer has been loaded, but `isCachedColumnBuffersLoaded` returns false, then we
+ // will clear the buffer and build a new plan. It is inefficient but doesn't affect
+ // correctness.
+ // 2) The buffer has been cleared, but `isCachedColumnBuffersLoaded` returns true, then we
+ // will keep it as it is. It means the physical plan has been re-compiled already in the
+ // other thread.
+ val buildNewPlan =
+ clearCache || !cd.cachedRepresentation.cacheBuilder.isCachedColumnBuffersLoaded
+ if (condition(cd.plan) && buildNewPlan) {
+ cd.cachedRepresentation.cacheBuilder.clearCache()
// Remove the cache entry before we create a new one, so that we can have a different
// physical plan.
it.remove()
val plan = spark.sessionState.executePlan(cd.plan).executedPlan
val newCache = InMemoryRelation(
- cacheBuilder = cd.cachedRepresentation.cacheBuilder.withCachedPlan(plan),
+ cacheBuilder = cd.cachedRepresentation
+ .cacheBuilder.copy(cachedPlan = plan)(_cachedColumnBuffers = null),
logicalPlan = cd.plan)
needToRecache += cd.copy(cachedRepresentation = newCache)
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 36ed016773b67..d22fe64024bb2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -185,21 +185,16 @@ case class FileSourceScanExec(
partitionSchema = relation.partitionSchema,
relation.sparkSession.sessionState.conf)
- @transient private lazy val selectedPartitions: Seq[PartitionDirectory] = {
+ private var metadataTime = 0L
+
+ @transient private lazy val selectedPartitions: Array[PartitionDirectory] = {
val optimizerMetadataTimeNs = relation.location.metadataOpsTimeNs.getOrElse(0L)
val startTime = System.nanoTime()
val ret = relation.location.listFiles(partitionFilters, dataFilters)
val timeTakenMs = ((System.nanoTime() - startTime) + optimizerMetadataTimeNs) / 1000 / 1000
-
- metrics("numFiles").add(ret.map(_.files.size.toLong).sum)
- metrics("metadataTime").add(timeTakenMs)
-
- val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
- SQLMetrics.postDriverMetricUpdates(sparkContext, executionId,
- metrics("numFiles") :: metrics("metadataTime") :: Nil)
-
+ metadataTime = timeTakenMs
ret
- }
+ }.toArray
override lazy val (outputPartitioning, outputOrdering): (Partitioning, Seq[SortOrder]) = {
val bucketSpec = if (relation.sparkSession.sessionState.conf.bucketingEnabled) {
@@ -308,6 +303,8 @@ case class FileSourceScanExec(
}
private lazy val inputRDD: RDD[InternalRow] = {
+ // Update metrics for taking effect in both code generation node and normal node.
+ updateDriverMetrics()
val readFile: (PartitionedFile) => Iterator[InternalRow] =
relation.fileFormat.buildReaderWithPartitionValues(
sparkSession = relation.sparkSession,
@@ -380,7 +377,7 @@ case class FileSourceScanExec(
private def createBucketedReadRDD(
bucketSpec: BucketSpec,
readFile: (PartitionedFile) => Iterator[InternalRow],
- selectedPartitions: Seq[PartitionDirectory],
+ selectedPartitions: Array[PartitionDirectory],
fsRelation: HadoopFsRelation): RDD[InternalRow] = {
logInfo(s"Planning with ${bucketSpec.numBuckets} buckets")
val filesGroupedToBuckets =
@@ -405,7 +402,7 @@ case class FileSourceScanExec(
}
val filePartitions = Seq.tabulate(bucketSpec.numBuckets) { bucketId =>
- FilePartition(bucketId, prunedFilesGroupedToBuckets.getOrElse(bucketId, Nil))
+ FilePartition(bucketId, prunedFilesGroupedToBuckets.getOrElse(bucketId, Array.empty))
}
new FileScanRDD(fsRelation.sparkSession, readFile, filePartitions)
@@ -421,7 +418,7 @@ case class FileSourceScanExec(
*/
private def createNonBucketedReadRDD(
readFile: (PartitionedFile) => Iterator[InternalRow],
- selectedPartitions: Seq[PartitionDirectory],
+ selectedPartitions: Array[PartitionDirectory],
fsRelation: HadoopFsRelation): RDD[InternalRow] = {
val defaultMaxSplitBytes =
fsRelation.sparkSession.sessionState.conf.filesMaxPartitionBytes
@@ -452,7 +449,7 @@ case class FileSourceScanExec(
partition.values, file.getPath.toUri.toString, 0, file.getLen, hosts))
}
}
- }.toArray.sortBy(_.length)(implicitly[Ordering[Long]].reverse)
+ }.sortBy(_.length)(implicitly[Ordering[Long]].reverse)
val partitions = new ArrayBuffer[FilePartition]
val currentFiles = new ArrayBuffer[PartitionedFile]
@@ -464,7 +461,7 @@ case class FileSourceScanExec(
val newPartition =
FilePartition(
partitions.size,
- currentFiles.toArray.toSeq) // Copy to a new Array.
+ currentFiles.toArray) // Copy to a new Array.
partitions += newPartition
}
currentFiles.clear()
@@ -524,6 +521,19 @@ case class FileSourceScanExec(
}
}
+ /**
+ * Send the updated metrics to driver, while this function calling, selectedPartitions has
+ * been initialized. See SPARK-26327 for more detail.
+ */
+ private def updateDriverMetrics() = {
+ metrics("numFiles").add(selectedPartitions.map(_.files.size.toLong).sum)
+ metrics("metadataTime").add(metadataTime)
+
+ val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
+ SQLMetrics.postDriverMetricUpdates(sparkContext, executionId,
+ metrics("numFiles") :: metrics("metadataTime") :: Nil)
+ }
+
override def doCanonicalize(): FileSourceScanExec = {
FileSourceScanExec(
relation,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
index 3ca03ab2939aa..45e5f415e8da1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
@@ -72,6 +72,11 @@ case class OptimizeMetadataOnlyQuery(catalog: SessionCatalog) extends Rule[Logic
})
}
if (isAllDistinctAgg) {
+ logWarning("Since configuration `spark.sql.optimizer.metadataOnly` is enabled, " +
+ "Spark will scan partition-level metadata without scanning data files. " +
+ "This could result in wrong results when the partition metadata exists but the " +
+ "inclusive data files are empty."
+ )
a.withNewChildren(Seq(replaceTableScanWithPartitionMetadata(child, rel, filters)))
} else {
a
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 64f49e2d0d4e6..37353b873227f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.execution.command.{DescribeTableCommand, ExecutedCommandExec, ShowTablesCommand}
import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ReuseExchange}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{BinaryType, DateType, DecimalType, TimestampType, _}
import org.apache.spark.util.Utils
@@ -77,7 +78,13 @@ class QueryExecution(val sparkSession: SparkSession, val logical: LogicalPlan) {
lazy val executedPlan: SparkPlan = prepareForExecution(sparkPlan)
/** Internal version of the RDD. Avoids copies and has no schema */
- lazy val toRdd: RDD[InternalRow] = executedPlan.execute()
+ lazy val toRdd: RDD[InternalRow] = {
+ if (sparkSession.sessionState.conf.getConf(SQLConf.USE_CONF_ON_RDD_OPERATION)) {
+ new SQLExecutionRDD(executedPlan.execute(), sparkSession.sessionState.conf)
+ } else {
+ executedPlan.execute()
+ }
+ }
/**
* Prepares a planned [[SparkPlan]] for execution by inserting shuffle operations and internal
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecutionRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecutionRDD.scala
new file mode 100644
index 0000000000000..307e64b6bff7d
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecutionRDD.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution
+
+import org.apache.spark.{Partition, TaskContext}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * It is just a wrapper over `sqlRDD`, which sets and makes effective all the configs from the
+ * captured `SQLConf`.
+ * Please notice that this means we may miss configurations set after the creation of this RDD and
+ * before its execution.
+ *
+ * @param sqlRDD the `RDD` generated by the SQL plan
+ * @param conf the `SQLConf` to apply to the execution of the SQL plan
+ */
+class SQLExecutionRDD(
+ var sqlRDD: RDD[InternalRow], conf: SQLConf) extends RDD[InternalRow](sqlRDD) {
+ private val sqlConfigs = conf.getAllConfs
+ private lazy val sqlConfExecutorSide = {
+ val newConf = new SQLConf()
+ sqlConfigs.foreach { case (k, v) => newConf.setConfString(k, v) }
+ newConf
+ }
+
+ override val partitioner = firstParent[InternalRow].partitioner
+
+ override def getPartitions: Array[Partition] = firstParent[InternalRow].partitions
+
+ override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
+ // If we are in the context of a tracked SQL operation, `SQLExecution.EXECUTION_ID_KEY` is set
+ // and we have nothing to do here. Otherwise, we use the `SQLConf` captured at the creation of
+ // this RDD.
+ if (context.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) == null) {
+ SQLConf.withExistingConf(sqlConfExecutorSide) {
+ firstParent[InternalRow].iterator(split, context)
+ }
+ } else {
+ firstParent[InternalRow].iterator(split, context)
+ }
+ }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
index 0dc16ba5ce281..63ad3630970ba 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
@@ -59,6 +59,14 @@ case class SortExec(
"peakMemory" -> SQLMetrics.createSizeMetric(sparkContext, "peak memory"),
"spillSize" -> SQLMetrics.createSizeMetric(sparkContext, "spill size"))
+ private[sql] var rowSorter: UnsafeExternalRowSorter = _
+
+ /**
+ * This method gets invoked only once for each SortExec instance to initialize an
+ * UnsafeExternalRowSorter, both `plan.execute` and code generation are using it.
+ * In the code generation code path, we need to call this function outside the class so we
+ * should make it public.
+ */
def createSorter(): UnsafeExternalRowSorter = {
val ordering = newOrdering(sortOrder, output)
@@ -84,13 +92,13 @@ case class SortExec(
}
val pageSize = SparkEnv.get.memoryManager.pageSizeBytes
- val sorter = UnsafeExternalRowSorter.create(
+ rowSorter = UnsafeExternalRowSorter.create(
schema, ordering, prefixComparator, prefixComputer, pageSize, canUseRadixSort)
if (testSpillFrequency > 0) {
- sorter.setTestSpillFrequency(testSpillFrequency)
+ rowSorter.setTestSpillFrequency(testSpillFrequency)
}
- sorter
+ rowSorter
}
protected override def doExecute(): RDD[InternalRow] = {
@@ -186,4 +194,17 @@ case class SortExec(
|$sorterVariable.insertRow((UnsafeRow)${row.value});
""".stripMargin
}
+
+ /**
+ * In SortExec, we overwrites cleanupResources to close UnsafeExternalRowSorter.
+ */
+ override protected[sql] def cleanupResources(): Unit = {
+ if (rowSorter != null) {
+ // There's possible for rowSorter is null here, for example, in the scenario of empty
+ // iterator in the current task, the downstream physical node(like SortMergeJoinExec) will
+ // trigger cleanupResources before rowSorter initialized in createSorter.
+ rowSorter.cleanupResources()
+ }
+ super.cleanupResources()
+ }
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
index 1f97993e20458..7646f9613efb3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
@@ -250,7 +250,9 @@ abstract class SparkPlan extends QueryPlan[SparkPlan] with Logging with Serializ
val codec = CompressionCodec.createCodec(SparkEnv.get.conf)
val bos = new ByteArrayOutputStream()
val out = new DataOutputStream(codec.compressedOutputStream(bos))
- while (iter.hasNext && (n < 0 || count < n)) {
+ // `iter.hasNext` may produce one row and buffer it, we should only call it when the limit is
+ // not hit.
+ while ((n < 0 || count < n) && iter.hasNext) {
val row = iter.next().asInstanceOf[UnsafeRow]
out.writeInt(row.getSizeInBytes)
row.writeToStream(out, buffer)
@@ -419,6 +421,15 @@ abstract class SparkPlan extends QueryPlan[SparkPlan] with Logging with Serializ
}
newOrdering(order, Seq.empty)
}
+
+ /**
+ * Cleans up the resources used by the physical operator (if any). In general, all the resources
+ * should be cleaned up when the task finishes but operators like SortMergeJoinExec and LimitExec
+ * may want eager cleanup to free up tight resources (e.g., memory).
+ */
+ protected[sql] def cleanupResources(): Unit = {
+ children.foreach(_.cleanupResources())
+ }
}
object SparkPlan {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
index 2a2315896831c..59ffd16381116 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql.execution
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties
-
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
import org.apache.spark.sql.execution.metric.SQLMetricInfo
@@ -28,11 +26,11 @@ import org.apache.spark.sql.execution.metric.SQLMetricInfo
* Stores information about a SQL SparkPlan.
*/
@DeveloperApi
-@JsonIgnoreProperties(Array("metadata")) // The metadata field was removed in Spark 2.3.
class SparkPlanInfo(
val nodeName: String,
val simpleString: String,
val children: Seq[SparkPlanInfo],
+ val metadata: Map[String, String],
val metrics: Seq[SQLMetricInfo]) {
override def hashCode(): Int = {
@@ -59,6 +57,12 @@ private[execution] object SparkPlanInfo {
new SQLMetricInfo(metric.name.getOrElse(key), metric.id, metric.metricType)
}
- new SparkPlanInfo(plan.nodeName, plan.simpleString, children.map(fromSparkPlan), metrics)
+ // dump the file scan metadata (e.g file path) to event log
+ val metadata = plan match {
+ case fileScan: FileSourceScanExec => fileScan.metadata
+ case _ => Map[String, String]()
+ }
+ new SparkPlanInfo(plan.nodeName, plan.simpleString, children.map(fromSparkPlan),
+ metadata, metrics)
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
index 89cb63784c0f4..95206e8ef1820 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
@@ -39,7 +39,7 @@ import org.apache.spark.sql.types.StructType
/**
* Concrete parser for Spark SQL statements.
*/
-class SparkSqlParser(conf: SQLConf) extends AbstractSqlParser {
+class SparkSqlParser(conf: SQLConf) extends AbstractSqlParser(conf) {
val astBuilder = new SparkSqlAstBuilder(conf)
private val substitutor = new VariableSubstitution(conf)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala
index 1fc4de9e56015..d06380e8657ce 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala
@@ -142,7 +142,7 @@ trait CodegenSupport extends SparkPlan {
* Note that `outputVars` and `row` can't both be null.
*/
final def consume(ctx: CodegenContext, outputVars: Seq[ExprCode], row: String = null): String = {
- val inputVars =
+ val inputVarsCandidate =
if (outputVars != null) {
assert(outputVars.length == output.length)
// outputVars will be used to generate the code for UnsafeRow, so we should copy them
@@ -156,6 +156,11 @@ trait CodegenSupport extends SparkPlan {
}
}
+ val inputVars = inputVarsCandidate match {
+ case stream: Stream[ExprCode] => stream.force
+ case other => other
+ }
+
val rowVar = prepareRowVar(ctx, row, outputVars)
// Set up the `currentVars` in the codegen context, as we generate the code of `inputVars`
@@ -284,6 +289,18 @@ trait CodegenSupport extends SparkPlan {
evaluateVars.toString()
}
+ /**
+ * Returns source code to evaluate the variables for non-deterministic expressions, and clear the
+ * code of evaluated variables, to prevent them to be evaluated twice.
+ */
+ protected def evaluateNondeterministicVariables(
+ attributes: Seq[Attribute],
+ variables: Seq[ExprCode],
+ expressions: Seq[NamedExpression]): String = {
+ val nondeterministicAttrs = expressions.filterNot(_.deterministic).map(_.toAttribute)
+ evaluateRequiredVariables(attributes, variables, AttributeSet(nondeterministicAttrs))
+ }
+
/**
* The subset of inputSet those should be evaluated before this plan.
*
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
index 98adba50b2973..794b0b74a7ea7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
@@ -473,10 +473,13 @@ case class HashAggregateExec(
val resultVars = resultExpressions.map { e =>
BindReferences.bindReference(e, inputAttrs).genCode(ctx)
}
+ val evaluateNondeterministicResults =
+ evaluateNondeterministicVariables(output, resultVars, resultExpressions)
s"""
$evaluateKeyVars
$evaluateBufferVars
$evaluateAggResults
+ $evaluateNondeterministicResults
${consume(ctx, resultVars)}
"""
} else if (modes.contains(Partial) || modes.contains(PartialMerge)) {
@@ -513,10 +516,15 @@ case class HashAggregateExec(
// generate result based on grouping key
ctx.INPUT_ROW = keyTerm
ctx.currentVars = null
- val eval = resultExpressions.map{ e =>
+ val resultVars = resultExpressions.map{ e =>
BindReferences.bindReference(e, groupingAttributes).genCode(ctx)
}
- consume(ctx, eval)
+ val evaluateNondeterministicResults =
+ evaluateNondeterministicVariables(output, resultVars, resultExpressions)
+ s"""
+ $evaluateNondeterministicResults
+ ${consume(ctx, resultVars)}
+ """
}
ctx.addNewFunction(funcName,
s"""
@@ -583,9 +591,9 @@ case class HashAggregateExec(
val thisPlan = ctx.addReferenceObj("plan", this)
- // Create a name for the iterator from the fast hash map.
- val iterTermForFastHashMap = if (isFastHashMapEnabled) {
- // Generates the fast hash map class and creates the fash hash map term.
+ // Create a name for the iterator from the fast hash map, and the code to create fast hash map.
+ val (iterTermForFastHashMap, createFastHashMap) = if (isFastHashMapEnabled) {
+ // Generates the fast hash map class and creates the fast hash map term.
val fastHashMapClassName = ctx.freshName("FastHashMap")
if (isVectorizedHashMapEnabled) {
val generatedMap = new VectorizedHashMapGenerator(ctx, aggregateExpressions,
@@ -593,25 +601,30 @@ case class HashAggregateExec(
ctx.addInnerClass(generatedMap)
// Inline mutable state since not many aggregation operations in a task
- fastHashMapTerm = ctx.addMutableState(fastHashMapClassName, "vectorizedHastHashMap",
- v => s"$v = new $fastHashMapClassName();", forceInline = true)
- ctx.addMutableState(s"java.util.Iterator", "vectorizedFastHashMapIter",
+ fastHashMapTerm = ctx.addMutableState(
+ fastHashMapClassName, "vectorizedFastHashMap", forceInline = true)
+ val iter = ctx.addMutableState(
+ "java.util.Iterator",
+ "vectorizedFastHashMapIter",
forceInline = true)
+ val create = s"$fastHashMapTerm = new $fastHashMapClassName();"
+ (iter, create)
} else {
val generatedMap = new RowBasedHashMapGenerator(ctx, aggregateExpressions,
fastHashMapClassName, groupingKeySchema, bufferSchema, bitMaxCapacity).generate()
ctx.addInnerClass(generatedMap)
// Inline mutable state since not many aggregation operations in a task
- fastHashMapTerm = ctx.addMutableState(fastHashMapClassName, "fastHashMap",
- v => s"$v = new $fastHashMapClassName(" +
- s"$thisPlan.getTaskMemoryManager(), $thisPlan.getEmptyAggregationBuffer());",
- forceInline = true)
- ctx.addMutableState(
+ fastHashMapTerm = ctx.addMutableState(
+ fastHashMapClassName, "fastHashMap", forceInline = true)
+ val iter = ctx.addMutableState(
"org.apache.spark.unsafe.KVIterator",
"fastHashMapIter", forceInline = true)
+ val create = s"$fastHashMapTerm = new $fastHashMapClassName(" +
+ s"$thisPlan.getTaskMemoryManager(), $thisPlan.getEmptyAggregationBuffer());"
+ (iter, create)
}
- }
+ } else ("", "")
// Create a name for the iterator from the regular hash map.
// Inline mutable state since not many aggregation operations in a task
@@ -619,8 +632,7 @@ case class HashAggregateExec(
"mapIter", forceInline = true)
// create hashMap
val hashMapClassName = classOf[UnsafeFixedWidthAggregationMap].getName
- hashMapTerm = ctx.addMutableState(hashMapClassName, "hashMap",
- v => s"$v = $thisPlan.createHashMap();", forceInline = true)
+ hashMapTerm = ctx.addMutableState(hashMapClassName, "hashMap", forceInline = true)
sorterTerm = ctx.addMutableState(classOf[UnsafeKVExternalSorter].getName, "sorter",
forceInline = true)
@@ -720,6 +732,8 @@ case class HashAggregateExec(
s"""
if (!$initAgg) {
$initAgg = true;
+ $createFastHashMap
+ $hashMapTerm = $thisPlan.createHashMap();
long $beforeAgg = System.nanoTime();
$doAggFuncName();
$aggTime.add((System.nanoTime() - $beforeAgg) / 1000000);
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationIterator.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationIterator.scala
index 43514f5271ac8..b88ddba8e48d3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationIterator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationIterator.scala
@@ -161,9 +161,9 @@ class ObjectAggregationIterator(
// The the hash map gets too large, makes a sorted spill and clear the map.
if (hashMap.size >= fallbackCountThreshold) {
logInfo(
- s"Aggregation hash map reaches threshold " +
+ s"Aggregation hash map size ${hashMap.size} reaches threshold " +
s"capacity ($fallbackCountThreshold entries), spilling and falling back to sort" +
- s" based aggregation. You may change the threshold by adjust option " +
+ " based aggregation. You may change the threshold by adjust option " +
SQLConf.OBJECT_AGG_SORT_BASED_FALLBACK_THRESHOLD.key
)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowConverters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowConverters.scala
index 1a48bc8398a63..2bf6a58b55658 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowConverters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowConverters.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql.execution.arrow
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, FileInputStream, OutputStream}
-import java.nio.channels.{Channels, SeekableByteChannel}
+import java.nio.channels.{Channels, ReadableByteChannel}
import scala.collection.JavaConverters._
@@ -31,6 +31,7 @@ import org.apache.arrow.vector.ipc.message.{ArrowRecordBatch, MessageSerializer}
import org.apache.spark.TaskContext
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.network.util.JavaUtils
+import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, SQLContext}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.types._
@@ -189,7 +190,7 @@ private[sql] object ArrowConverters {
}
/**
- * Create a DataFrame from a JavaRDD of serialized ArrowRecordBatches.
+ * Create a DataFrame from an RDD of serialized ArrowRecordBatches.
*/
private[sql] def toDataFrame(
arrowBatchRDD: JavaRDD[Array[Byte]],
@@ -221,7 +222,7 @@ private[sql] object ArrowConverters {
/**
* Read an Arrow stream input and return an iterator of serialized ArrowRecordBatches.
*/
- private[sql] def getBatchesFromStream(in: SeekableByteChannel): Iterator[Array[Byte]] = {
+ private[sql] def getBatchesFromStream(in: ReadableByteChannel): Iterator[Array[Byte]] = {
// Iterate over the serialized Arrow RecordBatch messages from a stream
new Iterator[Array[Byte]] {
@@ -271,7 +272,7 @@ private[sql] object ArrowConverters {
} else {
if (bodyLength > 0) {
// Skip message body if not a RecordBatch
- in.position(in.position() + bodyLength)
+ Channels.newInputStream(in).skip(bodyLength)
}
// Proceed to next message
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala
index 533097ac399e9..b1e8fb39ac9de 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowUtils.scala
@@ -131,11 +131,8 @@ object ArrowUtils {
} else {
Nil
}
- val pandasColsByPosition = if (conf.pandasGroupedMapAssignColumnssByPosition) {
- Seq(SQLConf.PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_POSITION.key -> "true")
- } else {
- Nil
- }
- Map(timeZoneConf ++ pandasColsByPosition: _*)
+ val pandasColsByName = Seq(SQLConf.PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME.key ->
+ conf.pandasGroupedMapAssignColumnsByName.toString)
+ Map(timeZoneConf ++ pandasColsByName: _*)
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
index 9434ceb7cd16c..27aa9b8e3b1e3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql.execution
+import scala.collection.mutable
import scala.concurrent.{ExecutionContext, Future}
import scala.concurrent.duration.Duration
@@ -27,7 +28,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.plans.physical._
import org.apache.spark.sql.execution.metric.SQLMetrics
-import org.apache.spark.sql.types.LongType
+import org.apache.spark.sql.types.{LongType, StructType}
import org.apache.spark.util.ThreadUtils
import org.apache.spark.util.random.{BernoulliCellSampler, PoissonSampler}
@@ -160,6 +161,7 @@ case class FilterExec(condition: Expression, child: SparkPlan)
// This is very perf sensitive.
// TODO: revisit this. We can consider reordering predicates as well.
val generatedIsNotNullChecks = new Array[Boolean](notNullPreds.length)
+ val extraIsNotNullAttrs = mutable.Set[Attribute]()
val generated = otherPreds.map { c =>
val nullChecks = c.references.map { r =>
val idx = notNullPreds.indexWhere { n => n.asInstanceOf[IsNotNull].child.semanticEquals(r)}
@@ -167,6 +169,9 @@ case class FilterExec(condition: Expression, child: SparkPlan)
generatedIsNotNullChecks(idx) = true
// Use the child's output. The nullability is what the child produced.
genPredicate(notNullPreds(idx), input, child.output)
+ } else if (notNullAttributes.contains(r.exprId) && !extraIsNotNullAttrs.contains(r)) {
+ extraIsNotNullAttrs += r
+ genPredicate(IsNotNull(r), input, child.output)
} else {
""
}
@@ -275,7 +280,9 @@ case class SampleExec(
child.asInstanceOf[CodegenSupport].produce(ctx, this)
}
- override def needCopyResult: Boolean = withReplacement
+ override def needCopyResult: Boolean = {
+ child.asInstanceOf[CodegenSupport].needCopyResult || withReplacement
+ }
override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
val numOutput = metricTerm(ctx, "numOutputRows")
@@ -563,9 +570,20 @@ case class RangeExec(range: org.apache.spark.sql.catalyst.plans.logical.Range)
* [[org.apache.spark.sql.catalyst.plans.logical.Union.maxRowsPerPartition]].
*/
case class UnionExec(children: Seq[SparkPlan]) extends SparkPlan {
- override def output: Seq[Attribute] =
- children.map(_.output).transpose.map(attrs =>
- attrs.head.withNullability(attrs.exists(_.nullable)))
+ // updating nullability to make all the children consistent
+ override def output: Seq[Attribute] = {
+ children.map(_.output).transpose.map { attrs =>
+ val firstAttr = attrs.head
+ val nullable = attrs.exists(_.nullable)
+ val newDt = attrs.map(_.dataType).reduce(StructType.merge)
+ if (firstAttr.dataType == newDt) {
+ firstAttr.withNullability(nullable)
+ } else {
+ AttributeReference(firstAttr.name, newDt, nullable, firstAttr.metadata)(
+ firstAttr.exprId, firstAttr.qualifier)
+ }
+ }
+ }
protected override def doExecute(): RDD[InternalRow] =
sparkContext.union(children.map(_.execute()))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
index 1a8fbaca53f59..8eecd7a80e0c9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
@@ -74,14 +74,8 @@ case class CachedRDDBuilder(
}
}
- def withCachedPlan(cachedPlan: SparkPlan): CachedRDDBuilder = {
- new CachedRDDBuilder(
- useCompression,
- batchSize,
- storageLevel,
- cachedPlan = cachedPlan,
- tableName
- )(_cachedColumnBuffers)
+ def isCachedColumnBuffersLoaded: Boolean = {
+ _cachedColumnBuffers != null
}
private def buildBuffers(): RDD[CachedBatch] = {
@@ -206,7 +200,7 @@ case class InMemoryRelation(
outputOrdering).asInstanceOf[this.type]
}
- override protected def otherCopyArgs: Seq[AnyRef] = Seq(statsOfPlanToCache)
+ override protected def otherCopyArgs: Seq[AnyRef] = Seq(statsOfPlanToCache, outputOrdering)
override def simpleString: String =
s"InMemoryRelation [${Utils.truncatedString(output, ", ")}], ${cacheBuilder.storageLevel}"
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzePartitionCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzePartitionCommand.scala
index 18fefa0a6f19f..9ddc470fbd33c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzePartitionCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AnalyzePartitionCommand.scala
@@ -110,7 +110,7 @@ case class AnalyzePartitionCommand(
val newTotalSize = CommandUtils.calculateLocationSize(
sessionState, tableMeta.identifier, p.storage.locationUri)
val newRowCount = rowCounts.get(p.spec)
- val newStats = CommandUtils.compareAndGetNewStats(tableMeta.stats, newTotalSize, newRowCount)
+ val newStats = CommandUtils.compareAndGetNewStats(p.stats, newTotalSize, newRowCount)
newStats.map(_ => p.copy(stats = newStats))
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
index df71bc9effb3e..479231a8be258 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
@@ -35,16 +35,14 @@ object CommandUtils extends Logging {
/** Change statistics after changing data by commands. */
def updateTableStats(sparkSession: SparkSession, table: CatalogTable): Unit = {
- if (table.stats.nonEmpty) {
- val catalog = sparkSession.sessionState.catalog
- if (sparkSession.sessionState.conf.autoSizeUpdateEnabled) {
- val newTable = catalog.getTableMetadata(table.identifier)
- val newSize = CommandUtils.calculateTotalSize(sparkSession, newTable)
- val newStats = CatalogStatistics(sizeInBytes = newSize)
- catalog.alterTableStats(table.identifier, Some(newStats))
- } else {
- catalog.alterTableStats(table.identifier, None)
- }
+ val catalog = sparkSession.sessionState.catalog
+ if (sparkSession.sessionState.conf.autoSizeUpdateEnabled) {
+ val newTable = catalog.getTableMetadata(table.identifier)
+ val newSize = CommandUtils.calculateTotalSize(sparkSession, newTable)
+ val newStats = CatalogStatistics(sizeInBytes = newSize)
+ catalog.alterTableStats(table.identifier, Some(newStats))
+ } else if (table.stats.nonEmpty) {
+ catalog.alterTableStats(table.identifier, None)
}
}
@@ -59,9 +57,7 @@ object CommandUtils extends Logging {
val paths = partitions.map(x => new Path(x.storage.locationUri.get))
val stagingDir = sessionState.conf.getConfString("hive.exec.stagingdir", ".hive-staging")
val pathFilter = new PathFilter with Serializable {
- override def accept(path: Path): Boolean = {
- DataSourceUtils.isDataPath(path) && !path.getName.startsWith(stagingDir)
- }
+ override def accept(path: Path): Boolean = isDataPath(path, stagingDir)
}
val fileStatusSeq = InMemoryFileIndex.bulkListLeafFiles(
paths, sessionState.newHadoopConf(), pathFilter, spark)
@@ -93,8 +89,7 @@ object CommandUtils extends Logging {
val size = if (fileStatus.isDirectory) {
fs.listStatus(path)
.map { status =>
- if (!status.getPath.getName.startsWith(stagingDir) &&
- DataSourceUtils.isDataPath(path)) {
+ if (isDataPath(status.getPath, stagingDir)) {
getPathSize(fs, status.getPath)
} else {
0L
@@ -153,4 +148,8 @@ object CommandUtils extends Logging {
}
newStats
}
+
+ private def isDataPath(path: Path, stagingDir: String): Boolean = {
+ !path.getName.startsWith(stagingDir) && DataSourceUtils.isDataPath(path)
+ }
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala
index 0a185b8472060..a1bb5af1ab723 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala
@@ -26,7 +26,6 @@ import org.apache.spark.sql.execution.SparkPlan
import org.apache.spark.sql.execution.datasources.BasicWriteJobStatsTracker
import org.apache.spark.sql.execution.datasources.FileFormatWriter
import org.apache.spark.sql.execution.metric.SQLMetric
-import org.apache.spark.sql.types.StructType
import org.apache.spark.util.SerializableConfiguration
/**
@@ -75,18 +74,4 @@ object DataWritingCommand {
attr.withName(outputName)
}
}
-
- /**
- * Returns schema of logical plan with provided names.
- * The length of provided names should be the same of the length of [[LogicalPlan.schema]].
- */
- def logicalPlanSchemaWithNames(
- query: LogicalPlan,
- names: Seq[String]): StructType = {
- assert(query.schema.length == names.length,
- "The length of provided names doesn't match the length of query schema.")
- StructType(query.schema.zip(names).map { case (structField, outputName) =>
- structField.copy(name = outputName)
- })
- }
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index 2eca1c40a5b3f..6dc8426902193 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -306,7 +306,8 @@ case class LoadDataCommand(
val loadPath = {
if (isLocal) {
val localFS = FileContext.getLocalFSFileContext()
- makeQualified(FsConstants.LOCAL_FS_URI, localFS.getWorkingDirectory(), new Path(path))
+ LoadDataCommand.makeQualified(FsConstants.LOCAL_FS_URI, localFS.getWorkingDirectory(),
+ new Path(path))
} else {
val loadPath = new Path(path)
// Follow Hive's behavior:
@@ -323,7 +324,7 @@ case class LoadDataCommand(
// by considering the wild card scenario in mind.as per old logic query param is
// been considered while creating URI instance and if path contains wild card char '?'
// the remaining charecters after '?' will be removed while forming URI instance
- makeQualified(defaultFS, uriPath, loadPath)
+ LoadDataCommand.makeQualified(defaultFS, uriPath, loadPath)
}
}
val fs = loadPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
@@ -363,7 +364,9 @@ case class LoadDataCommand(
CommandUtils.updateTableStats(sparkSession, targetTable)
Seq.empty[Row]
}
+}
+object LoadDataCommand {
/**
* Returns a qualified path object. Method ported from org.apache.hadoop.fs.Path class.
*
@@ -372,8 +375,9 @@ case class LoadDataCommand(
* @param path Path instance based on the path string specified by the user.
* @return qualified path object
*/
- private def makeQualified(defaultUri: URI, workingDir: Path, path: Path): Path = {
- val pathUri = if (path.isAbsolute()) path.toUri() else new Path(workingDir, path).toUri()
+ private[sql] def makeQualified(defaultUri: URI, workingDir: Path, path: Path): Path = {
+ val newPath = new Path(workingDir, path)
+ val pathUri = if (path.isAbsolute()) path.toUri() else newPath.toUri()
if (pathUri.getScheme == null || pathUri.getAuthority == null &&
defaultUri.getAuthority != null) {
val scheme = if (pathUri.getScheme == null) defaultUri.getScheme else pathUri.getScheme
@@ -383,14 +387,14 @@ case class LoadDataCommand(
pathUri.getAuthority
}
try {
- val newUri = new URI(scheme, authority, pathUri.getPath, pathUri.getFragment)
+ val newUri = new URI(scheme, authority, pathUri.getPath, null, pathUri.getFragment)
new Path(newUri)
} catch {
case e: URISyntaxException =>
throw new IllegalArgumentException(e)
}
} else {
- path
+ newPath
}
}
}
@@ -1058,7 +1062,7 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman
private def showDataSourceTableOptions(metadata: CatalogTable, builder: StringBuilder): Unit = {
builder ++= s"USING ${metadata.provider.get}\n"
- val dataSourceOptions = metadata.storage.properties.map {
+ val dataSourceOptions = SQLConf.get.redactOptions(metadata.storage.properties).map {
case (key, value) => s"${quoteIdentifier(key)} '${escapeSingleQuotedString(value)}'"
} ++ metadata.storage.locationUri.flatMap { location =>
if (metadata.tableType == MANAGED) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 5172f32ec7b9c..abc8515af89d8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable
import org.apache.spark.sql.catalyst.expressions.{Alias, SubqueryExpression}
import org.apache.spark.sql.catalyst.plans.QueryPlan
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, View}
-import org.apache.spark.sql.types.MetadataBuilder
+import org.apache.spark.sql.types.{MetadataBuilder, StructType}
import org.apache.spark.sql.util.SchemaUtils
@@ -233,14 +233,15 @@ case class CreateViewCommand(
throw new AnalysisException(
"It is not allowed to create a persisted view from the Dataset API")
}
-
- val newProperties = generateViewProperties(properties, session, analyzedPlan)
+ val aliasedSchema = aliasPlan(session, analyzedPlan).schema
+ val newProperties = generateViewProperties(
+ properties, session, analyzedPlan, aliasedSchema.fieldNames)
CatalogTable(
identifier = name,
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
- schema = aliasPlan(session, analyzedPlan).schema,
+ schema = aliasedSchema,
properties = newProperties,
viewText = originalText,
comment = comment
@@ -294,7 +295,8 @@ case class AlterViewAsCommand(
val viewIdent = viewMeta.identifier
checkCyclicViewReference(analyzedPlan, Seq(viewIdent), viewIdent)
- val newProperties = generateViewProperties(viewMeta.properties, session, analyzedPlan)
+ val newProperties = generateViewProperties(
+ viewMeta.properties, session, analyzedPlan, analyzedPlan.schema.fieldNames)
val updatedViewMeta = viewMeta.copy(
schema = analyzedPlan.schema,
@@ -355,13 +357,15 @@ object ViewHelper {
def generateViewProperties(
properties: Map[String, String],
session: SparkSession,
- analyzedPlan: LogicalPlan): Map[String, String] = {
+ analyzedPlan: LogicalPlan,
+ fieldNames: Array[String]): Map[String, String] = {
+ // for createViewCommand queryOutput may be different from fieldNames
val queryOutput = analyzedPlan.schema.fieldNames
// Generate the query column names, throw an AnalysisException if there exists duplicate column
// names.
SchemaUtils.checkColumnNameDuplication(
- queryOutput, "in the view definition", session.sessionState.conf.resolver)
+ fieldNames, "in the view definition", session.sessionState.conf.resolver)
// Generate the view default database name.
val viewDefaultDatabase = session.sessionState.catalog.getCurrentDatabase
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
index 90cec5e72c1a7..1cb69d7ea7634 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
@@ -18,6 +18,8 @@
package org.apache.spark.sql.execution.datasources
import org.apache.hadoop.fs.Path
+import org.json4s.NoTypeHints
+import org.json4s.jackson.Serialization
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.types._
@@ -39,6 +41,24 @@ object DataSourceUtils {
verifySchema(format, schema, isReadPath = true)
}
+ /**
+ * The key to use for storing partitionBy columns as options.
+ */
+ val PARTITIONING_COLUMNS_KEY = "__partition_columns"
+
+ /**
+ * Utility methods for converting partitionBy columns to options and back.
+ */
+ private implicit val formats = Serialization.formats(NoTypeHints)
+
+ def encodePartitioningColumns(columns: Seq[String]): String = {
+ Serialization.write(columns)
+ }
+
+ def decodePartitioningColumns(str: String): Seq[String] = {
+ Serialization.read[Seq[String]](str)
+ }
+
/**
* Verify if the schema is supported in datasource. This verification should be done
* in a driver side.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FailureSafeParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FailureSafeParser.scala
index 90e81661bae7a..e618f17236784 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FailureSafeParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FailureSafeParser.scala
@@ -29,8 +29,7 @@ class FailureSafeParser[IN](
rawParser: IN => Seq[InternalRow],
mode: ParseMode,
schema: StructType,
- columnNameOfCorruptRecord: String,
- isMultiLine: Boolean) {
+ columnNameOfCorruptRecord: String) {
private val corruptFieldIndex = schema.getFieldIndex(columnNameOfCorruptRecord)
private val actualSchema = StructType(schema.filterNot(_.name == columnNameOfCorruptRecord))
@@ -58,15 +57,9 @@ class FailureSafeParser[IN](
}
}
- private val skipParsing = !isMultiLine && mode == PermissiveMode && schema.isEmpty
-
def parse(input: IN): Iterator[InternalRow] = {
try {
- if (skipParsing) {
- Iterator.single(InternalRow.empty)
- } else {
- rawParser.apply(input).toIterator.map(row => toResultRow(Some(row), () => null))
- }
+ rawParser.apply(input).toIterator.map(row => toResultRow(Some(row), () => null))
} catch {
case e: BadRecordException => mode match {
case PermissiveMode =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala
index 6499328e89ce7..10733810b6416 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala
@@ -179,7 +179,8 @@ class DynamicPartitionDataWriter(
val partitionName = ScalaUDF(
ExternalCatalogUtils.getPartitionPathString _,
StringType,
- Seq(Literal(c.name), Cast(c, StringType, Option(description.timeZoneId))))
+ Seq(Literal(c.name), Cast(c, StringType, Option(description.timeZoneId))),
+ Seq(true, true))
if (i == 0) Seq(partitionName) else Seq(Literal(Path.SEPARATOR), partitionName)
})
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
index 7c6ab4bc922fe..538b294fcd094 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
@@ -162,15 +162,17 @@ object FileFormatWriter extends Logging {
rdd
}
+ val jobIdInstant = new Date().getTime
val ret = new Array[WriteTaskResult](rddWithNonEmptyPartitions.partitions.length)
sparkSession.sparkContext.runJob(
rddWithNonEmptyPartitions,
(taskContext: TaskContext, iter: Iterator[InternalRow]) => {
executeTask(
description = description,
+ jobIdInstant = jobIdInstant,
sparkStageId = taskContext.stageId(),
sparkPartitionId = taskContext.partitionId(),
- sparkAttemptNumber = taskContext.attemptNumber(),
+ sparkAttemptNumber = taskContext.taskAttemptId().toInt & Integer.MAX_VALUE,
committer,
iterator = iter)
},
@@ -183,15 +185,15 @@ object FileFormatWriter extends Logging {
val commitMsgs = ret.map(_.commitMsg)
committer.commitJob(job, commitMsgs)
- logInfo(s"Job ${job.getJobID} committed.")
+ logInfo(s"Write Job ${description.uuid} committed.")
processStats(description.statsTrackers, ret.map(_.summary.stats))
- logInfo(s"Finished processing stats for job ${job.getJobID}.")
+ logInfo(s"Finished processing stats for write job ${description.uuid}.")
// return a set of all the partition paths that were updated during this job
ret.map(_.summary.updatedPartitions).reduceOption(_ ++ _).getOrElse(Set.empty)
} catch { case cause: Throwable =>
- logError(s"Aborting job ${job.getJobID}.", cause)
+ logError(s"Aborting job ${description.uuid}.", cause)
committer.abortJob(job)
throw new SparkException("Job aborted.", cause)
}
@@ -200,13 +202,14 @@ object FileFormatWriter extends Logging {
/** Writes data out in a single Spark task. */
private def executeTask(
description: WriteJobDescription,
+ jobIdInstant: Long,
sparkStageId: Int,
sparkPartitionId: Int,
sparkAttemptNumber: Int,
committer: FileCommitProtocol,
iterator: Iterator[InternalRow]): WriteTaskResult = {
- val jobId = SparkHadoopWriterUtils.createJobID(new Date, sparkStageId)
+ val jobId = SparkHadoopWriterUtils.createJobID(new Date(jobIdInstant), sparkStageId)
val taskId = new TaskID(jobId, TaskType.MAP, sparkPartitionId)
val taskAttemptId = new TaskAttemptID(taskId, sparkAttemptNumber)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala
index 99fc78ff3e49b..0756e7eed747c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala
@@ -57,7 +57,7 @@ case class PartitionedFile(
* A collection of file blocks that should be read as a single task
* (possibly from multiple partitioned directories).
*/
-case class FilePartition(index: Int, files: Seq[PartitionedFile]) extends RDDPartition
+case class FilePartition(index: Int, files: Array[PartitionedFile]) extends RDDPartition
/**
* An RDD that scans a list of file partitions.
@@ -85,18 +85,10 @@ class FileScanRDD(
// If we do a coalesce, however, we are likely to compute multiple partitions in the same
// task and in the same thread, in which case we need to avoid override values written by
// previous partitions (SPARK-13071).
- private def updateBytesRead(): Unit = {
+ private def incTaskInputMetricsBytesRead(): Unit = {
inputMetrics.setBytesRead(existingBytesRead + getBytesReadCallback())
}
- // If we can't get the bytes read from the FS stats, fall back to the file size,
- // which may be inaccurate.
- private def updateBytesReadWithFileSize(): Unit = {
- if (currentFile != null) {
- inputMetrics.incBytesRead(currentFile.length)
- }
- }
-
private[this] val files = split.asInstanceOf[FilePartition].files.toIterator
private[this] var currentFile: PartitionedFile = null
private[this] var currentIterator: Iterator[Object] = null
@@ -112,14 +104,18 @@ class FileScanRDD(
val nextElement = currentIterator.next()
// TODO: we should have a better separation of row based and batch based scan, so that we
// don't need to run this `if` for every record.
+ val preNumRecordsRead = inputMetrics.recordsRead
if (nextElement.isInstanceOf[ColumnarBatch]) {
+ incTaskInputMetricsBytesRead()
inputMetrics.incRecordsRead(nextElement.asInstanceOf[ColumnarBatch].numRows())
} else {
+ // too costly to update every record
+ if (inputMetrics.recordsRead %
+ SparkHadoopUtil.UPDATE_INPUT_METRICS_INTERVAL_RECORDS == 0) {
+ incTaskInputMetricsBytesRead()
+ }
inputMetrics.incRecordsRead(1)
}
- if (inputMetrics.recordsRead % SparkHadoopUtil.UPDATE_INPUT_METRICS_INTERVAL_RECORDS == 0) {
- updateBytesRead()
- }
nextElement
}
@@ -139,7 +135,6 @@ class FileScanRDD(
/** Advances to the next file. Returns true if a new non-empty iterator is available. */
private def nextIterator(): Boolean = {
- updateBytesReadWithFileSize()
if (files.hasNext) {
currentFile = files.next()
logInfo(s"Reading File $currentFile")
@@ -207,8 +202,7 @@ class FileScanRDD(
}
override def close(): Unit = {
- updateBytesRead()
- updateBytesReadWithFileSize()
+ incTaskInputMetricsBytesRead()
InputFileBlockHolder.unset()
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
index cc8af7b92c454..7b0e4dbcc25f4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
@@ -126,33 +126,15 @@ abstract class PartitioningAwareFileIndex(
val caseInsensitiveOptions = CaseInsensitiveMap(parameters)
val timeZoneId = caseInsensitiveOptions.get(DateTimeUtils.TIMEZONE_OPTION)
.getOrElse(sparkSession.sessionState.conf.sessionLocalTimeZone)
- val inferredPartitionSpec = PartitioningUtils.parsePartitions(
+
+ val caseSensitive = sparkSession.sqlContext.conf.caseSensitiveAnalysis
+ PartitioningUtils.parsePartitions(
leafDirs,
typeInference = sparkSession.sessionState.conf.partitionColumnTypeInferenceEnabled,
basePaths = basePaths,
+ userSpecifiedSchema = userSpecifiedSchema,
+ caseSensitive = caseSensitive,
timeZoneId = timeZoneId)
- userSpecifiedSchema match {
- case Some(userProvidedSchema) if userProvidedSchema.nonEmpty =>
- val userPartitionSchema =
- combineInferredAndUserSpecifiedPartitionSchema(inferredPartitionSpec)
-
- // we need to cast into the data type that user specified.
- def castPartitionValuesToUserSchema(row: InternalRow) = {
- InternalRow((0 until row.numFields).map { i =>
- val dt = inferredPartitionSpec.partitionColumns.fields(i).dataType
- Cast(
- Literal.create(row.get(i, dt), dt),
- userPartitionSchema.fields(i).dataType,
- Option(timeZoneId)).eval()
- }: _*)
- }
-
- PartitionSpec(userPartitionSchema, inferredPartitionSpec.partitions.map { part =>
- part.copy(values = castPartitionValuesToUserSchema(part.values))
- })
- case _ =>
- inferredPartitionSpec
- }
}
private def prunePartitions(
@@ -233,25 +215,6 @@ abstract class PartitioningAwareFileIndex(
val name = path.getName
!((name.startsWith("_") && !name.contains("=")) || name.startsWith("."))
}
-
- /**
- * In the read path, only managed tables by Hive provide the partition columns properly when
- * initializing this class. All other file based data sources will try to infer the partitioning,
- * and then cast the inferred types to user specified dataTypes if the partition columns exist
- * inside `userSpecifiedSchema`, otherwise we can hit data corruption bugs like SPARK-18510, or
- * inconsistent data types as reported in SPARK-21463.
- * @param spec A partition inference result
- * @return The PartitionSchema resolved from inference and cast according to `userSpecifiedSchema`
- */
- private def combineInferredAndUserSpecifiedPartitionSchema(spec: PartitionSpec): StructType = {
- val equality = sparkSession.sessionState.conf.resolver
- val resolved = spec.partitionColumns.map { partitionField =>
- // SPARK-18510: try to get schema from userSpecifiedSchema, otherwise fallback to inferred
- userSpecifiedSchema.flatMap(_.find(f => equality(f.name, partitionField.name))).getOrElse(
- partitionField)
- }
- StructType(resolved)
- }
}
object PartitioningAwareFileIndex {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
index 3183fd30e5e0d..41b0bb67e9626 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
@@ -31,7 +31,7 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.{Resolver, TypeCoercion}
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Literal}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
import org.apache.spark.sql.types._
import org.apache.spark.sql.util.SchemaUtils
@@ -94,18 +94,41 @@ object PartitioningUtils {
paths: Seq[Path],
typeInference: Boolean,
basePaths: Set[Path],
+ userSpecifiedSchema: Option[StructType],
+ caseSensitive: Boolean,
timeZoneId: String): PartitionSpec = {
- parsePartitions(paths, typeInference, basePaths, DateTimeUtils.getTimeZone(timeZoneId))
+ parsePartitions(paths, typeInference, basePaths, userSpecifiedSchema,
+ caseSensitive, DateTimeUtils.getTimeZone(timeZoneId))
}
private[datasources] def parsePartitions(
paths: Seq[Path],
typeInference: Boolean,
basePaths: Set[Path],
+ userSpecifiedSchema: Option[StructType],
+ caseSensitive: Boolean,
timeZone: TimeZone): PartitionSpec = {
+ val userSpecifiedDataTypes = if (userSpecifiedSchema.isDefined) {
+ val nameToDataType = userSpecifiedSchema.get.fields.map(f => f.name -> f.dataType).toMap
+ if (!caseSensitive) {
+ CaseInsensitiveMap(nameToDataType)
+ } else {
+ nameToDataType
+ }
+ } else {
+ Map.empty[String, DataType]
+ }
+
+ // SPARK-26990: use user specified field names if case insensitive.
+ val userSpecifiedNames = if (userSpecifiedSchema.isDefined && !caseSensitive) {
+ CaseInsensitiveMap(userSpecifiedSchema.get.fields.map(f => f.name -> f.name).toMap)
+ } else {
+ Map.empty[String, String]
+ }
+
// First, we need to parse every partition's path and see if we can find partition values.
val (partitionValues, optDiscoveredBasePaths) = paths.map { path =>
- parsePartition(path, typeInference, basePaths, timeZone)
+ parsePartition(path, typeInference, basePaths, userSpecifiedDataTypes, timeZone)
}.unzip
// We create pairs of (path -> path's partition value) here
@@ -147,7 +170,9 @@ object PartitioningUtils {
columnNames.zip(literals).map { case (name, Literal(_, dataType)) =>
// We always assume partition columns are nullable since we've no idea whether null values
// will be appended in the future.
- StructField(name, dataType, nullable = true)
+ val resultName = userSpecifiedNames.getOrElse(name, name)
+ val resultDataType = userSpecifiedDataTypes.getOrElse(name, dataType)
+ StructField(resultName, resultDataType, nullable = true)
}
}
@@ -185,6 +210,7 @@ object PartitioningUtils {
path: Path,
typeInference: Boolean,
basePaths: Set[Path],
+ userSpecifiedDataTypes: Map[String, DataType],
timeZone: TimeZone): (Option[PartitionValues], Option[Path]) = {
val columns = ArrayBuffer.empty[(String, Literal)]
// Old Hadoop versions don't have `Path.isRoot`
@@ -206,7 +232,7 @@ object PartitioningUtils {
// Let's say currentPath is a path of "/table/a=1/", currentPath.getName will give us a=1.
// Once we get the string, we try to parse it and find the partition column and value.
val maybeColumn =
- parsePartitionColumn(currentPath.getName, typeInference, timeZone)
+ parsePartitionColumn(currentPath.getName, typeInference, userSpecifiedDataTypes, timeZone)
maybeColumn.foreach(columns += _)
// Now, we determine if we should stop.
@@ -239,6 +265,7 @@ object PartitioningUtils {
private def parsePartitionColumn(
columnSpec: String,
typeInference: Boolean,
+ userSpecifiedDataTypes: Map[String, DataType],
timeZone: TimeZone): Option[(String, Literal)] = {
val equalSignIndex = columnSpec.indexOf('=')
if (equalSignIndex == -1) {
@@ -250,7 +277,16 @@ object PartitioningUtils {
val rawColumnValue = columnSpec.drop(equalSignIndex + 1)
assert(rawColumnValue.nonEmpty, s"Empty partition column value in '$columnSpec'")
- val literal = inferPartitionColumnValue(rawColumnValue, typeInference, timeZone)
+ val literal = if (userSpecifiedDataTypes.contains(columnName)) {
+ // SPARK-26188: if user provides corresponding column schema, get the column value without
+ // inference, and then cast it as user specified data type.
+ val columnValue = inferPartitionColumnValue(rawColumnValue, false, timeZone)
+ val castedValue =
+ Cast(columnValue, userSpecifiedDataTypes(columnName), Option(timeZone.getID)).eval()
+ Literal.create(castedValue, userSpecifiedDataTypes(columnName))
+ } else {
+ inferPartitionColumnValue(rawColumnValue, typeInference, timeZone)
+ }
Some(columnName -> literal)
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
index 2b86054c0ffcb..62c3c16684e82 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
@@ -210,7 +210,11 @@ object TextInputCSVDataSource extends CSVDataSource {
// Note: if there are only comments in the first block, the header would probably
// be not extracted.
CSVUtils.extractHeader(lines, parser.options).foreach { header =>
- val schema = if (columnPruning) requiredSchema else dataSchema
+ val actualRequiredSchema =
+ StructType(requiredSchema.filterNot(_.name == parser.options.columnNameOfCorruptRecord))
+ val actualDataSchema =
+ StructType(dataSchema.filterNot(_.name == parser.options.columnNameOfCorruptRecord))
+ val schema = if (columnPruning) actualRequiredSchema else actualDataSchema
val columnNames = parser.tokenizer.parseLine(header)
CSVDataSource.checkHeaderColumnNames(
schema,
@@ -240,23 +244,25 @@ object TextInputCSVDataSource extends CSVDataSource {
sparkSession: SparkSession,
csv: Dataset[String],
maybeFirstLine: Option[String],
- parsedOptions: CSVOptions): StructType = maybeFirstLine match {
- case Some(firstLine) =>
- val firstRow = new CsvParser(parsedOptions.asParserSettings).parseLine(firstLine)
- val caseSensitive = sparkSession.sessionState.conf.caseSensitiveAnalysis
- val header = makeSafeHeader(firstRow, caseSensitive, parsedOptions)
- val sampled: Dataset[String] = CSVUtils.sample(csv, parsedOptions)
- val tokenRDD = sampled.rdd.mapPartitions { iter =>
- val filteredLines = CSVUtils.filterCommentAndEmpty(iter, parsedOptions)
- val linesWithoutHeader =
- CSVUtils.filterHeaderLine(filteredLines, firstLine, parsedOptions)
- val parser = new CsvParser(parsedOptions.asParserSettings)
- linesWithoutHeader.map(parser.parseLine)
- }
- CSVInferSchema.infer(tokenRDD, header, parsedOptions)
- case None =>
- // If the first line could not be read, just return the empty schema.
- StructType(Nil)
+ parsedOptions: CSVOptions): StructType = {
+ val csvParser = new CsvParser(parsedOptions.asParserSettings)
+ maybeFirstLine.map(csvParser.parseLine(_)) match {
+ case Some(firstRow) if firstRow != null =>
+ val caseSensitive = sparkSession.sessionState.conf.caseSensitiveAnalysis
+ val header = makeSafeHeader(firstRow, caseSensitive, parsedOptions)
+ val sampled: Dataset[String] = CSVUtils.sample(csv, parsedOptions)
+ val tokenRDD = sampled.rdd.mapPartitions { iter =>
+ val filteredLines = CSVUtils.filterCommentAndEmpty(iter, parsedOptions)
+ val linesWithoutHeader =
+ CSVUtils.filterHeaderLine(filteredLines, maybeFirstLine.get, parsedOptions)
+ val parser = new CsvParser(parsedOptions.asParserSettings)
+ linesWithoutHeader.map(parser.parseLine)
+ }
+ CSVInferSchema.infer(tokenRDD, header, parsedOptions)
+ case _ =>
+ // If the first line could not be read, just return the empty schema.
+ StructType(Nil)
+ }
}
private def createBaseDataset(
@@ -295,7 +301,11 @@ object MultiLineCSVDataSource extends CSVDataSource {
caseSensitive: Boolean,
columnPruning: Boolean): Iterator[InternalRow] = {
def checkHeader(header: Array[String]): Unit = {
- val schema = if (columnPruning) requiredSchema else dataSchema
+ val actualRequiredSchema =
+ StructType(requiredSchema.filterNot(_.name == parser.options.columnNameOfCorruptRecord))
+ val actualDataSchema =
+ StructType(dataSchema.filterNot(_.name == parser.options.columnNameOfCorruptRecord))
+ val schema = if (columnPruning) actualRequiredSchema else actualDataSchema
CSVDataSource.checkHeaderColumnNames(
schema,
header,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVOptions.scala
index fab8d62da0c1d..492a21be6df3b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVOptions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVOptions.scala
@@ -162,6 +162,21 @@ class CSVOptions(
*/
val enforceSchema = getBool("enforceSchema", default = true)
+
+ /**
+ * String representation of an empty value in read and in write.
+ */
+ val emptyValue = parameters.get("emptyValue")
+ /**
+ * The string is returned when CSV reader doesn't have any characters for input value,
+ * or an empty quoted string `""`. Default value is empty string.
+ */
+ val emptyValueInRead = emptyValue.getOrElse("")
+ /**
+ * The value is used instead of an empty string in write. Default value is `""`
+ */
+ val emptyValueInWrite = emptyValue.getOrElse("\"\"")
+
def asWriterSettings: CsvWriterSettings = {
val writerSettings = new CsvWriterSettings()
val format = writerSettings.getFormat
@@ -173,7 +188,7 @@ class CSVOptions(
writerSettings.setIgnoreLeadingWhitespaces(ignoreLeadingWhiteSpaceFlagInWrite)
writerSettings.setIgnoreTrailingWhitespaces(ignoreTrailingWhiteSpaceFlagInWrite)
writerSettings.setNullValue(nullValue)
- writerSettings.setEmptyValue("\"\"")
+ writerSettings.setEmptyValue(emptyValueInWrite)
writerSettings.setSkipEmptyLines(true)
writerSettings.setQuoteAllFields(quoteAll)
writerSettings.setQuoteEscapingEnabled(escapeQuotes)
@@ -194,7 +209,7 @@ class CSVOptions(
settings.setInputBufferSize(inputBufferSize)
settings.setMaxColumns(maxColumns)
settings.setNullValue(nullValue)
- settings.setEmptyValue("")
+ settings.setEmptyValue(emptyValueInRead)
settings.setMaxCharsPerColumn(maxCharsPerColumn)
settings.setUnescapedQuoteHandling(UnescapedQuoteHandling.STOP_AT_DELIMITER)
settings
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala
index 7ce65fa89b02d..b367b3d0ac642 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala
@@ -97,23 +97,25 @@ object CSVUtils {
*/
@throws[IllegalArgumentException]
def toChar(str: String): Char = {
- if (str.charAt(0) == '\\') {
- str.charAt(1)
- match {
- case 't' => '\t'
- case 'r' => '\r'
- case 'b' => '\b'
- case 'f' => '\f'
- case '\"' => '\"' // In case user changes quote char and uses \" as delimiter in options
- case '\'' => '\''
- case 'u' if str == """\u0000""" => '\u0000'
- case _ =>
- throw new IllegalArgumentException(s"Unsupported special character for delimiter: $str")
- }
- } else if (str.length == 1) {
- str.charAt(0)
- } else {
- throw new IllegalArgumentException(s"Delimiter cannot be more than one character: $str")
+ (str: Seq[Char]) match {
+ case Seq() => throw new IllegalArgumentException("Delimiter cannot be empty string")
+ case Seq('\\') => throw new IllegalArgumentException("Single backslash is prohibited." +
+ " It has special meaning as beginning of an escape sequence." +
+ " To get the backslash character, pass a string with two backslashes as the delimiter.")
+ case Seq(c) => c
+ case Seq('\\', 't') => '\t'
+ case Seq('\\', 'r') => '\r'
+ case Seq('\\', 'b') => '\b'
+ case Seq('\\', 'f') => '\f'
+ // In case user changes quote char and uses \" as delimiter in options
+ case Seq('\\', '\"') => '\"'
+ case Seq('\\', '\'') => '\''
+ case Seq('\\', '\\') => '\\'
+ case _ if str == """\u0000""" => '\u0000'
+ case Seq('\\', _) =>
+ throw new IllegalArgumentException(s"Unsupported special character for delimiter: $str")
+ case _ =>
+ throw new IllegalArgumentException(s"Delimiter cannot be more than one character: $str")
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityGenerator.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityGenerator.scala
index 4082a0df8ba75..3118091111a74 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityGenerator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityGenerator.scala
@@ -23,6 +23,7 @@ import com.univocity.parsers.csv.CsvWriter
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimestampParser
import org.apache.spark.sql.types._
private[csv] class UnivocityGenerator(
@@ -42,14 +43,15 @@ private[csv] class UnivocityGenerator(
private val valueConverters: Array[ValueConverter] =
schema.map(_.dataType).map(makeConverter).toArray
+ @transient private lazy val timestampParser = new TimestampParser(options.timestampFormat)
+
private def makeConverter(dataType: DataType): ValueConverter = dataType match {
case DateType =>
(row: InternalRow, ordinal: Int) =>
options.dateFormat.format(DateTimeUtils.toJavaDate(row.getInt(ordinal)))
case TimestampType =>
- (row: InternalRow, ordinal: Int) =>
- options.timestampFormat.format(DateTimeUtils.toJavaTimestamp(row.getLong(ordinal)))
+ (row: InternalRow, ordinal: Int) => timestampParser.format(row.getLong(ordinal))
case udt: UserDefinedType[_] => makeConverter(udt.sqlType)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityParser.scala
index e15af425b2649..e847e408c7f19 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UnivocityParser.scala
@@ -29,6 +29,7 @@ import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
import org.apache.spark.sql.catalyst.util.{BadRecordException, DateTimeUtils}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimestampParser
import org.apache.spark.sql.execution.datasources.FailureSafeParser
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
@@ -77,6 +78,8 @@ class UnivocityParser(
private val row = new GenericInternalRow(requiredSchema.length)
+ @transient private lazy val timestampParser = new TimestampParser(options.timestampFormat)
+
// Retrieve the raw record string.
private def getCurrentInput: UTF8String = {
UTF8String.fromString(tokenizer.getContext.currentParsedContent().stripLineEnd)
@@ -156,10 +159,7 @@ class UnivocityParser(
case _: TimestampType => (d: String) =>
nullSafeDatum(d, name, nullable, options) { datum =>
- // This one will lose microseconds parts.
- // See https://issues.apache.org/jira/browse/SPARK-10681.
- Try(options.timestampFormat.parse(datum).getTime * 1000L)
- .getOrElse {
+ Try(timestampParser.parse(datum)).getOrElse {
// If it fails to parse, then tries the way used in 2.0 and 1.x for backwards
// compatibility.
DateTimeUtils.stringToTime(datum).getTime * 1000L
@@ -203,11 +203,20 @@ class UnivocityParser(
}
}
+ private val doParse = if (options.columnPruning && requiredSchema.isEmpty) {
+ // If `columnPruning` enabled and partition attributes scanned only,
+ // `schema` gets empty.
+ (_: String) => InternalRow.empty
+ } else {
+ // parse if the columnPruning is disabled or requiredSchema is nonEmpty
+ (input: String) => convert(tokenizer.parseLine(input))
+ }
+
/**
* Parses a single CSV string and turns it into either one resulting row or no row (if the
* the record is malformed).
*/
- def parse(input: String): InternalRow = convert(tokenizer.parseLine(input))
+ def parse(input: String): InternalRow = doParse(input)
private val getToken = if (options.columnPruning) {
(tokens: Array[String], index: Int) => tokens(index)
@@ -216,7 +225,12 @@ class UnivocityParser(
}
private def convert(tokens: Array[String]): InternalRow = {
- if (tokens.length != parsedSchema.length) {
+ if (tokens == null) {
+ throw BadRecordException(
+ () => getCurrentInput,
+ () => None,
+ new RuntimeException("Malformed CSV record"))
+ } else if (tokens.length != parsedSchema.length) {
// If the number of tokens doesn't match the schema, we should treat it as a malformed record.
// However, we still have chance to parse some of the tokens, by adding extra null tokens in
// the tail if the number is smaller, or by dropping extra tokens if the number is larger.
@@ -285,8 +299,7 @@ private[csv] object UnivocityParser {
input => Seq(parser.convert(input)),
parser.options.parseMode,
schema,
- parser.options.columnNameOfCorruptRecord,
- parser.options.multiLine)
+ parser.options.columnNameOfCorruptRecord)
convertStream(inputStream, shouldDropHeader, tokenizer, checkHeader) { tokens =>
safeParser.parse(tokens)
}.flatten
@@ -334,8 +347,7 @@ private[csv] object UnivocityParser {
input => Seq(parser.parse(input)),
parser.options.parseMode,
schema,
- parser.options.columnNameOfCorruptRecord,
- parser.options.multiLine)
+ parser.options.columnNameOfCorruptRecord)
filteredLines.flatMap(safeParser.parse)
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
index 7dfbb9d8b5c05..5d1feaed81a9a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
@@ -87,7 +87,7 @@ class JDBCOptions(
if (subquery.isEmpty) {
throw new IllegalArgumentException(s"Option `$JDBC_QUERY_STRING` can not be empty.")
} else {
- s"(${subquery}) __SPARK_GEN_JDBC_SUBQUERY_NAME_${curId.getAndIncrement()}"
+ s"(${subquery}) SPARK_GEN_SUBQ_${curId.getAndIncrement()}"
}
}
@@ -137,20 +137,17 @@ class JDBCOptions(
|the partition columns using the supplied subquery alias to resolve any ambiguity.
|Example :
|spark.read.format("jdbc")
- | .option("dbtable", "(select c1, c2 from t1) as subq")
- | .option("partitionColumn", "subq.c1"
- | .load()
+ | .option("url", jdbcUrl)
+ | .option("dbtable", "(select c1, c2 from t1) as subq")
+ | .option("partitionColumn", "c1")
+ | .option("lowerBound", "1")
+ | .option("upperBound", "100")
+ | .option("numPartitions", "3")
+ | .load()
""".stripMargin
)
- val fetchSize = {
- val size = parameters.getOrElse(JDBC_BATCH_FETCH_SIZE, "0").toInt
- require(size >= 0,
- s"Invalid value `${size.toString}` for parameter " +
- s"`$JDBC_BATCH_FETCH_SIZE`. The minimum value is 0. When the value is 0, " +
- "the JDBC driver ignores the value and does the estimates.")
- size
- }
+ val fetchSize = parameters.getOrElse(JDBC_BATCH_FETCH_SIZE, "0").toInt
// ------------------------------------------------------------
// Optional parameters only for writing
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
index edea549748b47..c0f628ff04108 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
@@ -438,6 +438,10 @@ object JdbcUtils extends Logging {
(rs: ResultSet, row: InternalRow, pos: Int) =>
row.setShort(pos, rs.getShort(pos + 1))
+ case ByteType =>
+ (rs: ResultSet, row: InternalRow, pos: Int) =>
+ row.setByte(pos, rs.getByte(pos + 1))
+
case StringType =>
(rs: ResultSet, row: InternalRow, pos: Int) =>
// TODO(davies): use getBytes for better performance, if the encoding is UTF-8
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
index 76f58371ae264..d6c588894d7f8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
@@ -139,8 +139,7 @@ object TextInputJsonDataSource extends JsonDataSource {
input => parser.parse(input, textParser, textToUTF8String),
parser.options.parseMode,
schema,
- parser.options.columnNameOfCorruptRecord,
- parser.options.multiLine)
+ parser.options.columnNameOfCorruptRecord)
linesReader.flatMap(safeParser.parse)
}
@@ -224,8 +223,7 @@ object MultiLineJsonDataSource extends JsonDataSource {
input => parser.parse[InputStream](input, streamParser, partitionedFileString),
parser.options.parseMode,
schema,
- parser.options.columnNameOfCorruptRecord,
- parser.options.multiLine)
+ parser.options.columnNameOfCorruptRecord)
safeParser.parse(
CodecStreams.createInputStreamWithCloseResource(conf, new Path(new URI(file.filePath))))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
index 4ecc54bd2fd96..decd5c5f19258 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
@@ -37,28 +37,34 @@ class OrcDeserializer(
private val resultRow = new SpecificInternalRow(requiredSchema.map(_.dataType))
+ // `fieldWriters(index)` is
+ // - null if the respective source column is missing, since the output value
+ // is always null in this case
+ // - a function that updates target column `index` otherwise.
private val fieldWriters: Array[WritableComparable[_] => Unit] = {
requiredSchema.zipWithIndex
- // The value of missing columns are always null, do not need writers.
- .filterNot { case (_, index) => requestedColIds(index) == -1 }
.map { case (f, index) =>
- val writer = newWriter(f.dataType, new RowUpdater(resultRow))
- (value: WritableComparable[_]) => writer(index, value)
+ if (requestedColIds(index) == -1) {
+ null
+ } else {
+ val writer = newWriter(f.dataType, new RowUpdater(resultRow))
+ (value: WritableComparable[_]) => writer(index, value)
+ }
}.toArray
}
- private val validColIds = requestedColIds.filterNot(_ == -1)
-
def deserialize(orcStruct: OrcStruct): InternalRow = {
- var i = 0
- while (i < validColIds.length) {
- val value = orcStruct.getFieldValue(validColIds(i))
- if (value == null) {
- resultRow.setNullAt(i)
- } else {
- fieldWriters(i)(value)
+ var targetColumnIndex = 0
+ while (targetColumnIndex < fieldWriters.length) {
+ if (fieldWriters(targetColumnIndex) != null) {
+ val value = orcStruct.getFieldValue(requestedColIds(targetColumnIndex))
+ if (value == null) {
+ resultRow.setNullAt(targetColumnIndex)
+ } else {
+ fieldWriters(targetColumnIndex)(value)
+ }
}
- i += 1
+ targetColumnIndex += 1
}
resultRow
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index dbafc468c6c40..6bc0d4f79ce11 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql.execution.datasources.orc
+import org.apache.orc.storage.common.`type`.HiveDecimal
import org.apache.orc.storage.ql.io.sarg.{PredicateLeaf, SearchArgument}
import org.apache.orc.storage.ql.io.sarg.SearchArgument.Builder
import org.apache.orc.storage.ql.io.sarg.SearchArgumentFactory.newBuilder
@@ -67,6 +68,16 @@ private[sql] object OrcFilters {
}
}
+ // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
+ // in order to distinguish predicate pushdown for nested columns.
+ private def quoteAttributeNameIfNeeded(name: String) : String = {
+ if (!name.contains("`") && name.contains(".")) {
+ s"`$name`"
+ } else {
+ name
+ }
+ }
+
/**
* Create ORC filter as a SearchArgument instance.
*/
@@ -124,10 +135,7 @@ private[sql] object OrcFilters {
case FloatType | DoubleType =>
value.asInstanceOf[Number].doubleValue()
case _: DecimalType =>
- val decimal = value.asInstanceOf[java.math.BigDecimal]
- val decimalWritable = new HiveDecimalWritable(decimal.longValue)
- decimalWritable.mutateEnforcePrecisionScale(decimal.precision, decimal.scale)
- decimalWritable
+ new HiveDecimalWritable(HiveDecimal.create(value.asInstanceOf[java.math.BigDecimal]))
case _ => value
}
@@ -178,38 +186,47 @@ private[sql] object OrcFilters {
// wrapped by a "parent" predicate (`And`, `Or`, or `Not`).
case EqualTo(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
+ val quotedName = quoteAttributeNameIfNeeded(attribute)
val castedValue = castLiteralValue(value, dataTypeMap(attribute))
- Some(builder.startAnd().equals(attribute, getType(attribute), castedValue).end())
+ Some(builder.startAnd().equals(quotedName, getType(attribute), castedValue).end())
case EqualNullSafe(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
+ val quotedName = quoteAttributeNameIfNeeded(attribute)
val castedValue = castLiteralValue(value, dataTypeMap(attribute))
- Some(builder.startAnd().nullSafeEquals(attribute, getType(attribute), castedValue).end())
+ Some(builder.startAnd().nullSafeEquals(quotedName, getType(attribute), castedValue).end())
case LessThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
+ val quotedName = quoteAttributeNameIfNeeded(attribute)
val castedValue = castLiteralValue(value, dataTypeMap(attribute))
- Some(builder.startAnd().lessThan(attribute, getType(attribute), castedValue).end())
+ Some(builder.startAnd().lessThan(quotedName, getType(attribute), castedValue).end())
case LessThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
+ val quotedName = quoteAttributeNameIfNeeded(attribute)
val castedValue = castLiteralValue(value, dataTypeMap(attribute))
- Some(builder.startAnd().lessThanEquals(attribute, getType(attribute), castedValue).end())
+ Some(builder.startAnd().lessThanEquals(quotedName, getType(attribute), castedValue).end())
case GreaterThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
+ val quotedName = quoteAttributeNameIfNeeded(attribute)
val castedValue = castLiteralValue(value, dataTypeMap(attribute))
- Some(builder.startNot().lessThanEquals(attribute, getType(attribute), castedValue).end())
+ Some(builder.startNot().lessThanEquals(quotedName, getType(attribute), castedValue).end())
case GreaterThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
+ val quotedName = quoteAttributeNameIfNeeded(attribute)
val castedValue = castLiteralValue(value, dataTypeMap(attribute))
- Some(builder.startNot().lessThan(attribute, getType(attribute), castedValue).end())
+ Some(builder.startNot().lessThan(quotedName, getType(attribute), castedValue).end())
case IsNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
- Some(builder.startAnd().isNull(attribute, getType(attribute)).end())
+ val quotedName = quoteAttributeNameIfNeeded(attribute)
+ Some(builder.startAnd().isNull(quotedName, getType(attribute)).end())
case IsNotNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
- Some(builder.startNot().isNull(attribute, getType(attribute)).end())
+ val quotedName = quoteAttributeNameIfNeeded(attribute)
+ Some(builder.startNot().isNull(quotedName, getType(attribute)).end())
case In(attribute, values) if isSearchableType(dataTypeMap(attribute)) =>
+ val quotedName = quoteAttributeNameIfNeeded(attribute)
val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(attribute)))
- Some(builder.startAnd().in(attribute, getType(attribute),
+ Some(builder.startAnd().in(quotedName, getType(attribute),
castedValues.map(_.asInstanceOf[AnyRef]): _*).end())
case _ => None
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
index ac062fdc092ee..95fb25bf5addb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.execution.datasources.orc
+import java.util.Locale
+
import scala.collection.JavaConverters._
import org.apache.hadoop.conf.Configuration
@@ -27,7 +29,7 @@ import org.apache.spark.SparkException
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.internal.Logging
import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.analysis.{caseInsensitiveResolution, caseSensitiveResolution}
+import org.apache.spark.sql.catalyst.analysis.caseSensitiveResolution
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
import org.apache.spark.sql.types._
@@ -116,8 +118,29 @@ object OrcUtils extends Logging {
}
})
} else {
- val resolver = if (isCaseSensitive) caseSensitiveResolution else caseInsensitiveResolution
- Some(requiredSchema.fieldNames.map { name => orcFieldNames.indexWhere(resolver(_, name)) })
+ if (isCaseSensitive) {
+ Some(requiredSchema.fieldNames.map { name =>
+ orcFieldNames.indexWhere(caseSensitiveResolution(_, name))
+ })
+ } else {
+ // Do case-insensitive resolution only if in case-insensitive mode
+ val caseInsensitiveOrcFieldMap =
+ orcFieldNames.zipWithIndex.groupBy(_._1.toLowerCase(Locale.ROOT))
+ Some(requiredSchema.fieldNames.map { requiredFieldName =>
+ caseInsensitiveOrcFieldMap
+ .get(requiredFieldName.toLowerCase(Locale.ROOT))
+ .map { matchedOrcFields =>
+ if (matchedOrcFields.size > 1) {
+ // Need to fail if there is ambiguity, i.e. more than one field is matched.
+ val matchedOrcFieldsString = matchedOrcFields.map(_._1).mkString("[", ", ", "]")
+ throw new RuntimeException(s"""Found duplicate field(s) "$requiredFieldName": """
+ + s"$matchedOrcFieldsString in case-insensitive mode")
+ } else {
+ matchedOrcFields.head._2
+ }
+ }.getOrElse(-1)
+ })
+ }
}
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
index ea4f1592a7c2e..16cd570901c26 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
@@ -129,7 +129,7 @@ class ParquetFileFormat
conf.setEnum(ParquetOutputFormat.JOB_SUMMARY_LEVEL, JobSummaryLevel.NONE)
}
- if (ParquetOutputFormat.getJobSummaryLevel(conf) == JobSummaryLevel.NONE
+ if (ParquetOutputFormat.getJobSummaryLevel(conf) != JobSummaryLevel.NONE
&& !classOf[ParquetOutputCommitter].isAssignableFrom(committerClass)) {
// output summary is requested, but the class is not a Parquet Committer
logWarning(s"Committer $committerClass is not a ParquetOutputCommitter and cannot" +
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
index 0c286defb9406..7e420d36f4455 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
@@ -541,7 +541,7 @@ private[parquet] class ParquetFilters(
}
override def keep(value: Binary): Boolean = {
- UTF8String.fromBytes(value.getBytes).startsWith(
+ value != null && UTF8String.fromBytes(value.getBytes).startsWith(
UTF8String.fromBytes(strToBinary.getBytes))
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala
index 6a46b5f8edc54..840fcae8c6915 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.execution.datasources.parquet
-import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, Expression, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.planning.PhysicalOperation
import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project}
import org.apache.spark.sql.catalyst.rules.Rule
@@ -110,7 +110,35 @@ private[sql] object ParquetSchemaPruning extends Rule[LogicalPlan] {
val projectionRootFields = projects.flatMap(getRootFields)
val filterRootFields = filters.flatMap(getRootFields)
- (projectionRootFields ++ filterRootFields).distinct
+ // Kind of expressions don't need to access any fields of a root fields, e.g., `IsNotNull`.
+ // For them, if there are any nested fields accessed in the query, we don't need to add root
+ // field access of above expressions.
+ // For example, for a query `SELECT name.first FROM contacts WHERE name IS NOT NULL`,
+ // we don't need to read nested fields of `name` struct other than `first` field.
+ val (rootFields, optRootFields) = (projectionRootFields ++ filterRootFields)
+ .distinct.partition(!_.prunedIfAnyChildAccessed)
+
+ optRootFields.filter { opt =>
+ !rootFields.exists { root =>
+ root.field.name == opt.field.name && {
+ // Checking if current optional root field can be pruned.
+ // For each required root field, we merge it with the optional root field:
+ // 1. If this optional root field has nested fields and any nested field of it is used
+ // in the query, the merged field type must equal to the optional root field type.
+ // We can prune this optional root field. For example, for optional root field
+ // `struct>`, if its field
+ // `struct>` is used, we don't need to add this optional
+ // root field.
+ // 2. If this optional root field has no nested fields, the merged field type equals
+ // to the optional root field only if they are the same. If they are, we can prune
+ // this optional root field too.
+ val rootFieldType = StructType(Array(root.field))
+ val optFieldType = StructType(Array(opt.field))
+ val merged = optFieldType.merge(rootFieldType)
+ merged.sameType(optFieldType)
+ }
+ }
+ } ++ rootFields
}
/**
@@ -156,7 +184,7 @@ private[sql] object ParquetSchemaPruning extends Rule[LogicalPlan] {
// in the resulting schema may differ from their ordering in the logical relation's
// original schema
val mergedSchema = requestedRootFields
- .map { case RootField(field, _) => StructType(Array(field)) }
+ .map { case root: RootField => StructType(Array(root.field)) }
.reduceLeft(_ merge _)
val dataSchemaFieldNames = fileDataSchema.fieldNames.toSet
val mergedDataSchema =
@@ -199,6 +227,15 @@ private[sql] object ParquetSchemaPruning extends Rule[LogicalPlan] {
case att: Attribute =>
RootField(StructField(att.name, att.dataType, att.nullable), derivedFromAtt = true) :: Nil
case SelectedField(field) => RootField(field, derivedFromAtt = false) :: Nil
+ // Root field accesses by `IsNotNull` and `IsNull` are special cases as the expressions
+ // don't actually use any nested fields. These root field accesses might be excluded later
+ // if there are any nested fields accesses in the query plan.
+ case IsNotNull(SelectedField(field)) =>
+ RootField(field, derivedFromAtt = false, prunedIfAnyChildAccessed = true) :: Nil
+ case IsNull(SelectedField(field)) =>
+ RootField(field, derivedFromAtt = false, prunedIfAnyChildAccessed = true) :: Nil
+ case IsNotNull(_: Attribute) | IsNull(_: Attribute) =>
+ expr.children.flatMap(getRootFields).map(_.copy(prunedIfAnyChildAccessed = true))
case _ =>
expr.children.flatMap(getRootFields)
}
@@ -250,8 +287,11 @@ private[sql] object ParquetSchemaPruning extends Rule[LogicalPlan] {
}
/**
- * A "root" schema field (aka top-level, no-parent) and whether it was derived from
- * an attribute or had a proper child.
+ * This represents a "root" schema field (aka top-level, no-parent). `field` is the
+ * `StructField` for field name and datatype. `derivedFromAtt` indicates whether it
+ * was derived from an attribute or had a proper child. `prunedIfAnyChildAccessed` means
+ * whether this root field can be pruned if any of child field is used in the query.
*/
- private case class RootField(field: StructField, derivedFromAtt: Boolean)
+ private case class RootField(field: StructField, derivedFromAtt: Boolean,
+ prunedIfAnyChildAccessed: Boolean = false)
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala
index f62f7349d1da7..782829887c446 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala
@@ -17,22 +17,19 @@
package org.apache.spark.sql.execution.datasources.v2
-import org.apache.spark._
+import scala.reflect.ClassTag
+
+import org.apache.spark.{InterruptibleIterator, Partition, SparkContext, TaskContext}
import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.sources.v2.reader.{InputPartition, PartitionReader, PartitionReaderFactory}
+import org.apache.spark.sql.sources.v2.reader.InputPartition
-class DataSourceRDDPartition(val index: Int, val inputPartition: InputPartition)
+class DataSourceRDDPartition[T : ClassTag](val index: Int, val inputPartition: InputPartition[T])
extends Partition with Serializable
-// TODO: we should have 2 RDDs: an RDD[InternalRow] for row-based scan, an `RDD[ColumnarBatch]` for
-// columnar scan.
-class DataSourceRDD(
+class DataSourceRDD[T: ClassTag](
sc: SparkContext,
- @transient private val inputPartitions: Seq[InputPartition],
- partitionReaderFactory: PartitionReaderFactory,
- columnarReads: Boolean)
- extends RDD[InternalRow](sc, Nil) {
+ @transient private val inputPartitions: Seq[InputPartition[T]])
+ extends RDD[T](sc, Nil) {
override protected def getPartitions: Array[Partition] = {
inputPartitions.zipWithIndex.map {
@@ -40,21 +37,11 @@ class DataSourceRDD(
}.toArray
}
- private def castPartition(split: Partition): DataSourceRDDPartition = split match {
- case p: DataSourceRDDPartition => p
- case _ => throw new SparkException(s"[BUG] Not a DataSourceRDDPartition: $split")
- }
-
- override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
- val inputPartition = castPartition(split).inputPartition
- val reader: PartitionReader[_] = if (columnarReads) {
- partitionReaderFactory.createColumnarReader(inputPartition)
- } else {
- partitionReaderFactory.createReader(inputPartition)
- }
-
+ override def compute(split: Partition, context: TaskContext): Iterator[T] = {
+ val reader = split.asInstanceOf[DataSourceRDDPartition[T]].inputPartition
+ .createPartitionReader()
context.addTaskCompletionListener[Unit](_ => reader.close())
- val iter = new Iterator[Any] {
+ val iter = new Iterator[T] {
private[this] var valuePrepared = false
override def hasNext: Boolean = {
@@ -64,7 +51,7 @@ class DataSourceRDD(
valuePrepared
}
- override def next(): Any = {
+ override def next(): T = {
if (!hasNext) {
throw new java.util.NoSuchElementException("End of stream")
}
@@ -72,11 +59,10 @@ class DataSourceRDD(
reader.get()
}
}
- // TODO: SPARK-25083 remove the type erasure hack in data source scan
- new InterruptibleIterator(context, iter.asInstanceOf[Iterator[InternalRow]])
+ new InterruptibleIterator(context, iter)
}
override def getPreferredLocations(split: Partition): Seq[String] = {
- castPartition(split).inputPartition.preferredLocations()
+ split.asInstanceOf[DataSourceRDDPartition[T]].inputPartition.preferredLocations()
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Relation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Relation.scala
index f7e29593a6353..abc5fb979250a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Relation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Relation.scala
@@ -27,21 +27,21 @@ import org.apache.spark.sql.catalyst.analysis.{MultiInstanceRelation, NamedRelat
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression}
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
import org.apache.spark.sql.sources.DataSourceRegister
-import org.apache.spark.sql.sources.v2.{BatchReadSupportProvider, BatchWriteSupportProvider, DataSourceOptions, DataSourceV2}
-import org.apache.spark.sql.sources.v2.reader.{BatchReadSupport, ReadSupport, ScanConfigBuilder, SupportsReportStatistics}
-import org.apache.spark.sql.sources.v2.writer.BatchWriteSupport
+import org.apache.spark.sql.sources.v2.{DataSourceOptions, DataSourceV2, ReadSupport, WriteSupport}
+import org.apache.spark.sql.sources.v2.reader.{DataSourceReader, SupportsReportStatistics}
+import org.apache.spark.sql.sources.v2.writer.DataSourceWriter
import org.apache.spark.sql.types.StructType
/**
* A logical plan representing a data source v2 scan.
*
* @param source An instance of a [[DataSourceV2]] implementation.
- * @param options The options for this scan. Used to create fresh [[BatchWriteSupport]].
- * @param userSpecifiedSchema The user-specified schema for this scan.
+ * @param options The options for this scan. Used to create fresh [[DataSourceReader]].
+ * @param userSpecifiedSchema The user-specified schema for this scan. Used to create fresh
+ * [[DataSourceReader]].
*/
case class DataSourceV2Relation(
source: DataSourceV2,
- readSupport: BatchReadSupport,
output: Seq[AttributeReference],
options: Map[String, String],
tableIdent: Option[TableIdentifier] = None,
@@ -58,12 +58,13 @@ case class DataSourceV2Relation(
override def simpleString: String = "RelationV2 " + metadataString
- def newWriteSupport(): BatchWriteSupport = source.createWriteSupport(options, schema)
+ def newReader(): DataSourceReader = source.createReader(options, userSpecifiedSchema)
- override def computeStats(): Statistics = readSupport match {
+ def newWriter(): DataSourceWriter = source.createWriter(options, schema)
+
+ override def computeStats(): Statistics = newReader match {
case r: SupportsReportStatistics =>
- val statistics = r.estimateStatistics(readSupport.newScanConfigBuilder().build())
- Statistics(sizeInBytes = statistics.sizeInBytes().orElse(conf.defaultSizeInBytes))
+ Statistics(sizeInBytes = r.estimateStatistics.sizeInBytes().orElse(conf.defaultSizeInBytes))
case _ =>
Statistics(sizeInBytes = conf.defaultSizeInBytes)
}
@@ -84,8 +85,7 @@ case class StreamingDataSourceV2Relation(
output: Seq[AttributeReference],
source: DataSourceV2,
options: Map[String, String],
- readSupport: ReadSupport,
- scanConfigBuilder: ScanConfigBuilder)
+ reader: DataSourceReader)
extends LeafNode with MultiInstanceRelation with DataSourceV2StringFormat {
override def isStreaming: Boolean = true
@@ -99,8 +99,7 @@ case class StreamingDataSourceV2Relation(
// TODO: unify the equal/hashCode implementation for all data source v2 query plans.
override def equals(other: Any): Boolean = other match {
case other: StreamingDataSourceV2Relation =>
- output == other.output && readSupport.getClass == other.readSupport.getClass &&
- options == other.options
+ output == other.output && reader.getClass == other.reader.getClass && options == other.options
case _ => false
}
@@ -108,10 +107,9 @@ case class StreamingDataSourceV2Relation(
Seq(output, source, options).hashCode()
}
- override def computeStats(): Statistics = readSupport match {
+ override def computeStats(): Statistics = reader match {
case r: SupportsReportStatistics =>
- val statistics = r.estimateStatistics(scanConfigBuilder.build())
- Statistics(sizeInBytes = statistics.sizeInBytes().orElse(conf.defaultSizeInBytes))
+ Statistics(sizeInBytes = r.estimateStatistics.sizeInBytes().orElse(conf.defaultSizeInBytes))
case _ =>
Statistics(sizeInBytes = conf.defaultSizeInBytes)
}
@@ -119,19 +117,19 @@ case class StreamingDataSourceV2Relation(
object DataSourceV2Relation {
private implicit class SourceHelpers(source: DataSourceV2) {
- def asReadSupportProvider: BatchReadSupportProvider = {
+ def asReadSupport: ReadSupport = {
source match {
- case provider: BatchReadSupportProvider =>
- provider
+ case support: ReadSupport =>
+ support
case _ =>
throw new AnalysisException(s"Data source is not readable: $name")
}
}
- def asWriteSupportProvider: BatchWriteSupportProvider = {
+ def asWriteSupport: WriteSupport = {
source match {
- case provider: BatchWriteSupportProvider =>
- provider
+ case support: WriteSupport =>
+ support
case _ =>
throw new AnalysisException(s"Data source is not writable: $name")
}
@@ -146,26 +144,23 @@ object DataSourceV2Relation {
}
}
- def createReadSupport(
+ def createReader(
options: Map[String, String],
- userSpecifiedSchema: Option[StructType]): BatchReadSupport = {
+ userSpecifiedSchema: Option[StructType]): DataSourceReader = {
val v2Options = new DataSourceOptions(options.asJava)
userSpecifiedSchema match {
case Some(s) =>
- asReadSupportProvider.createBatchReadSupport(s, v2Options)
+ asReadSupport.createReader(s, v2Options)
case _ =>
- asReadSupportProvider.createBatchReadSupport(v2Options)
+ asReadSupport.createReader(v2Options)
}
}
- def createWriteSupport(
+ def createWriter(
options: Map[String, String],
- schema: StructType): BatchWriteSupport = {
- asWriteSupportProvider.createBatchWriteSupport(
- UUID.randomUUID().toString,
- schema,
- SaveMode.Append,
- new DataSourceOptions(options.asJava)).get
+ schema: StructType): DataSourceWriter = {
+ val v2Options = new DataSourceOptions(options.asJava)
+ asWriteSupport.createWriter(UUID.randomUUID.toString, schema, SaveMode.Append, v2Options).get
}
}
@@ -174,16 +169,15 @@ object DataSourceV2Relation {
options: Map[String, String],
tableIdent: Option[TableIdentifier] = None,
userSpecifiedSchema: Option[StructType] = None): DataSourceV2Relation = {
- val readSupport = source.createReadSupport(options, userSpecifiedSchema)
- val output = readSupport.fullSchema().toAttributes
+ val reader = source.createReader(options, userSpecifiedSchema)
val ident = tableIdent.orElse(tableFromOptions(options))
DataSourceV2Relation(
- source, readSupport, output, options, ident, userSpecifiedSchema)
+ source, reader.readSchema().toAttributes, options, ident, userSpecifiedSchema)
}
private def tableFromOptions(options: Map[String, String]): Option[TableIdentifier] = {
options
- .get(DataSourceOptions.TABLE_KEY)
- .map(TableIdentifier(_, options.get(DataSourceOptions.DATABASE_KEY)))
+ .get(DataSourceOptions.TABLE_KEY)
+ .map(TableIdentifier(_, options.get(DataSourceOptions.DATABASE_KEY)))
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2ScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2ScanExec.scala
index 04a97735d024d..c8494f97f1761 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2ScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2ScanExec.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.execution.datasources.v2
+import scala.collection.JavaConverters._
+
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions._
@@ -26,7 +28,8 @@ import org.apache.spark.sql.execution.{ColumnarBatchScan, LeafExecNode, WholeSta
import org.apache.spark.sql.execution.streaming.continuous._
import org.apache.spark.sql.sources.v2.DataSourceV2
import org.apache.spark.sql.sources.v2.reader._
-import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousPartitionReaderFactory, ContinuousReadSupport, MicroBatchReadSupport}
+import org.apache.spark.sql.sources.v2.reader.streaming.ContinuousReader
+import org.apache.spark.sql.vectorized.ColumnarBatch
/**
* Physical plan node for scanning data from a data source.
@@ -36,8 +39,7 @@ case class DataSourceV2ScanExec(
@transient source: DataSourceV2,
@transient options: Map[String, String],
@transient pushedFilters: Seq[Expression],
- @transient readSupport: ReadSupport,
- @transient scanConfig: ScanConfig)
+ @transient reader: DataSourceReader)
extends LeafExecNode with DataSourceV2StringFormat with ColumnarBatchScan {
override def simpleString: String = "ScanV2 " + metadataString
@@ -45,8 +47,7 @@ case class DataSourceV2ScanExec(
// TODO: unify the equal/hashCode implementation for all data source v2 query plans.
override def equals(other: Any): Boolean = other match {
case other: DataSourceV2ScanExec =>
- output == other.output && readSupport.getClass == other.readSupport.getClass &&
- options == other.options
+ output == other.output && reader.getClass == other.reader.getClass && options == other.options
case _ => false
}
@@ -54,39 +55,36 @@ case class DataSourceV2ScanExec(
Seq(output, source, options).hashCode()
}
- override def outputPartitioning: physical.Partitioning = readSupport match {
- case _ if partitions.length == 1 =>
+ override def outputPartitioning: physical.Partitioning = reader match {
+ case r: SupportsScanColumnarBatch if r.enableBatchRead() && batchPartitions.size == 1 =>
+ SinglePartition
+
+ case r: SupportsScanColumnarBatch if !r.enableBatchRead() && partitions.size == 1 =>
+ SinglePartition
+
+ case r if !r.isInstanceOf[SupportsScanColumnarBatch] && partitions.size == 1 =>
SinglePartition
case s: SupportsReportPartitioning =>
new DataSourcePartitioning(
- s.outputPartitioning(scanConfig), AttributeMap(output.map(a => a -> a.name)))
+ s.outputPartitioning(), AttributeMap(output.map(a => a -> a.name)))
case _ => super.outputPartitioning
}
- private lazy val partitions: Seq[InputPartition] = readSupport.planInputPartitions(scanConfig)
-
- private lazy val readerFactory = readSupport match {
- case r: BatchReadSupport => r.createReaderFactory(scanConfig)
- case r: MicroBatchReadSupport => r.createReaderFactory(scanConfig)
- case r: ContinuousReadSupport => r.createContinuousReaderFactory(scanConfig)
- case _ => throw new IllegalStateException("unknown read support: " + readSupport)
+ private lazy val partitions: Seq[InputPartition[InternalRow]] = {
+ reader.planInputPartitions().asScala
}
- // TODO: clean this up when we have dedicated scan plan for continuous streaming.
- override val supportsBatch: Boolean = {
- require(partitions.forall(readerFactory.supportColumnarReads) ||
- !partitions.exists(readerFactory.supportColumnarReads),
- "Cannot mix row-based and columnar input partitions.")
-
- partitions.exists(readerFactory.supportColumnarReads)
+ private lazy val batchPartitions: Seq[InputPartition[ColumnarBatch]] = reader match {
+ case r: SupportsScanColumnarBatch if r.enableBatchRead() =>
+ assert(!reader.isInstanceOf[ContinuousReader],
+ "continuous stream reader does not support columnar read yet.")
+ r.planBatchInputPartitions().asScala
}
- private lazy val inputRDD: RDD[InternalRow] = readSupport match {
- case _: ContinuousReadSupport =>
- assert(!supportsBatch,
- "continuous stream reader does not support columnar read yet.")
+ private lazy val inputRDD: RDD[InternalRow] = reader match {
+ case _: ContinuousReader =>
EpochCoordinatorRef.get(
sparkContext.getLocalProperty(ContinuousExecution.EPOCH_COORDINATOR_ID_KEY),
sparkContext.env)
@@ -95,17 +93,22 @@ case class DataSourceV2ScanExec(
sparkContext,
sqlContext.conf.continuousStreamingExecutorQueueSize,
sqlContext.conf.continuousStreamingExecutorPollIntervalMs,
- partitions,
- schema,
- readerFactory.asInstanceOf[ContinuousPartitionReaderFactory])
+ partitions).asInstanceOf[RDD[InternalRow]]
+
+ case r: SupportsScanColumnarBatch if r.enableBatchRead() =>
+ new DataSourceRDD(sparkContext, batchPartitions).asInstanceOf[RDD[InternalRow]]
case _ =>
- new DataSourceRDD(
- sparkContext, partitions, readerFactory.asInstanceOf[PartitionReaderFactory], supportsBatch)
+ new DataSourceRDD(sparkContext, partitions).asInstanceOf[RDD[InternalRow]]
}
override def inputRDDs(): Seq[RDD[InternalRow]] = Seq(inputRDD)
+ override val supportsBatch: Boolean = reader match {
+ case r: SupportsScanColumnarBatch if r.enableBatchRead() => true
+ case _ => false
+ }
+
override protected def needsUnsafeRowConversion: Boolean = false
override protected def doExecute(): RDD[InternalRow] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
index 9a3109e7c199e..9d97d3b58f30c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
@@ -26,8 +26,8 @@ import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan, Rep
import org.apache.spark.sql.execution.{FilterExec, ProjectExec, SparkPlan}
import org.apache.spark.sql.execution.datasources.DataSourceStrategy
import org.apache.spark.sql.execution.streaming.continuous.{ContinuousCoalesceExec, WriteToContinuousDataSource, WriteToContinuousDataSourceExec}
-import org.apache.spark.sql.sources.v2.reader._
-import org.apache.spark.sql.sources.v2.reader.streaming.ContinuousReadSupport
+import org.apache.spark.sql.sources.v2.reader.{DataSourceReader, SupportsPushDownFilters, SupportsPushDownRequiredColumns}
+import org.apache.spark.sql.sources.v2.reader.streaming.ContinuousReader
object DataSourceV2Strategy extends Strategy {
@@ -37,9 +37,9 @@ object DataSourceV2Strategy extends Strategy {
* @return pushed filter and post-scan filters.
*/
private def pushFilters(
- configBuilder: ScanConfigBuilder,
+ reader: DataSourceReader,
filters: Seq[Expression]): (Seq[Expression], Seq[Expression]) = {
- configBuilder match {
+ reader match {
case r: SupportsPushDownFilters =>
// A map from translated data source filters to original catalyst filter expressions.
val translatedFilterToExpr = mutable.HashMap.empty[sources.Filter, Expression]
@@ -71,43 +71,41 @@ object DataSourceV2Strategy extends Strategy {
/**
* Applies column pruning to the data source, w.r.t. the references of the given expressions.
*
- * @return the created `ScanConfig`(since column pruning is the last step of operator pushdown),
- * and new output attributes after column pruning.
+ * @return new output attributes after column pruning.
*/
// TODO: nested column pruning.
private def pruneColumns(
- configBuilder: ScanConfigBuilder,
+ reader: DataSourceReader,
relation: DataSourceV2Relation,
- exprs: Seq[Expression]): (ScanConfig, Seq[AttributeReference]) = {
- configBuilder match {
+ exprs: Seq[Expression]): Seq[AttributeReference] = {
+ reader match {
case r: SupportsPushDownRequiredColumns =>
val requiredColumns = AttributeSet(exprs.flatMap(_.references))
val neededOutput = relation.output.filter(requiredColumns.contains)
if (neededOutput != relation.output) {
r.pruneColumns(neededOutput.toStructType)
- val config = r.build()
val nameToAttr = relation.output.map(_.name).zip(relation.output).toMap
- config -> config.readSchema().toAttributes.map {
+ r.readSchema().toAttributes.map {
// We have to keep the attribute id during transformation.
a => a.withExprId(nameToAttr(a.name).exprId)
}
} else {
- r.build() -> relation.output
+ relation.output
}
- case _ => configBuilder.build() -> relation.output
+ case _ => relation.output
}
}
override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
case PhysicalOperation(project, filters, relation: DataSourceV2Relation) =>
- val configBuilder = relation.readSupport.newScanConfigBuilder()
+ val reader = relation.newReader()
// `pushedFilters` will be pushed down and evaluated in the underlying data sources.
// `postScanFilters` need to be evaluated after the scan.
// `postScanFilters` and `pushedFilters` can overlap, e.g. the parquet row group filter.
- val (pushedFilters, postScanFilters) = pushFilters(configBuilder, filters)
- val (config, output) = pruneColumns(configBuilder, relation, project ++ postScanFilters)
+ val (pushedFilters, postScanFilters) = pushFilters(reader, filters)
+ val output = pruneColumns(reader, relation, project ++ postScanFilters)
logInfo(
s"""
|Pushing operators to ${relation.source.getClass}
@@ -117,12 +115,7 @@ object DataSourceV2Strategy extends Strategy {
""".stripMargin)
val scan = DataSourceV2ScanExec(
- output,
- relation.source,
- relation.options,
- pushedFilters,
- relation.readSupport,
- config)
+ output, relation.source, relation.options, pushedFilters, reader)
val filterCondition = postScanFilters.reduceLeftOption(And)
val withFilter = filterCondition.map(FilterExec(_, scan)).getOrElse(scan)
@@ -131,26 +124,22 @@ object DataSourceV2Strategy extends Strategy {
ProjectExec(project, withFilter) :: Nil
case r: StreamingDataSourceV2Relation =>
- // TODO: support operator pushdown for streaming data sources.
- val scanConfig = r.scanConfigBuilder.build()
// ensure there is a projection, which will produce unsafe rows required by some operators
ProjectExec(r.output,
- DataSourceV2ScanExec(
- r.output, r.source, r.options, r.pushedFilters, r.readSupport, scanConfig)) :: Nil
+ DataSourceV2ScanExec(r.output, r.source, r.options, r.pushedFilters, r.reader)) :: Nil
case WriteToDataSourceV2(writer, query) =>
WriteToDataSourceV2Exec(writer, planLater(query)) :: Nil
case AppendData(r: DataSourceV2Relation, query, _) =>
- WriteToDataSourceV2Exec(r.newWriteSupport(), planLater(query)) :: Nil
+ WriteToDataSourceV2Exec(r.newWriter(), planLater(query)) :: Nil
case WriteToContinuousDataSource(writer, query) =>
WriteToContinuousDataSourceExec(writer, planLater(query)) :: Nil
case Repartition(1, false, child) =>
- val isContinuous = child.find {
- case s: StreamingDataSourceV2Relation => s.readSupport.isInstanceOf[ContinuousReadSupport]
- case _ => false
+ val isContinuous = child.collectFirst {
+ case StreamingDataSourceV2Relation(_, _, _, r: ContinuousReader) => r
}.isDefined
if (isContinuous) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Utils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Utils.scala
index e9cc3991155c4..5267f5f1580c3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Utils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Utils.scala
@@ -21,7 +21,6 @@ import java.util.regex.Pattern
import org.apache.spark.internal.Logging
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.sources.DataSourceRegister
import org.apache.spark.sql.sources.v2.{DataSourceV2, SessionConfigSupport}
private[sql] object DataSourceV2Utils extends Logging {
@@ -56,12 +55,4 @@ private[sql] object DataSourceV2Utils extends Logging {
case _ => Map.empty
}
-
- def failForUserSpecifiedSchema[T](ds: DataSourceV2): T = {
- val name = ds match {
- case register: DataSourceRegister => register.shortName()
- case _ => ds.getClass.getName
- }
- throw new UnsupportedOperationException(name + " source does not support user-specified schema")
- }
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
index c3f7b690ef636..ae8941a40a276 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
@@ -23,11 +23,15 @@ import org.apache.spark.{SparkEnv, SparkException, TaskContext}
import org.apache.spark.executor.CommitDeniedException
import org.apache.spark.internal.Logging
import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.execution.streaming.MicroBatchExecution
import org.apache.spark.sql.sources.v2.writer._
+import org.apache.spark.sql.types.StructType
import org.apache.spark.util.Utils
/**
@@ -35,8 +39,7 @@ import org.apache.spark.util.Utils
* specific logical plans, like [[org.apache.spark.sql.catalyst.plans.logical.AppendData]].
*/
@deprecated("Use specific logical plans like AppendData instead", "2.4.0")
-case class WriteToDataSourceV2(writeSupport: BatchWriteSupport, query: LogicalPlan)
- extends LogicalPlan {
+case class WriteToDataSourceV2(writer: DataSourceWriter, query: LogicalPlan) extends LogicalPlan {
override def children: Seq[LogicalPlan] = Seq(query)
override def output: Seq[Attribute] = Nil
}
@@ -44,48 +47,46 @@ case class WriteToDataSourceV2(writeSupport: BatchWriteSupport, query: LogicalPl
/**
* The physical plan for writing data into data source v2.
*/
-case class WriteToDataSourceV2Exec(writeSupport: BatchWriteSupport, query: SparkPlan)
- extends SparkPlan {
-
+case class WriteToDataSourceV2Exec(writer: DataSourceWriter, query: SparkPlan) extends SparkPlan {
override def children: Seq[SparkPlan] = Seq(query)
override def output: Seq[Attribute] = Nil
override protected def doExecute(): RDD[InternalRow] = {
- val writerFactory = writeSupport.createBatchWriterFactory()
- val useCommitCoordinator = writeSupport.useCommitCoordinator
+ val writeTask = writer.createWriterFactory()
+ val useCommitCoordinator = writer.useCommitCoordinator
val rdd = query.execute()
val messages = new Array[WriterCommitMessage](rdd.partitions.length)
- logInfo(s"Start processing data source write support: $writeSupport. " +
+ logInfo(s"Start processing data source writer: $writer. " +
s"The input RDD has ${messages.length} partitions.")
try {
sparkContext.runJob(
rdd,
(context: TaskContext, iter: Iterator[InternalRow]) =>
- DataWritingSparkTask.run(writerFactory, context, iter, useCommitCoordinator),
+ DataWritingSparkTask.run(writeTask, context, iter, useCommitCoordinator),
rdd.partitions.indices,
(index, message: WriterCommitMessage) => {
messages(index) = message
- writeSupport.onDataWriterCommit(message)
+ writer.onDataWriterCommit(message)
}
)
- logInfo(s"Data source write support $writeSupport is committing.")
- writeSupport.commit(messages)
- logInfo(s"Data source write support $writeSupport committed.")
+ logInfo(s"Data source writer $writer is committing.")
+ writer.commit(messages)
+ logInfo(s"Data source writer $writer committed.")
} catch {
case cause: Throwable =>
- logError(s"Data source write support $writeSupport is aborting.")
+ logError(s"Data source writer $writer is aborting.")
try {
- writeSupport.abort(messages)
+ writer.abort(messages)
} catch {
case t: Throwable =>
- logError(s"Data source write support $writeSupport failed to abort.")
+ logError(s"Data source writer $writer failed to abort.")
cause.addSuppressed(t)
throw new SparkException("Writing job failed.", cause)
}
- logError(s"Data source write support $writeSupport aborted.")
+ logError(s"Data source writer $writer aborted.")
cause match {
// Only wrap non fatal exceptions.
case NonFatal(e) => throw new SparkException("Writing job aborted.", e)
@@ -99,7 +100,7 @@ case class WriteToDataSourceV2Exec(writeSupport: BatchWriteSupport, query: Spark
object DataWritingSparkTask extends Logging {
def run(
- writerFactory: DataWriterFactory,
+ writeTask: DataWriterFactory[InternalRow],
context: TaskContext,
iter: Iterator[InternalRow],
useCommitCoordinator: Boolean): WriterCommitMessage = {
@@ -108,7 +109,8 @@ object DataWritingSparkTask extends Logging {
val partId = context.partitionId()
val taskId = context.taskAttemptId()
val attemptId = context.attemptNumber()
- val dataWriter = writerFactory.createWriter(partId, taskId)
+ val epochId = Option(context.getLocalProperty(MicroBatchExecution.BATCH_ID_KEY)).getOrElse("0")
+ val dataWriter = writeTask.createDataWriter(partId, taskId, epochId.toLong)
// write the data and commit this writer.
Utils.tryWithSafeFinallyAndFailureCallbacks(block = {
@@ -120,11 +122,11 @@ object DataWritingSparkTask extends Logging {
val coordinator = SparkEnv.get.outputCommitCoordinator
val commitAuthorized = coordinator.canCommit(stageId, stageAttempt, partId, attemptId)
if (commitAuthorized) {
- logInfo(s"Commit authorized for partition $partId (task $taskId, attempt $attemptId" +
+ logInfo(s"Commit authorized for partition $partId (task $taskId, attempt $attemptId, " +
s"stage $stageId.$stageAttempt)")
dataWriter.commit()
} else {
- val message = s"Commit denied for partition $partId (task $taskId, attempt $attemptId" +
+ val message = s"Commit denied for partition $partId (task $taskId, attempt $attemptId, " +
s"stage $stageId.$stageAttempt)"
logInfo(message)
// throwing CommitDeniedException will trigger the catch block for abort
@@ -136,17 +138,17 @@ object DataWritingSparkTask extends Logging {
dataWriter.commit()
}
- logInfo(s"Committed partition $partId (task $taskId, attempt $attemptId" +
+ logInfo(s"Committed partition $partId (task $taskId, attempt $attemptId, " +
s"stage $stageId.$stageAttempt)")
msg
})(catchBlock = {
// If there is an error, abort this writer
- logError(s"Aborting commit for partition $partId (task $taskId, attempt $attemptId" +
+ logError(s"Aborting commit for partition $partId (task $taskId, attempt $attemptId, " +
s"stage $stageId.$stageAttempt)")
dataWriter.abort()
- logError(s"Aborted commit for partition $partId (task $taskId, attempt $attemptId" +
+ logError(s"Aborted commit for partition $partId (task $taskId, attempt $attemptId, " +
s"stage $stageId.$stageAttempt)")
})
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
index d2d5011bbcb97..bdb9a3129f036 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
@@ -24,8 +24,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.physical._
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, ShuffledHashJoinExec,
- SortMergeJoinExec}
+import org.apache.spark.sql.execution.joins.{ShuffledHashJoinExec, SortMergeJoinExec}
import org.apache.spark.sql.internal.SQLConf
/**
@@ -221,25 +220,41 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
}
private def reorder(
- leftKeys: Seq[Expression],
- rightKeys: Seq[Expression],
+ leftKeys: IndexedSeq[Expression],
+ rightKeys: IndexedSeq[Expression],
expectedOrderOfKeys: Seq[Expression],
currentOrderOfKeys: Seq[Expression]): (Seq[Expression], Seq[Expression]) = {
- val leftKeysBuffer = ArrayBuffer[Expression]()
- val rightKeysBuffer = ArrayBuffer[Expression]()
- val pickedIndexes = mutable.Set[Int]()
- val keysAndIndexes = currentOrderOfKeys.zipWithIndex
+ if (expectedOrderOfKeys.size != currentOrderOfKeys.size) {
+ return (leftKeys, rightKeys)
+ }
+
+ // Build a lookup between an expression and the positions its holds in the current key seq.
+ val keyToIndexMap = mutable.Map.empty[Expression, mutable.BitSet]
+ currentOrderOfKeys.zipWithIndex.foreach {
+ case (key, index) =>
+ keyToIndexMap.getOrElseUpdate(key.canonicalized, mutable.BitSet.empty).add(index)
+ }
+
+ // Reorder the keys.
+ val leftKeysBuffer = new ArrayBuffer[Expression](leftKeys.size)
+ val rightKeysBuffer = new ArrayBuffer[Expression](rightKeys.size)
+ val iterator = expectedOrderOfKeys.iterator
+ while (iterator.hasNext) {
+ // Lookup the current index of this key.
+ keyToIndexMap.get(iterator.next().canonicalized) match {
+ case Some(indices) if indices.nonEmpty =>
+ // Take the first available index from the map.
+ val index = indices.firstKey
+ indices.remove(index)
- expectedOrderOfKeys.foreach(expression => {
- val index = keysAndIndexes.find { case (e, idx) =>
- // As we may have the same key used many times, we need to filter out its occurrence we
- // have already used.
- e.semanticEquals(expression) && !pickedIndexes.contains(idx)
- }.map(_._2).get
- pickedIndexes += index
- leftKeysBuffer.append(leftKeys(index))
- rightKeysBuffer.append(rightKeys(index))
- })
+ // Add the keys for that index to the reordered keys.
+ leftKeysBuffer += leftKeys(index)
+ rightKeysBuffer += rightKeys(index)
+ case _ =>
+ // The expression cannot be found, or we have exhausted all indices for that expression.
+ return (leftKeys, rightKeys)
+ }
+ }
(leftKeysBuffer, rightKeysBuffer)
}
@@ -249,20 +264,13 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
leftPartitioning: Partitioning,
rightPartitioning: Partitioning): (Seq[Expression], Seq[Expression]) = {
if (leftKeys.forall(_.deterministic) && rightKeys.forall(_.deterministic)) {
- leftPartitioning match {
- case HashPartitioning(leftExpressions, _)
- if leftExpressions.length == leftKeys.length &&
- leftKeys.forall(x => leftExpressions.exists(_.semanticEquals(x))) =>
- reorder(leftKeys, rightKeys, leftExpressions, leftKeys)
-
- case _ => rightPartitioning match {
- case HashPartitioning(rightExpressions, _)
- if rightExpressions.length == rightKeys.length &&
- rightKeys.forall(x => rightExpressions.exists(_.semanticEquals(x))) =>
- reorder(leftKeys, rightKeys, rightExpressions, rightKeys)
-
- case _ => (leftKeys, rightKeys)
- }
+ (leftPartitioning, rightPartitioning) match {
+ case (HashPartitioning(leftExpressions, _), _) =>
+ reorder(leftKeys.toIndexedSeq, rightKeys.toIndexedSeq, leftExpressions, leftKeys)
+ case (_, HashPartitioning(rightExpressions, _)) =>
+ reorder(leftKeys.toIndexedSeq, rightKeys.toIndexedSeq, rightExpressions, rightKeys)
+ case _ =>
+ (leftKeys, rightKeys)
}
} else {
(leftKeys, rightKeys)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
index 9576605b1a214..0eaa30ec57be5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
@@ -231,11 +231,6 @@ object ShuffleExchangeExec {
override def numPartitions: Int = 1
override def getPartition(key: Any): Int = 0
}
- case l: LocalPartitioning =>
- new Partitioner {
- override def numPartitions: Int = l.numPartitions
- override def getPartition(key: Any): Int = key.asInstanceOf[Int]
- }
case _ => sys.error(s"Exchange not implemented for $newPartitioning")
// TODO: Handle BroadcastPartitioning.
}
@@ -252,9 +247,6 @@ object ShuffleExchangeExec {
val projection = UnsafeProjection.create(h.partitionIdExpression :: Nil, outputAttributes)
row => projection(row).getInt(0)
case RangePartitioning(_, _) | SinglePartition => identity
- case _: LocalPartitioning =>
- val partitionId = TaskContext.get().partitionId()
- _ => partitionId
case _ => sys.error(s"Exchange not implemented for $newPartitioning")
}
@@ -277,7 +269,7 @@ object ShuffleExchangeExec {
}
// The comparator for comparing row hashcode, which should always be Integer.
val prefixComparator = PrefixComparators.LONG
- val canUseRadixSort = SparkEnv.get.conf.get(SQLConf.RADIX_SORT_ENABLED)
+
// The prefix computer generates row hashcode as the prefix, so we may decrease the
// probability that the prefixes are equal when input rows choose column values from a
// limited range.
@@ -299,7 +291,9 @@ object ShuffleExchangeExec {
prefixComparator,
prefixComputer,
pageSize,
- canUseRadixSort)
+ // We are comparing binary here, which does not support radix sort.
+ // See more details in SPARK-28699.
+ false)
sorter.sort(iter.asInstanceOf[Iterator[UnsafeRow]])
}
} else {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala
index a6f3ea47c8492..b25a34cc10d49 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala
@@ -48,8 +48,7 @@ case class BroadcastHashJoinExec(
extends BinaryExecNode with HashJoin with CodegenSupport {
override lazy val metrics = Map(
- "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
- "avgHashProbe" -> SQLMetrics.createAverageMetric(sparkContext, "avg hash probe"))
+ "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
override def requiredChildDistribution: Seq[Distribution] = {
val mode = HashedRelationBroadcastMode(buildKeys)
@@ -63,13 +62,12 @@ case class BroadcastHashJoinExec(
protected override def doExecute(): RDD[InternalRow] = {
val numOutputRows = longMetric("numOutputRows")
- val avgHashProbe = longMetric("avgHashProbe")
val broadcastRelation = buildPlan.executeBroadcast[HashedRelation]()
streamedPlan.execute().mapPartitions { streamedIter =>
val hashed = broadcastRelation.value.asReadOnlyCopy()
TaskContext.get().taskMetrics().incPeakExecutionMemory(hashed.estimatedSize)
- join(streamedIter, hashed, numOutputRows, avgHashProbe)
+ join(streamedIter, hashed, numOutputRows)
}
}
@@ -111,23 +109,6 @@ case class BroadcastHashJoinExec(
}
}
- /**
- * Returns the codes used to add a task completion listener to update avg hash probe
- * at the end of the task.
- */
- private def genTaskListener(avgHashProbe: String, relationTerm: String): String = {
- val listenerClass = classOf[TaskCompletionListener].getName
- val taskContextClass = classOf[TaskContext].getName
- s"""
- | $taskContextClass$$.MODULE$$.get().addTaskCompletionListener(new $listenerClass() {
- | @Override
- | public void onTaskCompletion($taskContextClass context) {
- | $avgHashProbe.set($relationTerm.getAverageProbesPerLookup());
- | }
- | });
- """.stripMargin
- }
-
/**
* Returns a tuple of Broadcast of HashedRelation and the variable name for it.
*/
@@ -137,15 +118,11 @@ case class BroadcastHashJoinExec(
val broadcast = ctx.addReferenceObj("broadcast", broadcastRelation)
val clsName = broadcastRelation.value.getClass.getName
- // At the end of the task, we update the avg hash probe.
- val avgHashProbe = metricTerm(ctx, "avgHashProbe")
-
// Inline mutable state since not many join operations in a task
val relationTerm = ctx.addMutableState(clsName, "relation",
v => s"""
| $v = (($clsName) $broadcast.value()).asReadOnlyCopy();
| incPeakExecutionMemory($v.estimatedSize());
- | ${genTaskListener(avgHashProbe, v)}
""".stripMargin, forceInline = true)
(broadcastRelation, relationTerm)
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
index dab873bf9b9a0..b197bf6c89981 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
@@ -194,8 +194,7 @@ trait HashJoin {
protected def join(
streamedIter: Iterator[InternalRow],
hashed: HashedRelation,
- numOutputRows: SQLMetric,
- avgHashProbe: SQLMetric): Iterator[InternalRow] = {
+ numOutputRows: SQLMetric): Iterator[InternalRow] = {
val joinedIter = joinType match {
case _: InnerLike =>
@@ -213,10 +212,6 @@ trait HashJoin {
s"BroadcastHashJoin should not take $x as the JoinType")
}
- // At the end of the task, we update the avg hash probe.
- TaskContext.get().addTaskCompletionListener[Unit](_ =>
- avgHashProbe.set(hashed.getAverageProbesPerLookup))
-
val resultProj = createResultProjection
joinedIter.map { r =>
numOutputRows += 1
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
index 86eb47a70f1ad..956430407db5e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
@@ -81,10 +81,6 @@ private[execution] sealed trait HashedRelation extends KnownSizeEstimation {
*/
def close(): Unit
- /**
- * Returns the average number of probes per key lookup.
- */
- def getAverageProbesPerLookup: Double
}
private[execution] object HashedRelation {
@@ -281,7 +277,6 @@ private[joins] class UnsafeHashedRelation(
read(() => in.readInt(), () => in.readLong(), in.readBytes)
}
- override def getAverageProbesPerLookup: Double = binaryMap.getAverageProbesPerLookup
}
private[joins] object UnsafeHashedRelation {
@@ -395,10 +390,6 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
// The number of unique keys.
private var numKeys = 0L
- // Tracking average number of probes per key lookup.
- private var numKeyLookups = 0L
- private var numProbes = 0L
-
// needed by serializer
def this() = {
this(
@@ -483,8 +474,6 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
*/
def getValue(key: Long, resultRow: UnsafeRow): UnsafeRow = {
if (isDense) {
- numKeyLookups += 1
- numProbes += 1
if (key >= minKey && key <= maxKey) {
val value = array((key - minKey).toInt)
if (value > 0) {
@@ -493,14 +482,11 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
}
} else {
var pos = firstSlot(key)
- numKeyLookups += 1
- numProbes += 1
while (array(pos + 1) != 0) {
if (array(pos) == key) {
return getRow(array(pos + 1), resultRow)
}
pos = nextSlot(pos)
- numProbes += 1
}
}
null
@@ -528,8 +514,6 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
*/
def get(key: Long, resultRow: UnsafeRow): Iterator[UnsafeRow] = {
if (isDense) {
- numKeyLookups += 1
- numProbes += 1
if (key >= minKey && key <= maxKey) {
val value = array((key - minKey).toInt)
if (value > 0) {
@@ -538,14 +522,11 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
}
} else {
var pos = firstSlot(key)
- numKeyLookups += 1
- numProbes += 1
while (array(pos + 1) != 0) {
if (array(pos) == key) {
return valueIter(array(pos + 1), resultRow)
}
pos = nextSlot(pos)
- numProbes += 1
}
}
null
@@ -585,11 +566,8 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
private def updateIndex(key: Long, address: Long): Unit = {
var pos = firstSlot(key)
assert(numKeys < array.length / 2)
- numKeyLookups += 1
- numProbes += 1
while (array(pos) != key && array(pos + 1) != 0) {
pos = nextSlot(pos)
- numProbes += 1
}
if (array(pos + 1) == 0) {
// this is the first value for this key, put the address in array.
@@ -721,8 +699,6 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
writeLong(maxKey)
writeLong(numKeys)
writeLong(numValues)
- writeLong(numKeyLookups)
- writeLong(numProbes)
writeLong(array.length)
writeLongArray(writeBuffer, array, array.length)
@@ -764,8 +740,6 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
maxKey = readLong()
numKeys = readLong()
numValues = readLong()
- numKeyLookups = readLong()
- numProbes = readLong()
val length = readLong().toInt
mask = length - 2
@@ -784,10 +758,6 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
read(() => in.readBoolean(), () => in.readLong(), in.readBytes)
}
- /**
- * Returns the average number of probes per key lookup.
- */
- def getAverageProbesPerLookup: Double = numProbes.toDouble / numKeyLookups
}
private[joins] class LongHashedRelation(
@@ -840,7 +810,6 @@ private[joins] class LongHashedRelation(
map = in.readObject().asInstanceOf[LongToUnsafeRowMap]
}
- override def getAverageProbesPerLookup: Double = map.getAverageProbesPerLookup
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashJoinExec.scala
index 2b59ed6e4d16b..524804d61e599 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashJoinExec.scala
@@ -42,8 +42,7 @@ case class ShuffledHashJoinExec(
override lazy val metrics = Map(
"numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
"buildDataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size of build side"),
- "buildTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to build hash map"),
- "avgHashProbe" -> SQLMetrics.createAverageMetric(sparkContext, "avg hash probe"))
+ "buildTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to build hash map"))
override def requiredChildDistribution: Seq[Distribution] =
HashClusteredDistribution(leftKeys) :: HashClusteredDistribution(rightKeys) :: Nil
@@ -63,10 +62,9 @@ case class ShuffledHashJoinExec(
protected override def doExecute(): RDD[InternalRow] = {
val numOutputRows = longMetric("numOutputRows")
- val avgHashProbe = longMetric("avgHashProbe")
streamedPlan.execute().zipPartitions(buildPlan.execute()) { (streamIter, buildIter) =>
val hashed = buildHashedRelation(buildIter)
- join(streamIter, hashed, numOutputRows, avgHashProbe)
+ join(streamIter, hashed, numOutputRows)
}
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
index f4b9d132122e4..416ed4c24eea2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
@@ -173,7 +173,8 @@ case class SortMergeJoinExec(
RowIterator.fromScala(leftIter),
RowIterator.fromScala(rightIter),
inMemoryThreshold,
- spillThreshold
+ spillThreshold,
+ cleanupResources
)
private[this] val joinRow = new JoinedRow
@@ -217,7 +218,8 @@ case class SortMergeJoinExec(
streamedIter = RowIterator.fromScala(leftIter),
bufferedIter = RowIterator.fromScala(rightIter),
inMemoryThreshold,
- spillThreshold
+ spillThreshold,
+ cleanupResources
)
val rightNullRow = new GenericInternalRow(right.output.length)
new LeftOuterIterator(
@@ -231,7 +233,8 @@ case class SortMergeJoinExec(
streamedIter = RowIterator.fromScala(rightIter),
bufferedIter = RowIterator.fromScala(leftIter),
inMemoryThreshold,
- spillThreshold
+ spillThreshold,
+ cleanupResources
)
val leftNullRow = new GenericInternalRow(left.output.length)
new RightOuterIterator(
@@ -265,7 +268,8 @@ case class SortMergeJoinExec(
RowIterator.fromScala(leftIter),
RowIterator.fromScala(rightIter),
inMemoryThreshold,
- spillThreshold
+ spillThreshold,
+ cleanupResources
)
private[this] val joinRow = new JoinedRow
@@ -300,7 +304,8 @@ case class SortMergeJoinExec(
RowIterator.fromScala(leftIter),
RowIterator.fromScala(rightIter),
inMemoryThreshold,
- spillThreshold
+ spillThreshold,
+ cleanupResources
)
private[this] val joinRow = new JoinedRow
@@ -342,7 +347,8 @@ case class SortMergeJoinExec(
RowIterator.fromScala(leftIter),
RowIterator.fromScala(rightIter),
inMemoryThreshold,
- spillThreshold
+ spillThreshold,
+ cleanupResources
)
private[this] val joinRow = new JoinedRow
@@ -622,6 +628,9 @@ case class SortMergeJoinExec(
(evaluateVariables(leftVars), "")
}
+ val thisPlan = ctx.addReferenceObj("plan", this)
+ val eagerCleanup = s"$thisPlan.cleanupResources();"
+
s"""
|while (findNextInnerJoinRows($leftInput, $rightInput)) {
| ${leftVarDecl.mkString("\n")}
@@ -635,6 +644,7 @@ case class SortMergeJoinExec(
| }
| if (shouldStop()) return;
|}
+ |$eagerCleanup
""".stripMargin
}
}
@@ -660,6 +670,7 @@ case class SortMergeJoinExec(
* @param inMemoryThreshold Threshold for number of rows guaranteed to be held in memory by
* internal buffer
* @param spillThreshold Threshold for number of rows to be spilled by internal buffer
+ * @param eagerCleanupResources the eager cleanup function to be invoked when no join row found
*/
private[joins] class SortMergeJoinScanner(
streamedKeyGenerator: Projection,
@@ -668,7 +679,8 @@ private[joins] class SortMergeJoinScanner(
streamedIter: RowIterator,
bufferedIter: RowIterator,
inMemoryThreshold: Int,
- spillThreshold: Int) {
+ spillThreshold: Int,
+ eagerCleanupResources: () => Unit) {
private[this] var streamedRow: InternalRow = _
private[this] var streamedRowKey: InternalRow = _
private[this] var bufferedRow: InternalRow = _
@@ -692,7 +704,8 @@ private[joins] class SortMergeJoinScanner(
def getBufferedMatches: ExternalAppendOnlyUnsafeRowArray = bufferedMatches
/**
- * Advances both input iterators, stopping when we have found rows with matching join keys.
+ * Advances both input iterators, stopping when we have found rows with matching join keys. If no
+ * join rows found, try to do the eager resources cleanup.
* @return true if matching rows have been found and false otherwise. If this returns true, then
* [[getStreamedRow]] and [[getBufferedMatches]] can be called to construct the join
* results.
@@ -702,7 +715,7 @@ private[joins] class SortMergeJoinScanner(
// Advance the streamed side of the join until we find the next row whose join key contains
// no nulls or we hit the end of the streamed iterator.
}
- if (streamedRow == null) {
+ val found = if (streamedRow == null) {
// We have consumed the entire streamed iterator, so there can be no more matches.
matchJoinKey = null
bufferedMatches.clear()
@@ -742,17 +755,19 @@ private[joins] class SortMergeJoinScanner(
true
}
}
+ if (!found) eagerCleanupResources()
+ found
}
/**
* Advances the streamed input iterator and buffers all rows from the buffered input that
- * have matching keys.
+ * have matching keys. If no join rows found, try to do the eager resources cleanup.
* @return true if the streamed iterator returned a row, false otherwise. If this returns true,
* then [[getStreamedRow]] and [[getBufferedMatches]] can be called to produce the outer
* join results.
*/
final def findNextOuterJoinRows(): Boolean = {
- if (!advancedStreamed()) {
+ val found = if (!advancedStreamed()) {
// We have consumed the entire streamed iterator, so there can be no more matches.
matchJoinKey = null
bufferedMatches.clear()
@@ -782,6 +797,8 @@ private[joins] class SortMergeJoinScanner(
// If there is a streamed input then we always return true
true
}
+ if (!found) eagerCleanupResources()
+ found
}
// --- Private methods --------------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala
index fb46970e38f3c..66bcda8913738 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/limit.scala
@@ -47,16 +47,13 @@ case class CollectLimitExec(limit: Int, child: SparkPlan) extends UnaryExecNode
}
/**
- * Take the first `limit` elements of each child partition, but do not collect or shuffle them.
+ * Helper trait which defines methods that are shared by both
+ * [[LocalLimitExec]] and [[GlobalLimitExec]].
*/
-case class LocalLimitExec(limit: Int, child: SparkPlan) extends UnaryExecNode with CodegenSupport {
-
+trait BaseLimitExec extends UnaryExecNode with CodegenSupport {
+ val limit: Int
override def output: Seq[Attribute] = child.output
- override def outputOrdering: Seq[SortOrder] = child.outputOrdering
-
- override def outputPartitioning: Partitioning = child.outputPartitioning
-
protected override def doExecute(): RDD[InternalRow] = child.execute().mapPartitions { iter =>
iter.take(limit)
}
@@ -96,93 +93,25 @@ case class LocalLimitExec(limit: Int, child: SparkPlan) extends UnaryExecNode wi
}
/**
- * Take the `limit` elements of the child output.
+ * Take the first `limit` elements of each child partition, but do not collect or shuffle them.
*/
-case class GlobalLimitExec(limit: Int, child: SparkPlan) extends UnaryExecNode {
+case class LocalLimitExec(limit: Int, child: SparkPlan) extends BaseLimitExec {
- override def output: Seq[Attribute] = child.output
+ override def outputOrdering: Seq[SortOrder] = child.outputOrdering
override def outputPartitioning: Partitioning = child.outputPartitioning
+}
- override def outputOrdering: Seq[SortOrder] = child.outputOrdering
+/**
+ * Take the first `limit` elements of the child's single output partition.
+ */
+case class GlobalLimitExec(limit: Int, child: SparkPlan) extends BaseLimitExec {
- private val serializer: Serializer = new UnsafeRowSerializer(child.output.size)
+ override def requiredChildDistribution: List[Distribution] = AllTuples :: Nil
- protected override def doExecute(): RDD[InternalRow] = {
- val childRDD = child.execute()
- val partitioner = LocalPartitioning(childRDD)
- val shuffleDependency = ShuffleExchangeExec.prepareShuffleDependency(
- childRDD, child.output, partitioner, serializer)
- val numberOfOutput: Seq[Long] = if (shuffleDependency.rdd.getNumPartitions != 0) {
- // submitMapStage does not accept RDD with 0 partition.
- // So, we will not submit this dependency.
- val submittedStageFuture = sparkContext.submitMapStage(shuffleDependency)
- submittedStageFuture.get().recordsByPartitionId.toSeq
- } else {
- Nil
- }
+ override def outputPartitioning: Partitioning = child.outputPartitioning
- // This is an optimization to evenly distribute limited rows across all partitions.
- // When enabled, Spark goes to take rows at each partition repeatedly until reaching
- // limit number. When disabled, Spark takes all rows at first partition, then rows
- // at second partition ..., until reaching limit number.
- val flatGlobalLimit = sqlContext.conf.limitFlatGlobalLimit
-
- val shuffled = new ShuffledRowRDD(shuffleDependency)
-
- val sumOfOutput = numberOfOutput.sum
- if (sumOfOutput <= limit) {
- shuffled
- } else if (!flatGlobalLimit) {
- var numRowTaken = 0
- val takeAmounts = numberOfOutput.map { num =>
- if (numRowTaken + num < limit) {
- numRowTaken += num.toInt
- num.toInt
- } else {
- val toTake = limit - numRowTaken
- numRowTaken += toTake
- toTake
- }
- }
- val broadMap = sparkContext.broadcast(takeAmounts)
- shuffled.mapPartitionsWithIndexInternal { case (index, iter) =>
- iter.take(broadMap.value(index).toInt)
- }
- } else {
- // We try to evenly require the asked limit number of rows across all child rdd's partitions.
- var rowsNeedToTake: Long = limit
- val takeAmountByPartition: Array[Long] = Array.fill[Long](numberOfOutput.length)(0L)
- val remainingRowsByPartition: Array[Long] = Array(numberOfOutput: _*)
-
- while (rowsNeedToTake > 0) {
- val nonEmptyParts = remainingRowsByPartition.count(_ > 0)
- // If the rows needed to take are less the number of non-empty partitions, take one row from
- // each non-empty partitions until we reach `limit` rows.
- // Otherwise, evenly divide the needed rows to each non-empty partitions.
- val takePerPart = math.max(1, rowsNeedToTake / nonEmptyParts)
- remainingRowsByPartition.zipWithIndex.foreach { case (num, index) =>
- // In case `rowsNeedToTake` < `nonEmptyParts`, we may run out of `rowsNeedToTake` during
- // the traversal, so we need to add this check.
- if (rowsNeedToTake > 0 && num > 0) {
- if (num >= takePerPart) {
- rowsNeedToTake -= takePerPart
- takeAmountByPartition(index) += takePerPart
- remainingRowsByPartition(index) -= takePerPart
- } else {
- rowsNeedToTake -= num
- takeAmountByPartition(index) += num
- remainingRowsByPartition(index) -= num
- }
- }
- }
- }
- val broadMap = sparkContext.broadcast(takeAmountByPartition)
- shuffled.mapPartitionsWithIndexInternal { case (index, iter) =>
- iter.take(broadMap.value(index).toInt)
- }
- }
- }
+ override def outputOrdering: Seq[SortOrder] = child.outputOrdering
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
index 18992d7a9f974..04623b1ab3c2f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
@@ -117,9 +117,9 @@ class ArrowPythonRunner(
startTime: Long,
env: SparkEnv,
worker: Socket,
- released: AtomicBoolean,
+ releasedOrClosed: AtomicBoolean,
context: TaskContext): Iterator[ColumnarBatch] = {
- new ReaderIterator(stream, writerThread, startTime, env, worker, released, context) {
+ new ReaderIterator(stream, writerThread, startTime, env, worker, releasedOrClosed, context) {
private val allocator = ArrowUtils.rootAllocator.newChildAllocator(
s"stdin reader for $pythonExec", 0, Long.MaxValue)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
index cc61faa7e7051..752d271c4cc35 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
@@ -59,9 +59,9 @@ class PythonUDFRunner(
startTime: Long,
env: SparkEnv,
worker: Socket,
- released: AtomicBoolean,
+ releasedOrClosed: AtomicBoolean,
context: TaskContext): Iterator[Array[Byte]] = {
- new ReaderIterator(stream, writerThread, startTime, env, worker, released, context) {
+ new ReaderIterator(stream, writerThread, startTime, env, worker, releasedOrClosed, context) {
protected override def read(): Array[Byte] = {
if (writerThread.exception.isDefined) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/RowQueue.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/RowQueue.scala
index e2fa6e7f504ba..d2820ff335ecf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/RowQueue.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/RowQueue.scala
@@ -21,9 +21,10 @@ import java.io._
import com.google.common.io.Closeables
-import org.apache.spark.SparkException
+import org.apache.spark.{SparkEnv, SparkException}
import org.apache.spark.io.NioBufferedFileInputStream
import org.apache.spark.memory.{MemoryConsumer, TaskMemoryManager}
+import org.apache.spark.serializer.SerializerManager
import org.apache.spark.sql.catalyst.expressions.UnsafeRow
import org.apache.spark.unsafe.Platform
import org.apache.spark.unsafe.memory.MemoryBlock
@@ -108,9 +109,13 @@ private[python] abstract class InMemoryRowQueue(val page: MemoryBlock, numFields
* A RowQueue that is backed by a file on disk. This queue will stop accepting new rows once any
* reader has begun reading from the queue.
*/
-private[python] case class DiskRowQueue(file: File, fields: Int) extends RowQueue {
- private var out = new DataOutputStream(
- new BufferedOutputStream(new FileOutputStream(file.toString)))
+private[python] case class DiskRowQueue(
+ file: File,
+ fields: Int,
+ serMgr: SerializerManager) extends RowQueue {
+
+ private var out = new DataOutputStream(serMgr.wrapForEncryption(
+ new BufferedOutputStream(new FileOutputStream(file.toString))))
private var unreadBytes = 0L
private var in: DataInputStream = _
@@ -131,7 +136,8 @@ private[python] case class DiskRowQueue(file: File, fields: Int) extends RowQueu
if (out != null) {
out.close()
out = null
- in = new DataInputStream(new NioBufferedFileInputStream(file))
+ in = new DataInputStream(serMgr.wrapForEncryption(
+ new NioBufferedFileInputStream(file)))
}
if (unreadBytes > 0) {
@@ -166,7 +172,8 @@ private[python] case class DiskRowQueue(file: File, fields: Int) extends RowQueu
private[python] case class HybridRowQueue(
memManager: TaskMemoryManager,
tempDir: File,
- numFields: Int)
+ numFields: Int,
+ serMgr: SerializerManager)
extends MemoryConsumer(memManager) with RowQueue {
// Each buffer should have at least one row
@@ -212,7 +219,7 @@ private[python] case class HybridRowQueue(
}
private def createDiskQueue(): RowQueue = {
- DiskRowQueue(File.createTempFile("buffer", "", tempDir), numFields)
+ DiskRowQueue(File.createTempFile("buffer", "", tempDir), numFields, serMgr)
}
private def createNewQueue(required: Long): RowQueue = {
@@ -279,3 +286,9 @@ private[python] case class HybridRowQueue(
}
}
}
+
+private[python] object HybridRowQueue {
+ def apply(taskMemoryMgr: TaskMemoryManager, file: File, fields: Int): HybridRowQueue = {
+ HybridRowQueue(taskMemoryMgr, file, fields, SparkEnv.get.serializerManager)
+ }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManager.scala
index 606ba250ad9d2..285af65f78ca1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CheckpointFileManager.scala
@@ -56,7 +56,7 @@ trait CheckpointFileManager {
* @param overwriteIfPossible If true, then the implementations must do a best-effort attempt to
* overwrite the file if it already exists. It should not throw
* any exception if the file exists. However, if false, then the
- * implementation must not overwrite if the file alraedy exists and
+ * implementation must not overwrite if the file already exists and
* must throw `FileAlreadyExistsException` in that case.
*/
def createAtomic(path: Path, overwriteIfPossible: Boolean): CancellableFSDataOutputStream
@@ -329,6 +329,8 @@ class FileContextBasedCheckpointFileManager(path: Path, hadoopConf: Configuratio
override def renameTempFile(srcPath: Path, dstPath: Path, overwriteIfPossible: Boolean): Unit = {
import Options.Rename._
fc.rename(srcPath, dstPath, if (overwriteIfPossible) OVERWRITE else NONE)
+ // TODO: this is a workaround of HADOOP-16255 - remove this when HADOOP-16255 is resolved
+ mayRemoveCrcFile(srcPath)
}
@@ -345,5 +347,17 @@ class FileContextBasedCheckpointFileManager(path: Path, hadoopConf: Configuratio
case _: LocalFs | _: RawLocalFs => true // LocalFs = RawLocalFs + ChecksumFs
case _ => false
}
+
+ private def mayRemoveCrcFile(path: Path): Unit = {
+ try {
+ val checksumFile = new Path(path.getParent, s".${path.getName}.crc")
+ if (exists(checksumFile)) {
+ // checksum file exists, deleting it
+ delete(checksumFile)
+ }
+ } catch {
+ case NonFatal(_) => // ignore, we are removing crc file as "best-effort"
+ }
+ }
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala
index b161651c4e6a3..6fa7ee0c38185 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala
@@ -36,10 +36,19 @@ case class EventTimeStats(var max: Long, var min: Long, var avg: Double, var cou
}
def merge(that: EventTimeStats): Unit = {
- this.max = math.max(this.max, that.max)
- this.min = math.min(this.min, that.min)
- this.count += that.count
- this.avg += (that.avg - this.avg) * that.count / this.count
+ if (that.count == 0) {
+ // no-op
+ } else if (this.count == 0) {
+ this.max = that.max
+ this.min = that.min
+ this.count = that.count
+ this.avg = that.avg
+ } else {
+ this.max = math.max(this.max, that.max)
+ this.min = math.min(this.min, that.min)
+ this.count += that.count
+ this.avg += (that.avg - this.avg) * that.count / this.count
+ }
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala
index 8628471fdb925..7b2ea9627a98e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala
@@ -117,7 +117,9 @@ class FileStreamSourceLog(
val batches =
(existedBatches ++ retrievedBatches).map(i => i._1 -> i._2.get).toArray.sortBy(_._1)
- HDFSMetadataLog.verifyBatchIds(batches.map(_._1), startId, endId)
+ if (startBatchId <= endBatchId) {
+ HDFSMetadataLog.verifyBatchIds(batches.map(_._1), startId, endId)
+ }
batches
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/GroupStateImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/GroupStateImpl.scala
index 7f65e3ea9dd5e..486368cfd3574 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/GroupStateImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/GroupStateImpl.scala
@@ -19,8 +19,6 @@ package org.apache.spark.sql.execution.streaming
import java.sql.Date
-import org.apache.commons.lang3.StringUtils
-
import org.apache.spark.sql.catalyst.plans.logical.{EventTimeTimeout, ProcessingTimeTimeout}
import org.apache.spark.sql.execution.streaming.GroupStateImpl._
import org.apache.spark.sql.streaming.{GroupState, GroupStateTimeout}
@@ -160,20 +158,7 @@ private[sql] class GroupStateImpl[S] private(
def getTimeoutTimestamp: Long = timeoutTimestamp
private def parseDuration(duration: String): Long = {
- if (StringUtils.isBlank(duration)) {
- throw new IllegalArgumentException(
- "Provided duration is null or blank.")
- }
- val intervalString = if (duration.startsWith("interval")) {
- duration
- } else {
- "interval " + duration
- }
- val cal = CalendarInterval.fromString(intervalString)
- if (cal == null) {
- throw new IllegalArgumentException(
- s"Provided duration ($duration) is not valid.")
- }
+ val cal = CalendarInterval.fromCaseInsensitiveString(duration)
if (cal.milliseconds < 0 || cal.months < 0) {
throw new IllegalArgumentException(s"Provided duration ($duration) is not positive")
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLog.scala
index bd0a46115ceb0..62d524ff19b69 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLog.scala
@@ -262,7 +262,8 @@ class HDFSMetadataLog[T <: AnyRef : ClassTag](sparkSession: SparkSession, path:
object HDFSMetadataLog {
/**
- * Verify if batchIds are continuous and between `startId` and `endId`.
+ * Verify if batchIds are continuous and between `startId` and `endId` (both inclusive and
+ * startId assumed to be <= endId).
*
* @param batchIds the sorted ids to verify.
* @param startId the start id. If it's set, batchIds should start with this id.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
index b1cafd67820c2..3bcc26b210028 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.execution.streaming
+import java.util.Optional
+
import scala.collection.JavaConverters._
import scala.collection.mutable.{Map => MutableMap}
@@ -26,9 +28,9 @@ import org.apache.spark.sql.catalyst.expressions.{Alias, CurrentBatchTimestamp,
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Project}
import org.apache.spark.sql.execution.SQLExecution
import org.apache.spark.sql.execution.datasources.v2.{StreamingDataSourceV2Relation, WriteToDataSourceV2}
-import org.apache.spark.sql.execution.streaming.sources.{MicroBatchWritSupport, RateControlMicroBatchReadSupport}
-import org.apache.spark.sql.sources.v2._
-import org.apache.spark.sql.sources.v2.reader.streaming.{MicroBatchReadSupport, Offset => OffsetV2}
+import org.apache.spark.sql.execution.streaming.sources.MicroBatchWriter
+import org.apache.spark.sql.sources.v2.{DataSourceOptions, DataSourceV2, MicroBatchReadSupport, StreamWriteSupport}
+import org.apache.spark.sql.sources.v2.reader.streaming.{MicroBatchReader, Offset => OffsetV2}
import org.apache.spark.sql.streaming.{OutputMode, ProcessingTime, Trigger}
import org.apache.spark.util.{Clock, Utils}
@@ -49,8 +51,8 @@ class MicroBatchExecution(
@volatile protected var sources: Seq[BaseStreamingSource] = Seq.empty
- private val readSupportToDataSourceMap =
- MutableMap.empty[MicroBatchReadSupport, (DataSourceV2, Map[String, String])]
+ private val readerToDataSourceMap =
+ MutableMap.empty[MicroBatchReader, (DataSourceV2, Map[String, String])]
private val triggerExecutor = trigger match {
case t: ProcessingTime => ProcessingTimeExecutor(t, triggerClock)
@@ -89,19 +91,20 @@ class MicroBatchExecution(
StreamingExecutionRelation(source, output)(sparkSession)
})
case s @ StreamingRelationV2(
- dataSourceV2: MicroBatchReadSupportProvider, sourceName, options, output, _) if
+ dataSourceV2: MicroBatchReadSupport, sourceName, options, output, _) if
!disabledSources.contains(dataSourceV2.getClass.getCanonicalName) =>
v2ToExecutionRelationMap.getOrElseUpdate(s, {
// Materialize source to avoid creating it in every batch
val metadataPath = s"$resolvedCheckpointRoot/sources/$nextSourceId"
- val readSupport = dataSourceV2.createMicroBatchReadSupport(
+ val reader = dataSourceV2.createMicroBatchReader(
+ Optional.empty(), // user specified schema
metadataPath,
new DataSourceOptions(options.asJava))
nextSourceId += 1
- readSupportToDataSourceMap(readSupport) = dataSourceV2 -> options
- logInfo(s"Using MicroBatchReadSupport [$readSupport] from " +
+ readerToDataSourceMap(reader) = dataSourceV2 -> options
+ logInfo(s"Using MicroBatchReader [$reader] from " +
s"DataSourceV2 named '$sourceName' [$dataSourceV2]")
- StreamingExecutionRelation(readSupport, output)(sparkSession)
+ StreamingExecutionRelation(reader, output)(sparkSession)
})
case s @ StreamingRelationV2(dataSourceV2, sourceName, _, output, v1Relation) =>
v2ToExecutionRelationMap.getOrElseUpdate(s, {
@@ -341,19 +344,19 @@ class MicroBatchExecution(
reportTimeTaken("getOffset") {
(s, s.getOffset)
}
- case s: RateControlMicroBatchReadSupport =>
- updateStatusMessage(s"Getting offsets from $s")
- reportTimeTaken("latestOffset") {
- val startOffset = availableOffsets
- .get(s).map(off => s.deserializeOffset(off.json))
- .getOrElse(s.initialOffset())
- (s, Option(s.latestOffset(startOffset)))
- }
- case s: MicroBatchReadSupport =>
+ case s: MicroBatchReader =>
updateStatusMessage(s"Getting offsets from $s")
- reportTimeTaken("latestOffset") {
- (s, Option(s.latestOffset()))
+ reportTimeTaken("setOffsetRange") {
+ // Once v1 streaming source execution is gone, we can refactor this away.
+ // For now, we set the range here to get the source to infer the available end offset,
+ // get that offset, and then set the range again when we later execute.
+ s.setOffsetRange(
+ toJava(availableOffsets.get(s).map(off => s.deserializeOffset(off.json))),
+ Optional.empty())
}
+
+ val currentOffset = reportTimeTaken("getEndOffset") { s.getEndOffset() }
+ (s, Option(currentOffset))
}.toMap
availableOffsets ++= latestOffsets.filter { case (_, o) => o.nonEmpty }.mapValues(_.get)
@@ -393,8 +396,8 @@ class MicroBatchExecution(
if (prevBatchOff.isDefined) {
prevBatchOff.get.toStreamProgress(sources).foreach {
case (src: Source, off) => src.commit(off)
- case (readSupport: MicroBatchReadSupport, off) =>
- readSupport.commit(readSupport.deserializeOffset(off.json))
+ case (reader: MicroBatchReader, off) =>
+ reader.commit(reader.deserializeOffset(off.json))
case (src, _) =>
throw new IllegalArgumentException(
s"Unknown source is found at constructNextBatch: $src")
@@ -438,34 +441,30 @@ class MicroBatchExecution(
s"${batch.queryExecution.logical}")
logDebug(s"Retrieving data from $source: $current -> $available")
Some(source -> batch.logicalPlan)
-
- // TODO(cloud-fan): for data source v2, the new batch is just a new `ScanConfigBuilder`, but
- // to be compatible with streaming source v1, we return a logical plan as a new batch here.
- case (readSupport: MicroBatchReadSupport, available)
- if committedOffsets.get(readSupport).map(_ != available).getOrElse(true) =>
- val current = committedOffsets.get(readSupport).map {
- off => readSupport.deserializeOffset(off.json)
- }
- val endOffset: OffsetV2 = available match {
- case v1: SerializedOffset => readSupport.deserializeOffset(v1.json)
+ case (reader: MicroBatchReader, available)
+ if committedOffsets.get(reader).map(_ != available).getOrElse(true) =>
+ val current = committedOffsets.get(reader).map(off => reader.deserializeOffset(off.json))
+ val availableV2: OffsetV2 = available match {
+ case v1: SerializedOffset => reader.deserializeOffset(v1.json)
case v2: OffsetV2 => v2
}
- val startOffset = current.getOrElse(readSupport.initialOffset)
- val scanConfigBuilder = readSupport.newScanConfigBuilder(startOffset, endOffset)
- logDebug(s"Retrieving data from $readSupport: $current -> $endOffset")
+ reader.setOffsetRange(
+ toJava(current),
+ Optional.of(availableV2))
+ logDebug(s"Retrieving data from $reader: $current -> $availableV2")
- val (source, options) = readSupport match {
+ val (source, options) = reader match {
// `MemoryStream` is special. It's for test only and doesn't have a `DataSourceV2`
// implementation. We provide a fake one here for explain.
case _: MemoryStream[_] => MemoryStreamDataSource -> Map.empty[String, String]
// Provide a fake value here just in case something went wrong, e.g. the reader gives
// a wrong `equals` implementation.
- case _ => readSupportToDataSourceMap.getOrElse(readSupport, {
+ case _ => readerToDataSourceMap.getOrElse(reader, {
FakeDataSourceV2 -> Map.empty[String, String]
})
}
- Some(readSupport -> StreamingDataSourceV2Relation(
- readSupport.fullSchema().toAttributes, source, options, readSupport, scanConfigBuilder))
+ Some(reader -> StreamingDataSourceV2Relation(
+ reader.readSchema().toAttributes, source, options, reader))
case _ => None
}
}
@@ -490,8 +489,11 @@ class MicroBatchExecution(
// Rewire the plan to use the new attributes that were returned by the source.
val newAttributePlan = newBatchesPlan transformAllExpressions {
case ct: CurrentTimestamp =>
+ // CurrentTimestamp is not TimeZoneAwareExpression while CurrentBatchTimestamp is.
+ // Without TimeZoneId, CurrentBatchTimestamp is unresolved. Here, we use an explicit
+ // dummy string to prevent UnresolvedException and to prevent to be used in the future.
CurrentBatchTimestamp(offsetSeqMetadata.batchTimestampMs,
- ct.dataType)
+ ct.dataType, Some("Dummy TimeZoneId"))
case cd: CurrentDate =>
CurrentBatchTimestamp(offsetSeqMetadata.batchTimestampMs,
cd.dataType, cd.timeZoneId)
@@ -499,18 +501,20 @@ class MicroBatchExecution(
val triggerLogicalPlan = sink match {
case _: Sink => newAttributePlan
- case s: StreamingWriteSupportProvider =>
- val writer = s.createStreamingWriteSupport(
+ case s: StreamWriteSupport =>
+ val writer = s.createStreamWriter(
s"$runId",
newAttributePlan.schema,
outputMode,
new DataSourceOptions(extraOptions.asJava))
- WriteToDataSourceV2(new MicroBatchWritSupport(currentBatchId, writer), newAttributePlan)
+ WriteToDataSourceV2(new MicroBatchWriter(currentBatchId, writer), newAttributePlan)
case _ => throw new IllegalArgumentException(s"unknown sink type for $sink")
}
sparkSessionToRunBatch.sparkContext.setLocalProperty(
MicroBatchExecution.BATCH_ID_KEY, currentBatchId.toString)
+ sparkSessionToRunBatch.sparkContext.setLocalProperty(
+ StreamExecution.IS_CONTINUOUS_PROCESSING, false.toString)
reportTimeTaken("queryPlanning") {
lastExecution = new IncrementalExecution(
@@ -531,7 +535,7 @@ class MicroBatchExecution(
SQLExecution.withNewExecutionId(sparkSessionToRunBatch, lastExecution) {
sink match {
case s: Sink => s.addBatch(currentBatchId, nextBatch)
- case _: StreamingWriteSupportProvider =>
+ case _: StreamWriteSupport =>
// This doesn't accumulate any data - it just forces execution of the microbatch writer.
nextBatch.collect()
}
@@ -555,6 +559,10 @@ class MicroBatchExecution(
awaitProgressLock.unlock()
}
}
+
+ private def toJava(scalaOption: Option[OffsetV2]): Optional[OffsetV2] = {
+ Optional.ofNullable(scalaOption.orNull)
+ }
}
object MicroBatchExecution {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
index d4b50655c7215..6a380ab89ff78 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{EventTimeWatermark, LogicalP
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.execution.QueryExecution
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec
-import org.apache.spark.sql.sources.v2.reader.streaming.MicroBatchReadSupport
+import org.apache.spark.sql.sources.v2.reader.streaming.MicroBatchReader
import org.apache.spark.sql.streaming._
import org.apache.spark.sql.streaming.StreamingQueryListener.QueryProgressEvent
import org.apache.spark.util.Clock
@@ -251,7 +251,7 @@ trait ProgressReporter extends Logging {
// Check whether the streaming query's logical plan has only V2 data sources
val allStreamingLeaves =
logicalPlan.collect { case s: StreamingExecutionRelation => s }
- allStreamingLeaves.forall { _.source.isInstanceOf[MicroBatchReadSupport] }
+ allStreamingLeaves.forall { _.source.isInstanceOf[MicroBatchReader] }
}
if (onlyDataSourceV2Sources) {
@@ -278,7 +278,7 @@ trait ProgressReporter extends Logging {
new IdentityHashMap[DataSourceV2ScanExec, DataSourceV2ScanExec]()
lastExecution.executedPlan.collectLeaves().foreach {
- case s: DataSourceV2ScanExec if s.readSupport.isInstanceOf[BaseStreamingSource] =>
+ case s: DataSourceV2ScanExec if s.reader.isInstanceOf[BaseStreamingSource] =>
uniqueStreamingExecLeavesMap.put(s, s)
case _ =>
}
@@ -286,7 +286,7 @@ trait ProgressReporter extends Logging {
val sourceToInputRowsTuples =
uniqueStreamingExecLeavesMap.values.asScala.map { execLeaf =>
val numRows = execLeaf.metrics.get("numOutputRows").map(_.value).getOrElse(0L)
- val source = execLeaf.readSupport.asInstanceOf[BaseStreamingSource]
+ val source = execLeaf.reader.asInstanceOf[BaseStreamingSource]
source -> numRows
}.toSeq
logDebug("Source -> # input rows\n\t" + sourceToInputRowsTuples.mkString("\n\t"))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
index a39bb715c9913..c1aa98a2f602a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
@@ -87,6 +87,7 @@ abstract class StreamExecution(
val resolvedCheckpointRoot = {
val checkpointPath = new Path(checkpointRoot)
val fs = checkpointPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
+ fs.mkdirs(checkpointPath)
checkpointPath.makeQualified(fs.getUri, fs.getWorkingDirectory).toUri.toString
}
@@ -175,6 +176,9 @@ abstract class StreamExecution(
lazy val streamMetrics = new MetricsReporter(
this, s"spark.streaming.${Option(name).getOrElse(id)}")
+ /** Isolated spark session to run the batches with. */
+ private val sparkSessionForStream = sparkSession.cloneSession()
+
/**
* The thread that runs the micro-batches of this stream. Note that this thread must be
* [[org.apache.spark.util.UninterruptibleThread]] to workaround KAFKA-1894: interrupting a
@@ -264,8 +268,6 @@ abstract class StreamExecution(
// force initialization of the logical plan so that the sources can be created
logicalPlan
- // Isolated spark session to run the batches with.
- val sparkSessionForStream = sparkSession.cloneSession()
// Adaptive execution can change num shuffle partitions, disallow
sparkSessionForStream.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
// Disable cost-based join optimization as we do not want stateful operations to be rearranged
@@ -529,6 +531,7 @@ abstract class StreamExecution(
object StreamExecution {
val QUERY_ID_KEY = "sql.streaming.queryId"
+ val IS_CONTINUOUS_PROCESSING = "__is_continuous_processing"
def isInterruptionException(e: Throwable): Boolean = e match {
// InterruptedIOException - thrown when an I/O operation is interrupted
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamMetadata.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamMetadata.scala
index 0bc54eac4ee8e..516afbea5d9de 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamMetadata.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamMetadata.scala
@@ -19,16 +19,18 @@ package org.apache.spark.sql.execution.streaming
import java.io.{InputStreamReader, OutputStreamWriter}
import java.nio.charset.StandardCharsets
+import java.util.ConcurrentModificationException
import scala.util.control.NonFatal
import org.apache.commons.io.IOUtils
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileSystem, FSDataInputStream, FSDataOutputStream, Path}
+import org.apache.hadoop.fs.{FileAlreadyExistsException, FSDataInputStream, Path}
import org.json4s.NoTypeHints
import org.json4s.jackson.Serialization
import org.apache.spark.internal.Logging
+import org.apache.spark.sql.execution.streaming.CheckpointFileManager.CancellableFSDataOutputStream
import org.apache.spark.sql.streaming.StreamingQuery
/**
@@ -70,19 +72,26 @@ object StreamMetadata extends Logging {
metadata: StreamMetadata,
metadataFile: Path,
hadoopConf: Configuration): Unit = {
- var output: FSDataOutputStream = null
+ var output: CancellableFSDataOutputStream = null
try {
- val fs = metadataFile.getFileSystem(hadoopConf)
- output = fs.create(metadataFile)
+ val fileManager = CheckpointFileManager.create(metadataFile.getParent, hadoopConf)
+ output = fileManager.createAtomic(metadataFile, overwriteIfPossible = false)
val writer = new OutputStreamWriter(output)
Serialization.write(metadata, writer)
writer.close()
} catch {
- case NonFatal(e) =>
+ case e: FileAlreadyExistsException =>
+ if (output != null) {
+ output.cancel()
+ }
+ throw new ConcurrentModificationException(
+ s"Multiple streaming queries are concurrently using $metadataFile", e)
+ case e: Throwable =>
+ if (output != null) {
+ output.cancel()
+ }
logError(s"Error writing stream metadata $metadata to $metadataFile", e)
throw e
- } finally {
- IOUtils.closeQuietly(output)
}
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala
index 4b696dfa57359..24195b5657e8a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingRelation.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
import org.apache.spark.sql.execution.LeafExecNode
import org.apache.spark.sql.execution.datasources.DataSource
-import org.apache.spark.sql.sources.v2.{ContinuousReadSupportProvider, DataSourceV2}
+import org.apache.spark.sql.sources.v2.{ContinuousReadSupport, DataSourceV2}
object StreamingRelation {
def apply(dataSource: DataSource): StreamingRelation = {
@@ -83,7 +83,7 @@ case class StreamingExecutionRelation(
// We have to pack in the V1 data source as a shim, for the case when a source implements
// continuous processing (which is always V2) but only has V1 microbatch support. We don't
-// know at read time whether the query is continuous or not, so we need to be able to
+// know at read time whether the query is conntinuous or not, so we need to be able to
// swap a V1 relation back in.
/**
* Used to link a [[DataSourceV2]] into a streaming
@@ -113,7 +113,7 @@ case class StreamingRelationV2(
* Used to link a [[DataSourceV2]] into a continuous processing execution.
*/
case class ContinuousExecutionRelation(
- source: ContinuousReadSupportProvider,
+ source: ContinuousReadSupport,
extraOptions: Map[String, String],
output: Seq[Attribute])(session: SparkSession)
extends LeafNode with MultiInstanceRelation {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/WatermarkTracker.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/WatermarkTracker.scala
index 7b30db44a2090..b726c4f5a3a69 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/WatermarkTracker.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/WatermarkTracker.scala
@@ -61,7 +61,7 @@ case object MinWatermark extends MultipleWatermarkPolicy {
}
/**
- * Policy to choose the *min* of the operator watermark values as the global watermark value. So the
+ * Policy to choose the *max* of the operator watermark values as the global watermark value. So the
* global watermark will advance if any of the individual operator watermarks has advanced.
* In other words, in a streaming query with multiple input streams and watermarks defined on all
* of them, the global watermark will advance as fast as the fastest input. So if there is watermark
@@ -106,10 +106,9 @@ case class WatermarkTracker(policy: MultipleWatermarkPolicy) extends Logging {
}
}
- // Update the global watermark to the minimum of all watermark nodes.
- // This is the safest option, because only the global watermark is fault-tolerant. Making
- // it the minimum of all individual watermarks guarantees it will never advance past where
- // any individual watermark operator would be if it were in a plan by itself.
+ // Update the global watermark accordingly to the chosen policy. To find all available policies
+ // and their semantics, please check the comments of
+ // `org.apache.spark.sql.execution.streaming.MultipleWatermarkPolicy` implementations.
val chosenGlobalWatermark = policy.chooseGlobalWatermark(operatorToWatermarkMap.values.toSeq)
if (chosenGlobalWatermark > globalWatermarkMs) {
logInfo(s"Updating event-time watermark from $globalWatermarkMs to $chosenGlobalWatermark ms")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
index 9c5c16f4f5d13..cfba1001c6de0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
@@ -18,10 +18,10 @@
package org.apache.spark.sql.execution.streaming
import org.apache.spark.sql._
-import org.apache.spark.sql.execution.streaming.sources.ConsoleWriteSupport
+import org.apache.spark.sql.execution.streaming.sources.ConsoleWriter
import org.apache.spark.sql.sources.{BaseRelation, CreatableRelationProvider, DataSourceRegister}
-import org.apache.spark.sql.sources.v2.{DataSourceOptions, DataSourceV2, StreamingWriteSupportProvider}
-import org.apache.spark.sql.sources.v2.writer.streaming.StreamingWriteSupport
+import org.apache.spark.sql.sources.v2.{DataSourceOptions, DataSourceV2, StreamWriteSupport}
+import org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter
import org.apache.spark.sql.streaming.OutputMode
import org.apache.spark.sql.types.StructType
@@ -31,16 +31,16 @@ case class ConsoleRelation(override val sqlContext: SQLContext, data: DataFrame)
}
class ConsoleSinkProvider extends DataSourceV2
- with StreamingWriteSupportProvider
+ with StreamWriteSupport
with DataSourceRegister
with CreatableRelationProvider {
- override def createStreamingWriteSupport(
+ override def createStreamWriter(
queryId: String,
schema: StructType,
mode: OutputMode,
- options: DataSourceOptions): StreamingWriteSupport = {
- new ConsoleWriteSupport(schema, options)
+ options: DataSourceOptions): StreamWriter = {
+ new ConsoleWriter(schema, options)
}
def createRelation(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousCoalesceRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousCoalesceRDD.scala
index aec756c0eb2a4..14046f6a99c24 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousCoalesceRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousCoalesceRDD.scala
@@ -110,8 +110,9 @@ class ContinuousCoalesceRDD(
context.getLocalProperty(ContinuousExecution.START_EPOCH_KEY).toLong)
while (!context.isInterrupted() && !context.isCompleted()) {
writer.write(prev.compute(prevSplit, context).asInstanceOf[Iterator[UnsafeRow]])
- // Note that current epoch is a non-inheritable thread local, so each writer thread
- // can properly increment its own epoch without affecting the main task thread.
+ // Note that current epoch is a inheritable thread local but makes another instance,
+ // so each writer thread can properly increment its own epoch without affecting
+ // the main task thread.
EpochTracker.incrementCurrentEpoch()
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousDataSourceRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousDataSourceRDD.scala
index b68f67e0b22d9..554a0b0573f4d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousDataSourceRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousDataSourceRDD.scala
@@ -21,13 +21,12 @@ import org.apache.spark._
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.sources.v2.reader._
-import org.apache.spark.sql.sources.v2.reader.streaming.ContinuousPartitionReaderFactory
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.sources.v2.reader.streaming.ContinuousInputPartitionReader
import org.apache.spark.util.NextIterator
class ContinuousDataSourceRDDPartition(
val index: Int,
- val inputPartition: InputPartition)
+ val inputPartition: InputPartition[InternalRow])
extends Partition with Serializable {
// This is semantically a lazy val - it's initialized once the first time a call to
@@ -50,22 +49,15 @@ class ContinuousDataSourceRDD(
sc: SparkContext,
dataQueueSize: Int,
epochPollIntervalMs: Long,
- private val inputPartitions: Seq[InputPartition],
- schema: StructType,
- partitionReaderFactory: ContinuousPartitionReaderFactory)
+ private val readerInputPartitions: Seq[InputPartition[InternalRow]])
extends RDD[InternalRow](sc, Nil) {
override protected def getPartitions: Array[Partition] = {
- inputPartitions.zipWithIndex.map {
+ readerInputPartitions.zipWithIndex.map {
case (inputPartition, index) => new ContinuousDataSourceRDDPartition(index, inputPartition)
}.toArray
}
- private def castPartition(split: Partition): ContinuousDataSourceRDDPartition = split match {
- case p: ContinuousDataSourceRDDPartition => p
- case _ => throw new SparkException(s"[BUG] Not a ContinuousDataSourceRDDPartition: $split")
- }
-
/**
* Initialize the shared reader for this partition if needed, then read rows from it until
* it returns null to signal the end of the epoch.
@@ -77,12 +69,10 @@ class ContinuousDataSourceRDD(
}
val readerForPartition = {
- val partition = castPartition(split)
+ val partition = split.asInstanceOf[ContinuousDataSourceRDDPartition]
if (partition.queueReader == null) {
- val partitionReader = partitionReaderFactory.createReader(
- partition.inputPartition)
- partition.queueReader = new ContinuousQueuedDataReader(
- partition.index, partitionReader, schema, context, dataQueueSize, epochPollIntervalMs)
+ partition.queueReader =
+ new ContinuousQueuedDataReader(partition, context, dataQueueSize, epochPollIntervalMs)
}
partition.queueReader
@@ -103,6 +93,17 @@ class ContinuousDataSourceRDD(
}
override def getPreferredLocations(split: Partition): Seq[String] = {
- castPartition(split).inputPartition.preferredLocations()
+ split.asInstanceOf[ContinuousDataSourceRDDPartition].inputPartition.preferredLocations()
+ }
+}
+
+object ContinuousDataSourceRDD {
+ private[continuous] def getContinuousReader(
+ reader: InputPartitionReader[InternalRow]): ContinuousInputPartitionReader[_] = {
+ reader match {
+ case r: ContinuousInputPartitionReader[InternalRow] => r
+ case _ =>
+ throw new IllegalStateException(s"Unknown continuous reader type ${reader.getClass}")
+ }
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
index 4ddebb33b79d1..0b332c247476d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
@@ -29,12 +29,13 @@ import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, CurrentDate, CurrentTimestamp}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.SQLExecution
-import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2ScanExec, StreamingDataSourceV2Relation}
+import org.apache.spark.sql.execution.datasources.v2.{StreamingDataSourceV2Relation, WriteToDataSourceV2}
import org.apache.spark.sql.execution.streaming.{ContinuousExecutionRelation, StreamingRelationV2, _}
import org.apache.spark.sql.sources.v2
-import org.apache.spark.sql.sources.v2.{ContinuousReadSupportProvider, DataSourceOptions, StreamingWriteSupportProvider}
-import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousReadSupport, PartitionOffset}
+import org.apache.spark.sql.sources.v2.{ContinuousReadSupport, DataSourceOptions, StreamWriteSupport}
+import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousReader, PartitionOffset}
import org.apache.spark.sql.streaming.{OutputMode, ProcessingTime, Trigger}
+import org.apache.spark.sql.types.StructType
import org.apache.spark.util.{Clock, Utils}
class ContinuousExecution(
@@ -42,7 +43,7 @@ class ContinuousExecution(
name: String,
checkpointRoot: String,
analyzedPlan: LogicalPlan,
- sink: StreamingWriteSupportProvider,
+ sink: StreamWriteSupport,
trigger: Trigger,
triggerClock: Clock,
outputMode: OutputMode,
@@ -52,7 +53,7 @@ class ContinuousExecution(
sparkSession, name, checkpointRoot, analyzedPlan, sink,
trigger, triggerClock, outputMode, deleteCheckpointOnStop) {
- @volatile protected var continuousSources: Seq[ContinuousReadSupport] = Seq()
+ @volatile protected var continuousSources: Seq[ContinuousReader] = Seq()
override protected def sources: Seq[BaseStreamingSource] = continuousSources
// For use only in test harnesses.
@@ -62,8 +63,7 @@ class ContinuousExecution(
val toExecutionRelationMap = MutableMap[StreamingRelationV2, ContinuousExecutionRelation]()
analyzedPlan.transform {
case r @ StreamingRelationV2(
- source: ContinuousReadSupportProvider, _, extraReaderOptions, output, _) =>
- // TODO: shall we create `ContinuousReadSupport` here instead of each reconfiguration?
+ source: ContinuousReadSupport, _, extraReaderOptions, output, _) =>
toExecutionRelationMap.getOrElseUpdate(r, {
ContinuousExecutionRelation(source, extraReaderOptions, output)(sparkSession)
})
@@ -148,7 +148,8 @@ class ContinuousExecution(
val metadataPath = s"$resolvedCheckpointRoot/sources/$nextSourceId"
nextSourceId += 1
- dataSource.createContinuousReadSupport(
+ dataSource.createContinuousReader(
+ java.util.Optional.empty[StructType](),
metadataPath,
new DataSourceOptions(extraReaderOptions.asJava))
}
@@ -159,9 +160,9 @@ class ContinuousExecution(
var insertedSourceId = 0
val withNewSources = logicalPlan transform {
case ContinuousExecutionRelation(source, options, output) =>
- val readSupport = continuousSources(insertedSourceId)
+ val reader = continuousSources(insertedSourceId)
insertedSourceId += 1
- val newOutput = readSupport.fullSchema().toAttributes
+ val newOutput = reader.readSchema().toAttributes
assert(output.size == newOutput.size,
s"Invalid reader: ${Utils.truncatedString(output, ",")} != " +
@@ -169,10 +170,9 @@ class ContinuousExecution(
replacements ++= output.zip(newOutput)
val loggedOffset = offsets.offsets(0)
- val realOffset = loggedOffset.map(off => readSupport.deserializeOffset(off.json))
- val startOffset = realOffset.getOrElse(readSupport.initialOffset)
- val scanConfigBuilder = readSupport.newScanConfigBuilder(startOffset)
- StreamingDataSourceV2Relation(newOutput, source, options, readSupport, scanConfigBuilder)
+ val realOffset = loggedOffset.map(off => reader.deserializeOffset(off.json))
+ reader.setStartOffset(java.util.Optional.ofNullable(realOffset.orNull))
+ StreamingDataSourceV2Relation(newOutput, source, options, reader)
}
// Rewire the plan to use the new attributes that were returned by the source.
@@ -185,13 +185,17 @@ class ContinuousExecution(
"CurrentTimestamp and CurrentDate not yet supported for continuous processing")
}
- val writer = sink.createStreamingWriteSupport(
+ val writer = sink.createStreamWriter(
s"$runId",
triggerLogicalPlan.schema,
outputMode,
new DataSourceOptions(extraOptions.asJava))
val withSink = WriteToContinuousDataSource(writer, triggerLogicalPlan)
+ val reader = withSink.collect {
+ case StreamingDataSourceV2Relation(_, _, _, r: ContinuousReader) => r
+ }.head
+
reportTimeTaken("queryPlanning") {
lastExecution = new IncrementalExecution(
sparkSessionForQuery,
@@ -204,11 +208,8 @@ class ContinuousExecution(
lastExecution.executedPlan // Force the lazy generation of execution plan
}
- val (readSupport, scanConfig) = lastExecution.executedPlan.collect {
- case scan: DataSourceV2ScanExec if scan.readSupport.isInstanceOf[ContinuousReadSupport] =>
- scan.readSupport.asInstanceOf[ContinuousReadSupport] -> scan.scanConfig
- }.head
-
+ sparkSessionForQuery.sparkContext.setLocalProperty(
+ StreamExecution.IS_CONTINUOUS_PROCESSING, true.toString)
sparkSessionForQuery.sparkContext.setLocalProperty(
ContinuousExecution.START_EPOCH_KEY, currentBatchId.toString)
// Add another random ID on top of the run ID, to distinguish epoch coordinators across
@@ -224,16 +225,14 @@ class ContinuousExecution(
// Use the parent Spark session for the endpoint since it's where this query ID is registered.
val epochEndpoint =
EpochCoordinatorRef.create(
- writer, readSupport, this, epochCoordinatorId, currentBatchId, sparkSession, SparkEnv.get)
+ writer, reader, this, epochCoordinatorId, currentBatchId, sparkSession, SparkEnv.get)
val epochUpdateThread = new Thread(new Runnable {
override def run: Unit = {
try {
triggerExecutor.execute(() => {
startTrigger()
- val shouldReconfigure = readSupport.needsReconfiguration(scanConfig) &&
- state.compareAndSet(ACTIVE, RECONFIGURING)
- if (shouldReconfigure) {
+ if (reader.needsReconfiguration() && state.compareAndSet(ACTIVE, RECONFIGURING)) {
if (queryExecutionThread.isAlive) {
queryExecutionThread.interrupt()
}
@@ -259,8 +258,9 @@ class ContinuousExecution(
epochUpdateThread.start()
reportTimeTaken("runContinuous") {
- SQLExecution.withNewExecutionId(
- sparkSessionForQuery, lastExecution)(lastExecution.toRdd)
+ SQLExecution.withNewExecutionId(sparkSessionForQuery, lastExecution) {
+ lastExecution.executedPlan.execute()
+ }
}
} catch {
case t: Throwable
@@ -268,14 +268,30 @@ class ContinuousExecution(
logInfo(s"Query $id ignoring exception from reconfiguring: $t")
// interrupted by reconfiguration - swallow exception so we can restart the query
} finally {
- epochEndpoint.askSync[Unit](StopContinuousExecutionWrites)
- SparkEnv.get.rpcEnv.stop(epochEndpoint)
-
- epochUpdateThread.interrupt()
- epochUpdateThread.join()
-
- stopSources()
- sparkSession.sparkContext.cancelJobGroup(runId.toString)
+ // The above execution may finish before getting interrupted, for example, a Spark job having
+ // 0 partitions will complete immediately. Then the interrupted status will sneak here.
+ //
+ // To handle this case, we do the two things here:
+ //
+ // 1. Clean up the resources in `queryExecutionThread.runUninterruptibly`. This may increase
+ // the waiting time of `stop` but should be minor because the operations here are very fast
+ // (just sending an RPC message in the same process and stopping a very simple thread).
+ // 2. Clear the interrupted status at the end so that it won't impact the `runContinuous`
+ // call. We may clear the interrupted status set by `stop`, but it doesn't affect the query
+ // termination because `runActivatedStream` will check `state` and exit accordingly.
+ queryExecutionThread.runUninterruptibly {
+ try {
+ epochEndpoint.askSync[Unit](StopContinuousExecutionWrites)
+ } finally {
+ SparkEnv.get.rpcEnv.stop(epochEndpoint)
+ epochUpdateThread.interrupt()
+ epochUpdateThread.join()
+ stopSources()
+ // The following line must be the last line because it may fail if SparkContext is stopped
+ sparkSession.sparkContext.cancelJobGroup(runId.toString)
+ }
+ }
+ Thread.interrupted()
}
}
@@ -283,12 +299,10 @@ class ContinuousExecution(
* Report ending partition offsets for the given reader at the given epoch.
*/
def addOffset(
- epoch: Long,
- readSupport: ContinuousReadSupport,
- partitionOffsets: Seq[PartitionOffset]): Unit = {
+ epoch: Long, reader: ContinuousReader, partitionOffsets: Seq[PartitionOffset]): Unit = {
assert(continuousSources.length == 1, "only one continuous source supported currently")
- val globalOffset = readSupport.mergeOffsets(partitionOffsets.toArray)
+ val globalOffset = reader.mergeOffsets(partitionOffsets.toArray)
val oldOffset = synchronized {
offsetLog.add(epoch, OffsetSeq.fill(globalOffset))
offsetLog.get(epoch - 1)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousQueuedDataReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousQueuedDataReader.scala
index 65c5fc63c2f46..ec1dabd7da3e9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousQueuedDataReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousQueuedDataReader.scala
@@ -25,9 +25,8 @@ import scala.util.control.NonFatal
import org.apache.spark.{SparkEnv, SparkException, TaskContext}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.UnsafeProjection
-import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousPartitionReader, PartitionOffset}
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.sources.v2.reader.{InputPartition, InputPartitionReader}
+import org.apache.spark.sql.sources.v2.reader.streaming.PartitionOffset
import org.apache.spark.util.ThreadUtils
/**
@@ -38,14 +37,15 @@ import org.apache.spark.util.ThreadUtils
* offsets across epochs. Each compute() should call the next() method here until null is returned.
*/
class ContinuousQueuedDataReader(
- partitionIndex: Int,
- reader: ContinuousPartitionReader[InternalRow],
- schema: StructType,
+ partition: ContinuousDataSourceRDDPartition,
context: TaskContext,
dataQueueSize: Int,
epochPollIntervalMs: Long) extends Closeable {
+ private val reader = partition.inputPartition.createPartitionReader()
+
// Important sequencing - we must get our starting point before the provider threads start running
- private var currentOffset: PartitionOffset = reader.getOffset
+ private var currentOffset: PartitionOffset =
+ ContinuousDataSourceRDD.getContinuousReader(reader).getOffset
/**
* The record types in the read buffer.
@@ -66,7 +66,7 @@ class ContinuousQueuedDataReader(
epochMarkerExecutor.scheduleWithFixedDelay(
epochMarkerGenerator, 0, epochPollIntervalMs, TimeUnit.MILLISECONDS)
- private val dataReaderThread = new DataReaderThread(schema)
+ private val dataReaderThread = new DataReaderThread
dataReaderThread.setDaemon(true)
dataReaderThread.start()
@@ -113,7 +113,7 @@ class ContinuousQueuedDataReader(
currentEntry match {
case EpochMarker =>
epochCoordEndpoint.send(ReportPartitionOffset(
- partitionIndex, EpochTracker.getCurrentEpoch.get, currentOffset))
+ partition.index, EpochTracker.getCurrentEpoch.get, currentOffset))
null
case ContinuousRow(row, offset) =>
currentOffset = offset
@@ -128,16 +128,16 @@ class ContinuousQueuedDataReader(
/**
* The data component of [[ContinuousQueuedDataReader]]. Pushes (row, offset) to the queue when
- * a new row arrives to the [[ContinuousPartitionReader]].
+ * a new row arrives to the [[InputPartitionReader]].
*/
- class DataReaderThread(schema: StructType) extends Thread(
+ class DataReaderThread extends Thread(
s"continuous-reader--${context.partitionId()}--" +
s"${context.getLocalProperty(ContinuousExecution.EPOCH_COORDINATOR_ID_KEY)}") with Logging {
@volatile private[continuous] var failureReason: Throwable = _
- private val toUnsafe = UnsafeProjection.create(schema)
override def run(): Unit = {
TaskContext.setTaskContext(context)
+ val baseReader = ContinuousDataSourceRDD.getContinuousReader(reader)
try {
while (!shouldStop()) {
if (!reader.next()) {
@@ -149,9 +149,8 @@ class ContinuousQueuedDataReader(
return
}
}
- // `InternalRow#copy` may not be properly implemented, for safety we convert to unsafe row
- // before copy here.
- queue.put(ContinuousRow(toUnsafe(reader.get()).copy(), reader.getOffset))
+
+ queue.put(ContinuousRow(reader.get().copy(), baseReader.getOffset))
}
} catch {
case _: InterruptedException =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousRateStreamSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousRateStreamSource.scala
index a6cde2b8a710f..bfb78fcc3adda 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousRateStreamSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousRateStreamSource.scala
@@ -17,22 +17,24 @@
package org.apache.spark.sql.execution.streaming.continuous
+import scala.collection.JavaConverters._
+
import org.json4s.DefaultFormats
import org.json4s.jackson.Serialization
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.execution.streaming.{RateStreamOffset, SimpleStreamingScanConfig, SimpleStreamingScanConfigBuilder, ValueRunTimeMsPair}
+import org.apache.spark.sql.execution.streaming.{RateStreamOffset, ValueRunTimeMsPair}
import org.apache.spark.sql.execution.streaming.sources.RateStreamProvider
import org.apache.spark.sql.sources.v2.DataSourceOptions
import org.apache.spark.sql.sources.v2.reader._
-import org.apache.spark.sql.sources.v2.reader.streaming._
+import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousInputPartitionReader, ContinuousReader, Offset, PartitionOffset}
import org.apache.spark.sql.types.StructType
case class RateStreamPartitionOffset(
partition: Int, currentValue: Long, currentTimeMs: Long) extends PartitionOffset
-class RateStreamContinuousReadSupport(options: DataSourceOptions) extends ContinuousReadSupport {
+class RateStreamContinuousReader(options: DataSourceOptions) extends ContinuousReader {
implicit val defaultFormats: DefaultFormats = DefaultFormats
val creationTime = System.currentTimeMillis()
@@ -54,18 +56,18 @@ class RateStreamContinuousReadSupport(options: DataSourceOptions) extends Contin
RateStreamOffset(Serialization.read[Map[Int, ValueRunTimeMsPair]](json))
}
- override def fullSchema(): StructType = RateStreamProvider.SCHEMA
+ override def readSchema(): StructType = RateStreamProvider.SCHEMA
- override def newScanConfigBuilder(start: Offset): ScanConfigBuilder = {
- new SimpleStreamingScanConfigBuilder(fullSchema(), start)
- }
+ private var offset: Offset = _
- override def initialOffset: Offset = createInitialOffset(numPartitions, creationTime)
+ override def setStartOffset(offset: java.util.Optional[Offset]): Unit = {
+ this.offset = offset.orElse(createInitialOffset(numPartitions, creationTime))
+ }
- override def planInputPartitions(config: ScanConfig): Array[InputPartition] = {
- val startOffset = config.asInstanceOf[SimpleStreamingScanConfig].start
+ override def getStartOffset(): Offset = offset
- val partitionStartMap = startOffset match {
+ override def planInputPartitions(): java.util.List[InputPartition[InternalRow]] = {
+ val partitionStartMap = offset match {
case off: RateStreamOffset => off.partitionToValueAndRunTimeMs
case off =>
throw new IllegalArgumentException(
@@ -88,12 +90,8 @@ class RateStreamContinuousReadSupport(options: DataSourceOptions) extends Contin
i,
numPartitions,
perPartitionRate)
- }.toArray
- }
-
- override def createContinuousReaderFactory(
- config: ScanConfig): ContinuousPartitionReaderFactory = {
- RateStreamContinuousReaderFactory
+ .asInstanceOf[InputPartition[InternalRow]]
+ }.asJava
}
override def commit(end: Offset): Unit = {}
@@ -120,23 +118,33 @@ case class RateStreamContinuousInputPartition(
partitionIndex: Int,
increment: Long,
rowsPerSecond: Double)
- extends InputPartition
-
-object RateStreamContinuousReaderFactory extends ContinuousPartitionReaderFactory {
- override def createReader(partition: InputPartition): ContinuousPartitionReader[InternalRow] = {
- val p = partition.asInstanceOf[RateStreamContinuousInputPartition]
- new RateStreamContinuousPartitionReader(
- p.startValue, p.startTimeMs, p.partitionIndex, p.increment, p.rowsPerSecond)
+ extends ContinuousInputPartition[InternalRow] {
+
+ override def createContinuousReader(
+ offset: PartitionOffset): InputPartitionReader[InternalRow] = {
+ val rateStreamOffset = offset.asInstanceOf[RateStreamPartitionOffset]
+ require(rateStreamOffset.partition == partitionIndex,
+ s"Expected partitionIndex: $partitionIndex, but got: ${rateStreamOffset.partition}")
+ new RateStreamContinuousInputPartitionReader(
+ rateStreamOffset.currentValue,
+ rateStreamOffset.currentTimeMs,
+ partitionIndex,
+ increment,
+ rowsPerSecond)
}
+
+ override def createPartitionReader(): InputPartitionReader[InternalRow] =
+ new RateStreamContinuousInputPartitionReader(
+ startValue, startTimeMs, partitionIndex, increment, rowsPerSecond)
}
-class RateStreamContinuousPartitionReader(
+class RateStreamContinuousInputPartitionReader(
startValue: Long,
startTimeMs: Long,
partitionIndex: Int,
increment: Long,
rowsPerSecond: Double)
- extends ContinuousPartitionReader[InternalRow] {
+ extends ContinuousInputPartitionReader[InternalRow] {
private var nextReadTime: Long = startTimeMs
private val readTimeIncrement: Long = (1000 / rowsPerSecond).toLong
@@ -148,8 +156,10 @@ class RateStreamContinuousPartitionReader(
nextReadTime += readTimeIncrement
try {
- while (System.currentTimeMillis < nextReadTime) {
- Thread.sleep(nextReadTime - System.currentTimeMillis)
+ var toWaitMs = nextReadTime - System.currentTimeMillis
+ while (toWaitMs > 0) {
+ Thread.sleep(toWaitMs)
+ toWaitMs = nextReadTime - System.currentTimeMillis
}
} catch {
case _: InterruptedException =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTextSocketSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTextSocketSource.scala
index 28ab2448a6633..56bfefd91aafc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTextSocketSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTextSocketSource.scala
@@ -20,9 +20,10 @@ package org.apache.spark.sql.execution.streaming.continuous
import java.io.{BufferedReader, InputStreamReader, IOException}
import java.net.Socket
import java.sql.Timestamp
-import java.util.Calendar
+import java.util.{Calendar, List => JList}
import javax.annotation.concurrent.GuardedBy
+import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
import org.json4s.{DefaultFormats, NoTypeHints}
@@ -33,26 +34,24 @@ import org.apache.spark.internal.Logging
import org.apache.spark.rpc.RpcEndpointRef
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.execution.streaming.{Offset => _, _}
+import org.apache.spark.sql.execution.streaming.{ContinuousRecordEndpoint, ContinuousRecordPartitionOffset, GetRecord}
import org.apache.spark.sql.execution.streaming.sources.TextSocketReader
import org.apache.spark.sql.sources.v2.DataSourceOptions
-import org.apache.spark.sql.sources.v2.reader._
-import org.apache.spark.sql.sources.v2.reader.streaming._
+import org.apache.spark.sql.sources.v2.reader.{InputPartition, InputPartitionReader}
+import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousInputPartitionReader, ContinuousReader, Offset, PartitionOffset}
import org.apache.spark.sql.types.StructType
import org.apache.spark.util.RpcUtils
/**
- * A ContinuousReadSupport that reads text lines through a TCP socket, designed only for tutorials
- * and debugging. This ContinuousReadSupport will *not* work in production applications due to
- * multiple reasons, including no support for fault recovery.
+ * A ContinuousReader that reads text lines through a TCP socket, designed only for tutorials and
+ * debugging. This ContinuousReader will *not* work in production applications due to multiple
+ * reasons, including no support for fault recovery.
*
* The driver maintains a socket connection to the host-port, keeps the received messages in
* buckets and serves the messages to the executors via a RPC endpoint.
*/
-class TextSocketContinuousReadSupport(options: DataSourceOptions)
- extends ContinuousReadSupport with Logging {
-
+class TextSocketContinuousReader(options: DataSourceOptions) extends ContinuousReader with Logging {
implicit val defaultFormats: DefaultFormats = DefaultFormats
private val host: String = options.get("host").get()
@@ -74,8 +73,7 @@ class TextSocketContinuousReadSupport(options: DataSourceOptions)
@GuardedBy("this")
private var currentOffset: Int = -1
- // Exposed for tests.
- private[spark] var startOffset: TextSocketOffset = _
+ private var startOffset: TextSocketOffset = _
private val recordEndpoint = new ContinuousRecordEndpoint(buckets, this)
@volatile private var endpointRef: RpcEndpointRef = _
@@ -96,16 +94,16 @@ class TextSocketContinuousReadSupport(options: DataSourceOptions)
TextSocketOffset(Serialization.read[List[Int]](json))
}
- override def initialOffset(): Offset = {
- startOffset = TextSocketOffset(List.fill(numPartitions)(0))
- startOffset
+ override def setStartOffset(offset: java.util.Optional[Offset]): Unit = {
+ this.startOffset = offset
+ .orElse(TextSocketOffset(List.fill(numPartitions)(0)))
+ .asInstanceOf[TextSocketOffset]
+ recordEndpoint.setStartOffsets(startOffset.offsets)
}
- override def newScanConfigBuilder(start: Offset): ScanConfigBuilder = {
- new SimpleStreamingScanConfigBuilder(fullSchema(), start)
- }
+ override def getStartOffset: Offset = startOffset
- override def fullSchema(): StructType = {
+ override def readSchema(): StructType = {
if (includeTimestamp) {
TextSocketReader.SCHEMA_TIMESTAMP
} else {
@@ -113,10 +111,8 @@ class TextSocketContinuousReadSupport(options: DataSourceOptions)
}
}
- override def planInputPartitions(config: ScanConfig): Array[InputPartition] = {
- val startOffset = config.asInstanceOf[SimpleStreamingScanConfig]
- .start.asInstanceOf[TextSocketOffset]
- recordEndpoint.setStartOffsets(startOffset.offsets)
+ override def planInputPartitions(): JList[InputPartition[InternalRow]] = {
+
val endpointName = s"TextSocketContinuousReaderEndpoint-${java.util.UUID.randomUUID()}"
endpointRef = recordEndpoint.rpcEnv.setupEndpoint(endpointName, recordEndpoint)
@@ -136,13 +132,10 @@ class TextSocketContinuousReadSupport(options: DataSourceOptions)
startOffset.offsets.zipWithIndex.map {
case (offset, i) =>
- TextSocketContinuousInputPartition(endpointName, i, offset, includeTimestamp)
- }.toArray
- }
+ TextSocketContinuousInputPartition(
+ endpointName, i, offset, includeTimestamp): InputPartition[InternalRow]
+ }.asJava
- override def createContinuousReaderFactory(
- config: ScanConfig): ContinuousPartitionReaderFactory = {
- TextSocketReaderFactory
}
override def commit(end: Offset): Unit = synchronized {
@@ -197,7 +190,7 @@ class TextSocketContinuousReadSupport(options: DataSourceOptions)
logWarning(s"Stream closed by $host:$port")
return
}
- TextSocketContinuousReadSupport.this.synchronized {
+ TextSocketContinuousReader.this.synchronized {
currentOffset += 1
val newData = (line,
Timestamp.valueOf(
@@ -228,30 +221,25 @@ case class TextSocketContinuousInputPartition(
driverEndpointName: String,
partitionId: Int,
startOffset: Int,
- includeTimestamp: Boolean) extends InputPartition
-
-
-object TextSocketReaderFactory extends ContinuousPartitionReaderFactory {
+ includeTimestamp: Boolean)
+extends InputPartition[InternalRow] {
- override def createReader(partition: InputPartition): ContinuousPartitionReader[InternalRow] = {
- val p = partition.asInstanceOf[TextSocketContinuousInputPartition]
- new TextSocketContinuousPartitionReader(
- p.driverEndpointName, p.partitionId, p.startOffset, p.includeTimestamp)
- }
+ override def createPartitionReader(): InputPartitionReader[InternalRow] =
+ new TextSocketContinuousInputPartitionReader(driverEndpointName, partitionId, startOffset,
+ includeTimestamp)
}
-
/**
* Continuous text socket input partition reader.
*
* Polls the driver endpoint for new records.
*/
-class TextSocketContinuousPartitionReader(
+class TextSocketContinuousInputPartitionReader(
driverEndpointName: String,
partitionId: Int,
startOffset: Int,
includeTimestamp: Boolean)
- extends ContinuousPartitionReader[InternalRow] {
+ extends ContinuousInputPartitionReader[InternalRow] {
private val endpoint = RpcUtils.makeDriverRef(
driverEndpointName,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTrigger.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTrigger.scala
index 90e1766c4d9f1..e214b3ce17d92 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTrigger.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTrigger.scala
@@ -21,8 +21,6 @@ import java.util.concurrent.TimeUnit
import scala.concurrent.duration.Duration
-import org.apache.commons.lang3.StringUtils
-
import org.apache.spark.annotation.{Experimental, InterfaceStability}
import org.apache.spark.sql.streaming.{ProcessingTime, Trigger}
import org.apache.spark.unsafe.types.CalendarInterval
@@ -38,18 +36,7 @@ case class ContinuousTrigger(intervalMs: Long) extends Trigger {
private[sql] object ContinuousTrigger {
def apply(interval: String): ContinuousTrigger = {
- if (StringUtils.isBlank(interval)) {
- throw new IllegalArgumentException(
- "interval cannot be null or blank.")
- }
- val cal = if (interval.startsWith("interval")) {
- CalendarInterval.fromString(interval)
- } else {
- CalendarInterval.fromString("interval " + interval)
- }
- if (cal == null) {
- throw new IllegalArgumentException(s"Invalid interval: $interval")
- }
+ val cal = CalendarInterval.fromCaseInsensitiveString(interval)
if (cal.months > 0) {
throw new IllegalArgumentException(s"Doesn't support month or year interval: $interval")
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousWriteRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousWriteRDD.scala
index a08411d746abe..967dbe24a3705 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousWriteRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousWriteRDD.scala
@@ -20,8 +20,7 @@ package org.apache.spark.sql.execution.streaming.continuous
import org.apache.spark.{Partition, SparkEnv, TaskContext}
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.sources.v2.writer.DataWriter
-import org.apache.spark.sql.sources.v2.writer.streaming.StreamingDataWriterFactory
+import org.apache.spark.sql.sources.v2.writer.{DataWriter, DataWriterFactory}
import org.apache.spark.util.Utils
/**
@@ -32,7 +31,7 @@ import org.apache.spark.util.Utils
*
* We keep repeating prev.compute() and writing new epochs until the query is shut down.
*/
-class ContinuousWriteRDD(var prev: RDD[InternalRow], writerFactory: StreamingDataWriterFactory)
+class ContinuousWriteRDD(var prev: RDD[InternalRow], writeTask: DataWriterFactory[InternalRow])
extends RDD[Unit](prev) {
override val partitioner = prev.partitioner
@@ -51,7 +50,7 @@ class ContinuousWriteRDD(var prev: RDD[InternalRow], writerFactory: StreamingDat
Utils.tryWithSafeFinallyAndFailureCallbacks(block = {
try {
val dataIterator = prev.compute(split, context)
- dataWriter = writerFactory.createWriter(
+ dataWriter = writeTask.createDataWriter(
context.partitionId(),
context.taskAttemptId(),
EpochTracker.getCurrentEpoch.get)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/EpochCoordinator.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/EpochCoordinator.scala
index 2238ce26e7b46..8877ebeb26735 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/EpochCoordinator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/EpochCoordinator.scala
@@ -23,9 +23,9 @@ import org.apache.spark.SparkEnv
import org.apache.spark.internal.Logging
import org.apache.spark.rpc.{RpcCallContext, RpcEndpointRef, RpcEnv, ThreadSafeRpcEndpoint}
import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousReadSupport, PartitionOffset}
+import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousReader, PartitionOffset}
import org.apache.spark.sql.sources.v2.writer.WriterCommitMessage
-import org.apache.spark.sql.sources.v2.writer.streaming.StreamingWriteSupport
+import org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter
import org.apache.spark.util.RpcUtils
private[continuous] sealed trait EpochCoordinatorMessage extends Serializable
@@ -82,15 +82,15 @@ private[sql] object EpochCoordinatorRef extends Logging {
* Create a reference to a new [[EpochCoordinator]].
*/
def create(
- writeSupport: StreamingWriteSupport,
- readSupport: ContinuousReadSupport,
+ writer: StreamWriter,
+ reader: ContinuousReader,
query: ContinuousExecution,
epochCoordinatorId: String,
startEpoch: Long,
session: SparkSession,
env: SparkEnv): RpcEndpointRef = synchronized {
val coordinator = new EpochCoordinator(
- writeSupport, readSupport, query, startEpoch, session, env.rpcEnv)
+ writer, reader, query, startEpoch, session, env.rpcEnv)
val ref = env.rpcEnv.setupEndpoint(endpointName(epochCoordinatorId), coordinator)
logInfo("Registered EpochCoordinator endpoint")
ref
@@ -115,8 +115,8 @@ private[sql] object EpochCoordinatorRef extends Logging {
* have both committed and reported an end offset for a given epoch.
*/
private[continuous] class EpochCoordinator(
- writeSupport: StreamingWriteSupport,
- readSupport: ContinuousReadSupport,
+ writer: StreamWriter,
+ reader: ContinuousReader,
query: ContinuousExecution,
startEpoch: Long,
session: SparkSession,
@@ -198,7 +198,7 @@ private[continuous] class EpochCoordinator(
s"and is ready to be committed. Committing epoch $epoch.")
// Sequencing is important here. We must commit to the writer before recording the commit
// in the query, or we will end up dropping the commit if we restart in the middle.
- writeSupport.commit(epoch, messages.toArray)
+ writer.commit(epoch, messages.toArray)
query.commit(epoch)
}
@@ -220,7 +220,7 @@ private[continuous] class EpochCoordinator(
partitionOffsets.collect { case ((e, _), o) if e == epoch => o }
if (thisEpochOffsets.size == numReaderPartitions) {
logDebug(s"Epoch $epoch has offsets reported from all partitions: $thisEpochOffsets")
- query.addOffset(epoch, readSupport, thisEpochOffsets.toSeq)
+ query.addOffset(epoch, reader, thisEpochOffsets.toSeq)
resolveCommitsAtEpoch(epoch)
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/EpochTracker.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/EpochTracker.scala
index bc0ae428d4521..631ae4806d2f4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/EpochTracker.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/EpochTracker.scala
@@ -26,8 +26,15 @@ import java.util.concurrent.atomic.AtomicLong
object EpochTracker {
// The current epoch. Note that this is a shared reference; ContinuousWriteRDD.compute() will
// update the underlying AtomicLong as it finishes epochs. Other code should only read the value.
- private val currentEpoch: ThreadLocal[AtomicLong] = new ThreadLocal[AtomicLong] {
- override def initialValue() = new AtomicLong(-1)
+ private val currentEpoch: InheritableThreadLocal[AtomicLong] = {
+ new InheritableThreadLocal[AtomicLong] {
+ override protected def childValue(parent: AtomicLong): AtomicLong = {
+ // Note: make another instance so that changes in the parent epoch aren't reflected in
+ // those in the children threads. This is required at `ContinuousCoalesceRDD`.
+ new AtomicLong(parent.get)
+ }
+ override def initialValue() = new AtomicLong(-1)
+ }
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSource.scala
index 7ad21cc304e7c..943c731a70529 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSource.scala
@@ -19,13 +19,13 @@ package org.apache.spark.sql.execution.streaming.continuous
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.sources.v2.writer.streaming.StreamingWriteSupport
+import org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter
/**
* The logical plan for writing data in a continuous stream.
*/
case class WriteToContinuousDataSource(
- writeSupport: StreamingWriteSupport, query: LogicalPlan) extends LogicalPlan {
+ writer: StreamWriter, query: LogicalPlan) extends LogicalPlan {
override def children: Seq[LogicalPlan] = Seq(query)
override def output: Seq[Attribute] = Nil
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSourceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSourceExec.scala
index c216b61383856..927d3a84e296b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSourceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/WriteToContinuousDataSourceExec.scala
@@ -26,21 +26,21 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.execution.SparkPlan
import org.apache.spark.sql.execution.streaming.StreamExecution
-import org.apache.spark.sql.sources.v2.writer.streaming.StreamingWriteSupport
+import org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter
/**
- * The physical plan for writing data into a continuous processing [[StreamingWriteSupport]].
+ * The physical plan for writing data into a continuous processing [[StreamWriter]].
*/
-case class WriteToContinuousDataSourceExec(writeSupport: StreamingWriteSupport, query: SparkPlan)
+case class WriteToContinuousDataSourceExec(writer: StreamWriter, query: SparkPlan)
extends SparkPlan with Logging {
override def children: Seq[SparkPlan] = Seq(query)
override def output: Seq[Attribute] = Nil
override protected def doExecute(): RDD[InternalRow] = {
- val writerFactory = writeSupport.createStreamingWriterFactory()
+ val writerFactory = writer.createWriterFactory()
val rdd = new ContinuousWriteRDD(query.execute(), writerFactory)
- logInfo(s"Start processing data source write support: $writeSupport. " +
+ logInfo(s"Start processing data source writer: $writer. " +
s"The input RDD has ${rdd.partitions.length} partitions.")
EpochCoordinatorRef.get(
sparkContext.getLocalProperty(ContinuousExecution.EPOCH_COORDINATOR_ID_KEY),
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
index adf52aba21a04..f81abdcc3711a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
@@ -17,9 +17,12 @@
package org.apache.spark.sql.execution.streaming
+import java.{util => ju}
+import java.util.Optional
import java.util.concurrent.atomic.AtomicInteger
import javax.annotation.concurrent.GuardedBy
+import scala.collection.JavaConverters._
import scala.collection.mutable.{ArrayBuffer, ListBuffer}
import scala.util.control.NonFatal
@@ -31,8 +34,8 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, UnsafeRow}
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.EstimationUtils
import org.apache.spark.sql.catalyst.streaming.InternalOutputModes._
-import org.apache.spark.sql.sources.v2.reader._
-import org.apache.spark.sql.sources.v2.reader.streaming.{MicroBatchReadSupport, Offset => OffsetV2}
+import org.apache.spark.sql.sources.v2.reader.{InputPartition, InputPartitionReader}
+import org.apache.spark.sql.sources.v2.reader.streaming.{MicroBatchReader, Offset => OffsetV2}
import org.apache.spark.sql.streaming.OutputMode
import org.apache.spark.sql.types.StructType
import org.apache.spark.util.Utils
@@ -64,7 +67,7 @@ abstract class MemoryStreamBase[A : Encoder](sqlContext: SQLContext) extends Bas
addData(data.toTraversable)
}
- def fullSchema(): StructType = encoder.schema
+ def readSchema(): StructType = encoder.schema
protected def logicalPlan: LogicalPlan
@@ -77,7 +80,7 @@ abstract class MemoryStreamBase[A : Encoder](sqlContext: SQLContext) extends Bas
* available.
*/
case class MemoryStream[A : Encoder](id: Int, sqlContext: SQLContext)
- extends MemoryStreamBase[A](sqlContext) with MicroBatchReadSupport with Logging {
+ extends MemoryStreamBase[A](sqlContext) with MicroBatchReader with Logging {
protected val logicalPlan: LogicalPlan =
StreamingExecutionRelation(this, attributes)(sqlContext.sparkSession)
@@ -119,22 +122,24 @@ case class MemoryStream[A : Encoder](id: Int, sqlContext: SQLContext)
override def toString: String = s"MemoryStream[${Utils.truncatedString(output, ",")}]"
- override def deserializeOffset(json: String): OffsetV2 = LongOffset(json.toLong)
+ override def setOffsetRange(start: Optional[OffsetV2], end: Optional[OffsetV2]): Unit = {
+ synchronized {
+ startOffset = start.orElse(LongOffset(-1)).asInstanceOf[LongOffset]
+ endOffset = end.orElse(currentOffset).asInstanceOf[LongOffset]
+ }
+ }
- override def initialOffset: OffsetV2 = LongOffset(-1)
+ override def deserializeOffset(json: String): OffsetV2 = LongOffset(json.toLong)
- override def latestOffset(): OffsetV2 = {
- if (currentOffset.offset == -1) null else currentOffset
+ override def getStartOffset: OffsetV2 = synchronized {
+ if (startOffset.offset == -1) null else startOffset
}
- override def newScanConfigBuilder(start: OffsetV2, end: OffsetV2): ScanConfigBuilder = {
- new SimpleStreamingScanConfigBuilder(fullSchema(), start, Some(end))
+ override def getEndOffset: OffsetV2 = synchronized {
+ if (endOffset.offset == -1) null else endOffset
}
- override def planInputPartitions(config: ScanConfig): Array[InputPartition] = {
- val sc = config.asInstanceOf[SimpleStreamingScanConfig]
- val startOffset = sc.start.asInstanceOf[LongOffset]
- val endOffset = sc.end.get.asInstanceOf[LongOffset]
+ override def planInputPartitions(): ju.List[InputPartition[InternalRow]] = {
synchronized {
// Compute the internal batch numbers to fetch: [startOrdinal, endOrdinal)
val startOrdinal = startOffset.offset.toInt + 1
@@ -151,15 +156,11 @@ case class MemoryStream[A : Encoder](id: Int, sqlContext: SQLContext)
logDebug(generateDebugString(newBlocks.flatten, startOrdinal, endOrdinal))
newBlocks.map { block =>
- new MemoryStreamInputPartition(block)
- }.toArray
+ new MemoryStreamInputPartition(block): InputPartition[InternalRow]
+ }.asJava
}
}
- override def createReaderFactory(config: ScanConfig): PartitionReaderFactory = {
- MemoryStreamReaderFactory
- }
-
private def generateDebugString(
rows: Seq[UnsafeRow],
startOrdinal: Int,
@@ -200,12 +201,10 @@ case class MemoryStream[A : Encoder](id: Int, sqlContext: SQLContext)
}
-class MemoryStreamInputPartition(val records: Array[UnsafeRow]) extends InputPartition
-
-object MemoryStreamReaderFactory extends PartitionReaderFactory {
- override def createReader(partition: InputPartition): PartitionReader[InternalRow] = {
- val records = partition.asInstanceOf[MemoryStreamInputPartition].records
- new PartitionReader[InternalRow] {
+class MemoryStreamInputPartition(records: Array[UnsafeRow])
+ extends InputPartition[InternalRow] {
+ override def createPartitionReader(): InputPartitionReader[InternalRow] = {
+ new InputPartitionReader[InternalRow] {
private var currentIndex = -1
override def next(): Boolean = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriteSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriter.scala
similarity index 86%
rename from sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriteSupport.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriter.scala
index 833e62f35ede1..fd45ba509091e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriteSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ConsoleWriter.scala
@@ -19,15 +19,16 @@ package org.apache.spark.sql.execution.streaming.sources
import org.apache.spark.internal.Logging
import org.apache.spark.sql.{Dataset, SparkSession}
+import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
import org.apache.spark.sql.sources.v2.DataSourceOptions
-import org.apache.spark.sql.sources.v2.writer.WriterCommitMessage
-import org.apache.spark.sql.sources.v2.writer.streaming.{StreamingDataWriterFactory, StreamingWriteSupport}
+import org.apache.spark.sql.sources.v2.writer.{DataWriterFactory, WriterCommitMessage}
+import org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter
import org.apache.spark.sql.types.StructType
/** Common methods used to create writes for the the console sink */
-class ConsoleWriteSupport(schema: StructType, options: DataSourceOptions)
- extends StreamingWriteSupport with Logging {
+class ConsoleWriter(schema: StructType, options: DataSourceOptions)
+ extends StreamWriter with Logging {
// Number of rows to display, by default 20 rows
protected val numRowsToShow = options.getInt("numRows", 20)
@@ -38,7 +39,7 @@ class ConsoleWriteSupport(schema: StructType, options: DataSourceOptions)
assert(SparkSession.getActiveSession.isDefined)
protected val spark = SparkSession.getActiveSession.get
- def createStreamingWriterFactory(): StreamingDataWriterFactory = PackedRowWriterFactory
+ def createWriterFactory(): DataWriterFactory[InternalRow] = PackedRowWriterFactory
override def commit(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {
// We have to print a "Batch" label for the epoch for compatibility with the pre-data source V2
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ContinuousMemoryStream.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ContinuousMemoryStream.scala
index dbcc4483e5770..4a32217f149bd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ContinuousMemoryStream.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ContinuousMemoryStream.scala
@@ -17,22 +17,26 @@
package org.apache.spark.sql.execution.streaming.sources
+import java.{util => ju}
+import java.util.Optional
import java.util.concurrent.atomic.AtomicInteger
import javax.annotation.concurrent.GuardedBy
+import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
import org.json4s.NoTypeHints
import org.json4s.jackson.Serialization
import org.apache.spark.{SparkEnv, TaskContext}
-import org.apache.spark.rpc.RpcEndpointRef
+import org.apache.spark.rpc.{RpcCallContext, RpcEndpointRef, RpcEnv, ThreadSafeRpcEndpoint}
import org.apache.spark.sql.{Encoder, SQLContext}
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.execution.streaming.{Offset => _, _}
-import org.apache.spark.sql.sources.v2.{ContinuousReadSupportProvider, DataSourceOptions}
-import org.apache.spark.sql.sources.v2.reader.{InputPartition, ScanConfig, ScanConfigBuilder}
-import org.apache.spark.sql.sources.v2.reader.streaming._
+import org.apache.spark.sql.execution.streaming._
+import org.apache.spark.sql.sources.v2.{ContinuousReadSupport, DataSourceOptions}
+import org.apache.spark.sql.sources.v2.reader.InputPartition
+import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousInputPartitionReader, ContinuousReader, Offset, PartitionOffset}
+import org.apache.spark.sql.types.StructType
import org.apache.spark.util.RpcUtils
/**
@@ -44,9 +48,7 @@ import org.apache.spark.util.RpcUtils
* the specified offset within the list, or null if that offset doesn't yet have a record.
*/
class ContinuousMemoryStream[A : Encoder](id: Int, sqlContext: SQLContext, numPartitions: Int = 2)
- extends MemoryStreamBase[A](sqlContext)
- with ContinuousReadSupportProvider with ContinuousReadSupport {
-
+ extends MemoryStreamBase[A](sqlContext) with ContinuousReader with ContinuousReadSupport {
private implicit val formats = Serialization.formats(NoTypeHints)
protected val logicalPlan =
@@ -57,6 +59,9 @@ class ContinuousMemoryStream[A : Encoder](id: Int, sqlContext: SQLContext, numPa
@GuardedBy("this")
private val records = Seq.fill(numPartitions)(new ListBuffer[A])
+ @GuardedBy("this")
+ private var startOffset: ContinuousMemoryStreamOffset = _
+
private val recordEndpoint = new ContinuousRecordEndpoint(records, this)
@volatile private var endpointRef: RpcEndpointRef = _
@@ -70,8 +75,15 @@ class ContinuousMemoryStream[A : Encoder](id: Int, sqlContext: SQLContext, numPa
ContinuousMemoryStreamOffset((0 until numPartitions).map(i => (i, records(i).size)).toMap)
}
- override def initialOffset(): Offset = {
- ContinuousMemoryStreamOffset((0 until numPartitions).map(i => (i, 0)).toMap)
+ override def setStartOffset(start: Optional[Offset]): Unit = synchronized {
+ // Inferred initial offset is position 0 in each partition.
+ startOffset = start.orElse {
+ ContinuousMemoryStreamOffset((0 until numPartitions).map(i => (i, 0)).toMap)
+ }.asInstanceOf[ContinuousMemoryStreamOffset]
+ }
+
+ override def getStartOffset: Offset = synchronized {
+ startOffset
}
override def deserializeOffset(json: String): ContinuousMemoryStreamOffset = {
@@ -86,40 +98,34 @@ class ContinuousMemoryStream[A : Encoder](id: Int, sqlContext: SQLContext, numPa
)
}
- override def newScanConfigBuilder(start: Offset): ScanConfigBuilder = {
- new SimpleStreamingScanConfigBuilder(fullSchema(), start)
- }
-
- override def planInputPartitions(config: ScanConfig): Array[InputPartition] = {
- val startOffset = config.asInstanceOf[SimpleStreamingScanConfig]
- .start.asInstanceOf[ContinuousMemoryStreamOffset]
+ override def planInputPartitions(): ju.List[InputPartition[InternalRow]] = {
synchronized {
val endpointName = s"ContinuousMemoryStreamRecordEndpoint-${java.util.UUID.randomUUID()}-$id"
endpointRef =
recordEndpoint.rpcEnv.setupEndpoint(endpointName, recordEndpoint)
startOffset.partitionNums.map {
- case (part, index) => ContinuousMemoryStreamInputPartition(endpointName, part, index)
- }.toArray
+ case (part, index) =>
+ new ContinuousMemoryStreamInputPartition(
+ endpointName, part, index): InputPartition[InternalRow]
+ }.toList.asJava
}
}
- override def createContinuousReaderFactory(
- config: ScanConfig): ContinuousPartitionReaderFactory = {
- ContinuousMemoryStreamReaderFactory
- }
-
override def stop(): Unit = {
if (endpointRef != null) recordEndpoint.rpcEnv.stop(endpointRef)
}
override def commit(end: Offset): Unit = {}
- // ContinuousReadSupportProvider implementation
+ // ContinuousReadSupport implementation
// This is necessary because of how StreamTest finds the source for AddDataMemory steps.
- override def createContinuousReadSupport(
+ def createContinuousReader(
+ schema: Optional[StructType],
checkpointLocation: String,
- options: DataSourceOptions): ContinuousReadSupport = this
+ options: DataSourceOptions): ContinuousReader = {
+ this
+ }
}
object ContinuousMemoryStream {
@@ -135,16 +141,12 @@ object ContinuousMemoryStream {
/**
* An input partition for continuous memory stream.
*/
-case class ContinuousMemoryStreamInputPartition(
+class ContinuousMemoryStreamInputPartition(
driverEndpointName: String,
partition: Int,
- startOffset: Int) extends InputPartition
-
-object ContinuousMemoryStreamReaderFactory extends ContinuousPartitionReaderFactory {
- override def createReader(partition: InputPartition): ContinuousPartitionReader[InternalRow] = {
- val p = partition.asInstanceOf[ContinuousMemoryStreamInputPartition]
- new ContinuousMemoryStreamPartitionReader(p.driverEndpointName, p.partition, p.startOffset)
- }
+ startOffset: Int) extends InputPartition[InternalRow] {
+ override def createPartitionReader: ContinuousMemoryStreamInputPartitionReader =
+ new ContinuousMemoryStreamInputPartitionReader(driverEndpointName, partition, startOffset)
}
/**
@@ -152,10 +154,10 @@ object ContinuousMemoryStreamReaderFactory extends ContinuousPartitionReaderFact
*
* Polls the driver endpoint for new records.
*/
-class ContinuousMemoryStreamPartitionReader(
+class ContinuousMemoryStreamInputPartitionReader(
driverEndpointName: String,
partition: Int,
- startOffset: Int) extends ContinuousPartitionReader[InternalRow] {
+ startOffset: Int) extends ContinuousInputPartitionReader[InternalRow] {
private val endpoint = RpcUtils.makeDriverRef(
driverEndpointName,
SparkEnv.get.conf,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriteSupportProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterProvider.scala
similarity index 77%
rename from sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriteSupportProvider.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterProvider.scala
index 4218fd51ad206..bfbb5fe90f2ab 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriteSupportProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterProvider.scala
@@ -17,14 +17,15 @@
package org.apache.spark.sql.execution.streaming.sources
+import org.apache.spark.SparkException
import org.apache.spark.sql.{ForeachWriter, SparkSession}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.catalyst.expressions.UnsafeRow
import org.apache.spark.sql.execution.python.PythonForeachWriter
-import org.apache.spark.sql.sources.v2.{DataSourceOptions, StreamingWriteSupportProvider}
-import org.apache.spark.sql.sources.v2.writer.{DataWriter, WriterCommitMessage}
-import org.apache.spark.sql.sources.v2.writer.streaming.{StreamingDataWriterFactory, StreamingWriteSupport}
+import org.apache.spark.sql.sources.v2.{DataSourceOptions, StreamWriteSupport}
+import org.apache.spark.sql.sources.v2.writer.{DataWriter, DataWriterFactory, WriterCommitMessage}
+import org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter
import org.apache.spark.sql.streaming.OutputMode
import org.apache.spark.sql.types.StructType
@@ -37,21 +38,20 @@ import org.apache.spark.sql.types.StructType
* a [[ExpressionEncoder]] or a direct converter function.
* @tparam T The expected type of the sink.
*/
-case class ForeachWriteSupportProvider[T](
+case class ForeachWriterProvider[T](
writer: ForeachWriter[T],
- converter: Either[ExpressionEncoder[T], InternalRow => T])
- extends StreamingWriteSupportProvider {
+ converter: Either[ExpressionEncoder[T], InternalRow => T]) extends StreamWriteSupport {
- override def createStreamingWriteSupport(
+ override def createStreamWriter(
queryId: String,
schema: StructType,
mode: OutputMode,
- options: DataSourceOptions): StreamingWriteSupport = {
- new StreamingWriteSupport {
+ options: DataSourceOptions): StreamWriter = {
+ new StreamWriter {
override def commit(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}
override def abort(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}
- override def createStreamingWriterFactory(): StreamingDataWriterFactory = {
+ override def createWriterFactory(): DataWriterFactory[InternalRow] = {
val rowConverter: InternalRow => T = converter match {
case Left(enc) =>
val boundEnc = enc.resolveAndBind(
@@ -69,16 +69,16 @@ case class ForeachWriteSupportProvider[T](
}
}
-object ForeachWriteSupportProvider {
+object ForeachWriterProvider {
def apply[T](
writer: ForeachWriter[T],
- encoder: ExpressionEncoder[T]): ForeachWriteSupportProvider[_] = {
+ encoder: ExpressionEncoder[T]): ForeachWriterProvider[_] = {
writer match {
case pythonWriter: PythonForeachWriter =>
- new ForeachWriteSupportProvider[UnsafeRow](
+ new ForeachWriterProvider[UnsafeRow](
pythonWriter, Right((x: InternalRow) => x.asInstanceOf[UnsafeRow]))
case _ =>
- new ForeachWriteSupportProvider[T](writer, Left(encoder))
+ new ForeachWriterProvider[T](writer, Left(encoder))
}
}
}
@@ -86,8 +86,8 @@ object ForeachWriteSupportProvider {
case class ForeachWriterFactory[T](
writer: ForeachWriter[T],
rowConverter: InternalRow => T)
- extends StreamingDataWriterFactory {
- override def createWriter(
+ extends DataWriterFactory[InternalRow] {
+ override def createDataWriter(
partitionId: Int,
taskId: Long,
epochId: Long): ForeachDataWriter[T] = {
@@ -113,6 +113,7 @@ class ForeachDataWriter[T](
// If open returns false, we should skip writing rows.
private val opened = writer.open(partitionId, epochId)
+ private var closeCalled: Boolean = false
override def write(record: InternalRow): Unit = {
if (!opened) return
@@ -121,17 +122,26 @@ class ForeachDataWriter[T](
writer.process(rowConverter(record))
} catch {
case t: Throwable =>
- writer.close(t)
+ closeWriter(t)
throw t
}
}
override def commit(): WriterCommitMessage = {
- writer.close(null)
+ closeWriter(null)
ForeachWriterCommitMessage
}
- override def abort(): Unit = {}
+ override def abort(): Unit = {
+ closeWriter(new SparkException("Foreach writer has been aborted due to a task failure"))
+ }
+
+ private def closeWriter(errorOrNull: Throwable): Unit = {
+ if (!closeCalled) {
+ closeCalled = true
+ writer.close(errorOrNull)
+ }
+ }
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/MicroBatchWritSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/MicroBatchWriter.scala
similarity index 50%
rename from sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/MicroBatchWritSupport.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/MicroBatchWriter.scala
index 9f88416871f8e..2d43a7bb77872 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/MicroBatchWritSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/MicroBatchWriter.scala
@@ -18,34 +18,20 @@
package org.apache.spark.sql.execution.streaming.sources
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.sources.v2.writer.{BatchWriteSupport, DataWriter, DataWriterFactory, WriterCommitMessage}
-import org.apache.spark.sql.sources.v2.writer.streaming.{StreamingDataWriterFactory, StreamingWriteSupport}
+import org.apache.spark.sql.sources.v2.writer.{DataSourceWriter, DataWriterFactory, WriterCommitMessage}
+import org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter
/**
- * A [[BatchWriteSupport]] used to hook V2 stream writers into a microbatch plan. It implements
- * the non-streaming interface, forwarding the epoch ID determined at construction to a wrapped
- * streaming write support.
+ * A [[DataSourceWriter]] used to hook V2 stream writers into a microbatch plan. It implements
+ * the non-streaming interface, forwarding the batch ID determined at construction to a wrapped
+ * streaming writer.
*/
-class MicroBatchWritSupport(eppchId: Long, val writeSupport: StreamingWriteSupport)
- extends BatchWriteSupport {
-
+class MicroBatchWriter(batchId: Long, val writer: StreamWriter) extends DataSourceWriter {
override def commit(messages: Array[WriterCommitMessage]): Unit = {
- writeSupport.commit(eppchId, messages)
+ writer.commit(batchId, messages)
}
- override def abort(messages: Array[WriterCommitMessage]): Unit = {
- writeSupport.abort(eppchId, messages)
- }
+ override def abort(messages: Array[WriterCommitMessage]): Unit = writer.abort(batchId, messages)
- override def createBatchWriterFactory(): DataWriterFactory = {
- new MicroBatchWriterFactory(eppchId, writeSupport.createStreamingWriterFactory())
- }
-}
-
-class MicroBatchWriterFactory(epochId: Long, streamingWriterFactory: StreamingDataWriterFactory)
- extends DataWriterFactory {
-
- override def createWriter(partitionId: Int, taskId: Long): DataWriter[InternalRow] = {
- streamingWriterFactory.createWriter(partitionId, taskId, epochId)
- }
+ override def createWriterFactory(): DataWriterFactory[InternalRow] = writer.createWriterFactory()
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/PackedRowWriterFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/PackedRowWriterFactory.scala
index ac3c71cc222b1..f26e11d842b29 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/PackedRowWriterFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/PackedRowWriterFactory.scala
@@ -21,18 +21,17 @@ import scala.collection.mutable
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.sources.v2.writer.{BatchWriteSupport, DataWriter, DataWriterFactory, WriterCommitMessage}
-import org.apache.spark.sql.sources.v2.writer.streaming.StreamingDataWriterFactory
+import org.apache.spark.sql.sources.v2.writer.{DataSourceWriter, DataWriter, DataWriterFactory, WriterCommitMessage}
/**
* A simple [[DataWriterFactory]] whose tasks just pack rows into the commit message for delivery
- * to a [[BatchWriteSupport]] on the driver.
+ * to a [[DataSourceWriter]] on the driver.
*
* Note that, because it sends all rows to the driver, this factory will generally be unsuitable
* for production-quality sinks. It's intended for use in tests.
*/
-case object PackedRowWriterFactory extends StreamingDataWriterFactory {
- override def createWriter(
+case object PackedRowWriterFactory extends DataWriterFactory[InternalRow] {
+ override def createDataWriter(
partitionId: Int,
taskId: Long,
epochId: Long): DataWriter[InternalRow] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamMicroBatchReadSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamMicroBatchReader.scala
similarity index 78%
rename from sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamMicroBatchReadSupport.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamMicroBatchReader.scala
index f5364047adff1..9e0d954932163 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamMicroBatchReadSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamMicroBatchReader.scala
@@ -19,24 +19,27 @@ package org.apache.spark.sql.execution.streaming.sources
import java.io._
import java.nio.charset.StandardCharsets
+import java.util.Optional
import java.util.concurrent.TimeUnit
+import scala.collection.JavaConverters._
+
import org.apache.commons.io.IOUtils
import org.apache.spark.internal.Logging
import org.apache.spark.network.util.JavaUtils
-import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.{Row, SparkSession}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.execution.streaming._
import org.apache.spark.sql.sources.v2.DataSourceOptions
import org.apache.spark.sql.sources.v2.reader._
-import org.apache.spark.sql.sources.v2.reader.streaming.{MicroBatchReadSupport, Offset}
+import org.apache.spark.sql.sources.v2.reader.streaming.{MicroBatchReader, Offset}
import org.apache.spark.sql.types.StructType
import org.apache.spark.util.{ManualClock, SystemClock}
-class RateStreamMicroBatchReadSupport(options: DataSourceOptions, checkpointLocation: String)
- extends MicroBatchReadSupport with Logging {
+class RateStreamMicroBatchReader(options: DataSourceOptions, checkpointLocation: String)
+ extends MicroBatchReader with Logging {
import RateStreamProvider._
private[sources] val clock = {
@@ -103,30 +106,38 @@ class RateStreamMicroBatchReadSupport(options: DataSourceOptions, checkpointLoca
@volatile private var lastTimeMs: Long = creationTimeMs
- override def initialOffset(): Offset = LongOffset(0L)
+ private var start: LongOffset = _
+ private var end: LongOffset = _
- override def latestOffset(): Offset = {
- val now = clock.getTimeMillis()
- if (lastTimeMs < now) {
- lastTimeMs = now
- }
- LongOffset(TimeUnit.MILLISECONDS.toSeconds(lastTimeMs - creationTimeMs))
- }
+ override def readSchema(): StructType = SCHEMA
- override def deserializeOffset(json: String): Offset = {
- LongOffset(json.toLong)
+ override def setOffsetRange(start: Optional[Offset], end: Optional[Offset]): Unit = {
+ this.start = start.orElse(LongOffset(0L)).asInstanceOf[LongOffset]
+ this.end = end.orElse {
+ val now = clock.getTimeMillis()
+ if (lastTimeMs < now) {
+ lastTimeMs = now
+ }
+ LongOffset(TimeUnit.MILLISECONDS.toSeconds(lastTimeMs - creationTimeMs))
+ }.asInstanceOf[LongOffset]
}
- override def fullSchema(): StructType = SCHEMA
+ override def getStartOffset(): Offset = {
+ if (start == null) throw new IllegalStateException("start offset not set")
+ start
+ }
+ override def getEndOffset(): Offset = {
+ if (end == null) throw new IllegalStateException("end offset not set")
+ end
+ }
- override def newScanConfigBuilder(start: Offset, end: Offset): ScanConfigBuilder = {
- new SimpleStreamingScanConfigBuilder(fullSchema(), start, Some(end))
+ override def deserializeOffset(json: String): Offset = {
+ LongOffset(json.toLong)
}
- override def planInputPartitions(config: ScanConfig): Array[InputPartition] = {
- val sc = config.asInstanceOf[SimpleStreamingScanConfig]
- val startSeconds = sc.start.asInstanceOf[LongOffset].offset
- val endSeconds = sc.end.get.asInstanceOf[LongOffset].offset
+ override def planInputPartitions(): java.util.List[InputPartition[InternalRow]] = {
+ val startSeconds = LongOffset.convert(start).map(_.offset).getOrElse(0L)
+ val endSeconds = LongOffset.convert(end).map(_.offset).getOrElse(0L)
assert(startSeconds <= endSeconds, s"startSeconds($startSeconds) > endSeconds($endSeconds)")
if (endSeconds > maxSeconds) {
throw new ArithmeticException("Integer overflow. Max offset with " +
@@ -142,7 +153,7 @@ class RateStreamMicroBatchReadSupport(options: DataSourceOptions, checkpointLoca
s"rangeStart: $rangeStart, rangeEnd: $rangeEnd")
if (rangeStart == rangeEnd) {
- return Array.empty
+ return List.empty.asJava
}
val localStartTimeMs = creationTimeMs + TimeUnit.SECONDS.toMillis(startSeconds)
@@ -159,11 +170,8 @@ class RateStreamMicroBatchReadSupport(options: DataSourceOptions, checkpointLoca
(0 until numPartitions).map { p =>
new RateStreamMicroBatchInputPartition(
p, numPartitions, rangeStart, rangeEnd, localStartTimeMs, relativeMsPerValue)
- }.toArray
- }
-
- override def createReaderFactory(config: ScanConfig): PartitionReaderFactory = {
- RateStreamMicroBatchReaderFactory
+ : InputPartition[InternalRow]
+ }.toList.asJava
}
override def commit(end: Offset): Unit = {}
@@ -175,29 +183,26 @@ class RateStreamMicroBatchReadSupport(options: DataSourceOptions, checkpointLoca
s"numPartitions=${options.get(NUM_PARTITIONS).orElse("default")}"
}
-case class RateStreamMicroBatchInputPartition(
+class RateStreamMicroBatchInputPartition(
partitionId: Int,
numPartitions: Int,
rangeStart: Long,
rangeEnd: Long,
localStartTimeMs: Long,
- relativeMsPerValue: Double) extends InputPartition
+ relativeMsPerValue: Double) extends InputPartition[InternalRow] {
-object RateStreamMicroBatchReaderFactory extends PartitionReaderFactory {
- override def createReader(partition: InputPartition): PartitionReader[InternalRow] = {
- val p = partition.asInstanceOf[RateStreamMicroBatchInputPartition]
- new RateStreamMicroBatchPartitionReader(p.partitionId, p.numPartitions, p.rangeStart,
- p.rangeEnd, p.localStartTimeMs, p.relativeMsPerValue)
- }
+ override def createPartitionReader(): InputPartitionReader[InternalRow] =
+ new RateStreamMicroBatchInputPartitionReader(partitionId, numPartitions, rangeStart, rangeEnd,
+ localStartTimeMs, relativeMsPerValue)
}
-class RateStreamMicroBatchPartitionReader(
+class RateStreamMicroBatchInputPartitionReader(
partitionId: Int,
numPartitions: Int,
rangeStart: Long,
rangeEnd: Long,
localStartTimeMs: Long,
- relativeMsPerValue: Double) extends PartitionReader[InternalRow] {
+ relativeMsPerValue: Double) extends InputPartitionReader[InternalRow] {
private var count: Long = 0
override def next(): Boolean = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamProvider.scala
index 6942dfbfe0ecf..6bdd492f0cb35 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/RateStreamProvider.scala
@@ -17,11 +17,14 @@
package org.apache.spark.sql.execution.streaming.sources
+import java.util.Optional
+
import org.apache.spark.network.util.JavaUtils
-import org.apache.spark.sql.execution.streaming.continuous.RateStreamContinuousReadSupport
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.streaming.continuous.RateStreamContinuousReader
import org.apache.spark.sql.sources.DataSourceRegister
import org.apache.spark.sql.sources.v2._
-import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousReadSupport, MicroBatchReadSupport}
+import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousReader, MicroBatchReader}
import org.apache.spark.sql.types._
/**
@@ -39,12 +42,13 @@ import org.apache.spark.sql.types._
* be resource constrained, and `numPartitions` can be tweaked to help reach the desired speed.
*/
class RateStreamProvider extends DataSourceV2
- with MicroBatchReadSupportProvider with ContinuousReadSupportProvider with DataSourceRegister {
+ with MicroBatchReadSupport with ContinuousReadSupport with DataSourceRegister {
import RateStreamProvider._
- override def createMicroBatchReadSupport(
+ override def createMicroBatchReader(
+ schema: Optional[StructType],
checkpointLocation: String,
- options: DataSourceOptions): MicroBatchReadSupport = {
+ options: DataSourceOptions): MicroBatchReader = {
if (options.get(ROWS_PER_SECOND).isPresent) {
val rowsPerSecond = options.get(ROWS_PER_SECOND).get().toLong
if (rowsPerSecond <= 0) {
@@ -70,14 +74,17 @@ class RateStreamProvider extends DataSourceV2
}
}
- new RateStreamMicroBatchReadSupport(options, checkpointLocation)
+ if (schema.isPresent) {
+ throw new AnalysisException("The rate source does not support a user-specified schema.")
+ }
+
+ new RateStreamMicroBatchReader(options, checkpointLocation)
}
- override def createContinuousReadSupport(
+ override def createContinuousReader(
+ schema: Optional[StructType],
checkpointLocation: String,
- options: DataSourceOptions): ContinuousReadSupport = {
- new RateStreamContinuousReadSupport(options)
- }
+ options: DataSourceOptions): ContinuousReader = new RateStreamContinuousReader(options)
override def shortName(): String = "rate"
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memoryV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memoryV2.scala
index c50dc7bcb8da1..cb76e8650339e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memoryV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memoryV2.scala
@@ -32,9 +32,9 @@ import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, Statistics}
import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.EstimationUtils
import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.{Append, Complete, Update}
import org.apache.spark.sql.execution.streaming.{MemorySinkBase, Sink}
-import org.apache.spark.sql.sources.v2.{DataSourceOptions, DataSourceV2, StreamingWriteSupportProvider}
+import org.apache.spark.sql.sources.v2.{DataSourceOptions, DataSourceV2, StreamWriteSupport}
import org.apache.spark.sql.sources.v2.writer._
-import org.apache.spark.sql.sources.v2.writer.streaming.{StreamingDataWriterFactory, StreamingWriteSupport}
+import org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter
import org.apache.spark.sql.streaming.OutputMode
import org.apache.spark.sql.types.StructType
@@ -42,15 +42,13 @@ import org.apache.spark.sql.types.StructType
* A sink that stores the results in memory. This [[Sink]] is primarily intended for use in unit
* tests and does not provide durability.
*/
-class MemorySinkV2 extends DataSourceV2 with StreamingWriteSupportProvider
- with MemorySinkBase with Logging {
-
- override def createStreamingWriteSupport(
+class MemorySinkV2 extends DataSourceV2 with StreamWriteSupport with MemorySinkBase with Logging {
+ override def createStreamWriter(
queryId: String,
schema: StructType,
mode: OutputMode,
- options: DataSourceOptions): StreamingWriteSupport = {
- new MemoryStreamingWriteSupport(this, mode, schema)
+ options: DataSourceOptions): StreamWriter = {
+ new MemoryStreamWriter(this, mode, schema)
}
private case class AddedData(batchId: Long, data: Array[Row])
@@ -122,13 +120,10 @@ class MemorySinkV2 extends DataSourceV2 with StreamingWriteSupportProvider
case class MemoryWriterCommitMessage(partition: Int, data: Seq[Row])
extends WriterCommitMessage {}
-class MemoryStreamingWriteSupport(
- val sink: MemorySinkV2, outputMode: OutputMode, schema: StructType)
- extends StreamingWriteSupport {
+class MemoryStreamWriter(val sink: MemorySinkV2, outputMode: OutputMode, schema: StructType)
+ extends StreamWriter {
- override def createStreamingWriterFactory: MemoryWriterFactory = {
- MemoryWriterFactory(outputMode, schema)
- }
+ override def createWriterFactory: MemoryWriterFactory = MemoryWriterFactory(outputMode, schema)
override def commit(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {
val newRows = messages.flatMap {
@@ -143,19 +138,13 @@ class MemoryStreamingWriteSupport(
}
case class MemoryWriterFactory(outputMode: OutputMode, schema: StructType)
- extends DataWriterFactory with StreamingDataWriterFactory {
+ extends DataWriterFactory[InternalRow] {
- override def createWriter(
- partitionId: Int,
- taskId: Long): DataWriter[InternalRow] = {
- new MemoryDataWriter(partitionId, outputMode, schema)
- }
-
- override def createWriter(
+ override def createDataWriter(
partitionId: Int,
taskId: Long,
epochId: Long): DataWriter[InternalRow] = {
- createWriter(partitionId, taskId)
+ new MemoryDataWriter(partitionId, outputMode, schema)
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/socket.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/socket.scala
index b2a573eae504a..874c479db95d5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/socket.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/socket.scala
@@ -20,10 +20,11 @@ package org.apache.spark.sql.execution.streaming.sources
import java.io.{BufferedReader, InputStreamReader, IOException}
import java.net.Socket
import java.text.SimpleDateFormat
-import java.util.{Calendar, Locale}
+import java.util.{Calendar, List => JList, Locale, Optional}
import java.util.concurrent.atomic.AtomicBoolean
import javax.annotation.concurrent.GuardedBy
+import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
import scala.util.{Failure, Success, Try}
@@ -31,15 +32,16 @@ import org.apache.spark.internal.Logging
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.execution.streaming.{LongOffset, SimpleStreamingScanConfig, SimpleStreamingScanConfigBuilder}
-import org.apache.spark.sql.execution.streaming.continuous.TextSocketContinuousReadSupport
+import org.apache.spark.sql.execution.streaming.LongOffset
+import org.apache.spark.sql.execution.streaming.continuous.TextSocketContinuousReader
import org.apache.spark.sql.sources.DataSourceRegister
-import org.apache.spark.sql.sources.v2.{ContinuousReadSupportProvider, DataSourceOptions, DataSourceV2, MicroBatchReadSupportProvider}
-import org.apache.spark.sql.sources.v2.reader._
-import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousReadSupport, MicroBatchReadSupport, Offset}
+import org.apache.spark.sql.sources.v2.{ContinuousReadSupport, DataSourceOptions, DataSourceV2, MicroBatchReadSupport}
+import org.apache.spark.sql.sources.v2.reader.{InputPartition, InputPartitionReader}
+import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousReader, MicroBatchReader, Offset}
import org.apache.spark.sql.types.{StringType, StructField, StructType, TimestampType}
import org.apache.spark.unsafe.types.UTF8String
+// Shared object for micro-batch and continuous reader
object TextSocketReader {
val SCHEMA_REGULAR = StructType(StructField("value", StringType) :: Nil)
val SCHEMA_TIMESTAMP = StructType(StructField("value", StringType) ::
@@ -48,12 +50,14 @@ object TextSocketReader {
}
/**
- * A MicroBatchReadSupport that reads text lines through a TCP socket, designed only for tutorials
- * and debugging. This MicroBatchReadSupport will *not* work in production applications due to
- * multiple reasons, including no support for fault recovery.
+ * A MicroBatchReader that reads text lines through a TCP socket, designed only for tutorials and
+ * debugging. This MicroBatchReader will *not* work in production applications due to multiple
+ * reasons, including no support for fault recovery.
*/
-class TextSocketMicroBatchReadSupport(options: DataSourceOptions)
- extends MicroBatchReadSupport with Logging {
+class TextSocketMicroBatchReader(options: DataSourceOptions) extends MicroBatchReader with Logging {
+
+ private var startOffset: Offset = _
+ private var endOffset: Offset = _
private val host: String = options.get("host").get()
private val port: Int = options.get("port").get().toInt
@@ -99,7 +103,7 @@ class TextSocketMicroBatchReadSupport(options: DataSourceOptions)
logWarning(s"Stream closed by $host:$port")
return
}
- TextSocketMicroBatchReadSupport.this.synchronized {
+ TextSocketMicroBatchReader.this.synchronized {
val newData = (
UTF8String.fromString(line),
DateTimeUtils.fromMillis(Calendar.getInstance().getTimeInMillis)
@@ -116,15 +120,24 @@ class TextSocketMicroBatchReadSupport(options: DataSourceOptions)
readThread.start()
}
- override def initialOffset(): Offset = LongOffset(-1L)
+ override def setOffsetRange(start: Optional[Offset], end: Optional[Offset]): Unit = synchronized {
+ startOffset = start.orElse(LongOffset(-1L))
+ endOffset = end.orElse(currentOffset)
+ }
- override def latestOffset(): Offset = currentOffset
+ override def getStartOffset(): Offset = {
+ Option(startOffset).getOrElse(throw new IllegalStateException("start offset not set"))
+ }
+
+ override def getEndOffset(): Offset = {
+ Option(endOffset).getOrElse(throw new IllegalStateException("end offset not set"))
+ }
override def deserializeOffset(json: String): Offset = {
LongOffset(json.toLong)
}
- override def fullSchema(): StructType = {
+ override def readSchema(): StructType = {
if (options.getBoolean("includeTimestamp", false)) {
TextSocketReader.SCHEMA_TIMESTAMP
} else {
@@ -132,14 +145,12 @@ class TextSocketMicroBatchReadSupport(options: DataSourceOptions)
}
}
- override def newScanConfigBuilder(start: Offset, end: Offset): ScanConfigBuilder = {
- new SimpleStreamingScanConfigBuilder(fullSchema(), start, Some(end))
- }
+ override def planInputPartitions(): JList[InputPartition[InternalRow]] = {
+ assert(startOffset != null && endOffset != null,
+ "start offset and end offset should already be set before create read tasks.")
- override def planInputPartitions(config: ScanConfig): Array[InputPartition] = {
- val sc = config.asInstanceOf[SimpleStreamingScanConfig]
- val startOrdinal = sc.start.asInstanceOf[LongOffset].offset.toInt + 1
- val endOrdinal = sc.end.get.asInstanceOf[LongOffset].offset.toInt + 1
+ val startOrdinal = LongOffset.convert(startOffset).get.offset.toInt + 1
+ val endOrdinal = LongOffset.convert(endOffset).get.offset.toInt + 1
// Internal buffer only holds the batches after lastOffsetCommitted
val rawList = synchronized {
@@ -161,29 +172,26 @@ class TextSocketMicroBatchReadSupport(options: DataSourceOptions)
slices(idx % numPartitions).append(r)
}
- slices.map(TextSocketInputPartition)
- }
+ (0 until numPartitions).map { i =>
+ val slice = slices(i)
+ new InputPartition[InternalRow] {
+ override def createPartitionReader(): InputPartitionReader[InternalRow] =
+ new InputPartitionReader[InternalRow] {
+ private var currentIdx = -1
- override def createReaderFactory(config: ScanConfig): PartitionReaderFactory = {
- new PartitionReaderFactory {
- override def createReader(partition: InputPartition): PartitionReader[InternalRow] = {
- val slice = partition.asInstanceOf[TextSocketInputPartition].slice
- new PartitionReader[InternalRow] {
- private var currentIdx = -1
+ override def next(): Boolean = {
+ currentIdx += 1
+ currentIdx < slice.size
+ }
- override def next(): Boolean = {
- currentIdx += 1
- currentIdx < slice.size
- }
+ override def get(): InternalRow = {
+ InternalRow(slice(currentIdx)._1, slice(currentIdx)._2)
+ }
- override def get(): InternalRow = {
- InternalRow(slice(currentIdx)._1, slice(currentIdx)._2)
+ override def close(): Unit = {}
}
-
- override def close(): Unit = {}
- }
}
- }
+ }.toList.asJava
}
override def commit(end: Offset): Unit = synchronized {
@@ -219,11 +227,8 @@ class TextSocketMicroBatchReadSupport(options: DataSourceOptions)
override def toString: String = s"TextSocketV2[host: $host, port: $port]"
}
-case class TextSocketInputPartition(slice: ListBuffer[(UTF8String, Long)]) extends InputPartition
-
class TextSocketSourceProvider extends DataSourceV2
- with MicroBatchReadSupportProvider with ContinuousReadSupportProvider
- with DataSourceRegister with Logging {
+ with MicroBatchReadSupport with ContinuousReadSupport with DataSourceRegister with Logging {
private def checkParameters(params: DataSourceOptions): Unit = {
logWarning("The socket source should not be used for production applications! " +
@@ -243,18 +248,27 @@ class TextSocketSourceProvider extends DataSourceV2
}
}
- override def createMicroBatchReadSupport(
+ override def createMicroBatchReader(
+ schema: Optional[StructType],
checkpointLocation: String,
- options: DataSourceOptions): MicroBatchReadSupport = {
+ options: DataSourceOptions): MicroBatchReader = {
checkParameters(options)
- new TextSocketMicroBatchReadSupport(options)
+ if (schema.isPresent) {
+ throw new AnalysisException("The socket source does not support a user-specified schema.")
+ }
+
+ new TextSocketMicroBatchReader(options)
}
- override def createContinuousReadSupport(
+ override def createContinuousReader(
+ schema: Optional[StructType],
checkpointLocation: String,
- options: DataSourceOptions): ContinuousReadSupport = {
+ options: DataSourceOptions): ContinuousReader = {
checkParameters(options)
- new TextSocketContinuousReadSupport(options)
+ if (schema.isPresent) {
+ throw new AnalysisException("The socket source does not support a user-specified schema.")
+ }
+ new TextSocketContinuousReader(options)
}
/** String that represents the format that this data source provider uses. */
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/HDFSBackedStateStoreProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/HDFSBackedStateStoreProvider.scala
index 92a2480e8b017..a63b05ac3e0b2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/HDFSBackedStateStoreProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/HDFSBackedStateStoreProvider.scala
@@ -43,7 +43,7 @@ import org.apache.spark.util.{SizeEstimator, Utils}
/**
* An implementation of [[StateStoreProvider]] and [[StateStore]] in which all the data is backed
- * by files in a HDFS-compatible file system. All updates to the store has to be done in sets
+ * by files in an HDFS-compatible file system. All updates to the store has to be done in sets
* transactionally, and each set of updates increments the store's version. These versions can
* be used to re-execute the updates (by retries in RDD operations) on the correct version of
* the store, and regenerate the store version.
@@ -79,7 +79,7 @@ private[state] class HDFSBackedStateStoreProvider extends StateStoreProvider wit
// java.util.ConcurrentModificationException
type MapType = java.util.concurrent.ConcurrentHashMap[UnsafeRow, UnsafeRow]
- /** Implementation of [[StateStore]] API which is backed by a HDFS-compatible file system */
+ /** Implementation of [[StateStore]] API which is backed by an HDFS-compatible file system */
class HDFSBackedStateStore(val version: Long, mapToUpdate: MapType)
extends StateStore {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDD.scala
index 3f11b8f79943c..4a69a48fed75f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStoreRDD.scala
@@ -23,6 +23,7 @@ import scala.reflect.ClassTag
import org.apache.spark.{Partition, TaskContext}
import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.execution.streaming.StreamExecution
import org.apache.spark.sql.execution.streaming.continuous.EpochTracker
import org.apache.spark.sql.internal.SessionState
import org.apache.spark.sql.types.StructType
@@ -74,9 +75,14 @@ class StateStoreRDD[T: ClassTag, U: ClassTag](
// If we're in continuous processing mode, we should get the store version for the current
// epoch rather than the one at planning time.
- val currentVersion = EpochTracker.getCurrentEpoch match {
- case None => storeVersion
- case Some(value) => value
+ val isContinuous = Option(ctxt.getLocalProperty(StreamExecution.IS_CONTINUOUS_PROCESSING))
+ .map(_.toBoolean).getOrElse(false)
+ val currentVersion = if (isContinuous) {
+ val epoch = EpochTracker.getCurrentEpoch
+ assert(epoch.isDefined, "Current epoch must be defined for continuous processing streams.")
+ epoch.get
+ } else {
+ storeVersion
}
store = StateStore.get(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListener.scala
index d254af400a7cf..111ac3111f7cd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListener.scala
@@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.execution.ui
-import java.util.Date
+import java.util.{Date, NoSuchElementException}
import java.util.concurrent.ConcurrentHashMap
import java.util.function.Function
@@ -77,7 +77,29 @@ class SQLAppStatusListener(
val executionId = executionIdString.toLong
val jobId = event.jobId
- val exec = getOrCreateExecution(executionId)
+ val exec = Option(liveExecutions.get(executionId))
+ .orElse {
+ try {
+ // Should not overwrite the kvstore with new entry, if it already has the SQLExecution
+ // data corresponding to the execId.
+ val sqlStoreData = kvstore.read(classOf[SQLExecutionUIData], executionId)
+ val executionData = new LiveExecutionData(executionId)
+ executionData.description = sqlStoreData.description
+ executionData.details = sqlStoreData.details
+ executionData.physicalPlanDescription = sqlStoreData.physicalPlanDescription
+ executionData.metrics = sqlStoreData.metrics
+ executionData.submissionTime = sqlStoreData.submissionTime
+ executionData.completionTime = sqlStoreData.completionTime
+ executionData.jobs = sqlStoreData.jobs
+ executionData.stages = sqlStoreData.stages
+ executionData.metricsValues = sqlStoreData.metricValues
+ executionData.endEvents = sqlStoreData.jobs.size + 1
+ liveExecutions.put(executionId, executionData)
+ Some(executionData)
+ } catch {
+ case _: NoSuchElementException => None
+ }
+ }.getOrElse(getOrCreateExecution(executionId))
// Record the accumulator IDs for the stages of this job, so that the code that keeps
// track of the metrics knows which accumulators to look at.
@@ -159,7 +181,6 @@ class SQLAppStatusListener(
}
private def aggregateMetrics(exec: LiveExecutionData): Map[Long, String] = {
- val metricIds = exec.metrics.map(_.accumulatorId).sorted
val metricTypes = exec.metrics.map { m => (m.accumulatorId, m.metricType) }.toMap
val metrics = exec.stages.toSeq
.flatMap { stageId => Option(stageMetrics.get(stageId)) }
@@ -167,10 +188,10 @@ class SQLAppStatusListener(
.flatMap { metrics => metrics.ids.zip(metrics.values) }
val aggregatedMetrics = (metrics ++ exec.driverAccumUpdates.toSeq)
- .filter { case (id, _) => metricIds.contains(id) }
+ .filter { case (id, _) => metricTypes.contains(id) }
.groupBy(_._1)
.map { case (id, values) =>
- id -> SQLMetrics.stringValue(metricTypes(id), values.map(_._2).toSeq)
+ id -> SQLMetrics.stringValue(metricTypes(id), values.map(_._2))
}
// Check the execution again for whether the aggregated metrics data has been calculated.
@@ -276,16 +297,20 @@ class SQLAppStatusListener(
exec.endEvents += 1
update(exec)
- // Remove stale LiveStageMetrics objects for stages that are not active anymore.
- val activeStages = liveExecutions.values().asScala.flatMap { other =>
- if (other != exec) other.stages else Nil
- }.toSet
- stageMetrics.keySet().asScala
- .filter(!activeStages.contains(_))
- .foreach(stageMetrics.remove)
+ removeStaleMetricsData(exec)
}
}
+ private def removeStaleMetricsData(exec: LiveExecutionData): Unit = {
+ // Remove stale LiveStageMetrics objects for stages that are not active anymore.
+ val activeStages = liveExecutions.values().asScala.flatMap { other =>
+ if (other != exec) other.stages else Nil
+ }.toSet
+ stageMetrics.keySet().asScala
+ .filter(!activeStages.contains(_))
+ .foreach(stageMetrics.remove)
+ }
+
private def onDriverAccumUpdates(event: SparkListenerDriverAccumUpdates): Unit = {
val SparkListenerDriverAccumUpdates(executionId, accumUpdates) = event
Option(liveExecutions.get(executionId)).foreach { exec =>
@@ -312,6 +337,7 @@ class SQLAppStatusListener(
val now = System.nanoTime()
if (exec.endEvents >= exec.jobs.size + 1) {
exec.write(kvstore, now)
+ removeStaleMetricsData(exec)
liveExecutions.remove(exec.executionId)
} else if (force) {
exec.write(kvstore, now)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
index 697757f8a73ce..eb956c4b3e888 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
@@ -73,19 +73,24 @@ case class UserDefinedFunction protected[sql] (
*/
@scala.annotation.varargs
def apply(exprs: Column*): Column = {
- if (inputTypes.isDefined && nullableTypes.isDefined) {
- require(inputTypes.get.length == nullableTypes.get.length)
+ // TODO: make sure this class is only instantiated through `SparkUserDefinedFunction.create()`
+ // and `nullableTypes` is always set.
+ if (nullableTypes.isEmpty) {
+ nullableTypes = Some(ScalaReflection.getParameterTypeNullability(f))
+ }
+ if (inputTypes.isDefined) {
+ assert(inputTypes.get.length == nullableTypes.get.length)
}
Column(ScalaUDF(
f,
dataType,
exprs.map(_.expr),
+ nullableTypes.get,
inputTypes.getOrElse(Nil),
udfName = _nameOption,
nullable = _nullable,
- udfDeterministic = _deterministic,
- nullableTypes = nullableTypes.getOrElse(Nil)))
+ udfDeterministic = _deterministic))
}
private def copyAll(): UserDefinedFunction = {
@@ -146,9 +151,14 @@ private[sql] object SparkUserDefinedFunction {
def create(
f: AnyRef,
dataType: DataType,
- inputSchemas: Option[Seq[ScalaReflection.Schema]]): UserDefinedFunction = {
- val udf = new UserDefinedFunction(f, dataType, inputSchemas.map(_.map(_.dataType)))
- udf.nullableTypes = inputSchemas.map(_.map(_.nullable))
+ inputSchemas: Seq[Option[ScalaReflection.Schema]]): UserDefinedFunction = {
+ val inputTypes = if (inputSchemas.contains(None)) {
+ None
+ } else {
+ Some(inputSchemas.map(_.get.dataType))
+ }
+ val udf = new UserDefinedFunction(f, dataType, inputTypes)
+ udf.nullableTypes = Some(inputSchemas.map(_.map(_.nullable).getOrElse(true)))
udf
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/Window.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/Window.scala
index cd819bab1b14c..14dec8f0810f2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/Window.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/Window.scala
@@ -215,52 +215,10 @@ object Window {
}
/**
- * Creates a [[WindowSpec]] with the frame boundaries defined,
- * from `start` (inclusive) to `end` (inclusive).
- *
- * Both `start` and `end` are relative to the current row. For example, "lit(0)" means
- * "current row", while "lit(-1)" means one off before the current row, and "lit(5)" means the
- * five off after the current row.
- *
- * Users should use `unboundedPreceding()`, `unboundedFollowing()`, and `currentRow()` from
- * [[org.apache.spark.sql.functions]] to specify special boundary values, literals are not
- * transformed to [[org.apache.spark.sql.catalyst.expressions.SpecialFrameBoundary]]s.
- *
- * A range-based boundary is based on the actual value of the ORDER BY
- * expression(s). An offset is used to alter the value of the ORDER BY expression, for
- * instance if the current order by expression has a value of 10 and the lower bound offset
- * is -3, the resulting lower bound for the current row will be 10 - 3 = 7. This however puts a
- * number of constraints on the ORDER BY expressions: there can be only one expression and this
- * expression must have a numerical/date/timestamp data type. An exception can be made when the
- * offset is unbounded, because no value modification is needed, in this case multiple and
- * non-numerical/date/timestamp data type ORDER BY expression are allowed.
- *
- * {{{
- * import org.apache.spark.sql.expressions.Window
- * val df = Seq((1, "a"), (1, "a"), (2, "a"), (1, "b"), (2, "b"), (3, "b"))
- * .toDF("id", "category")
- * val byCategoryOrderedById =
- * Window.partitionBy('category).orderBy('id).rangeBetween(currentRow(), lit(1))
- * df.withColumn("sum", sum('id) over byCategoryOrderedById).show()
- *
- * +---+--------+---+
- * | id|category|sum|
- * +---+--------+---+
- * | 1| b| 3|
- * | 2| b| 5|
- * | 3| b| 3|
- * | 1| a| 4|
- * | 1| a| 4|
- * | 2| a| 2|
- * +---+--------+---+
- * }}}
- *
- * @param start boundary start, inclusive. The frame is unbounded if the expression is
- * [[org.apache.spark.sql.catalyst.expressions.UnboundedPreceding]].
- * @param end boundary end, inclusive. The frame is unbounded if the expression is
- * [[org.apache.spark.sql.catalyst.expressions.UnboundedFollowing]].
+ * This function has been deprecated in Spark 2.4. See SPARK-25842 for more information.
* @since 2.3.0
*/
+ @deprecated("Use the version with Long parameter types", "2.4.0")
def rangeBetween(start: Column, end: Column): WindowSpec = {
spec.rangeBetween(start, end)
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala
index 4c41aa3c5fb67..0cc43a58237df 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala
@@ -210,51 +210,10 @@ class WindowSpec private[sql](
}
/**
- * Defines the frame boundaries, from `start` (inclusive) to `end` (inclusive).
- *
- * Both `start` and `end` are relative to the current row. For example, "lit(0)" means
- * "current row", while "lit(-1)" means one off before the current row, and "lit(5)" means the
- * five off after the current row.
- *
- * Users should use `unboundedPreceding()`, `unboundedFollowing()`, and `currentRow()` from
- * [[org.apache.spark.sql.functions]] to specify special boundary values, literals are not
- * transformed to [[org.apache.spark.sql.catalyst.expressions.SpecialFrameBoundary]]s.
- *
- * A range-based boundary is based on the actual value of the ORDER BY
- * expression(s). An offset is used to alter the value of the ORDER BY expression, for
- * instance if the current order by expression has a value of 10 and the lower bound offset
- * is -3, the resulting lower bound for the current row will be 10 - 3 = 7. This however puts a
- * number of constraints on the ORDER BY expressions: there can be only one expression and this
- * expression must have a numerical/date/timestamp data type. An exception can be made when the
- * offset is unbounded, because no value modification is needed, in this case multiple and
- * non-numerical/date/timestamp data type ORDER BY expression are allowed.
- *
- * {{{
- * import org.apache.spark.sql.expressions.Window
- * val df = Seq((1, "a"), (1, "a"), (2, "a"), (1, "b"), (2, "b"), (3, "b"))
- * .toDF("id", "category")
- * val byCategoryOrderedById =
- * Window.partitionBy('category).orderBy('id).rangeBetween(currentRow(), lit(1))
- * df.withColumn("sum", sum('id) over byCategoryOrderedById).show()
- *
- * +---+--------+---+
- * | id|category|sum|
- * +---+--------+---+
- * | 1| b| 3|
- * | 2| b| 5|
- * | 3| b| 3|
- * | 1| a| 4|
- * | 1| a| 4|
- * | 2| a| 2|
- * +---+--------+---+
- * }}}
- *
- * @param start boundary start, inclusive. The frame is unbounded if the expression is
- * [[org.apache.spark.sql.catalyst.expressions.UnboundedPreceding]].
- * @param end boundary end, inclusive. The frame is unbounded if the expression is
- * [[org.apache.spark.sql.catalyst.expressions.UnboundedFollowing]].
+ * This function has been deprecated in Spark 2.4. See SPARK-25842 for more information.
* @since 2.3.0
*/
+ @deprecated("Use the version with Long parameter types", "2.4.0")
def rangeBetween(start: Column, end: Column): WindowSpec = {
new WindowSpec(
partitionSpec,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 10b67d7a1ca54..f059add514e5b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -830,30 +830,30 @@ object functions {
// Window functions
//////////////////////////////////////////////////////////////////////////////////////////////
/**
- * Window function: returns the special frame boundary that represents the first row in the
- * window partition.
+ * This function has been deprecated in Spark 2.4. See SPARK-25842 for more information.
*
* @group window_funcs
* @since 2.3.0
*/
+ @deprecated("Use Window.unboundedPreceding", "2.4.0")
def unboundedPreceding(): Column = Column(UnboundedPreceding)
/**
- * Window function: returns the special frame boundary that represents the last row in the
- * window partition.
+ * This function has been deprecated in Spark 2.4. See SPARK-25842 for more information.
*
* @group window_funcs
* @since 2.3.0
*/
+ @deprecated("Use Window.unboundedFollowing", "2.4.0")
def unboundedFollowing(): Column = Column(UnboundedFollowing)
/**
- * Window function: returns the special frame boundary that represents the current row in the
- * window partition.
+ * This function has been deprecated in Spark 2.4. See SPARK-25842 for more information.
*
* @group window_funcs
* @since 2.3.0
*/
+ @deprecated("Use Window.currentRow", "2.4.0")
def currentRow(): Column = Column(CurrentRow)
/**
@@ -3265,6 +3265,11 @@ object functions {
/**
* Returns an array containing all the elements in `x` from index `start` (or starting from the
* end if `start` is negative) with the specified `length`.
+ *
+ * @param x the array column to be sliced
+ * @param start the starting index
+ * @param length the length of the slice
+ *
* @group collection_funcs
* @since 2.4.0
*/
@@ -3602,14 +3607,24 @@ object functions {
}
/**
- * Parses a column containing a JSON string and infers its schema.
+ * Parses a JSON string and infers its schema in DDL format.
*
- * @param e a string column containing JSON data.
+ * @param json a JSON string.
*
* @group collection_funcs
* @since 2.4.0
*/
- def schema_of_json(e: Column): Column = withExpr(new SchemaOfJson(e.expr))
+ def schema_of_json(json: String): Column = schema_of_json(lit(json))
+
+ /**
+ * Parses a JSON string and infers its schema in DDL format.
+ *
+ * @param json a string literal containing a JSON string.
+ *
+ * @group collection_funcs
+ * @since 2.4.0
+ */
+ def schema_of_json(json: Column): Column = withExpr(new SchemaOfJson(json.expr))
/**
* (Scala-specific) Converts a column containing a `StructType`, `ArrayType` or
@@ -3780,13 +3795,6 @@ object functions {
*/
def map_values(e: Column): Column = withExpr { MapValues(e.expr) }
- /**
- * Returns an unordered array of all entries in the given map.
- * @group collection_funcs
- * @since 2.4.0
- */
- def map_entries(e: Column): Column = withExpr { MapEntries(e.expr) }
-
/**
* Returns a map created from the given array of entries.
* @group collection_funcs
@@ -3819,7 +3827,7 @@ object functions {
(0 to 10).foreach { x =>
val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"})
val typeTags = (1 to x).map(i => s"A$i: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _)
- val inputSchemas = (1 to x).foldRight("Nil")((i, s) => {s"ScalaReflection.schemaFor(typeTag[A$i]) :: $s"})
+ val inputSchemas = (1 to x).foldRight("Nil")((i, s) => {s"Try(ScalaReflection.schemaFor(typeTag[A$i])).toOption :: $s"})
println(s"""
|/**
| * Defines a Scala closure of $x arguments as user-defined function (UDF).
@@ -3832,7 +3840,7 @@ object functions {
| */
|def udf[$typeTags](f: Function$x[$types]): UserDefinedFunction = {
| val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- | val inputSchemas = Try($inputTypes).toOption
+ | val inputSchemas = $inputSchemas
| val udf = SparkUserDefinedFunction.create(f, dataType, inputSchemas)
| if (nullable) udf else udf.asNonNullable()
|}""".stripMargin)
@@ -3856,7 +3864,7 @@ object functions {
| */
|def udf(f: UDF$i[$extTypeArgs], returnType: DataType): UserDefinedFunction = {
| val func = f$anyCast.call($anyParams)
- | SparkUserDefinedFunction.create($funcCall, returnType, inputSchemas = None)
+ | SparkUserDefinedFunction.create($funcCall, returnType, inputSchemas = Seq.fill($i)(None))
|}""".stripMargin)
}
@@ -3877,7 +3885,7 @@ object functions {
*/
def udf[RT: TypeTag](f: Function0[RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas = Try(Nil).toOption
+ val inputSchemas = Nil
val udf = SparkUserDefinedFunction.create(f, dataType, inputSchemas)
if (nullable) udf else udf.asNonNullable()
}
@@ -3893,7 +3901,7 @@ object functions {
*/
def udf[RT: TypeTag, A1: TypeTag](f: Function1[A1, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1]) :: Nil).toOption
+ val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Nil
val udf = SparkUserDefinedFunction.create(f, dataType, inputSchemas)
if (nullable) udf else udf.asNonNullable()
}
@@ -3909,7 +3917,7 @@ object functions {
*/
def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag](f: Function2[A1, A2, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1]) :: ScalaReflection.schemaFor(typeTag[A2]) :: Nil).toOption
+ val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Nil
val udf = SparkUserDefinedFunction.create(f, dataType, inputSchemas)
if (nullable) udf else udf.asNonNullable()
}
@@ -3925,7 +3933,7 @@ object functions {
*/
def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](f: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1]) :: ScalaReflection.schemaFor(typeTag[A2]) :: ScalaReflection.schemaFor(typeTag[A3]) :: Nil).toOption
+ val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Nil
val udf = SparkUserDefinedFunction.create(f, dataType, inputSchemas)
if (nullable) udf else udf.asNonNullable()
}
@@ -3941,7 +3949,7 @@ object functions {
*/
def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](f: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1]) :: ScalaReflection.schemaFor(typeTag[A2]) :: ScalaReflection.schemaFor(typeTag[A3]) :: ScalaReflection.schemaFor(typeTag[A4]) :: Nil).toOption
+ val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Nil
val udf = SparkUserDefinedFunction.create(f, dataType, inputSchemas)
if (nullable) udf else udf.asNonNullable()
}
@@ -3957,7 +3965,7 @@ object functions {
*/
def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](f: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1]) :: ScalaReflection.schemaFor(typeTag[A2]) :: ScalaReflection.schemaFor(typeTag[A3]) :: ScalaReflection.schemaFor(typeTag[A4]) :: ScalaReflection.schemaFor(typeTag[A5]) :: Nil).toOption
+ val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Nil
val udf = SparkUserDefinedFunction.create(f, dataType, inputSchemas)
if (nullable) udf else udf.asNonNullable()
}
@@ -3973,7 +3981,7 @@ object functions {
*/
def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1]) :: ScalaReflection.schemaFor(typeTag[A2]) :: ScalaReflection.schemaFor(typeTag[A3]) :: ScalaReflection.schemaFor(typeTag[A4]) :: ScalaReflection.schemaFor(typeTag[A5]) :: ScalaReflection.schemaFor(typeTag[A6]) :: Nil).toOption
+ val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Nil
val udf = SparkUserDefinedFunction.create(f, dataType, inputSchemas)
if (nullable) udf else udf.asNonNullable()
}
@@ -3989,7 +3997,7 @@ object functions {
*/
def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1]) :: ScalaReflection.schemaFor(typeTag[A2]) :: ScalaReflection.schemaFor(typeTag[A3]) :: ScalaReflection.schemaFor(typeTag[A4]) :: ScalaReflection.schemaFor(typeTag[A5]) :: ScalaReflection.schemaFor(typeTag[A6]) :: ScalaReflection.schemaFor(typeTag[A7]) :: Nil).toOption
+ val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A7])).toOption :: Nil
val udf = SparkUserDefinedFunction.create(f, dataType, inputSchemas)
if (nullable) udf else udf.asNonNullable()
}
@@ -4005,7 +4013,7 @@ object functions {
*/
def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1]) :: ScalaReflection.schemaFor(typeTag[A2]) :: ScalaReflection.schemaFor(typeTag[A3]) :: ScalaReflection.schemaFor(typeTag[A4]) :: ScalaReflection.schemaFor(typeTag[A5]) :: ScalaReflection.schemaFor(typeTag[A6]) :: ScalaReflection.schemaFor(typeTag[A7]) :: ScalaReflection.schemaFor(typeTag[A8]) :: Nil).toOption
+ val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A7])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A8])).toOption :: Nil
val udf = SparkUserDefinedFunction.create(f, dataType, inputSchemas)
if (nullable) udf else udf.asNonNullable()
}
@@ -4021,7 +4029,7 @@ object functions {
*/
def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1]) :: ScalaReflection.schemaFor(typeTag[A2]) :: ScalaReflection.schemaFor(typeTag[A3]) :: ScalaReflection.schemaFor(typeTag[A4]) :: ScalaReflection.schemaFor(typeTag[A5]) :: ScalaReflection.schemaFor(typeTag[A6]) :: ScalaReflection.schemaFor(typeTag[A7]) :: ScalaReflection.schemaFor(typeTag[A8]) :: ScalaReflection.schemaFor(typeTag[A9]) :: Nil).toOption
+ val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A7])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A8])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A9])).toOption :: Nil
val udf = SparkUserDefinedFunction.create(f, dataType, inputSchemas)
if (nullable) udf else udf.asNonNullable()
}
@@ -4037,7 +4045,7 @@ object functions {
*/
def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1]) :: ScalaReflection.schemaFor(typeTag[A2]) :: ScalaReflection.schemaFor(typeTag[A3]) :: ScalaReflection.schemaFor(typeTag[A4]) :: ScalaReflection.schemaFor(typeTag[A5]) :: ScalaReflection.schemaFor(typeTag[A6]) :: ScalaReflection.schemaFor(typeTag[A7]) :: ScalaReflection.schemaFor(typeTag[A8]) :: ScalaReflection.schemaFor(typeTag[A9]) :: ScalaReflection.schemaFor(typeTag[A10]) :: Nil).toOption
+ val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A7])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A8])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A9])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A10])).toOption :: Nil
val udf = SparkUserDefinedFunction.create(f, dataType, inputSchemas)
if (nullable) udf else udf.asNonNullable()
}
@@ -4057,7 +4065,7 @@ object functions {
*/
def udf(f: UDF0[_], returnType: DataType): UserDefinedFunction = {
val func = f.asInstanceOf[UDF0[Any]].call()
- SparkUserDefinedFunction.create(() => func, returnType, inputSchemas = None)
+ SparkUserDefinedFunction.create(() => func, returnType, inputSchemas = Seq.fill(0)(None))
}
/**
@@ -4071,7 +4079,7 @@ object functions {
*/
def udf(f: UDF1[_, _], returnType: DataType): UserDefinedFunction = {
val func = f.asInstanceOf[UDF1[Any, Any]].call(_: Any)
- SparkUserDefinedFunction.create(func, returnType, inputSchemas = None)
+ SparkUserDefinedFunction.create(func, returnType, inputSchemas = Seq.fill(1)(None))
}
/**
@@ -4085,7 +4093,7 @@ object functions {
*/
def udf(f: UDF2[_, _, _], returnType: DataType): UserDefinedFunction = {
val func = f.asInstanceOf[UDF2[Any, Any, Any]].call(_: Any, _: Any)
- SparkUserDefinedFunction.create(func, returnType, inputSchemas = None)
+ SparkUserDefinedFunction.create(func, returnType, inputSchemas = Seq.fill(2)(None))
}
/**
@@ -4099,7 +4107,7 @@ object functions {
*/
def udf(f: UDF3[_, _, _, _], returnType: DataType): UserDefinedFunction = {
val func = f.asInstanceOf[UDF3[Any, Any, Any, Any]].call(_: Any, _: Any, _: Any)
- SparkUserDefinedFunction.create(func, returnType, inputSchemas = None)
+ SparkUserDefinedFunction.create(func, returnType, inputSchemas = Seq.fill(3)(None))
}
/**
@@ -4113,7 +4121,7 @@ object functions {
*/
def udf(f: UDF4[_, _, _, _, _], returnType: DataType): UserDefinedFunction = {
val func = f.asInstanceOf[UDF4[Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any)
- SparkUserDefinedFunction.create(func, returnType, inputSchemas = None)
+ SparkUserDefinedFunction.create(func, returnType, inputSchemas = Seq.fill(4)(None))
}
/**
@@ -4127,7 +4135,7 @@ object functions {
*/
def udf(f: UDF5[_, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
val func = f.asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any)
- SparkUserDefinedFunction.create(func, returnType, inputSchemas = None)
+ SparkUserDefinedFunction.create(func, returnType, inputSchemas = Seq.fill(5)(None))
}
/**
@@ -4141,7 +4149,7 @@ object functions {
*/
def udf(f: UDF6[_, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
val func = f.asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
- SparkUserDefinedFunction.create(func, returnType, inputSchemas = None)
+ SparkUserDefinedFunction.create(func, returnType, inputSchemas = Seq.fill(6)(None))
}
/**
@@ -4155,7 +4163,7 @@ object functions {
*/
def udf(f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
val func = f.asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
- SparkUserDefinedFunction.create(func, returnType, inputSchemas = None)
+ SparkUserDefinedFunction.create(func, returnType, inputSchemas = Seq.fill(7)(None))
}
/**
@@ -4169,7 +4177,7 @@ object functions {
*/
def udf(f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
val func = f.asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
- SparkUserDefinedFunction.create(func, returnType, inputSchemas = None)
+ SparkUserDefinedFunction.create(func, returnType, inputSchemas = Seq.fill(8)(None))
}
/**
@@ -4183,7 +4191,7 @@ object functions {
*/
def udf(f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
val func = f.asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
- SparkUserDefinedFunction.create(func, returnType, inputSchemas = None)
+ SparkUserDefinedFunction.create(func, returnType, inputSchemas = Seq.fill(9)(None))
}
/**
@@ -4197,7 +4205,7 @@ object functions {
*/
def udf(f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
val func = f.asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
- SparkUserDefinedFunction.create(func, returnType, inputSchemas = None)
+ SparkUserDefinedFunction.create(func, returnType, inputSchemas = Seq.fill(10)(None))
}
// scalastyle:on parameter.number
@@ -4216,7 +4224,9 @@ object functions {
* @since 2.0.0
*/
def udf(f: AnyRef, dataType: DataType): UserDefinedFunction = {
- SparkUserDefinedFunction.create(f, dataType, inputSchemas = None)
+ // TODO: should call SparkUserDefinedFunction.create() instead but inputSchemas is currently
+ // unavailable. We may need to create type-safe overloaded versions of udf() methods.
+ new UserDefinedFunction(f, dataType, inputTypes = None)
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index 3a0db7e16c23a..9c1a15c46acd9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -308,13 +308,14 @@ private[sql] trait WithTestConf { self: BaseSessionStateBuilder =>
def overrideConfs: Map[String, String]
override protected lazy val conf: SQLConf = {
+ val overrideConfigurations = overrideConfs
val conf = parentState.map(_.conf.clone()).getOrElse {
new SQLConf {
clear()
override def clear(): Unit = {
super.clear()
// Make sure we start with the default test configs even after clear
- overrideConfs.foreach { case (key, value) => setConfString(key, value) }
+ overrideConfigurations.foreach { case (key, value) => setConfString(key, value) }
}
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
index eca612f06f9bb..bd25a6437033c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
@@ -74,8 +74,10 @@ object HiveSerDe {
def sourceToSerDe(source: String): Option[HiveSerDe] = {
val key = source.toLowerCase(Locale.ROOT) match {
case s if s.startsWith("org.apache.spark.sql.parquet") => "parquet"
+ case s if s.startsWith("org.apache.spark.sql.execution.datasources.parquet") => "parquet"
case s if s.startsWith("org.apache.spark.sql.orc") => "orc"
case s if s.startsWith("org.apache.spark.sql.hive.orc") => "orc"
+ case s if s.startsWith("org.apache.spark.sql.execution.datasources.orc") => "orc"
case s if s.equals("orcfile") => "orc"
case s if s.equals("parquetfile") => "parquet"
case s if s.equals("avrofile") => "avro"
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index 5b6160e2b408f..4d2be13c48414 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -193,14 +193,3 @@ object SharedState extends Logging {
}
}
}
-
-
-/**
- * URL class loader that exposes the `addURL` and `getURLs` methods in URLClassLoader.
- * This class loader cannot be closed (its `close` method is a no-op).
- */
-private[sql] class NonClosableMutableURLClassLoader(parent: ClassLoader)
- extends MutableURLClassLoader(Array.empty, parent) {
-
- override def close(): Unit = {}
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
index da787b4859a73..805f73dee141b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
@@ -30,7 +30,11 @@ private object MsSqlServerDialect extends JdbcDialect {
// String is recommend by Microsoft SQL Server for datetimeoffset types in non-MS clients
Option(StringType)
} else {
- None
+ sqlType match {
+ case java.sql.Types.SMALLINT => Some(ShortType)
+ case java.sql.Types.REAL => Some(FloatType)
+ case _ => None
+ }
}
}
@@ -38,6 +42,8 @@ private object MsSqlServerDialect extends JdbcDialect {
case TimestampType => Some(JdbcType("DATETIME", java.sql.Types.TIMESTAMP))
case StringType => Some(JdbcType("NVARCHAR(MAX)", java.sql.Types.NVARCHAR))
case BooleanType => Some(JdbcType("BIT", java.sql.Types.BIT))
+ case BinaryType => Some(JdbcType("VARBINARY(MAX)", java.sql.Types.VARBINARY))
+ case ShortType => Some(JdbcType("SMALLINT", java.sql.Types.SMALLINT))
case _ => None
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
index f8d2bc8e0f13f..5be45c973a5f2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
@@ -60,7 +60,10 @@ private object PostgresDialect extends JdbcDialect {
case "bytea" => Some(BinaryType)
case "timestamp" | "timestamptz" | "time" | "timetz" => Some(TimestampType)
case "date" => Some(DateType)
- case "numeric" | "decimal" => Some(DecimalType.bounded(precision, scale))
+ case "numeric" | "decimal" if precision > 0 => Some(DecimalType.bounded(precision, scale))
+ case "numeric" | "decimal" =>
+ // SPARK-26538: handle numeric without explicit precision and scale.
+ Some(DecimalType. SYSTEM_DEFAULT)
case _ => None
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
index 39e9e1ad426be..a9cb5e8fc8e85 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.streaming
-import java.util.Locale
+import java.util.{Locale, Optional}
import scala.collection.JavaConverters._
@@ -26,10 +26,11 @@ import org.apache.spark.internal.Logging
import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, SparkSession}
import org.apache.spark.sql.execution.command.DDLUtils
import org.apache.spark.sql.execution.datasources.DataSource
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Utils
import org.apache.spark.sql.execution.streaming.{StreamingRelation, StreamingRelationV2}
import org.apache.spark.sql.sources.StreamSourceProvider
-import org.apache.spark.sql.sources.v2.{ContinuousReadSupportProvider, DataSourceOptions, MicroBatchReadSupportProvider}
-import org.apache.spark.sql.sources.v2.reader.streaming.{ContinuousReadSupport, MicroBatchReadSupport}
+import org.apache.spark.sql.sources.v2.{ContinuousReadSupport, DataSourceOptions, MicroBatchReadSupport}
+import org.apache.spark.sql.sources.v2.reader.streaming.MicroBatchReader
import org.apache.spark.sql.types.StructType
import org.apache.spark.util.Utils
@@ -158,7 +159,6 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
}
val ds = DataSource.lookupDataSource(source, sparkSession.sqlContext.conf).newInstance()
- val options = new DataSourceOptions(extraOptions.asJava)
// We need to generate the V1 data source so we can pass it to the V2 relation as a shim.
// We can't be sure at this point whether we'll actually want to use V2, since we don't know the
// writer or whether the query is continuous.
@@ -172,50 +172,44 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
case _ => None
}
ds match {
- case s: MicroBatchReadSupportProvider =>
- var tempReadSupport: MicroBatchReadSupport = null
+ case s: MicroBatchReadSupport =>
+ val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
+ ds = s, conf = sparkSession.sessionState.conf)
+ val options = sessionOptions ++ extraOptions
+ val dataSourceOptions = new DataSourceOptions(options.asJava)
+ var tempReader: MicroBatchReader = null
val schema = try {
- val tmpCheckpointPath = Utils.createTempDir(namePrefix = s"tempCP").getCanonicalPath
- tempReadSupport = if (userSpecifiedSchema.isDefined) {
- s.createMicroBatchReadSupport(userSpecifiedSchema.get, tmpCheckpointPath, options)
- } else {
- s.createMicroBatchReadSupport(tmpCheckpointPath, options)
- }
- tempReadSupport.fullSchema()
+ tempReader = s.createMicroBatchReader(
+ Optional.ofNullable(userSpecifiedSchema.orNull),
+ Utils.createTempDir(namePrefix = s"temporaryReader").getCanonicalPath,
+ dataSourceOptions)
+ tempReader.readSchema()
} finally {
// Stop tempReader to avoid side-effect thing
- if (tempReadSupport != null) {
- tempReadSupport.stop()
- tempReadSupport = null
+ if (tempReader != null) {
+ tempReader.stop()
+ tempReader = null
}
}
Dataset.ofRows(
sparkSession,
StreamingRelationV2(
- s, source, extraOptions.toMap,
+ s, source, options,
schema.toAttributes, v1Relation)(sparkSession))
- case s: ContinuousReadSupportProvider =>
- var tempReadSupport: ContinuousReadSupport = null
- val schema = try {
- val tmpCheckpointPath = Utils.createTempDir(namePrefix = s"tempCP").getCanonicalPath
- tempReadSupport = if (userSpecifiedSchema.isDefined) {
- s.createContinuousReadSupport(userSpecifiedSchema.get, tmpCheckpointPath, options)
- } else {
- s.createContinuousReadSupport(tmpCheckpointPath, options)
- }
- tempReadSupport.fullSchema()
- } finally {
- // Stop tempReader to avoid side-effect thing
- if (tempReadSupport != null) {
- tempReadSupport.stop()
- tempReadSupport = null
- }
- }
+ case s: ContinuousReadSupport =>
+ val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
+ ds = s, conf = sparkSession.sessionState.conf)
+ val options = sessionOptions ++ extraOptions
+ val dataSourceOptions = new DataSourceOptions(options.asJava)
+ val tempReader = s.createContinuousReader(
+ Optional.ofNullable(userSpecifiedSchema.orNull),
+ Utils.createTempDir(namePrefix = s"temporaryReader").getCanonicalPath,
+ dataSourceOptions)
Dataset.ofRows(
sparkSession,
StreamingRelationV2(
- s, source, extraOptions.toMap,
- schema.toAttributes, v1Relation)(sparkSession))
+ s, source, options,
+ tempReader.readSchema().toAttributes, v1Relation)(sparkSession))
case _ =>
// Code path for data source v1.
Dataset.ofRows(sparkSession, StreamingRelation(v1DataSource))
@@ -327,6 +321,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
* whitespaces from values being read should be skipped.
* `nullValue` (default empty string): sets the string representation of a null value. Since
* 2.0.1, this applies to all supported types including the string type.
+ * `emptyValue` (default empty string): sets the string representation of an empty value.
* `nanValue` (default `NaN`): sets the string representation of a non-number" value.
* `positiveInf` (default `Inf`): sets the string representation of a positive infinity
* value.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
index 7866e4f70f14b..ec7eb1d3deccd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
@@ -21,16 +21,17 @@ import java.util.Locale
import scala.collection.JavaConverters._
-import org.apache.spark.annotation.InterfaceStability
+import org.apache.spark.annotation.{InterfaceStability, Since}
import org.apache.spark.api.java.function.VoidFunction2
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.streaming.InternalOutputModes
import org.apache.spark.sql.execution.command.DDLUtils
import org.apache.spark.sql.execution.datasources.DataSource
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Utils
import org.apache.spark.sql.execution.streaming._
import org.apache.spark.sql.execution.streaming.continuous.ContinuousTrigger
import org.apache.spark.sql.execution.streaming.sources._
-import org.apache.spark.sql.sources.v2.StreamingWriteSupportProvider
+import org.apache.spark.sql.sources.v2.StreamWriteSupport
/**
* Interface used to write a streaming `Dataset` to external storage systems (e.g. file systems,
@@ -45,14 +46,16 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
/**
* Specifies how data of a streaming DataFrame/Dataset is written to a streaming sink.
- * - `OutputMode.Append()`: only the new rows in the streaming DataFrame/Dataset will be
- * written to the sink
- * - `OutputMode.Complete()`: all the rows in the streaming DataFrame/Dataset will be written
- * to the sink every time these is some updates
- * - `OutputMode.Update()`: only the rows that were updated in the streaming DataFrame/Dataset
- * will be written to the sink every time there are some updates. If
- * the query doesn't contain aggregations, it will be equivalent to
- * `OutputMode.Append()` mode.
+ *
+ * `OutputMode.Append()`: only the new rows in the streaming DataFrame/Dataset will be
+ * written to the sink.
+ * `OutputMode.Complete()`: all the rows in the streaming DataFrame/Dataset will be written
+ * to the sink every time there are some updates.
+ * `OutputMode.Update()`: only the rows that were updated in the streaming
+ * DataFrame/Dataset will be written to the sink every time there are some updates.
+ * If the query doesn't contain aggregations, it will be equivalent to
+ * `OutputMode.Append()` mode.
+ *
*
* @since 2.0.0
*/
@@ -63,13 +66,16 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
/**
* Specifies how data of a streaming DataFrame/Dataset is written to a streaming sink.
- * - `append`: only the new rows in the streaming DataFrame/Dataset will be written to
- * the sink
- * - `complete`: all the rows in the streaming DataFrame/Dataset will be written to the sink
- * every time these is some updates
- * - `update`: only the rows that were updated in the streaming DataFrame/Dataset will
- * be written to the sink every time there are some updates. If the query doesn't
- * contain aggregations, it will be equivalent to `append` mode.
+ *
+ * `append`: only the new rows in the streaming DataFrame/Dataset will be written to
+ * the sink.
+ * `complete`: all the rows in the streaming DataFrame/Dataset will be written to the sink
+ * every time there are some updates.
+ * `update`: only the rows that were updated in the streaming DataFrame/Dataset will
+ * be written to the sink every time there are some updates. If the query doesn't
+ * contain aggregations, it will be equivalent to `append` mode.
+ *
+ *
* @since 2.0.0
*/
def outputMode(outputMode: String): DataStreamWriter[T] = {
@@ -130,8 +136,10 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
* laid out on the file system similar to Hive's partitioning scheme. As an example, when we
* partition a dataset by year and then month, the directory layout would look like:
*
- * - year=2016/month=01/
- * - year=2016/month=02/
+ *
+ * year=2016/month=01/
+ * year=2016/month=02/
+ *
*
* Partitioning is one of the most widely used techniques to optimize physical data layout.
* It provides a coarse-grained index for skipping unnecessary data reads when queries have
@@ -270,7 +278,7 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
query
} else if (source == "foreach") {
assertNotPartitioned("foreach")
- val sink = ForeachWriteSupportProvider[T](foreachWriter, ds.exprEnc)
+ val sink = ForeachWriterProvider[T](foreachWriter, ds.exprEnc)
df.sparkSession.sessionState.streamingQueryManager.startQuery(
extraOptions.get("queryName"),
extraOptions.get("checkpointLocation"),
@@ -298,23 +306,27 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
} else {
val ds = DataSource.lookupDataSource(source, df.sparkSession.sessionState.conf)
val disabledSources = df.sparkSession.sqlContext.conf.disabledV2StreamingWriters.split(",")
+ var options = extraOptions.toMap
val sink = ds.newInstance() match {
- case w: StreamingWriteSupportProvider
- if !disabledSources.contains(w.getClass.getCanonicalName) => w
+ case w: StreamWriteSupport if !disabledSources.contains(w.getClass.getCanonicalName) =>
+ val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
+ w, df.sparkSession.sessionState.conf)
+ options = sessionOptions ++ extraOptions
+ w
case _ =>
val ds = DataSource(
df.sparkSession,
className = source,
- options = extraOptions.toMap,
+ options = options,
partitionColumns = normalizedParCols.getOrElse(Nil))
ds.createSink(outputMode)
}
df.sparkSession.sessionState.streamingQueryManager.startQuery(
- extraOptions.get("queryName"),
- extraOptions.get("checkpointLocation"),
+ options.get("queryName"),
+ options.get("checkpointLocation"),
df,
- extraOptions.toMap,
+ options,
sink,
outputMode,
useTempCheckpointLocation = source == "console",
@@ -374,7 +386,7 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
* @since 2.4.0
*/
@InterfaceStability.Evolving
- def foreachBatch(function: VoidFunction2[Dataset[T], Long]): DataStreamWriter[T] = {
+ def foreachBatch(function: VoidFunction2[Dataset[T], java.lang.Long]): DataStreamWriter[T] = {
foreachBatch((batchDs: Dataset[T], batchId: Long) => function.call(batchDs, batchId))
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ProcessingTime.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ProcessingTime.scala
index a033575d3d38f..9cf7c330f125b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ProcessingTime.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ProcessingTime.scala
@@ -21,8 +21,6 @@ import java.util.concurrent.TimeUnit
import scala.concurrent.duration.Duration
-import org.apache.commons.lang3.StringUtils
-
import org.apache.spark.annotation.InterfaceStability
import org.apache.spark.unsafe.types.CalendarInterval
@@ -76,18 +74,7 @@ object ProcessingTime {
*/
@deprecated("use Trigger.ProcessingTime(interval)", "2.2.0")
def apply(interval: String): ProcessingTime = {
- if (StringUtils.isBlank(interval)) {
- throw new IllegalArgumentException(
- "interval cannot be null or blank.")
- }
- val cal = if (interval.startsWith("interval")) {
- CalendarInterval.fromString(interval)
- } else {
- CalendarInterval.fromString("interval " + interval)
- }
- if (cal == null) {
- throw new IllegalArgumentException(s"Invalid interval: $interval")
- }
+ val cal = CalendarInterval.fromCaseInsensitiveString(interval)
if (cal.months > 0) {
throw new IllegalArgumentException(s"Doesn't support month or year interval: $interval")
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
index 6aa82b89ede81..ace87b01c3b62 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
@@ -79,7 +79,7 @@ object StreamingQueryListener {
/**
* Event representing the start of a query
- * @param id An unique query id that persists across restarts. See `StreamingQuery.id()`.
+ * @param id A unique query id that persists across restarts. See `StreamingQuery.id()`.
* @param runId A query id that is unique for every start/restart. See `StreamingQuery.runId()`.
* @param name User-specified name of the query, null if not specified.
* @since 2.1.0
@@ -101,7 +101,7 @@ object StreamingQueryListener {
/**
* Event representing that termination of a query.
*
- * @param id An unique query id that persists across restarts. See `StreamingQuery.id()`.
+ * @param id A unique query id that persists across restarts. See `StreamingQuery.id()`.
* @param runId A query id that is unique for every start/restart. See `StreamingQuery.runId()`.
* @param exception The exception message of the query if the query was terminated
* with an exception. Otherwise, it will be `None`.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala
index cd52d991d55c9..1e27e4b7b706f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.streaming
import java.util.UUID
import javax.annotation.concurrent.GuardedBy
+import scala.collection.JavaConverters._
import scala.collection.mutable
import org.apache.hadoop.fs.Path
@@ -34,7 +35,7 @@ import org.apache.spark.sql.execution.streaming.continuous.{ContinuousExecution,
import org.apache.spark.sql.execution.streaming.state.StateStoreCoordinatorRef
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.StaticSQLConf.STREAMING_QUERY_LISTENERS
-import org.apache.spark.sql.sources.v2.StreamingWriteSupportProvider
+import org.apache.spark.sql.sources.v2.StreamWriteSupport
import org.apache.spark.util.{Clock, SystemClock, Utils}
/**
@@ -196,6 +197,11 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
listenerBus.removeListener(listener)
}
+ /** Only exposed for testing. */
+ private[sql] def listListeners(): Array[StreamingQueryListener] = {
+ listenerBus.listeners.asScala.toArray
+ }
+
/** Post a listener event */
private[sql] def postListenerEvent(event: StreamingQueryListener.Event): Unit = {
listenerBus.post(event)
@@ -256,7 +262,7 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
}
(sink, trigger) match {
- case (v2Sink: StreamingWriteSupportProvider, trigger: ContinuousTrigger) =>
+ case (v2Sink: StreamWriteSupport, trigger: ContinuousTrigger) =>
if (sparkSession.sessionState.conf.isUnsupportedOperationCheckEnabled) {
UnsupportedOperationChecker.checkForContinuous(analyzedPlan, outputMode)
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/progress.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/progress.scala
index f2173aa1e59c2..6803f5cad38cd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/progress.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/progress.scala
@@ -73,7 +73,7 @@ class StateOperatorProgress private[sql](
* a trigger. Each event relates to processing done for a single trigger of the streaming
* query. Events are emitted even when no new data is available to be processed.
*
- * @param id An unique query id that persists across restarts. See `StreamingQuery.id()`.
+ * @param id A unique query id that persists across restarts. See `StreamingQuery.id()`.
* @param runId A query id that is unique for every start/restart. See `StreamingQuery.runId()`.
* @param name User-specified name of the query, null if not specified.
* @param timestamp Beginning time of the trigger in ISO8601 format, i.e. UTC timestamps.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
index 2b46233e1a5df..abfc6773e361f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
@@ -108,6 +108,9 @@ class ExecutionListenerManager private extends Logging {
listeners.clear()
}
+ /** Only exposed for testing. */
+ private[sql] def listListeners(): Array[QueryExecutionListener] = listeners.toArray
+
/**
* Get an identical copy of this listener manager.
*/
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanWithArraySuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanWithArraySuite.java
new file mode 100644
index 0000000000000..70dd11067253e
--- /dev/null
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanWithArraySuite.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.org.apache.spark.sql;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoder;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.test.TestSparkSession;
+
+public class JavaBeanWithArraySuite {
+
+ private static final List RECORDS = new ArrayList<>();
+
+ static {
+ RECORDS.add(new Record(1, Arrays.asList(new Interval(111, 211), new Interval(121, 221))));
+ RECORDS.add(new Record(2, Arrays.asList(new Interval(112, 212), new Interval(122, 222))));
+ RECORDS.add(new Record(3, Arrays.asList(new Interval(113, 213), new Interval(123, 223))));
+ }
+
+ private TestSparkSession spark;
+
+ @Before
+ public void setUp() {
+ spark = new TestSparkSession();
+ }
+
+ @After
+ public void tearDown() {
+ spark.stop();
+ spark = null;
+ }
+
+ @Test
+ public void testBeanWithArrayFieldDeserialization() {
+
+ Encoder encoder = Encoders.bean(Record.class);
+
+ Dataset dataset = spark
+ .read()
+ .format("json")
+ .schema("id int, intervals array>")
+ .load("src/test/resources/test-data/with-array-fields.json")
+ .as(encoder);
+
+ List records = dataset.collectAsList();
+ Assert.assertEquals(records, RECORDS);
+ }
+
+ public static class Record {
+
+ private int id;
+ private List intervals;
+
+ public Record() { }
+
+ Record(int id, List intervals) {
+ this.id = id;
+ this.intervals = intervals;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ public List getIntervals() {
+ return intervals;
+ }
+
+ public void setIntervals(List intervals) {
+ this.intervals = intervals;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Record)) return false;
+ Record other = (Record) obj;
+ return (other.id == this.id) && other.intervals.equals(this.intervals);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("{ id: %d, intervals: %s }", id, intervals);
+ }
+ }
+
+ public static class Interval {
+
+ private long startTime;
+ private long endTime;
+
+ public Interval() { }
+
+ Interval(long startTime, long endTime) {
+ this.startTime = startTime;
+ this.endTime = endTime;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ public long getEndTime() {
+ return endTime;
+ }
+
+ public void setEndTime(long endTime) {
+ this.endTime = endTime;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Interval)) return false;
+ Interval other = (Interval) obj;
+ return (other.startTime == this.startTime) && (other.endTime == this.endTime);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("[%d,%d]", startTime, endTime);
+ }
+ }
+}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/execution/sort/RecordBinaryComparatorSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/execution/sort/RecordBinaryComparatorSuite.java
index 97f3dc588ecc5..c832d07d58479 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/execution/sort/RecordBinaryComparatorSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/execution/sort/RecordBinaryComparatorSuite.java
@@ -32,6 +32,7 @@
import org.apache.spark.util.collection.unsafe.sort.*;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -271,7 +272,7 @@ public void testBinaryComparatorWhenSubtractionIsDivisibleByMaxIntValue() throws
insertRow(row1);
insertRow(row2);
- assert(compare(0, 1) < 0);
+ assert(compare(0, 1) > 0);
}
@Test
@@ -319,4 +320,48 @@ public void testBinaryComparatorWhenOnlyTheLastColumnDiffers() throws Exception
assert(compare(0, 1) < 0);
}
+
+ @Test
+ public void testCompareLongsAsLittleEndian() {
+ long arrayOffset = Platform.LONG_ARRAY_OFFSET + 4;
+
+ long[] arr1 = new long[2];
+ Platform.putLong(arr1, arrayOffset, 0x0100000000000000L);
+ long[] arr2 = new long[2];
+ Platform.putLong(arr2, arrayOffset + 4, 0x0000000000000001L);
+ // leftBaseOffset is not aligned while rightBaseOffset is aligned,
+ // it will start by comparing long
+ int result1 = binaryComparator.compare(arr1, arrayOffset, 8, arr2, arrayOffset + 4, 8);
+
+ long[] arr3 = new long[2];
+ Platform.putLong(arr3, arrayOffset, 0x0100000000000000L);
+ long[] arr4 = new long[2];
+ Platform.putLong(arr4, arrayOffset, 0x0000000000000001L);
+ // both left and right offset is not aligned, it will start with byte-by-byte comparison
+ int result2 = binaryComparator.compare(arr3, arrayOffset, 8, arr4, arrayOffset, 8);
+
+ Assert.assertEquals(result1, result2);
+ }
+
+ @Test
+ public void testCompareLongsAsUnsigned() {
+ long arrayOffset = Platform.LONG_ARRAY_OFFSET + 4;
+
+ long[] arr1 = new long[2];
+ Platform.putLong(arr1, arrayOffset + 4, 0xa000000000000000L);
+ long[] arr2 = new long[2];
+ Platform.putLong(arr2, arrayOffset + 4, 0x0000000000000000L);
+ // both leftBaseOffset and rightBaseOffset are aligned, so it will start by comparing long
+ int result1 = binaryComparator.compare(arr1, arrayOffset + 4, 8, arr2, arrayOffset + 4, 8);
+
+ long[] arr3 = new long[2];
+ Platform.putLong(arr3, arrayOffset, 0xa000000000000000L);
+ long[] arr4 = new long[2];
+ Platform.putLong(arr4, arrayOffset, 0x0000000000000000L);
+ // both leftBaseOffset and rightBaseOffset are not aligned,
+ // so it will start with byte-by-byte comparison
+ int result2 = binaryComparator.compare(arr3, arrayOffset, 8, arr4, arrayOffset, 8);
+
+ Assert.assertEquals(result1, result2);
+ }
}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaAdvancedDataSourceV2.java b/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaAdvancedDataSourceV2.java
index 5602310219a74..e4cead9df429c 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaAdvancedDataSourceV2.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaAdvancedDataSourceV2.java
@@ -24,71 +24,29 @@
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
import org.apache.spark.sql.sources.Filter;
import org.apache.spark.sql.sources.GreaterThan;
-import org.apache.spark.sql.sources.v2.BatchReadSupportProvider;
import org.apache.spark.sql.sources.v2.DataSourceOptions;
import org.apache.spark.sql.sources.v2.DataSourceV2;
+import org.apache.spark.sql.sources.v2.ReadSupport;
import org.apache.spark.sql.sources.v2.reader.*;
import org.apache.spark.sql.types.StructType;
-public class JavaAdvancedDataSourceV2 implements DataSourceV2, BatchReadSupportProvider {
+public class JavaAdvancedDataSourceV2 implements DataSourceV2, ReadSupport {
- public class ReadSupport extends JavaSimpleReadSupport {
- @Override
- public ScanConfigBuilder newScanConfigBuilder() {
- return new AdvancedScanConfigBuilder();
- }
-
- @Override
- public InputPartition[] planInputPartitions(ScanConfig config) {
- Filter[] filters = ((AdvancedScanConfigBuilder) config).filters;
- List res = new ArrayList<>();
-
- Integer lowerBound = null;
- for (Filter filter : filters) {
- if (filter instanceof GreaterThan) {
- GreaterThan f = (GreaterThan) filter;
- if ("i".equals(f.attribute()) && f.value() instanceof Integer) {
- lowerBound = (Integer) f.value();
- break;
- }
- }
- }
-
- if (lowerBound == null) {
- res.add(new JavaRangeInputPartition(0, 5));
- res.add(new JavaRangeInputPartition(5, 10));
- } else if (lowerBound < 4) {
- res.add(new JavaRangeInputPartition(lowerBound + 1, 5));
- res.add(new JavaRangeInputPartition(5, 10));
- } else if (lowerBound < 9) {
- res.add(new JavaRangeInputPartition(lowerBound + 1, 10));
- }
-
- return res.stream().toArray(InputPartition[]::new);
- }
-
- @Override
- public PartitionReaderFactory createReaderFactory(ScanConfig config) {
- StructType requiredSchema = ((AdvancedScanConfigBuilder) config).requiredSchema;
- return new AdvancedReaderFactory(requiredSchema);
- }
- }
-
- public static class AdvancedScanConfigBuilder implements ScanConfigBuilder, ScanConfig,
- SupportsPushDownFilters, SupportsPushDownRequiredColumns {
+ public class Reader implements DataSourceReader, SupportsPushDownRequiredColumns,
+ SupportsPushDownFilters {
// Exposed for testing.
public StructType requiredSchema = new StructType().add("i", "int").add("j", "int");
public Filter[] filters = new Filter[0];
@Override
- public void pruneColumns(StructType requiredSchema) {
- this.requiredSchema = requiredSchema;
+ public StructType readSchema() {
+ return requiredSchema;
}
@Override
- public StructType readSchema() {
- return requiredSchema;
+ public void pruneColumns(StructType requiredSchema) {
+ this.requiredSchema = requiredSchema;
}
@Override
@@ -121,54 +79,79 @@ public Filter[] pushedFilters() {
}
@Override
- public ScanConfig build() {
- return this;
+ public List> planInputPartitions() {
+ List> res = new ArrayList<>();
+
+ Integer lowerBound = null;
+ for (Filter filter : filters) {
+ if (filter instanceof GreaterThan) {
+ GreaterThan f = (GreaterThan) filter;
+ if ("i".equals(f.attribute()) && f.value() instanceof Integer) {
+ lowerBound = (Integer) f.value();
+ break;
+ }
+ }
+ }
+
+ if (lowerBound == null) {
+ res.add(new JavaAdvancedInputPartition(0, 5, requiredSchema));
+ res.add(new JavaAdvancedInputPartition(5, 10, requiredSchema));
+ } else if (lowerBound < 4) {
+ res.add(new JavaAdvancedInputPartition(lowerBound + 1, 5, requiredSchema));
+ res.add(new JavaAdvancedInputPartition(5, 10, requiredSchema));
+ } else if (lowerBound < 9) {
+ res.add(new JavaAdvancedInputPartition(lowerBound + 1, 10, requiredSchema));
+ }
+
+ return res;
}
}
- static class AdvancedReaderFactory implements PartitionReaderFactory {
- StructType requiredSchema;
+ static class JavaAdvancedInputPartition implements InputPartition,
+ InputPartitionReader {
+ private int start;
+ private int end;
+ private StructType requiredSchema;
- AdvancedReaderFactory(StructType requiredSchema) {
+ JavaAdvancedInputPartition(int start, int end, StructType requiredSchema) {
+ this.start = start;
+ this.end = end;
this.requiredSchema = requiredSchema;
}
@Override
- public PartitionReader createReader(InputPartition partition) {
- JavaRangeInputPartition p = (JavaRangeInputPartition) partition;
- return new PartitionReader() {
- private int current = p.start - 1;
-
- @Override
- public boolean next() throws IOException {
- current += 1;
- return current < p.end;
- }
+ public InputPartitionReader createPartitionReader() {
+ return new JavaAdvancedInputPartition(start - 1, end, requiredSchema);
+ }
- @Override
- public InternalRow get() {
- Object[] values = new Object[requiredSchema.size()];
- for (int i = 0; i < values.length; i++) {
- if ("i".equals(requiredSchema.apply(i).name())) {
- values[i] = current;
- } else if ("j".equals(requiredSchema.apply(i).name())) {
- values[i] = -current;
- }
- }
- return new GenericInternalRow(values);
+ @Override
+ public boolean next() {
+ start += 1;
+ return start < end;
+ }
+
+ @Override
+ public InternalRow get() {
+ Object[] values = new Object[requiredSchema.size()];
+ for (int i = 0; i < values.length; i++) {
+ if ("i".equals(requiredSchema.apply(i).name())) {
+ values[i] = start;
+ } else if ("j".equals(requiredSchema.apply(i).name())) {
+ values[i] = -start;
}
+ }
+ return new GenericInternalRow(values);
+ }
- @Override
- public void close() throws IOException {
+ @Override
+ public void close() throws IOException {
- }
- };
}
}
@Override
- public BatchReadSupport createBatchReadSupport(DataSourceOptions options) {
- return new ReadSupport();
+ public DataSourceReader createReader(DataSourceOptions options) {
+ return new Reader();
}
}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaBatchDataSourceV2.java b/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaBatchDataSourceV2.java
new file mode 100644
index 0000000000000..97d6176d02559
--- /dev/null
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaBatchDataSourceV2.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.org.apache.spark.sql.sources.v2;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector;
+import org.apache.spark.sql.sources.v2.DataSourceOptions;
+import org.apache.spark.sql.sources.v2.DataSourceV2;
+import org.apache.spark.sql.sources.v2.ReadSupport;
+import org.apache.spark.sql.sources.v2.reader.*;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.StructType;
+import org.apache.spark.sql.vectorized.ColumnVector;
+import org.apache.spark.sql.vectorized.ColumnarBatch;
+
+
+public class JavaBatchDataSourceV2 implements DataSourceV2, ReadSupport {
+
+ class Reader implements DataSourceReader, SupportsScanColumnarBatch {
+ private final StructType schema = new StructType().add("i", "int").add("j", "int");
+
+ @Override
+ public StructType readSchema() {
+ return schema;
+ }
+
+ @Override
+ public List> planBatchInputPartitions() {
+ return java.util.Arrays.asList(
+ new JavaBatchInputPartition(0, 50), new JavaBatchInputPartition(50, 90));
+ }
+ }
+
+ static class JavaBatchInputPartition
+ implements InputPartition, InputPartitionReader {
+ private int start;
+ private int end;
+
+ private static final int BATCH_SIZE = 20;
+
+ private OnHeapColumnVector i;
+ private OnHeapColumnVector j;
+ private ColumnarBatch batch;
+
+ JavaBatchInputPartition(int start, int end) {
+ this.start = start;
+ this.end = end;
+ }
+
+ @Override
+ public InputPartitionReader createPartitionReader() {
+ this.i = new OnHeapColumnVector(BATCH_SIZE, DataTypes.IntegerType);
+ this.j = new OnHeapColumnVector(BATCH_SIZE, DataTypes.IntegerType);
+ ColumnVector[] vectors = new ColumnVector[2];
+ vectors[0] = i;
+ vectors[1] = j;
+ this.batch = new ColumnarBatch(vectors);
+ return this;
+ }
+
+ @Override
+ public boolean next() {
+ i.reset();
+ j.reset();
+ int count = 0;
+ while (start < end && count < BATCH_SIZE) {
+ i.putInt(count, start);
+ j.putInt(count, -start);
+ start += 1;
+ count += 1;
+ }
+
+ if (count == 0) {
+ return false;
+ } else {
+ batch.setNumRows(count);
+ return true;
+ }
+ }
+
+ @Override
+ public ColumnarBatch get() {
+ return batch;
+ }
+
+ @Override
+ public void close() throws IOException {
+ batch.close();
+ }
+ }
+
+
+ @Override
+ public DataSourceReader createReader(DataSourceOptions options) {
+ return new Reader();
+ }
+}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaColumnarDataSourceV2.java b/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaColumnarDataSourceV2.java
deleted file mode 100644
index 28a9330398310..0000000000000
--- a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaColumnarDataSourceV2.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package test.org.apache.spark.sql.sources.v2;
-
-import java.io.IOException;
-
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector;
-import org.apache.spark.sql.sources.v2.BatchReadSupportProvider;
-import org.apache.spark.sql.sources.v2.DataSourceOptions;
-import org.apache.spark.sql.sources.v2.DataSourceV2;
-import org.apache.spark.sql.sources.v2.reader.*;
-import org.apache.spark.sql.types.DataTypes;
-import org.apache.spark.sql.vectorized.ColumnVector;
-import org.apache.spark.sql.vectorized.ColumnarBatch;
-
-
-public class JavaColumnarDataSourceV2 implements DataSourceV2, BatchReadSupportProvider {
-
- class ReadSupport extends JavaSimpleReadSupport {
-
- @Override
- public InputPartition[] planInputPartitions(ScanConfig config) {
- InputPartition[] partitions = new InputPartition[2];
- partitions[0] = new JavaRangeInputPartition(0, 50);
- partitions[1] = new JavaRangeInputPartition(50, 90);
- return partitions;
- }
-
- @Override
- public PartitionReaderFactory createReaderFactory(ScanConfig config) {
- return new ColumnarReaderFactory();
- }
- }
-
- static class ColumnarReaderFactory implements PartitionReaderFactory {
- private static final int BATCH_SIZE = 20;
-
- @Override
- public boolean supportColumnarReads(InputPartition partition) {
- return true;
- }
-
- @Override
- public PartitionReader createReader(InputPartition partition) {
- throw new UnsupportedOperationException("");
- }
-
- @Override
- public PartitionReader createColumnarReader(InputPartition partition) {
- JavaRangeInputPartition p = (JavaRangeInputPartition) partition;
- OnHeapColumnVector i = new OnHeapColumnVector(BATCH_SIZE, DataTypes.IntegerType);
- OnHeapColumnVector j = new OnHeapColumnVector(BATCH_SIZE, DataTypes.IntegerType);
- ColumnVector[] vectors = new ColumnVector[2];
- vectors[0] = i;
- vectors[1] = j;
- ColumnarBatch batch = new ColumnarBatch(vectors);
-
- return new PartitionReader() {
- private int current = p.start;
-
- @Override
- public boolean next() throws IOException {
- i.reset();
- j.reset();
- int count = 0;
- while (current < p.end && count < BATCH_SIZE) {
- i.putInt(count, current);
- j.putInt(count, -current);
- current += 1;
- count += 1;
- }
-
- if (count == 0) {
- return false;
- } else {
- batch.setNumRows(count);
- return true;
- }
- }
-
- @Override
- public ColumnarBatch get() {
- return batch;
- }
-
- @Override
- public void close() throws IOException {
- batch.close();
- }
- };
- }
- }
-
- @Override
- public BatchReadSupport createBatchReadSupport(DataSourceOptions options) {
- return new ReadSupport();
- }
-}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaPartitionAwareDataSource.java b/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaPartitionAwareDataSource.java
index 18a11dde82198..2d21324f5ece3 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaPartitionAwareDataSource.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaPartitionAwareDataSource.java
@@ -19,34 +19,38 @@
import java.io.IOException;
import java.util.Arrays;
+import java.util.List;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
-import org.apache.spark.sql.sources.v2.*;
+import org.apache.spark.sql.sources.v2.DataSourceOptions;
+import org.apache.spark.sql.sources.v2.DataSourceV2;
+import org.apache.spark.sql.sources.v2.ReadSupport;
import org.apache.spark.sql.sources.v2.reader.*;
import org.apache.spark.sql.sources.v2.reader.partitioning.ClusteredDistribution;
import org.apache.spark.sql.sources.v2.reader.partitioning.Distribution;
import org.apache.spark.sql.sources.v2.reader.partitioning.Partitioning;
+import org.apache.spark.sql.types.StructType;
-public class JavaPartitionAwareDataSource implements DataSourceV2, BatchReadSupportProvider {
+public class JavaPartitionAwareDataSource implements DataSourceV2, ReadSupport {
- class ReadSupport extends JavaSimpleReadSupport implements SupportsReportPartitioning {
+ class Reader implements DataSourceReader, SupportsReportPartitioning {
+ private final StructType schema = new StructType().add("a", "int").add("b", "int");
@Override
- public InputPartition[] planInputPartitions(ScanConfig config) {
- InputPartition[] partitions = new InputPartition[2];
- partitions[0] = new SpecificInputPartition(new int[]{1, 1, 3}, new int[]{4, 4, 6});
- partitions[1] = new SpecificInputPartition(new int[]{2, 4, 4}, new int[]{6, 2, 2});
- return partitions;
+ public StructType readSchema() {
+ return schema;
}
@Override
- public PartitionReaderFactory createReaderFactory(ScanConfig config) {
- return new SpecificReaderFactory();
+ public List> planInputPartitions() {
+ return java.util.Arrays.asList(
+ new SpecificInputPartition(new int[]{1, 1, 3}, new int[]{4, 4, 6}),
+ new SpecificInputPartition(new int[]{2, 4, 4}, new int[]{6, 2, 2}));
}
@Override
- public Partitioning outputPartitioning(ScanConfig config) {
+ public Partitioning outputPartitioning() {
return new MyPartitioning();
}
}
@@ -62,53 +66,50 @@ public int numPartitions() {
public boolean satisfy(Distribution distribution) {
if (distribution instanceof ClusteredDistribution) {
String[] clusteredCols = ((ClusteredDistribution) distribution).clusteredColumns;
- return Arrays.asList(clusteredCols).contains("i");
+ return Arrays.asList(clusteredCols).contains("a");
}
return false;
}
}
- static class SpecificInputPartition implements InputPartition {
- int[] i;
- int[] j;
+ static class SpecificInputPartition implements InputPartition,
+ InputPartitionReader {
+
+ private int[] i;
+ private int[] j;
+ private int current = -1;
SpecificInputPartition(int[] i, int[] j) {
assert i.length == j.length;
this.i = i;
this.j = j;
}
- }
- static class SpecificReaderFactory implements PartitionReaderFactory {
+ @Override
+ public boolean next() throws IOException {
+ current += 1;
+ return current < i.length;
+ }
+
+ @Override
+ public InternalRow get() {
+ return new GenericInternalRow(new Object[] {i[current], j[current]});
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
@Override
- public PartitionReader createReader(InputPartition partition) {
- SpecificInputPartition p = (SpecificInputPartition) partition;
- return new PartitionReader() {
- private int current = -1;
-
- @Override
- public boolean next() throws IOException {
- current += 1;
- return current < p.i.length;
- }
-
- @Override
- public InternalRow get() {
- return new GenericInternalRow(new Object[] {p.i[current], p.j[current]});
- }
-
- @Override
- public void close() throws IOException {
-
- }
- };
+ public InputPartitionReader createPartitionReader() {
+ return this;
}
}
@Override
- public BatchReadSupport createBatchReadSupport(DataSourceOptions options) {
- return new ReadSupport();
+ public DataSourceReader createReader(DataSourceOptions options) {
+ return new Reader();
}
}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaSchemaRequiredDataSource.java b/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaSchemaRequiredDataSource.java
index cc9ac04a0dad3..6fd6a44d2c4d5 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaSchemaRequiredDataSource.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaSchemaRequiredDataSource.java
@@ -17,39 +17,43 @@
package test.org.apache.spark.sql.sources.v2;
-import org.apache.spark.sql.sources.v2.BatchReadSupportProvider;
+import java.util.List;
+
+import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.sources.v2.DataSourceOptions;
import org.apache.spark.sql.sources.v2.DataSourceV2;
-import org.apache.spark.sql.sources.v2.reader.*;
+import org.apache.spark.sql.sources.v2.ReadSupport;
+import org.apache.spark.sql.sources.v2.reader.DataSourceReader;
+import org.apache.spark.sql.sources.v2.reader.InputPartition;
import org.apache.spark.sql.types.StructType;
-public class JavaSchemaRequiredDataSource implements DataSourceV2, BatchReadSupportProvider {
+public class JavaSchemaRequiredDataSource implements DataSourceV2, ReadSupport {
- class ReadSupport extends JavaSimpleReadSupport {
+ class Reader implements DataSourceReader {
private final StructType schema;
- ReadSupport(StructType schema) {
+ Reader(StructType schema) {
this.schema = schema;
}
@Override
- public StructType fullSchema() {
+ public StructType readSchema() {
return schema;
}
@Override
- public InputPartition[] planInputPartitions(ScanConfig config) {
- return new InputPartition[0];
+ public List> planInputPartitions() {
+ return java.util.Collections.emptyList();
}
}
@Override
- public BatchReadSupport createBatchReadSupport(DataSourceOptions options) {
+ public DataSourceReader createReader(DataSourceOptions options) {
throw new IllegalArgumentException("requires a user-supplied schema");
}
@Override
- public BatchReadSupport createBatchReadSupport(StructType schema, DataSourceOptions options) {
- return new ReadSupport(schema);
+ public DataSourceReader createReader(StructType schema, DataSourceOptions options) {
+ return new Reader(schema);
}
}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaSimpleDataSourceV2.java b/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaSimpleDataSourceV2.java
index 2cdbba84ec4a4..274dc3745bcf9 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaSimpleDataSourceV2.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaSimpleDataSourceV2.java
@@ -17,26 +17,72 @@
package test.org.apache.spark.sql.sources.v2;
-import org.apache.spark.sql.sources.v2.BatchReadSupportProvider;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
import org.apache.spark.sql.sources.v2.DataSourceV2;
import org.apache.spark.sql.sources.v2.DataSourceOptions;
-import org.apache.spark.sql.sources.v2.reader.*;
+import org.apache.spark.sql.sources.v2.ReadSupport;
+import org.apache.spark.sql.sources.v2.reader.InputPartitionReader;
+import org.apache.spark.sql.sources.v2.reader.InputPartition;
+import org.apache.spark.sql.sources.v2.reader.DataSourceReader;
+import org.apache.spark.sql.types.StructType;
+
+public class JavaSimpleDataSourceV2 implements DataSourceV2, ReadSupport {
+
+ class Reader implements DataSourceReader {
+ private final StructType schema = new StructType().add("i", "int").add("j", "int");
+
+ @Override
+ public StructType readSchema() {
+ return schema;
+ }
+
+ @Override
+ public List> planInputPartitions() {
+ return java.util.Arrays.asList(
+ new JavaSimpleInputPartition(0, 5),
+ new JavaSimpleInputPartition(5, 10));
+ }
+ }
+
+ static class JavaSimpleInputPartition implements InputPartition,
+ InputPartitionReader {
-public class JavaSimpleDataSourceV2 implements DataSourceV2, BatchReadSupportProvider {
+ private int start;
+ private int end;
- class ReadSupport extends JavaSimpleReadSupport {
+ JavaSimpleInputPartition(int start, int end) {
+ this.start = start;
+ this.end = end;
+ }
+
+ @Override
+ public InputPartitionReader createPartitionReader() {
+ return new JavaSimpleInputPartition(start - 1, end);
+ }
@Override
- public InputPartition[] planInputPartitions(ScanConfig config) {
- InputPartition[] partitions = new InputPartition[2];
- partitions[0] = new JavaRangeInputPartition(0, 5);
- partitions[1] = new JavaRangeInputPartition(5, 10);
- return partitions;
+ public boolean next() {
+ start += 1;
+ return start < end;
+ }
+
+ @Override
+ public InternalRow get() {
+ return new GenericInternalRow(new Object[] {start, -start});
+ }
+
+ @Override
+ public void close() throws IOException {
+
}
}
@Override
- public BatchReadSupport createBatchReadSupport(DataSourceOptions options) {
- return new ReadSupport();
+ public DataSourceReader createReader(DataSourceOptions options) {
+ return new Reader();
}
}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaSimpleReadSupport.java b/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaSimpleReadSupport.java
deleted file mode 100644
index 685f9b9747e85..0000000000000
--- a/sql/core/src/test/java/test/org/apache/spark/sql/sources/v2/JavaSimpleReadSupport.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package test.org.apache.spark.sql.sources.v2;
-
-import java.io.IOException;
-
-import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
-import org.apache.spark.sql.sources.v2.reader.*;
-import org.apache.spark.sql.types.StructType;
-
-abstract class JavaSimpleReadSupport implements BatchReadSupport {
-
- @Override
- public StructType fullSchema() {
- return new StructType().add("i", "int").add("j", "int");
- }
-
- @Override
- public ScanConfigBuilder newScanConfigBuilder() {
- return new JavaNoopScanConfigBuilder(fullSchema());
- }
-
- @Override
- public PartitionReaderFactory createReaderFactory(ScanConfig config) {
- return new JavaSimpleReaderFactory();
- }
-}
-
-class JavaNoopScanConfigBuilder implements ScanConfigBuilder, ScanConfig {
-
- private StructType schema;
-
- JavaNoopScanConfigBuilder(StructType schema) {
- this.schema = schema;
- }
-
- @Override
- public ScanConfig build() {
- return this;
- }
-
- @Override
- public StructType readSchema() {
- return schema;
- }
-}
-
-class JavaSimpleReaderFactory implements PartitionReaderFactory {
-
- @Override
- public PartitionReader createReader(InputPartition partition) {
- JavaRangeInputPartition p = (JavaRangeInputPartition) partition;
- return new PartitionReader() {
- private int current = p.start - 1;
-
- @Override
- public boolean next() throws IOException {
- current += 1;
- return current < p.end;
- }
-
- @Override
- public InternalRow get() {
- return new GenericInternalRow(new Object[] {current, -current});
- }
-
- @Override
- public void close() throws IOException {
-
- }
- };
- }
-}
-
-class JavaRangeInputPartition implements InputPartition {
- int start;
- int end;
-
- JavaRangeInputPartition(int start, int end) {
- this.start = start;
- this.end = end;
- }
-}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/streaming/JavaDataStreamReaderWriterSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/streaming/JavaDataStreamReaderWriterSuite.java
new file mode 100644
index 0000000000000..48cdb2642d830
--- /dev/null
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/streaming/JavaDataStreamReaderWriterSuite.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.org.apache.spark.sql.streaming;
+
+import java.io.File;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.spark.api.java.function.VoidFunction2;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.ForeachWriter;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.streaming.StreamingQuery;
+import org.apache.spark.sql.test.TestSparkSession;
+import org.apache.spark.util.Utils;
+
+public class JavaDataStreamReaderWriterSuite {
+ private SparkSession spark;
+ private String input;
+
+ @Before
+ public void setUp() {
+ spark = new TestSparkSession();
+ input = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "input").toString();
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ Utils.deleteRecursively(new File(input));
+ } finally {
+ spark.stop();
+ spark = null;
+ }
+ }
+
+ @Test
+ public void testForeachBatchAPI() {
+ StreamingQuery query = spark
+ .readStream()
+ .textFile(input)
+ .writeStream()
+ .foreachBatch(new VoidFunction2, Long>() {
+ @Override
+ public void call(Dataset v1, Long v2) throws Exception {}
+ })
+ .start();
+ query.stop();
+ }
+
+ @Test
+ public void testForeachAPI() {
+ StreamingQuery query = spark
+ .readStream()
+ .textFile(input)
+ .writeStream()
+ .foreach(new ForeachWriter() {
+ @Override
+ public boolean open(long partitionId, long epochId) {
+ return true;
+ }
+
+ @Override
+ public void process(String value) {}
+
+ @Override
+ public void close(Throwable errorOrNull) {}
+ })
+ .start();
+ query.stop();
+ }
+}
diff --git a/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
index a36b0cfa6ff18..46b38bed1c0fb 100644
--- a/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++ b/sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -9,6 +9,6 @@ org.apache.spark.sql.streaming.sources.FakeReadMicroBatchOnly
org.apache.spark.sql.streaming.sources.FakeReadContinuousOnly
org.apache.spark.sql.streaming.sources.FakeReadBothModes
org.apache.spark.sql.streaming.sources.FakeReadNeitherMode
-org.apache.spark.sql.streaming.sources.FakeWriteSupportProvider
+org.apache.spark.sql.streaming.sources.FakeWrite
org.apache.spark.sql.streaming.sources.FakeNoWrite
-org.apache.spark.sql.streaming.sources.FakeWriteSupportProviderV1Fallback
+org.apache.spark.sql.streaming.sources.FakeWriteV1Fallback
diff --git a/sql/core/src/test/resources/sql-tests/inputs/count.sql b/sql/core/src/test/resources/sql-tests/inputs/count.sql
new file mode 100644
index 0000000000000..9f9ee4a873d4f
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/count.sql
@@ -0,0 +1,27 @@
+-- Test data.
+CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES
+(1, 1), (1, 2), (2, 1), (1, 1), (null, 2), (1, null), (null, null)
+AS testData(a, b);
+
+-- count with single expression
+SELECT
+ count(*), count(1), count(null), count(a), count(b), count(a + b), count((a, b))
+FROM testData;
+
+-- distinct count with single expression
+SELECT
+ count(DISTINCT 1),
+ count(DISTINCT null),
+ count(DISTINCT a),
+ count(DISTINCT b),
+ count(DISTINCT (a + b)),
+ count(DISTINCT (a, b))
+FROM testData;
+
+-- count with multiple expressions
+SELECT count(a, b), count(b, a), count(testData.*) FROM testData;
+
+-- distinct count with multiple expressions
+SELECT
+ count(DISTINCT a, b), count(DISTINCT b, a), count(DISTINCT *), count(DISTINCT testData.*)
+FROM testData;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index 4950a4b7a4e5a..547c2bef02b24 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -27,36 +27,3 @@ select current_date = current_date(), current_timestamp = current_timestamp(), a
select a, b from ttf2 order by a, current_date;
select weekday('2007-02-03'), weekday('2009-07-30'), weekday('2017-05-27'), weekday(null), weekday('1582-10-15 13:10:15');
-
-select from_utc_timestamp('2015-07-24 00:00:00', 'PST');
-
-select from_utc_timestamp('2015-01-24 00:00:00', 'PST');
-
-select from_utc_timestamp(null, 'PST');
-
-select from_utc_timestamp('2015-07-24 00:00:00', null);
-
-select from_utc_timestamp(null, null);
-
-select from_utc_timestamp(cast(0 as timestamp), 'PST');
-
-select from_utc_timestamp(cast('2015-01-24' as date), 'PST');
-
-select to_utc_timestamp('2015-07-24 00:00:00', 'PST');
-
-select to_utc_timestamp('2015-01-24 00:00:00', 'PST');
-
-select to_utc_timestamp(null, 'PST');
-
-select to_utc_timestamp('2015-07-24 00:00:00', null);
-
-select to_utc_timestamp(null, null);
-
-select to_utc_timestamp(cast(0 as timestamp), 'PST');
-
-select to_utc_timestamp(cast('2015-01-24' as date), 'PST');
-
--- SPARK-23715: the input of to/from_utc_timestamp can not have timezone
-select from_utc_timestamp('2000-10-10 00:00:00+00:00', 'PST');
-
-select to_utc_timestamp('2000-10-10 00:00:00+00:00', 'PST');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/group-by-ordinal.sql b/sql/core/src/test/resources/sql-tests/inputs/group-by-ordinal.sql
index 928f766b4add2..3144833b608be 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/group-by-ordinal.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/group-by-ordinal.sql
@@ -38,7 +38,9 @@ select a, b, sum(b) from data group by 3;
select a, b, sum(b) + 2 from data group by 3;
-- negative case: nondeterministic expression
-select a, rand(0), sum(b) from data group by a, 2;
+select a, rand(0), sum(b)
+from
+(select /*+ REPARTITION(1) */ a, b from data) group by a, 2;
-- negative case: star
select * from data group by a, b, 1;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/group-by.sql b/sql/core/src/test/resources/sql-tests/inputs/group-by.sql
index 2c18d6aaabdba..433db71527437 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/group-by.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/group-by.sql
@@ -73,3 +73,10 @@ where b.z != b.z;
-- SPARK-24369 multiple distinct aggregations having the same argument set
SELECT corr(DISTINCT x, y), corr(DISTINCT y, x), count(*)
FROM (VALUES (1, 1), (2, 2), (2, 2)) t(x, y);
+
+-- SPARK-25708 HAVING without GROUP BY means global aggregate
+SELECT 1 FROM range(10) HAVING true;
+
+SELECT 1 FROM range(10) HAVING MAX(id) > 0;
+
+SELECT id FROM range(10) HAVING id > 0;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
index 02ad5e3538689..504306470faea 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
@@ -60,26 +60,3 @@ select zip_with(array('a', 'b', 'c'), array('d', 'e', 'f'), (x, y) -> concat(x,
-- Zip with array coalesce
select zip_with(array('a'), array('d', null, 'f'), (x, y) -> coalesce(x, y)) as v;
-
-create or replace temporary view nested as values
- (1, map(1, 1, 2, 2, 3, 3)),
- (2, map(4, 4, 5, 5, 6, 6))
- as t(x, ys);
-
--- Identity Transform Keys in a map
-select transform_keys(ys, (k, v) -> k) as v from nested;
-
--- Transform Keys in a map by adding constant
-select transform_keys(ys, (k, v) -> k + 1) as v from nested;
-
--- Transform Keys in a map using values
-select transform_keys(ys, (k, v) -> k + v) as v from nested;
-
--- Identity Transform values in a map
-select transform_values(ys, (k, v) -> v) as v from nested;
-
--- Transform values in a map by adding constant
-select transform_values(ys, (k, v) -> v + 1) as v from nested;
-
--- Transform values in a map using values
-select transform_values(ys, (k, v) -> k + v) as v from nested;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
index 0f22c0eeed581..e391e93f9c68f 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
@@ -56,3 +56,8 @@ select from_json('[{"a": 1}, 2]', 'array>');
select to_json(array('1', '2', '3'));
select to_json(array(array(1, 2, 3), array(4)));
+select schema_of_json(null);
+CREATE TEMPORARY VIEW jsonTable(jsonField, a) AS SELECT * FROM VALUES ('{"a": 1, "b": 2}', 'a');
+SELECT schema_of_json(jsonField) FROM jsonTable;
+-- Clean up
+DROP VIEW IF EXISTS jsonTable;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/limit.sql b/sql/core/src/test/resources/sql-tests/inputs/limit.sql
index e33cd819f281f..b4c73cf33e53a 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/limit.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/limit.sql
@@ -1,5 +1,3 @@
--- Disable global limit parallel
-set spark.sql.limit.flatGlobalLimit=false;
-- limit on various data types
SELECT * FROM testdata LIMIT 2;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/pivot.sql b/sql/core/src/test/resources/sql-tests/inputs/pivot.sql
index 1f607b334dc18..c2ecd97e2b02f 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/pivot.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/pivot.sql
@@ -287,3 +287,14 @@ PIVOT (
sum(earnings)
FOR (course, m) IN (('dotNET', map('1', 1)), ('Java', map('2', 2)))
);
+
+-- grouping columns output in the same order as input
+-- correctly handle pivot columns with different cases
+SELECT * FROM (
+ SELECT course, earnings, "a" as a, "z" as z, "b" as b, "y" as y, "c" as c, "x" as x, "d" as d, "w" as w
+ FROM courseSales
+)
+PIVOT (
+ sum(Earnings)
+ FOR Course IN ('dotNET', 'Java')
+);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql
index 22f3eafd6a02d..f06be5fc6aa58 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql
@@ -83,7 +83,7 @@ GROUP BY t1a,
t3a,
t3b,
t3c
-ORDER BY t1a DESC, t3b DESC;
+ORDER BY t1a DESC, t3b DESC, t3c ASC;
-- TC 01.03
SELECT Count(DISTINCT(t1a))
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql
index a862e0985b20c..a40ee082ba3b9 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql
@@ -1,9 +1,6 @@
-- A test suite for IN LIMIT in parent side, subquery, and both predicate subquery
-- It includes correlated cases.
--- Disable global limit optimization
-set spark.sql.limit.flatGlobalLimit=false;
-
create temporary view t1 as select * from values
("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'),
("val1b", 8S, 16, 19L, float(17.0), 25D, 26E2, timestamp '2014-05-04 01:01:00.000', date '2014-05-04'),
@@ -100,4 +97,4 @@ WHERE t1d NOT IN (SELECT t2d
LIMIT 1)
GROUP BY t1b
ORDER BY t1b NULLS last
-LIMIT 1;
+LIMIT 1;
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql
index 4f8ca8bfb27c1..bebc18a61894b 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql
@@ -128,7 +128,7 @@ GROUP BY t1b,
HAVING t1d NOT IN (SELECT t2d
FROM t2
WHERE t1d = t2d)
-ORDER BY t1b DESC;
+ORDER BY t1b DESC, t1d ASC;
-- TC 01.05
SELECT COUNT(DISTINCT(t1a)),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/mapZipWith.sql b/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/mapZipWith.sql
deleted file mode 100644
index 1727ee725db2e..0000000000000
--- a/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/mapZipWith.sql
+++ /dev/null
@@ -1,78 +0,0 @@
-CREATE TEMPORARY VIEW various_maps AS SELECT * FROM VALUES (
- map(true, false),
- map(2Y, 1Y),
- map(2S, 1S),
- map(2, 1),
- map(2L, 1L),
- map(922337203685477897945456575809789456, 922337203685477897945456575809789456),
- map(9.22337203685477897945456575809789456, 9.22337203685477897945456575809789456),
- map(2.0D, 1.0D),
- map(float(2.0), float(1.0)),
- map(date '2016-03-14', date '2016-03-13'),
- map(timestamp '2016-11-15 20:54:00.000', timestamp '2016-11-12 20:54:00.000'),
- map('true', 'false', '2', '1'),
- map('2016-03-14', '2016-03-13'),
- map('2016-11-15 20:54:00.000', '2016-11-12 20:54:00.000'),
- map('922337203685477897945456575809789456', 'text'),
- map(array(1L, 2L), array(1L, 2L)), map(array(1, 2), array(1, 2)),
- map(struct(1S, 2L), struct(1S, 2L)), map(struct(1, 2), struct(1, 2))
-) AS various_maps(
- boolean_map,
- tinyint_map,
- smallint_map,
- int_map,
- bigint_map,
- decimal_map1, decimal_map2,
- double_map,
- float_map,
- date_map,
- timestamp_map,
- string_map1, string_map2, string_map3, string_map4,
- array_map1, array_map2,
- struct_map1, struct_map2
-);
-
-SELECT map_zip_with(tinyint_map, smallint_map, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(smallint_map, int_map, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(int_map, bigint_map, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(double_map, float_map, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(decimal_map1, decimal_map2, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(decimal_map1, int_map, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(decimal_map1, double_map, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(decimal_map2, int_map, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(decimal_map2, double_map, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(string_map1, int_map, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(string_map2, date_map, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(timestamp_map, string_map3, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(decimal_map1, string_map4, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(array_map1, array_map2, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
-
-SELECT map_zip_with(struct_map1, struct_map2, (k, v1, v2) -> struct(k, v1, v2)) m
-FROM various_maps;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/udaf-regrfunctions.sql b/sql/core/src/test/resources/sql-tests/inputs/udaf-regrfunctions.sql
deleted file mode 100644
index 92c7e26e3add2..0000000000000
--- a/sql/core/src/test/resources/sql-tests/inputs/udaf-regrfunctions.sql
+++ /dev/null
@@ -1,56 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one or more
--- contributor license agreements. See the NOTICE file distributed with
--- this work for additional information regarding copyright ownership.
--- The ASF licenses this file to You under the Apache License, Version 2.0
--- (the "License"); you may not use this file except in compliance with
--- the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-CREATE OR REPLACE TEMPORARY VIEW t1 AS SELECT * FROM VALUES
- (101, 1, 1, 1),
- (201, 2, 1, 1),
- (301, 3, 1, 1),
- (401, 4, 1, 11),
- (501, 5, 1, null),
- (601, 6, null, 1),
- (701, 6, null, null),
- (102, 1, 2, 2),
- (202, 2, 1, 2),
- (302, 3, 2, 1),
- (402, 4, 2, 12),
- (502, 5, 2, null),
- (602, 6, null, 2),
- (702, 6, null, null),
- (103, 1, 3, 3),
- (203, 2, 1, 3),
- (303, 3, 3, 1),
- (403, 4, 3, 13),
- (503, 5, 3, null),
- (603, 6, null, 3),
- (703, 6, null, null),
- (104, 1, 4, 4),
- (204, 2, 1, 4),
- (304, 3, 4, 1),
- (404, 4, 4, 14),
- (504, 5, 4, null),
- (604, 6, null, 4),
- (704, 6, null, null),
- (800, 7, 1, 1)
-as t1(id, px, y, x);
-
-select px, var_pop(x), var_pop(y), corr(y,x), covar_samp(y,x), covar_pop(y,x), regr_count(y,x),
- regr_slope(y,x), regr_intercept(y,x), regr_r2(y,x), regr_sxx(y,x), regr_syy(y,x), regr_sxy(y,x),
- regr_avgx(y,x), regr_avgy(y,x), regr_count(y,x)
-from t1 group by px order by px;
-
-
-select id, regr_count(y,x) over (partition by px) from t1 order by id;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/window.sql b/sql/core/src/test/resources/sql-tests/inputs/window.sql
index cda4db4b449fe..faab4c61c8640 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/window.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/window.sql
@@ -109,3 +109,9 @@ last_value(false, false) OVER w AS last_value_contain_null
FROM testData
WINDOW w AS ()
ORDER BY cate, val;
+
+-- parentheses around window reference
+SELECT cate, sum(val) OVER (w)
+FROM testData
+WHERE val is not null
+WINDOW w AS (PARTITION BY cate ORDER BY val);
diff --git a/sql/core/src/test/resources/sql-tests/results/count.sql.out b/sql/core/src/test/resources/sql-tests/results/count.sql.out
new file mode 100644
index 0000000000000..b8a86d4c44594
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/count.sql.out
@@ -0,0 +1,55 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 5
+
+
+-- !query 0
+CREATE OR REPLACE TEMPORARY VIEW testData AS SELECT * FROM VALUES
+(1, 1), (1, 2), (2, 1), (1, 1), (null, 2), (1, null), (null, null)
+AS testData(a, b)
+-- !query 0 schema
+struct<>
+-- !query 0 output
+
+
+
+-- !query 1
+SELECT
+ count(*), count(1), count(null), count(a), count(b), count(a + b), count((a, b))
+FROM testData
+-- !query 1 schema
+struct
+-- !query 1 output
+7 7 0 5 5 4 7
+
+
+-- !query 2
+SELECT
+ count(DISTINCT 1),
+ count(DISTINCT null),
+ count(DISTINCT a),
+ count(DISTINCT b),
+ count(DISTINCT (a + b)),
+ count(DISTINCT (a, b))
+FROM testData
+-- !query 2 schema
+struct
+-- !query 2 output
+1 0 2 2 2 6
+
+
+-- !query 3
+SELECT count(a, b), count(b, a), count(testData.*) FROM testData
+-- !query 3 schema
+struct
+-- !query 3 output
+4 4 4
+
+
+-- !query 4
+SELECT
+ count(DISTINCT a, b), count(DISTINCT b, a), count(DISTINCT *), count(DISTINCT testData.*)
+FROM testData
+-- !query 4 schema
+struct
+-- !query 4 output
+3 3 3 3
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index 9eede305dbdcc..63aa00426ea32 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 26
+-- Number of queries: 10
-- !query 0
@@ -89,131 +89,3 @@ select weekday('2007-02-03'), weekday('2009-07-30'), weekday('2017-05-27'), week
struct
-- !query 9 output
5 3 5 NULL 4
-
-
--- !query 10
-select from_utc_timestamp('2015-07-24 00:00:00', 'PST')
--- !query 10 schema
-struct
--- !query 10 output
-2015-07-23 17:00:00
-
-
--- !query 11
-select from_utc_timestamp('2015-01-24 00:00:00', 'PST')
--- !query 11 schema
-struct