Merge a08582a into b61a318

uwescience · Jan 12, 2017 · 3558e59 · 3558e59
2 parents b61a318 + a08582a
commit 3558e59
Show file tree

Hide file tree

Showing 159 changed files with 5,787 additions and 1,061 deletions.
diff --git a/jsonQueries/pythonUDF/ingest_blob.json b/jsonQueries/pythonUDF/ingest_blob.json
@@ -0,0 +1,18 @@
+{
+  "relationKey" : {
+    "userName" : "public",
+    "programName" : "adhoc",
+    "relationName" : "raw"
+  },
+  "schema" : {
+    "columnTypes" : ["LONG_TYPE", "LONG_TYPE","LONG_TYPE","BLOB_TYPE"],
+    "columnNames" : ["id", "subjid","imgid" ,"image"]
+  },
+  "s3Source" : {
+    "dataType" : "S3",
+    "s3Uri" : "s3://imagedb-data/dmridatasample.csv"
+  },
+  "delimiter": ",",
+   "workers": [1,2]
+ }
+
diff --git a/jsonQueries/pythonUDF/udfApply.json b/jsonQueries/pythonUDF/udfApply.json
@@ -0,0 +1,68 @@
+{
+      "fragments":[
+        {
+          "operators": [
+            {
+              "opType": "TableScan",
+              "opId": 0,
+              "relationKey": {
+                "userName": "public",
+                "relationName": "raw",
+                "programName": "adhoc"
+              }
+            },
+            {
+
+               "opType": "Apply",
+               "opId": 1,
+               "emitExpressions": [
+                 {
+                   "outputName": "udfOutput",
+                   "rootExpressionOperator": {
+                     "type": "PYUDF",
+                     "name": "flatmapApplyTest",
+                     "outputType":"BLOB_TYPE",
+                     "addCounter":"False",
+                     "children": [
+                       {
+                         "type": "VARIABLE",
+                         "columnIdx": 3
+                       }
+                     ]
+                   }
+                 },
+                   {
+                   "outputName":"subjectid",
+                   "rootExpressionOperator":{
+                      "type":"VARIABLE",
+                      "columnIdx":2}
+                 },
+                 {"outputName":"imageid",
+                 "rootExpressionOperator":{
+                    "type":"VARIABLE",
+                    "columnIdx":1}
+               }
+               ],
+               "argChild": 0
+                },
+                {
+                  "opType": "DbInsert",
+                  "opId": 2,
+                  "relationKey": {
+                    "userName": "public",
+                    "relationName": "results",
+                    "programName": "adhoc"
+              },
+              "argChild": 1,
+              "argOverwriteTable": true
+                                                }
+
+             ]
+         }
+       ],
+
+   "language":"myrial",
+   "logicalRa":"Store(public:adhoc:result)[Apply(PyUDF(simpleApplyTest,))]]",
+
+   "rawQuery":"Simple Apply test for PythonUDFs"
+}
diff --git a/jsonQueries/pythonUDF/udfFlatmapApply.json b/jsonQueries/pythonUDF/udfFlatmapApply.json
@@ -0,0 +1,68 @@
+{
+      "fragments":[
+        {
+          "operators": [
+            {
+              "opType": "TableScan",
+              "opId": 0,
+              "relationKey": {
+                "userName": "public",
+                "relationName": "raw",
+                "programName": "adhoc"
+              }
+            },
+            {
+
+               "opType": "Apply",
+               "opId": 1,
+               "emitExpressions": [
+                 {
+                   "outputName": "udfOutput",
+                   "rootExpressionOperator": {
+                     "type": "PYUDF",
+                     "name": "flatmapApplyTest",
+                     "outputType":"BLOB_TYPE",
+                     "addCounter":"False",
+                     "children": [
+                       {
+                         "type": "VARIABLE",
+                         "columnIdx": 3
+                       }
+                     ]
+                   }
+                 },
+                   {
+                   "outputName":"subjectid",
+                   "rootExpressionOperator":{
+                      "type":"VARIABLE",
+                      "columnIdx":2}
+                 },
+                 {"outputName":"imageid",
+                 "rootExpressionOperator":{
+                    "type":"VARIABLE",
+                    "columnIdx":1}
+               }
+               ],
+               "argChild": 0
+                },
+                {
+                  "opType": "DbInsert",
+                  "opId": 2,
+                  "relationKey": {
+                    "userName": "public",
+                    "relationName": "results",
+                    "programName": "adhoc"
+              },
+              "argChild": 1,
+              "argOverwriteTable": true
+                                                }
+
+             ]
+         }
+       ],
+
+   "language":"myrial",
+   "logicalRa":"Store(public:adhoc:result)[Apply(PyUDF(flatmapApplyTest))]]",
+
+   "rawQuery":"Flatmap Apply test for PythonUDFs"
+}
diff --git a/protobuf/column.proto b/protobuf/column.proto
@@ -22,6 +22,7 @@ message ColumnMessage {
     STRING	= 4;
     BOOLEAN	= 5;
     DATETIME	= 6;
+    BLOB =7;
   }
 
   required ColumnMessage.Type type = 1;		// See enum above
@@ -33,6 +34,7 @@ message ColumnMessage {
   optional StringColumnMessage string_column = 7; 
   optional BooleanColumnMessage boolean_column = 8; 
   optional DateTimeColumnMessage date_column = 9; 
+  optional BlobColumnMessage blob_column = 10;
 }
 
 message IntColumnMessage {
@@ -64,3 +66,9 @@ message BooleanColumnMessage {
 message DateTimeColumnMessage {
   required bytes data = 1;
 }
+
+message BlobColumnMessage {
+   required bytes data = 1;
+   repeated int32 start_indices = 2;
+   repeated int32 end_indices = 3;
+ }