Skip to content

Commit

Permalink
DRILL-7196: Queries are still runnable on disabled plugins
Browse files Browse the repository at this point in the history 
- Storage client is not created anymore for disabled plugins
- GET "/storage/{name}.json" endpoint now working with
  plugin configuration directly, without client instantination.
  It have increased UI responsitivity.
- Hbase and mongo base test classes refactored to honor enabled
  plugin attribute
- Fixed path contructor for mongo test datasets:
  Now it is cross-platform
- Fixed test json files format which using plugin definitions
- Code cleanup
  • Loading branch information
@dgrinchenko @sachouche
dgrinchenko authored and sachouche committed May 29, 2019
1 parent 20b58ab commit e8b921b
Show file tree
Hide file tree
Showing 107 changed files with 3,231 additions and 3,142 deletions.
51 changes: 26 additions & 25 deletions common/src/test/resources/basic_physical.json
View file
@@ -1,40 +1,41 @@
{ {
head:{ "head":{
type:"APACHE_DRILL_PHYSICAL", "type":"APACHE_DRILL_PHYSICAL",
version:"1", "version":"1",
generator:{ "generator":{
type:"manual" "type":"manual"
} }
}, },
storage:{ "storage":{
fs1:{ "fs1":{
type:"mock" "type":"mock",
"enabled": true
} }
}, },
graph:[ "graph":[
{ {
@id:1, "@id":1,
pop:"scan", "pop":"scan",
storageengine:"fs1", "storageengine":"fs1",
entries:[ "entries":[
{"test1"} {"test1"}
], ],
output:[ "output":[
{ "name":"key", mode: "VECTOR", type:"SINT32"}, { "name":"key", "mode": "VECTOR", "type":"SINT32"},
{ "name":"value", mode: "VECTOR", type:"SINT32"} { "name":"value", "mode": "VECTOR", "type":"SINT32"}
] ]
}, },
{ {
@id:2, "@id":2,
child:1, "child":1,
pop: "store", "pop": "store",
mode: "SYSTEM_CHOICE", "mode": "SYSTEM_CHOICE",
storageengine: "fs1", "storageengine": "fs1",
entries:[ "entries":[
{ {
path:"/sort/sorted/${partition_number}.seq", "path":"/sort/sorted/${partition_number}.seq",
key:"Text", "key":"Text",
type:"JAVA_SEQUENCE" "type":"JAVA_SEQUENCE"
} }
] ]
} }
Expand Down
48 changes: 24 additions & 24 deletions common/src/test/resources/dsort-logical.json
View file
@@ -1,38 +1,38 @@
{ {
head:{ type:"apache_drill_logical_plan", version:"1", generator:{ type:"manual", info:"na"}}}, "head":{ "type":"apache_drill_logical_plan", "version":"1", "generator":{ "type":"manual", "info":"na"}},
storage:[ { type:"fs", name:"fs1", root:"file:///" }], "storage":[ { "type":"fs", "name":"fs1", "root":"file:///", "enabled": true}],
query:[ { op: "sequence", sequence: [ "query":[ { "op": "sequence", "sequence": [
{ {
op:"scan", "op":"scan",
storageengine:"fs1", "storageengine":"fs1",
ref: "data", "ref": "data",
selection: { "selection": {
path: "/sort/unsorted/*.seq", "path": "/sort/unsorted/*.seq",
type: "JAVA_SEQUENCE" "type": "JAVA_SEQUENCE"
} }
}, },
{ {
op: "order", "op": "order",
orderings: [ "orderings": [
{order: "desc", expr: "data.key" } {"order": "desc", "expr": "data.key" }
] ]
}, },
{ {
op: "project", "op": "project",
projections: [ "projections": [
{ ref: "output.key", expr: "data.key" }, { "ref": "output.key", "expr": "data.key" },
{ ref: "output.value", expr: "data.value" } { "ref": "output.value", "expr": "data.value" }
] ]
}, },
{ {
op: "store", "op": "store",
storageengine: "fs1", "storageengine": "fs1",
target: { "target": {
path: "/sort/sorted/${partition}.seq", "path": "/sort/sorted/${partition}.seq",
type: "JAVA_SEQUENCE", "type": "JAVA_SEQUENCE",
partition: { "partition": {
type: "ORDERED", "type": "ORDERED",
exprs: ["key"] "exprs": ["key"]
} }
} }
} }
Expand Down
4 changes: 2 additions & 2 deletions common/src/test/resources/jdbc_plan.json
View file
Expand Up @@ -4,8 +4,8 @@
"generator":{"type":"manual","info":"na"} "generator":{"type":"manual","info":"na"}
}, },
"storage":{ "storage":{
"donuts-json":{"type":"classpath"}, "donuts-json":{"type":"classpath", "enabled": true},
"queue":{"type":"queue"} "queue":{"type":"queue", "enabled": true}
}, },
"query":[ "query":[
{"op":"sequence", {"op":"sequence",
Expand Down
159 changes: 81 additions & 78 deletions common/src/test/resources/simple_plan.json
View file
@@ -1,129 +1,132 @@
{ {
head:{ "head":{
type:"apache_drill_logical_plan", "type":"apache_drill_logical_plan",
version:"1", "version":"1",
generator:{ "generator":{
type:"manual", "type":"manual",
info:"na" "info":"na"
} }
}, },
storage:{ "storage":[
logs: { {
type:"text", "type":"text",
file: "local://logs/*.log", "file": "local://logs/*.log",
compress:"gzip", "compress":"gzip",
line-delimiter:"\n", "line-delimiter":"\n",
record-maker:{ "record-maker":{
type:"first-row", "type":"first-row",
delimiter:"," "delimiter":","
} },
"enabled": true
}, },
{ {
type:"mongo", "type":"mongo",
name:"users", "name":"users",
connection:"mongodb://blue:red@localhost/users" "connection":"mongodb://blue:red@localhost/users",
"enabled": true
}, },
{ {
type:"mysql", "type":"mysql",
name:"mysql", "name":"mysql",
connection:"jdbc:mysql://localhost/main" "connection":"jdbc:mysql://localhost/main",
"enabled": true
} }
], ],
query:[ "query":[
{ {
@id:"1", "@id": 1 ,
op:"scan", "op":"scan",
memo:"initial_scan", "memo":"initial_scan",
storageengine:"local-logs", "storageengine":"local-logs",
selection: {} "selection": {}
}, },
{ {
@id:"2", "@id": 2 ,
input:"1", "input": 1 ,
memo:"transform1", "memo":"transform1",
op:"transform", "op":"transform",
transforms:[ "transforms":[
{ {
ref:"userId", "ref":"userId",
expr:"regex_like('activity.cookie', \"persistent=([^;]*)\")" "expr":"regex_like('activity.cookie', \"persistent=([^;]*)\")"
}, },
{ {
ref:"session", "ref":"session",
expr:"regex_like('activity.cookie', \"session=([^;]*)\")" "expr":"regex_like('activity.cookie', \"session=([^;]*)\")"
} }
] ]
}, },
{ {
@id:"3", "@id": 3 ,
input:"2", "input": 2 ,
memo:"transform2", "memo":"transform2",
op:"transform", "op":"transform",
transforms:[ "transforms":[
{ {
ref:"userId", "ref":"userId",
expr:"regex_like('activity.cookie', \"persistent=([^;]*)\")" "expr":"regex_like('activity.cookie', \"persistent=([^;]*)\")"
}, },
{ {
ref:"session", "ref":"session",
expr:"regex_like('activity.cookie', \"session=([^;]*)\")" "expr":"regex_like('activity.cookie', \"session=([^;]*)\")"
} }
] ]
}, },
{ {
@id:"7", "@id": 7 ,
input:"3", "input": 3 ,
op:"sequence", "op":"sequence",
do:[ "do":[
{ {
op:"transform", "op":"transform",
memo:"seq_transform", "memo":"seq_transform",
transforms:[ "transforms":[
{ {
ref:"happy", "ref":"happy",
expr:"regex_like('ep2', \"dink\")" "expr":"regex_like('ep2', \"dink\")"
} }
] ]
} }
, ,
{ {
op:"transform", "op":"transform",
memo:"last_transform", "memo":"last_transform",
transforms:[ "transforms":[
{ {
ref:"abc", "ref":"abc",
expr:"123" "expr":"123"
} }
] ]
} }
] ]
}, },
{ {
@id:"10", "@id": 10,
input:"3", "input": 3,
op:"transform", "op":"transform",
memo:"t3", "memo":"t3",
transforms:[ "transforms":[
{ {
ref:"happy", "ref":"happy",
expr:"regex_like('ep2', \"dink\")" "expr":"regex_like('ep2', \"dink\")"
} }
] ]
}, },
{ {
@id:12, "@id":12,
op:"join", "op":"join",
type: "inner", "type": "inner",
left:"7", "left":"7",
right:"10", "right":"10",
conditions: [{relationship:"==", left: "1", right: "1" }] "conditions": [{"relationship":"==", "left": "1", "right": "1" }]
} }
, ,
{ {
input: 12, "input": 12,
op: "store", "op": "store",
memo: "output sink", "memo": "output sink",
target: { "target": {
file: "console:///stdout" "file": "console:///stdout"
} }
} }
] ]
Expand Down
5 changes: 3 additions & 2 deletions common/src/test/resources/storage_engine_plan.json
View file
Expand Up @@ -9,8 +9,9 @@
}, },
"storage" : { "storage" : {
"mock-engine": { "mock-engine": {
type : "mock", "type" : "mock",
url : "http://www.apache.org/" "url" : "http://www.apache.org/",
"enabled": true
} }
}, },
"query" : [ "query" : [
Expand Down

0 comments on commit e8b921b

Please sign in to comment.