-
Notifications
You must be signed in to change notification settings - Fork 6
/
createDataFrame.js
108 lines (91 loc) · 2.65 KB
/
createDataFrame.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
import joint from 'jointjs';
import * as config from '../../config';
import DefaultShape, {generatePorts} from '../../../../core/graph/DefaultShape';
import NodeTemplate from '../../../../core/graph/NodeTemplate';
const NAME = 'Create DataFrame';
const NODE_TYPE = 'dfCreateDataFrame';
const NO_INPUT_NODES = 0;
const NO_OUTPUT_NODES = 1;
const INPUT_DATA_TYPE = null;
const OUTPUT_DATA_TYPE = 'df';
const IS_NODE_HIDEN = false;
const PREFIX = 'createDataFrame(';
const WIDTH = 150;
const PARAMS = [
{
name: 'data',
description: 'An RDD of any kind of SQL data representation(e.g. row, tuple, int, boolean, etc.), or list, or pandas.DataFrame.',
required: true,
},
{
name: 'schema',
description: 'A pyspark.sql.types.DataType or a datatype string or a list of column names, default is None. The data type string format equals to pyspark.sql.types.DataType.simpleString, except that top level struct type can omit the struct<> and atomic types use typeName() as their format, e.g. use byte instead of tinyint for pyspark.sql.types.ByteType. We can also use int as a short name for IntegerType.',
required: false,
template: 'schema=None',
selectionStart: '7',
selectionEnd: 'all'
},
];
///////////////////////////////////////////////////////////
const FULL_NODE_TYPE = 'spark.' + NODE_TYPE;
const ports = [...generatePorts('in', NO_INPUT_NODES), ...generatePorts('out', NO_OUTPUT_NODES)];
const MODEL = DefaultShape.extend({
defaults: joint.util.deepSupplement({
type: FULL_NODE_TYPE,
size:{
width: WIDTH
},
attrs: {
text : { text: NAME },
rect : {
width: WIDTH,
fill: config.DF_NODES_FILL
}
},
dfGui: {
description: NAME,
},
ports: {
items: ports
}
}, DefaultShape.prototype.defaults)
});
if(!joint.shapes['spark']) joint.shapes['spark'] = {};
joint.shapes['spark'][NODE_TYPE] = MODEL;
export default class CreateDataFrame extends NodeTemplate{
static getType(){
return FULL_NODE_TYPE;
}
static getName(){
return NAME;
}
static getModel(){
return MODEL.bind(joint);
}
static isNodeHidden(){
return IS_NODE_HIDEN;
}
static getCodePrefix(langId){
return PREFIX;
}
static getCodeSuffix(langId){
return ")";
}
static getCodeParameters(langId){
return PARAMS;
}
static getOutputDataType(langId){
return OUTPUT_DATA_TYPE;
}
static isInputDataTypeValid(dataType, langId){
if(INPUT_DATA_TYPE){
if(typeof INPUT_DATA_TYPE == 'string')
return dataType == INPUT_DATA_TYPE;
return INPUT_DATA_TYPE.has(dataType);
}
return false;
}
static getWidth() {
return WIDTH;
}
}