-
Notifications
You must be signed in to change notification settings - Fork 46
/
google_dt.py
139 lines (99 loc) · 5.34 KB
/
google_dt.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
'''
@author Raj Singh
@file ion/processes/data/transforms/viz/google_dt.py
@description Convert CDM data to Google datatabbles
'''
from pyon.ion.transform import TransformFunction
from pyon.service.service import BaseService
from pyon.core.exception import BadRequest
from pyon.public import IonObject, RT, log
from ion.services.dm.utility.granule.granule import build_granule
from ion.services.dm.utility.granule.record_dictionary import RecordDictionaryTool
from prototype.sci_data.stream_defs import SBE37_CDM_stream_definition, SBE37_RAW_stream_definition
from coverage_model.parameter import ParameterDictionary, ParameterContext
from coverage_model.parameter_types import QuantityType
from coverage_model.basic_types import AxisTypeEnum
import numpy as np
from pyon.util.containers import get_safe
class VizTransformGoogleDT(TransformFunction):
"""
This class is used for converting incoming data from CDM format to JSON style Google DataTables
Note: One behaviour that this class is expected to achieve specifically is to determine if its supposed
to work as a realtime transform (exists indefinitely and maintains a sliding window of data) or as
a replay transform (one-shot).
[2] This transform behaves as an instantaneous forwarder. There is no waiting for the entire stream
to create the complete datatable. As the granules come in, they are translated to the datatable
'components'. Components, because we are not creating the actual datatable in this code. That's the job
of the viz service to put all the components of a datatable together in JSON format before sending it
to the client
[3] The time stamp in the incoming stream can't be converted to the datetime object here because
the Raw stream definition only expects regular primitives (strings, floats, ints etc)
"""
outgoing_stream_def = SBE37_RAW_stream_definition()
incoming_stream_def = SBE37_CDM_stream_definition()
def __init__(self):
super(VizTransformGoogleDT, self).__init__()
### Parameter dictionaries
self.define_parameter_dictionary()
def define_parameter_dictionary(self):
viz_product_type_ctxt = ParameterContext('viz_product_type', param_type=QuantityType(value_encoding=np.str))
viz_product_type_ctxt.uom = 'unknown'
viz_product_type_ctxt.fill_value = 0x0
data_description_ctxt = ParameterContext('data_description', param_type=QuantityType(value_encoding=np.int8))
data_description_ctxt.uom = 'unknown'
data_description_ctxt.fill_value = 0x0
data_content_ctxt = ParameterContext('data_content', param_type=QuantityType(value_encoding=np.int8))
data_content_ctxt.uom = 'unknown'
data_content_ctxt.fill_value = 0x0
# Define the parameter dictionary objects
self.gdt_paramdict = ParameterDictionary()
self.gdt_paramdict.add_context(viz_product_type_ctxt)
self.gdt_paramdict.add_context(data_description_ctxt)
self.gdt_paramdict.add_context(data_content_ctxt)
print " >>>>>>>>>>>>>>>>> HERE <<<<<<<<<<<<<<<<<<<"
return
def execute(self, granule):
log.debug('(Google DT transform): Received Viz Data Packet' )
#init stuff
varTuple = []
dataDescription = []
dataTableContent = []
rdt = RecordDictionaryTool.load_from_granule(granule)
vardict = {}
vardict['time'] = get_safe(rdt, 'time')
vardict['conductivity'] = get_safe(rdt, 'conductivity')
vardict['pressure'] = get_safe(rdt, 'pressure')
vardict['temperature'] = get_safe(rdt, 'temp')
vardict['longitude'] = get_safe(rdt, 'lon')
vardict['latitude'] = get_safe(rdt, 'lat')
vardict['height'] = get_safe(rdt, 'height')
arrLen = len(vardict['time']) # Figure out how many values are present in the granule
#iinit the dataTable
# create data description from the variables in the message
dataDescription = [('time', 'float', 'time')]
# split the data string to extract variable names
for varname in vardict.keys(): #psd.list_field_names():
if varname == 'time':
continue
dataDescription.append((varname, 'number', varname))
# Add the records to the datatable
for i in xrange(arrLen):
varTuple = []
for varname,_,_ in dataDescription:
if vardict[varname] == None or len(vardict[varname]) == 0:
val = 0.0
else:
val = float(vardict[varname][i])
varTuple.append(val)
# Append the tuples to the data table
if len(varTuple) > 0:
dataTableContent.append(varTuple)
# Create output dictionary from the param dict
out_rdt = RecordDictionaryTool(param_dictionary=self.gdt_paramdict)
# Prepare granule content
out_rdt['viz_product_type'] = np.array(['google_dt'])
out_rdt['data_description'] = np.array([dataDescription])
out_rdt['data_content'] = np.array([dataTableContent])
log.debug('Google DT transform: Sending a granule')
out_granule = build_granule(data_producer_id='google_dt_transform', param_dictionary = self.gdt_paramdict, record_dictionary=out_rdt)
return out_granule