Skip to content

Commit

Permalink
Merge pull request #2001 from lukecampbell/qcprocess
Browse files Browse the repository at this point in the history 
Adds gradient test to qc processor and stubs for local range
  • Loading branch information
@lukecampbell
lukecampbell committed Apr 10, 2014
2 parents 2a31ef3 + 6fc0cc9 commit e8f76d2
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 3 deletions.
56 changes: 55 additions & 1 deletion ion/processes/data/transforms/qc_post_processing.py
View file
Expand Up @@ -265,12 +265,29 @@ def run_qc(self, data_product, reference_designator, parameter, qc_mapping):
self.process_trend_test(coverage, parameter, input_name, ord_n, nstd)

elif alg.lower() == 'spketst':
log.error("Runnign Spike Test")
log.error("Running Spike Test")
row = self.recent_row(lookup_table['spike_test'])
acc = row['accuracy']
N = row['range_multiplier']
L = row['window_length']
self.process_spike_test(coverage, parameter, input_name, acc, N, L)

elif alg.lower() == "gradtst":
log.error("Running Gradient Test")
row = self.recent_row(lookup_table["gradient_test"])
ddatdx = row["ddatdx"]
mindx = row["mindx"]
startdat = row["startdat"]
if isinstance(startdat, basestring) and not startdat:
startdat = np.nan
if isinstance(mindx, basestring) and not mindx:
mindx = np.nan
toldat = row["toldat"]
self.process_gradient_test(coverage, parameter, input_name, ddatdx, mindx, startdat, toldat)

elif alg.lower() == 'loclrng':
log.error("Coming soon!")

finally:
coverage.close()

Expand Down Expand Up @@ -353,12 +370,49 @@ def process_spike_test(self, coverage, parameter, input_name, acc, N, L):
coverage.temporal_parameter_name : time_array,
parameter.name : qc_array
}

def process_gradient_test(self, coverage, parameter, input_name, ddatdx, mindx, startdat, toldat):
qc_array = coverage.get_parameter_values(parameter.name)
indexes = np.where(qc_array == -88)[0]

from ion_functions.qc.qc_functions import dataqc_gradienttest_wrapper
value_array = coverage.get_parameter_values(input_name)
time_array = coverage.get_parameter_values(coverage.temporal_parameter_name)

qc_array = dataqc_gradienttest_wrapper(value_array, time_array, ddatdx, mindx, startdat, toldat)

return_dictionary = {
coverage.temporal_parameter_name : time_array[indexes],
parameter.name : qc_array[indexes]
}

log.error("Normally I'd set these in the coverage model...")
log.error(return_dictionary)

def process_local_range_test(self, coverage, parameter, input_name, datlim, datlimz):
qc_array = coverage.get_parameter_values(parameter.name)
indexes = np.where(qc_array == -88)[0]

from ion_functions.qc.qc_functions import dataqc_localrangetest
value_array = coverage.get_parameter_values(input_name)
# z_parameter_name needs to come from, I guess the column headings...
# I also need to deal with the case where there are multiple axes...
# I don't have a good feeling about this.
z_parameter_name = None
z_array = coverage.get_parameter_values(z_parameter_name)

qc_array = dataqc_localrangetest(value_array, z_array, datlim, datlimz)
return_dictionary = {
coverage.temporal_parameter_name : time_array[indexes],
parameter.name : qc_array[indexes]
}
log.error("Normally I'd set these in the coverage model...")
log.error(return_dictionary)





def get_dataset(self, data_product):
dataset_ids, _ = self.resource_registry.find_objects(data_product, PRED.hasDataset, id_only=True)
if not dataset_ids:
Expand Down
50 changes: 48 additions & 2 deletions ion/services/dm/test/test_dm_extended.py
View file
Expand Up @@ -1516,6 +1516,9 @@ def make_tempwat(self, data_product_id):
return data_product_id

def add_tempwat_qc(self, data_product_id):
#--------------------------------------------------------------------------------
# Global Range
#--------------------------------------------------------------------------------
tempwat_qc = ParameterContext(name='tempwat_glblrng_qc',
parameter_type='quantity',
value_encoding='int8',
Expand All @@ -1524,6 +1527,9 @@ def add_tempwat_qc(self, data_product_id):
fill_value=-88)
tempwat_qc_id = self.dataset_management.create_parameter(tempwat_qc)
self.data_product_management.add_parameter_to_data_product(tempwat_qc_id, data_product_id)
#--------------------------------------------------------------------------------
# Trend Test
#--------------------------------------------------------------------------------
tempwat_qc = ParameterContext(name='tempwat_trndtst_qc',
parameter_type='quantity',
value_encoding='int8',
Expand All @@ -1532,6 +1538,9 @@ def add_tempwat_qc(self, data_product_id):
fill_value=-88)
tempwat_qc_id = self.dataset_management.create_parameter(tempwat_qc)
self.data_product_management.add_parameter_to_data_product(tempwat_qc_id, data_product_id)
#--------------------------------------------------------------------------------
# Spike Test
#--------------------------------------------------------------------------------
tempwat_qc = ParameterContext(name='tempwat_spketst_qc',
parameter_type='quantity',
value_encoding='int8',
Expand All @@ -1540,6 +1549,31 @@ def add_tempwat_qc(self, data_product_id):
fill_value=-88)
tempwat_qc_id = self.dataset_management.create_parameter(tempwat_qc)
self.data_product_management.add_parameter_to_data_product(tempwat_qc_id, data_product_id)
#--------------------------------------------------------------------------------
# Gradient Test
#--------------------------------------------------------------------------------
tempwat_qc = ParameterContext(name='tempwat_gradtst_qc',
parameter_type='quantity',
value_encoding='int8',
units='1',
ooi_short_name='TEMPWAT_GRADTST_QC',
fill_value=-88)
tempwat_qc_id = self.dataset_management.create_parameter(tempwat_qc)
self.data_product_management.add_parameter_to_data_product(tempwat_qc_id, data_product_id)
#--------------------------------------------------------------------------------
# Local Range Test
#--------------------------------------------------------------------------------
tempwat_qc = ParameterContext(name='tempwat_loclrng_qc',
parameter_type='quantity',
value_encoding='int8',
units='1',
ooi_short_name='TEMPWAT_LOCLRNG_QC',
fill_value=-88)
tempwat_qc_id = self.dataset_management.create_parameter(tempwat_qc)
self.data_product_management.add_parameter_to_data_product(tempwat_qc_id, data_product_id)
#--------------------------------------------------------------------------------
# Stuck Value
#--------------------------------------------------------------------------------
preswat_qc = ParameterContext(name='preswat_stuckvl_qc',
parameter_type='quantity',
value_encoding='int8',
Expand Down Expand Up @@ -1643,8 +1677,20 @@ def test_qc_stuff(self):
"ts_created":1396371094.658708,
"accuracy":0.0001
}
]
},
],
"gradient_test" : [
{
"toldat": 0.1,
"xunits" : "s",
"mindx" : 10,
"author" : "Boon",
"startdat" : "",
"ddatdx" : [-50.0, 50.0],
"units" : "deg_C",
"ts_created" : 1396371094.658695
}
]
},
"PRESWAT":{
"stuck_value":[
{
Expand Down

0 comments on commit e8f76d2

Please sign in to comment.