Merge branch 'next'

database/code/meta/imp-import/imp_parse_objects.sql

@@ -16,8 +16,7 @@ BEGIN
     SET body = p.body, 
         name =  CASE WHEN object_type IN ('source' ,'output', 'group', 'token') THEN p.body->>(object_type || '_name')
                 WHEN object_type IN ('output_template','source_template') THEN p.body->>'object_name'
-        ELSE p.body->>'name' END,
-        hash = md5(io.body_text)
+        ELSE p.body->>'name' END
     FROM parsed p WHERE p.import_object_id = io.import_object_id;
 
     INSERT INTO log.actor_log (log_id, message, actor_path, severity, insert_datetime)

database/code/meta/imp-import/impc_parse_raw_attribute.sql

@@ -14,6 +14,9 @@ BEGIN
         v_parsed := regexp_split_to_array(in_raw->>0, '\s+(AS\s+|)','i');
         v_attribute_name := v_parsed[1];
         v_data_type := lower(v_parsed[2]);
+        IF v_data_type IS NULL THEN
+            RETURN jsonb_build_object('error', format('Unable to parse raw attribute data type from %s', in_raw));
+        END IF;
         v_datatype_schema := meta.u_get_schema_from_type(null, v_data_type);
 
     ELSEIF jsonb_typeof(in_raw) = 'object' THEN -- parse object
@@ -27,11 +30,11 @@ BEGIN
 -- validate name and type
 
     IF v_attribute_name IS NULL THEN
-        RETURN jsonb_build_object('error', format('Invalid raw attribute %s', in_raw));
+        RETURN jsonb_build_object('error', format('Unable to parse raw attribute name from %s', in_raw));
     END IF;
 
     IF v_data_type NOT IN (SELECT hive_type from meta.attribute_type) THEN
-        RETURN jsonb_build_object('error', format('Invalid raw attribute datatype %s', in_raw));
+        RETURN jsonb_build_object('error', format('Invalid raw attribute datatype `%s`', v_data_type));
     END IF;
 
     RETURN jsonb_build_object('name', lower(v_attribute_name), 'data_type', v_data_type, 'datatype_schema', v_datatype_schema);

database/code/meta/u-utility/enr-query-generator/u_enr_query_add_enrichment.sql

@@ -71,7 +71,7 @@ FOR v_parameter IN
     INSERT INTO elements ( type, expression, alias, attribute_id, parent_ids, data_type)
     VALUES ( 'enrichment', v_expression, in_enr.attribute_name, in_enr.enrichment_id, v_parent_element_ids,
             --Check for explicit casts or numeric/decimal types that need to be cast to 38,12
-    (SELECT CASE WHEN COALESCE(NULLIF(in_enr.cast_datatype,''), in_enr.datatype) IN ('decimal', 'numeric') OR (COALESCE(in_enr.cast_datatype,'') <> '' AND in_enr.cast_datatype <> in_enr.datatype) THEN at.hive_ddl_type END FROM meta.attribute_type at WHERE at.hive_type = COALESCE(NULLIF(in_enr.cast_datatype,''), in_enr.datatype) ))
+    (SELECT CASE WHEN COALESCE(NULLIF(in_enr.cast_datatype,''), in_enr.datatype) = 'decimal' OR (COALESCE(in_enr.cast_datatype,'') <> '' AND in_enr.cast_datatype <> in_enr.datatype) THEN at.hive_ddl_type END FROM meta.attribute_type at WHERE at.hive_type = COALESCE(NULLIF(in_enr.cast_datatype,''), in_enr.datatype) ))
     RETURNING id )
     SELECT id INTO v_ret_element_id
     FROM cte;

database/code/meta/u-utility/u_output_query_column_select.sql

@@ -11,10 +11,12 @@ DECLARE
 BEGIN
 
     IF osc IS NULL THEN
-        RETURN CASE in_hive_type
-            WHEN 'struct' THEN 'struct()'
-            WHEN 'array' THEN 'array()'
-            ELSE 'CAST(null as ' || COALESCE(osc.datatype,'string') || ')'
+        RETURN 
+            CASE 
+            WHEN in_hive_type = 'struct' THEN 'struct()'
+            WHEN in_hive_type = 'array' THEN 'array()'
+            WHEN in_hive_type IS NOT NULL THEN 'CAST(null as ' || in_hive_type || ')'
+            ELSE 'null'
             END;
     END IF;
 

database/schema/10 aggregateTable.sql

@@ -18,7 +18,7 @@ VALUES ('any', 'Returns true if at least one value of `expr` is true.', FALSE, '
     ('bool_or', 'Returns true if at least one value of `expr` is true.', FALSE, 'boolean'),
     ('collect_list', 'Collects and returns a list of non-unique elements', FALSE, 'array'),
     ('collect_set', 'Collects and returns a set of unique elements', FALSE, 'array'),
-    ('corr', 'Returns Pearson coefficient of correlation between a set of number pairs.', TRUE, 'numeric'),
+    ('corr', 'Returns Pearson coefficient of correlation between a set of number pairs.', TRUE, 'decimal'),
     ('count', 'Returns the number of rows for which the column expression is non-null.', FALSE, 'long'),
     ('count_if', 'Returns the number of `TRUE` values for the expression.', FALSE, 'long'),
 ('count_min_sketch', 'Returns a count-min sketch of a column with the given esp, confidence and seed. The result is an array of bytes, which can be deserialized to a `CountMinSketch` before usage. Count-min sketch is a probabilistic data structure used for cardinality estimation using sub-linear space.', FALSE, 'string'),