Added idea of optional things to spec checks

cedadev · Dec 6, 2022 · c4fe779 · c4fe779
1 parent 533cc1e
commit c4fe779
Show file tree

Hide file tree

Showing 53 changed files with 800 additions and 777 deletions.
diff --git a/checksit/generic.py b/checksit/generic.py
@@ -70,8 +70,14 @@ def check_var_exists(dct, variables):
     errors = []
 
     for var in variables:
-        if var not in dct["variables"].keys():
-            errors.append(f"[variable**************:{var}]: Does not exist in file.")
+        if ':__OPTIONAL__' in var:
+            var = var.split(':')[0]
+            if var not in dct["variables"].keys():
+                # add warning/advisory
+                pass
+        else:
+            if var not in dct["variables"].keys():
+                errors.append(f"[variable**************:{var}]: Does not exist in file.")
 
     return errors
 
@@ -85,22 +91,39 @@ def check_dim_exists(dct, dimensions):
     errors = []
 
     for dim in dimensions:
-        if dim not in dct["dimensions"].keys():
-            errors.append(f"[dimension**************:{dim}]: Does not exist in file.")
+        if ':__OPTIONAL__' in dim:
+            dim = dim.split(':')[0]
+            if dim not in dct["dimensions"].keys():
+                # add warning/advisory
+                pass
+        else:
+            if dim not in dct["dimensions"].keys():
+                errors.append(f"[dimension**************:{dim}]: Does not exist in file.")
 
     return errors 
 
+
 def check_var(dct, variables, defined_attrs):
     """
     Check variables exist and have attributes defined.
     """
     errors = []
     for var in variables:
-        if var not in dct["variables"].keys():
-            errors.append(f"[variable**************:{var}]: Does not exist in file.")
+        if ':__OPTIONAL__' in var:
+            var = var.split(':')[0]
+            if var not in dct["variables"].keys():
+                # add warning/advisory
+                pass
+            else:
+                for attr in defined_attrs:
+                    if is_undefined(dct["variables"][var].get(attr)):
+                        errors.append(f"[variable**************:{var}]: Attribute '{attr}' must have a valid definition.")
         else:
-            for attr in defined_attrs:
-                if is_undefined(dct["variables"][var].get(attr)):
-                    errors.append(f"[variable**************:{var}]: Attribute '{attr}' must have a valid definition.")            
+            if var not in dct["variables"].keys():
+                errors.append(f"[variable**************:{var}]: Does not exist in file.")
+            else:
+                for attr in defined_attrs:
+                    if is_undefined(dct["variables"][var].get(attr)):
+                        errors.append(f"[variable**************:{var}]: Attribute '{attr}' must have a valid definition.")            
 
     return errors
diff --git a/make_specs.py b/make_specs.py
@@ -234,15 +234,15 @@
                 f.write(f'var-requires{key_number}:\n')
                 f.write('  func: checksit.generic.check_var\n  params:\n    variables:\n')
                 for var in product_info[key_number]:
-                    f.write(f'      - {var}\n')
+                    f.write(f'      - {var}:__OPTIONAL__\n')
                 f.write('    defined_attrs:\n')
                 for attr in product_attrs_info[key_number]:
                     f.write(f'      - {attr}\n')
             f.write('\n')
         if prod_dims_exist:
             f.write('dims-requires:\n  func: checksit.generic.check_dim_exists\n  params:\n    dimensions:\n')
             for dim in product_dims:
-                f.write(f'      - {dim}\n')
+                f.write(f'      - {dim}:__OPTIONAL__\n')
         if prod_attrs_exist:
             f.write('\nrequired-global-attrs:\n  func: checksit.generic.check_global_attrs\n  params:\n    vocab_attrs:\n')
             for attr, rule in attr_rules.items():

diff --git a/specs/groups/amof-acoustic-backscatter-winds.yml b/specs/groups/amof-acoustic-backscatter-winds.yml
@@ -2,7 +2,7 @@ var-requires0:
   func: checksit.generic.check_var
   params:
     variables:
-      - altitude
+      - altitude:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -18,12 +18,12 @@ var-requires1:
   func: checksit.generic.check_var
   params:
     variables:
-      - sound_intensity_level_in_air
-      - wind_speed
-      - wind_from_direction
-      - eastward_wind
-      - northward_wind
-      - upward_air_velocity
+      - sound_intensity_level_in_air:__OPTIONAL__
+      - wind_speed:__OPTIONAL__
+      - wind_from_direction:__OPTIONAL__
+      - eastward_wind:__OPTIONAL__
+      - northward_wind:__OPTIONAL__
+      - upward_air_velocity:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -39,9 +39,9 @@ var-requires2:
   func: checksit.generic.check_var
   params:
     variables:
-      - divergence_of_eastward_wind
-      - divergence_of_northward_wind
-      - divergence_of_upward_air_velocity
+      - divergence_of_eastward_wind:__OPTIONAL__
+      - divergence_of_northward_wind:__OPTIONAL__
+      - divergence_of_upward_air_velocity:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -56,12 +56,12 @@ var-requires3:
   func: checksit.generic.check_var
   params:
     variables:
-      - qc_flag_mean_winds
-      - qc_flag_wind_component_eastward
-      - qc_flag_wind_component_northward
-      - qc_flag_wind_component_upward_air_velocity
-      - qc_flag_backscatter
-      - qc_flag_background_noise
+      - qc_flag_mean_winds:__OPTIONAL__
+      - qc_flag_wind_component_eastward:__OPTIONAL__
+      - qc_flag_wind_component_northward:__OPTIONAL__
+      - qc_flag_wind_component_upward_air_velocity:__OPTIONAL__
+      - qc_flag_backscatter:__OPTIONAL__
+      - qc_flag_background_noise:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -74,4 +74,4 @@ dims-requires:
   func: checksit.generic.check_dim_exists
   params:
     dimensions:
-      - altitude
+      - altitude:__OPTIONAL__
diff --git a/specs/groups/amof-aerosol-backscatter-radial-winds.yml b/specs/groups/amof-aerosol-backscatter-radial-winds.yml
@@ -2,11 +2,11 @@ var-requires0:
   func: checksit.generic.check_var
   params:
     variables:
-      - range
-      - sensor_azimuth_angle_instrument_frame
-      - sensor_view_angle_instrument_frame
-      - sensor_azimuth_angle_earth_frame
-      - sensor_view_angle_earth_frame
+      - range:__OPTIONAL__
+      - sensor_azimuth_angle_instrument_frame:__OPTIONAL__
+      - sensor_view_angle_instrument_frame:__OPTIONAL__
+      - sensor_azimuth_angle_earth_frame:__OPTIONAL__
+      - sensor_view_angle_earth_frame:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -20,7 +20,7 @@ var-requires1:
   func: checksit.generic.check_var
   params:
     variables:
-      - radial_velocity_of_scatterers_away_from_instrument
+      - radial_velocity_of_scatterers_away_from_instrument:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -36,8 +36,8 @@ var-requires2:
   func: checksit.generic.check_var
   params:
     variables:
-      - attenuated_aerosol_backscatter_coefficient
-      - signal_to_noise_ratio_plus_1
+      - attenuated_aerosol_backscatter_coefficient:__OPTIONAL__
+      - signal_to_noise_ratio_plus_1:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -52,8 +52,8 @@ var-requires3:
   func: checksit.generic.check_var
   params:
     variables:
-      - qc_flag_radial_velocity_of_scatterers_away_from_instrument
-      - qc_flag_backscatter
+      - qc_flag_radial_velocity_of_scatterers_away_from_instrument:__OPTIONAL__
+      - qc_flag_backscatter:__OPTIONAL__
     defined_attrs:
       - type
       - units
@@ -65,8 +65,8 @@ dims-requires:
   func: checksit.generic.check_dim_exists
   params:
     dimensions:
-      - index_of_angle
-      - index_of_range
+      - index_of_angle:__OPTIONAL__
+      - index_of_range:__OPTIONAL__
 
 required-global-attrs:
   func: checksit.generic.check_global_attrs

diff --git a/specs/groups/amof-aerosol-backscatter.yml b/specs/groups/amof-aerosol-backscatter.yml
@@ -2,7 +2,7 @@ var-requires0:
   func: checksit.generic.check_var
   params:
     variables:
-      - altitude
+      - altitude:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -18,16 +18,16 @@ var-requires1:
   func: checksit.generic.check_var
   params:
     variables:
-      - attenuated_aerosol_backscatter_coefficient
-      - range_squared_corrected_backscatter_power
-      - laser_pulse_energy
-      - laser_temperature
-      - profile_pulses
-      - profile_scaling
-      - window_contamination
-      - window_transmittance
-      - background_light
-      - backscatter_sum
+      - attenuated_aerosol_backscatter_coefficient:__OPTIONAL__
+      - range_squared_corrected_backscatter_power:__OPTIONAL__
+      - laser_pulse_energy:__OPTIONAL__
+      - laser_temperature:__OPTIONAL__
+      - profile_pulses:__OPTIONAL__
+      - profile_scaling:__OPTIONAL__
+      - window_contamination:__OPTIONAL__
+      - window_transmittance:__OPTIONAL__
+      - background_light:__OPTIONAL__
+      - backscatter_sum:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -42,8 +42,8 @@ var-requires2:
   func: checksit.generic.check_var
   params:
     variables:
-      - sensor_zenith_angle
-      - sensor_azimuth_angle
+      - sensor_zenith_angle:__OPTIONAL__
+      - sensor_azimuth_angle:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -59,7 +59,7 @@ var-requires3:
   func: checksit.generic.check_var
   params:
     variables:
-      - qc_flag
+      - qc_flag:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -72,7 +72,7 @@ dims-requires:
   func: checksit.generic.check_dim_exists
   params:
     dimensions:
-      - altitude
+      - altitude:__OPTIONAL__
 
 required-global-attrs:
   func: checksit.generic.check_global_attrs

diff --git a/specs/groups/amof-aerosol-concentration.yml b/specs/groups/amof-aerosol-concentration.yml
@@ -2,7 +2,7 @@ var-requires0:
   func: checksit.generic.check_var
   params:
     variables:
-      - number_concentration_of_ambient_aerosol_particles_in_air
+      - number_concentration_of_ambient_aerosol_particles_in_air:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -17,7 +17,7 @@ var-requires1:
   func: checksit.generic.check_var
   params:
     variables:
-      - qc_flag
+      - qc_flag:__OPTIONAL__
     defined_attrs:
       - type
       - dimension

diff --git a/specs/groups/amof-aerosol-extinction.yml b/specs/groups/amof-aerosol-extinction.yml
@@ -2,7 +2,7 @@ var-requires0:
   func: checksit.generic.check_var
   params:
     variables:
-      - altitude
+      - altitude:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -18,8 +18,8 @@ var-requires1:
   func: checksit.generic.check_var
   params:
     variables:
-      - volume_extinction_coefficient_in_air_due_to_ambient_aerosol_particles_355
-      - volume_extinction_coefficient_in_air_due_to_ambient_aerosol_particles_316
+      - volume_extinction_coefficient_in_air_due_to_ambient_aerosol_particles_355:__OPTIONAL__
+      - volume_extinction_coefficient_in_air_due_to_ambient_aerosol_particles_316:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -33,8 +33,8 @@ var-requires2:
   func: checksit.generic.check_var
   params:
     variables:
-      - qc_flag_355
-      - qc_flag_316
+      - qc_flag_355:__OPTIONAL__
+      - qc_flag_316:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -47,4 +47,4 @@ dims-requires:
   func: checksit.generic.check_dim_exists
   params:
     dimensions:
-      - altitude
+      - altitude:__OPTIONAL__
diff --git a/specs/groups/amof-aerosol-no3-so4-nh3-org-concentration.yml b/specs/groups/amof-aerosol-no3-so4-nh3-org-concentration.yml
@@ -2,10 +2,10 @@ var-requires0:
   func: checksit.generic.check_var
   params:
     variables:
-      - mass_concentration_of_nitrate_in_ambient_aerosol_particles_in_air
-      - mass_concentration_of_sulfate_in_ambient_aerosol_particles_in_air
-      - mass_concentration_of_ammonia_in_ambient_aerosol_particles_in_air
-      - mass_concentration_of_organics_in_ambient_aerosol_particles_in_air
+      - mass_concentration_of_nitrate_in_ambient_aerosol_particles_in_air:__OPTIONAL__
+      - mass_concentration_of_sulfate_in_ambient_aerosol_particles_in_air:__OPTIONAL__
+      - mass_concentration_of_ammonia_in_ambient_aerosol_particles_in_air:__OPTIONAL__
+      - mass_concentration_of_organics_in_ambient_aerosol_particles_in_air:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -20,10 +20,10 @@ var-requires1:
   func: checksit.generic.check_var
   params:
     variables:
-      - qc_flag_no3
-      - qc_flag_so4
-      - qc_flag_nh3
-      - qc_flag_org
+      - qc_flag_no3:__OPTIONAL__
+      - qc_flag_so4:__OPTIONAL__
+      - qc_flag_nh3:__OPTIONAL__
+      - qc_flag_org:__OPTIONAL__
     defined_attrs:
       - type
       - dimension

diff --git a/specs/groups/amof-aerosol-optical-depth.yml b/specs/groups/amof-aerosol-optical-depth.yml
@@ -2,7 +2,7 @@ var-requires0:
   func: checksit.generic.check_var
   params:
     variables:
-      - instrument_channel_wavelength
+      - instrument_channel_wavelength:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -16,8 +16,8 @@ var-requires1:
   func: checksit.generic.check_var
   params:
     variables:
-      - angstrom_exponent_of_ambient_aerosol_in_air
-      - atmosphere_optical_thickness_due_to_ambient_aerosol_particles
+      - angstrom_exponent_of_ambient_aerosol_in_air:__OPTIONAL__
+      - atmosphere_optical_thickness_due_to_ambient_aerosol_particles:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -32,7 +32,7 @@ var-requires2:
   func: checksit.generic.check_var
   params:
     variables:
-      - qc_flag
+      - qc_flag:__OPTIONAL__
     defined_attrs:
       - type
       - dimension
@@ -45,4 +45,4 @@ dims-requires:
   func: checksit.generic.check_dim_exists
   params:
     dimensions:
-      - index
+      - index:__OPTIONAL__