Config file checking

declad/config.py

@@ -1,7 +1,85 @@
 import yaml
+import sys
 
 class Config(dict):
+    _valid_keys = set( [
+        "copy_command_template",
+        "create_dirs_command_template",
+        "debug_enabled",
+        "default_category",
+        "delete_command_template",
+        "destination_root_path",
+        "destination_server",
+        "download_command_template",
+        "graphite.host",
+        "graphite.interval",
+        "graphite.port",
+        "graphite.namespace",
+        "graphite.bin",
+        "keep_interval",
+        "log",
+        "low_water_mark",
+        "meta_suffix",
+        "metacat_dataset",
+        "metacat_namespace",
+        "metacat_url",
+        "max_movers",
+        "quarantine_location",
+        "rel_path_function",
+        "rel_path_pattern",
+        "retry_cooldown",
+        "rucio.activity",
+        "rucio.dataset_did_template",
+        "rucio.declare_to_rucio",
+        "rucio.target_rses",
+        "rucio.drop_rse",
+        "sam_location_template",
+        "samweb.user",
+        "samweb.url",
+        "samweb.token",
+        "samweb.cert",
+        "samweb.key",
+        "scanner.type",
+        "scanner.replace_location",
+        "scanner.path",
+        "scanner.interval",
+        "scanner.location",
+        "scanner.ls_command_template",
+        "scanner.metadata_extractor_log",
+        "scanner.metadata_extractor",
+        "scanner.parse_re",
+        "scanner.filename_patterns",
+        "scanner.filename_pattern",
+        "scanner.server",
+        "scanner.timeout",
+        "source_root_path",
+        "source_server",
+        "transfer_timeout",
+        "web_gui.prefix",
+        "web_gui.site_title",
+        "web_gui.port",
+    ])
 
-	def __init__(self, config_file):
-		cfg = yaml.load(open(config_file, "r"), Loader=yaml.SafeLoader)
-		dict.__init__(self, cfg.items())
+    def __init__(self, config_file):
+        with open(config_file, "r") as cff:
+            cfg = yaml.load(cff , Loader=yaml.SafeLoader)
+        self.check(cfg)
+        dict.__init__(self, cfg.items())
+
+    def check(self, cfg ):
+        unknown_keys= set()
+        for k,v in cfg.items():
+            if isinstance(v,dict):
+                for k2, v2 in v.items():
+                    if not f"{k}.{k2}" in Config._valid_keys:
+                        unknown_keys.add( f"{k}.{k2}" )
+            else:
+                if not k in Config._valid_keys:
+                    unknown_keys.add( k )
+        if unknown_keys:
+           raise KeyError(f"Unknown keys: {','.join(unknown_keys)} in config")
+
+
+if __name__ == "__main__":
+     # run as config file checker...
+     cf = Config(sys.argv[1])

declad/config_sample.yaml

@@ -5,17 +5,21 @@ scanner:
     filename_pattern:   "*.hdf5"
     # meta_suffix:        .json           # optional
     ls_command_template:    "xrdfs $server ls -l $location"         # $server and $location will be replaced in run time
-    parse_re:               "^(?P<type>[a-z-])\S+\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}\s+(?P<size>\d+)\s+(?P<path>\S+)$"
+    parse_re:               "^(?P<type>[a-z-])\\S+\\s+\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\s+(?P<size>\\d+)\\s+(?P<path>\\S+)$"
     timeout:                30      # seconds
 
 log: logs/declad.log
 max_movers: 10                          # default 10
-queue_capacity: 100                     # stop scanning for input files if the queue is backlogged. default - no limit
 retry_cooldown: 3600                    # file retry interval
 keep_interval: 86400                    # interval to keep file processing log in memory
 
 metacat_dataset:    dune:all                # namespace:name
-rucio_dataset_did_template: "%(run_type)s:%(run_type)s_%(run_number)s"          # Python %-operation template, applied to the file metadata dict
+rucio:
+  dataset_did_template: "%(run_type)s:%(run_type)s_%(run_number)s"          # Python %-operation template, applied to the file metadata dict
+  target_rses:            # RSEs to create replication rule to
+    - FNAL_DCACHE
+    - CERN_CASTOR
+  drop_rse:   CERN_EOS    # Rucio name of the RSE where the data arrives being copied from the dropbox
 
 source_server:        root://dropbox.host:port                 
 destination_server:   root://dest.host:port
@@ -25,18 +29,14 @@ create_dirs_command_template:   "xrdfs $server mkdir -p $path"
 copy_command_template:          "xrdcp --force --silent --tpc $src_url $dst_url"
 quarantine_location:            /eos/experiment/neutplatform/protodune/dune/ivm/quarantine
 
-drop_rse:   CERN_EOS            # Rucio name of the RSE where the data arrives being copied from the dropbox
-replication_targets:            # RSEs to create replication rule to
-    - FNAL_DCACHE
-    - CERN_CASTOR
 
 transfer_timeout: 120           # default 120
 keep_interval:  86400           # default 24 hours
 
 
 metacat_url:    https://metacat.fnal.gov:9443/dune_meta_demo/app/data
-rucio_url:      https://dune-rucio.fnal.gov
-samweb_url:     https://samweb.fnal.gov:8483/sam/dune/api           # optionel. If omited, do not declare to SAM
+samweb:
+  url:     https://samweb.fnal.gov:8483/sam/dune/api    # optional. If omited, do not declare to SAM
 
 web_gui:
     port: 8080

declad/dune_config.yaml

@@ -57,7 +57,6 @@ download_command_template:      "xrdcp --force --silent root://$server/$src_path
 quarantine_location:            /dune/scratch/dunepro/ingest/quarantine
 
 metacat_url:    https://metacat.fnal.gov:9443/dune_meta_prod/app
-rucio_url:      https://dune-rucio.fnal.gov
 
 sam_location_template: dcache:/pnfs/dune/persistent/staging/$dst_rel_dir
 

declad/version.py

@@ -1,3 +1,3 @@
 # will be overwritten by Makefile
 
-Version="2.1.1"
+Version="2.2.0"