diff --git a/declad/config.py b/declad/config.py index 136a3fd..04e87e8 100644 --- a/declad/config.py +++ b/declad/config.py @@ -1,7 +1,85 @@ import yaml +import sys class Config(dict): + _valid_keys = set( [ + "copy_command_template", + "create_dirs_command_template", + "debug_enabled", + "default_category", + "delete_command_template", + "destination_root_path", + "destination_server", + "download_command_template", + "graphite.host", + "graphite.interval", + "graphite.port", + "graphite.namespace", + "graphite.bin", + "keep_interval", + "log", + "low_water_mark", + "meta_suffix", + "metacat_dataset", + "metacat_namespace", + "metacat_url", + "max_movers", + "quarantine_location", + "rel_path_function", + "rel_path_pattern", + "retry_cooldown", + "rucio.activity", + "rucio.dataset_did_template", + "rucio.declare_to_rucio", + "rucio.target_rses", + "rucio.drop_rse", + "sam_location_template", + "samweb.user", + "samweb.url", + "samweb.token", + "samweb.cert", + "samweb.key", + "scanner.type", + "scanner.replace_location", + "scanner.path", + "scanner.interval", + "scanner.location", + "scanner.ls_command_template", + "scanner.metadata_extractor_log", + "scanner.metadata_extractor", + "scanner.parse_re", + "scanner.filename_patterns", + "scanner.filename_pattern", + "scanner.server", + "scanner.timeout", + "source_root_path", + "source_server", + "transfer_timeout", + "web_gui.prefix", + "web_gui.site_title", + "web_gui.port", + ]) - def __init__(self, config_file): - cfg = yaml.load(open(config_file, "r"), Loader=yaml.SafeLoader) - dict.__init__(self, cfg.items()) + def __init__(self, config_file): + with open(config_file, "r") as cff: + cfg = yaml.load(cff , Loader=yaml.SafeLoader) + self.check(cfg) + dict.__init__(self, cfg.items()) + + def check(self, cfg ): + unknown_keys= set() + for k,v in cfg.items(): + if isinstance(v,dict): + for k2, v2 in v.items(): + if not f"{k}.{k2}" in Config._valid_keys: + unknown_keys.add( f"{k}.{k2}" ) + else: + if not k in Config._valid_keys: + unknown_keys.add( k ) + if unknown_keys: + raise KeyError(f"Unknown keys: {','.join(unknown_keys)} in config") + + +if __name__ == "__main__": + # run as config file checker... + cf = Config(sys.argv[1]) diff --git a/declad/config_sample.yaml b/declad/config_sample.yaml index 23ea03c..dad1436 100644 --- a/declad/config_sample.yaml +++ b/declad/config_sample.yaml @@ -5,17 +5,21 @@ scanner: filename_pattern: "*.hdf5" # meta_suffix: .json # optional ls_command_template: "xrdfs $server ls -l $location" # $server and $location will be replaced in run time - parse_re: "^(?P[a-z-])\S+\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}\s+(?P\d+)\s+(?P\S+)$" + parse_re: "^(?P[a-z-])\\S+\\s+\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\s+(?P\\d+)\\s+(?P\\S+)$" timeout: 30 # seconds log: logs/declad.log max_movers: 10 # default 10 -queue_capacity: 100 # stop scanning for input files if the queue is backlogged. default - no limit retry_cooldown: 3600 # file retry interval keep_interval: 86400 # interval to keep file processing log in memory metacat_dataset: dune:all # namespace:name -rucio_dataset_did_template: "%(run_type)s:%(run_type)s_%(run_number)s" # Python %-operation template, applied to the file metadata dict +rucio: + dataset_did_template: "%(run_type)s:%(run_type)s_%(run_number)s" # Python %-operation template, applied to the file metadata dict + target_rses: # RSEs to create replication rule to + - FNAL_DCACHE + - CERN_CASTOR + drop_rse: CERN_EOS # Rucio name of the RSE where the data arrives being copied from the dropbox source_server: root://dropbox.host:port destination_server: root://dest.host:port @@ -25,18 +29,14 @@ create_dirs_command_template: "xrdfs $server mkdir -p $path" copy_command_template: "xrdcp --force --silent --tpc $src_url $dst_url" quarantine_location: /eos/experiment/neutplatform/protodune/dune/ivm/quarantine -drop_rse: CERN_EOS # Rucio name of the RSE where the data arrives being copied from the dropbox -replication_targets: # RSEs to create replication rule to - - FNAL_DCACHE - - CERN_CASTOR transfer_timeout: 120 # default 120 keep_interval: 86400 # default 24 hours metacat_url: https://metacat.fnal.gov:9443/dune_meta_demo/app/data -rucio_url: https://dune-rucio.fnal.gov -samweb_url: https://samweb.fnal.gov:8483/sam/dune/api # optionel. If omited, do not declare to SAM +samweb: + url: https://samweb.fnal.gov:8483/sam/dune/api # optional. If omited, do not declare to SAM web_gui: port: 8080 diff --git a/declad/dune_config.yaml b/declad/dune_config.yaml index 70a0c46..090192e 100644 --- a/declad/dune_config.yaml +++ b/declad/dune_config.yaml @@ -57,7 +57,6 @@ download_command_template: "xrdcp --force --silent root://$server/$src_path quarantine_location: /dune/scratch/dunepro/ingest/quarantine metacat_url: https://metacat.fnal.gov:9443/dune_meta_prod/app -rucio_url: https://dune-rucio.fnal.gov sam_location_template: dcache:/pnfs/dune/persistent/staging/$dst_rel_dir diff --git a/declad/version.py b/declad/version.py index 86ee6fc..3947335 100644 --- a/declad/version.py +++ b/declad/version.py @@ -1,3 +1,3 @@ # will be overwritten by Makefile -Version="2.1.1" +Version="2.2.0"