metadata.yamlΒΆ
set in config
metadata:
fixes: # Fix common issues with the metadata
keys: # Misspelled keys
"DUNE.requestid": "dune.requestid"
missing: # Default values for missing keys
"core.file_content_status": "good"
"retention.status": "active"
"retention.class": "unknown"
# Misspelled values
"core.data_tier":
"pandora_info": "pandora-info"
"trigger-primitives": "trigprim"
"core.data_stream":
"Test": "test"
"Cosmics": "cosmics"
"pdstl": "pedestal" # Not completely sure about this one
"core.file_format":
"tfile": "root" #
types: # These keys must have the correct types
"core.application.family": 'str'
"core.application.name": 'str'
"core.application.version": 'str'
"core.data_stream": 'str'
"core.data_tier": 'str'
"core.end_time": 'float'
"core.event_count": 'int'
"core.events": 'list'
"core.file_content_status": 'str'
"core.file_format": 'str'
"core.file_type": 'str'
"core.first_event_number": 'int'
"core.last_event_number": 'int'
"core.run_type": 'str'
"core.runs": 'list'
"core.runs_subruns": 'list'
"core.start_time": 'float'
"dune.daq_test": 'bool'
"dune.config_file": 'str'
"dune_mc.gen_fcl_filename": 'str'
"dune_mc.geometry_version": 'str'
"retention.status": 'str'
"retention.class": 'str'
restricted: # These keys must have a valid value
"core.run_type":
- "test"
- "protodune-sp"
- "protodune-dp"
- "iceberg"
- "np04_vst"
- "hd-coldbox"
- "vd-coldbox"
- "vd-coldbox-bottom"
- "vd-coldbox-top"
- "protodune-hd" # old script
- "hd-protodune"
- "vd-protodune"
- "vd-protodune-pds"
- "dc4-vd-coldbox-bottom"
- "dc4-vd-coldbox-top"
- "dc4-hd-protodune" # old script
- "neardet"
- "neardet-lar"
- "neardet-2x2" # old script
- "neardet-2x2-lar" # old script
- "neardet-2x2-lar-charge"
- "neardet-2x2-lar-light"
- "neardet-2x2-minerva" # old script
- "fardet" # old script
- "fardet-sp" # old script
- "fardet-hd" # old script
- "fardet-vd" # old script
- "311" # old script
- "311_dp_light" # old script
# Other keys from metacat
- "physics"
- "ehn1-beam-np04"
# - "justin-tutorial"
"core.file_type":
- "mc"
- "detector"
- "importedDetector" # old script, minerva
# Other keys from metacat
# - "unknown"
# - "test-data"
# - "binary"
# - "photon_detector"
# - "test"
# - "dc1simdaq"
# - "data"
"core.data_stream":
- "test"
- "out1"
- "out2"
- "noise"
- "cosmics"
- "calibration"
- "physics"
- "commissioning"
- "pedestal"
- "study"
- "trigprim"
- "pdstl"
- "linjc"
- "numib"
- "numip"
- "numil"
# Other keys from metacat
- "g4beamline"
"core.data_tier":
- "simulated"
- "raw"
- "hit-reconstructed"
- "full-reconstructed"
- "generated"
- "detector-simulated"
- "root-tuple"
- "root-hist"
- "decoded-raw"
- "pandora-info"
- "reco-recalibrated"
- "root-tuple-virtual"
- "trigprim"
- "sliced" # old script
- "dc1input" # old script
- "binary-raw" # old script
- "sam-user" # old script
# Other keys from metacat
# - "test"
# - "stage1"
# - "stage2"
"core.file_format":
- "root"
- "artroot"
- "binary"
- "rootntuple"
- "hdf5"
- "tar"
# - "unknown"
"dune.output_status":
- "confirmed"
required: # These keys are required for all files
- "core.data_stream"
- "core.data_tier"
# - "core.end_time"
- "core.file_content_status"
- "core.file_format"
- "core.file_type"
- "core.run_type"
- "core.runs"
# - "core.start_time"
- "retention.status"
- "retention.class"
conditional: # These keys are required for some file types
"'{core.file_type}' == 'mc'":
- "dune_mc.gen_fcl_filename"
- "dune_mc.geometry_version"
"'{core.data_tier}' == 'raw'": &raw_requirements
- "core.event_count"
- "core.first_event_number"
- "core.last_event_number"
"'{core.data_tier}' == 'binary-raw'": *raw_requirements
"'{core.data_tier}' == 'trigprim'": *raw_requirements
"'{core.data_tier}' == 'root-tuple'": &tuple_requirements
- "dune.config_file"
- "core.application.family"
- "core.application.name"
- "core.application.version"
"'{core.data_tier}' == 'root-tuple-virtual'": *tuple_requirements
optional: [] # These metadata keys are optional (overrides required and conditional keys)
consistent: # These metadata keys must be the same for all input files
- "core.run_type"
- "core.file_type"
- "core.file_format"
- "core.data_tier"
- "core.data_stream"
- "core.application.name"
- "dune.campaign"
- "dune.requestid"
- "dune.config_file" # Not checked in old merging
- "core.application.version" # Not checked in old merging
- "dune.output_status"
# - "core.runs"
merging: # Special merging behavior for metadata keys, options are:
# unique only save key if all values are the same
# all save a list of all values
# min save the minimum value
# max save the maximum value
# sum save the sum of all values
# union save the union of all values
# skip ignore the key
default: unique
"core.first_event_number": min
"core.last_event_number": max
"core.event_count": sum
"core.events": union
"core.runs": union
"core.runs_subruns": union
"dune.workflow": subset
# Skip values that don't make sense to merge
"core.start_time": skip
"core.end_time": skip
"Offline.options": skip
"Offline.machine": skip
overrides: # Set keys to specific values
"retention.status": "active"
abbreviations: # Try to shorten long output file names
"extensions": # Automatically remove these extensions from all keys
- "fcl"
- "json"
- "yaml"
- "root"
"substitutions": # Substitute these values in all keys
", ": ","
".": "-"
" ": "_"
"/": "-"
"core.run_type":
"protodune-sp": "pd-sp"
"protodune-dp": "pd-dp"
"hd-coldbox": "cb-hd"
"vd-coldbox": "cb-vd"
"vd-coldbox-bottom": "cb-vd-b"
"vd-coldbox-top": "cb-vd-t"
"protodune-hd": "pd-hd"
"hd-protodune": "pd-hd"
"vd-protodune": "pd-vd"
"vd-protodune-pds": "pd-vd-pds"
"vd-protodune-arapucas": "pd-vd-arapucas"
"dc4-vd-coldbox-bottom": "cb-dc4-vd-b"
"dc4-vd-coldbox-top": "cb-dc4-vd-t"
"dc4-hd-protodune": "pd-dc4-hd"
"neardet": "nd"
"neardet-lar": "nd-lar"
"neardet-2x2": "nd-2x2"
"neardet-2x2-lar": "nd-2x2-lar"
"neardet-2x2-lar-charge": "nd-2x2-lar-charge"
"neardet-2x2-lar-light": "nd-2x2-lar-light"
"neardet-2x2-minerva": "nd-2x2-minerva"
"fardet": "fd"
"fardet-sp": "fd-sp"
"fardet-hd": "fd-hd"
"fardet-vd": "fd-vd"
"fardet-dp": "fd-dp"
"fardet-moo": "fd-moo"
"311_dp_light": "311-dp-light"
"physics": "phys"
"protodune": "pd"
"protodune-vst": "pd-vst"
"core.file_type":
"detector": "det"
"importedDetector": "imp-det"
"binary": "bin"
"photon_detector": "photo-det"
"core.data_stream":
"calibration": "calib"
"physics": "phys"
"commissioning": "comm"
"pedestal": "pdstl"
"g4beamline": "g4"
"core.data_tier":
"simulated": "sim"
"hit-reconstructed": "hit-reco"
"full-reconstructed": "full-reco"
"generated": "gen"
"detector-simulated": "det-sim"
"root-tuple": "r-tuple"
"root-hist": "r-hist"
"decoded-raw": "dec-raw"
"pandora-info": "pandora"
"reco-recalibrated": "reco-recal"
"root-tuple-virtual": "r-tuple-v"
"binary-raw": "binary"
"sam-user": "sam"
"core.file_format":
"artroot": "art"
"binary": "bin"
"rootntuple": "tuple"