DL0 Data Products#
Examples of using Product for DL0 (raw archived) data product metadata.
import json
import textwrap
import uuid
from pathlib import Path
import ctao_datamodel as dm
import ctao_datamodel.models.dataproducts as dp
import yaml
from astropy import units as u
from astropy.coordinates import EarthLocation, SkyCoord
from astropy.table import Table
from astropy.time import Time
from astropydantic import AstroPydanticICRS, AstroPydanticQuantity, AstroPydanticTime
from IPython.display import HTML
Some common DL0 info#
For these examples, we will make some common metadata
EXAMPLES = []
curation = dp.Curation(
release="CTAO/DL0",
reference="https://unknown",
license="cc-by-sa",
license_url="https://creativecommons.org/licenses/by-sa/4.0/",
copyright="CTAO ERIC",
rights=dp.DataRights.SECURE,
)
model = dp.DataModel(
name="CTAO",
version="v1.0",
url="https://docs.ctao.org/data-model/dl0",
)
contact = dp.Contact(
name="CTAO HelpDesk", # for DL0, do we have another contact?
email="help@ctao.org",
organization="CTAO",
)
adh_activity = dp.Activity(
process=dp.ObservatoryProcess.ACQUISITION,
configuration_id="ACADA-Something?", # the ACADA config ID?
name="ACADA",
description="Acquisition of observtion data",
id=uuid.uuid1(),
start=Time("2025-11-28 13:45:12.62"),
software=dp.Software(
name="ACADA-ADH",
version="v0.1.2",
url="https://some/ref",
),
)
Examples:#
DL0/Event/Telescope/AirShower#
TARGET_DIR = SkyCoord.from_name("Crab")
WOBBLE_OFFSET = 0.9 * u.deg
POINTING_DIR = SkyCoord(
ra=TARGET_DIR.ra, dec=TARGET_DIR.dec + WOBBLE_OFFSET, frame=TARGET_DIR.frame
)
EPOCH = Time(0, format="mjd", scale="utc") # MJD epoch
T_START = (Time("2022-05-10T03:18:01.00") - EPOCH).tai.to_value(u.s)
T_STOP = (Time("2022-05-10T03:18:01.00") - EPOCH + 30 * u.min).tai.to_value(u.s)
LOCATION = EarthLocation.of_site(
"Roque de los Muchachos"
) # should use real CTAO North coords
/usr/local/lib/python3.13/site-packages/erfa/core.py:133: ErfaWarning: ERFA function "utctai" yielded 1 of "dubious year (Note 3)"
warn(f'ERFA function "{func_name}" yielded {wmsg}', ErfaWarning)
dl0_shower = dp.Product(
description="Raw Events from a single telescope, chunked.",
creation_time=Time.now(),
curation=curation,
data=dp.ProductType(
level=dp.DataLevel.DL0,
division=dp.DataDivision.EVENT,
association=dp.DataAssociation.TELESCOPE,
type=dp.DataType.AIRSHOWER,
),
instance=dp.InstanceIdentifier(
obs_id=1000012345,
ae_id=2,
ae_class="TEL",
subarray_id=5,
chunk_id=3,
site_id=dm.models.common.SiteID.CTAO_NORTH, # optional, as it is linked to the obs_id
data_source=dp.ACADADataSource(component="SDH", id=1),
# observing_period_id="2027", linked to obs_id, so needed?
observing_night="2022-05-09",
# calibration_service_id # should maybe be context here, not identifier?
),
model=model,
contact=contact,
activity=adh_activity,
acquisition=dp.Acquisition(
sb_id=1029394,
),
)
EXAMPLES.append(dl0_shower)
DL0/Event/Telescope/MuonCandidate#
dl0_muon = dp.Product(
description="Raw Muon candidate events from a single telescope, chunked.",
creation_time=Time.now(),
curation=curation,
data=dp.ProductType(
level=dp.DataLevel.DL0,
division=dp.DataDivision.EVENT,
association=dp.DataAssociation.TELESCOPE,
type=dp.DataType.MUON_CANDIDATE,
),
instance=dp.InstanceIdentifier(
obs_id=1000012345,
observing_night="2022-05-09",
ae_id=2,
ae_class="TEL",
subarray_id=5,
chunk_id=1,
site_id=dm.models.common.SiteID.CTAO_NORTH, # optional, as it is linked to the obs_id
data_source=dp.ACADADataSource(component="SDH", id=1),
),
model=model,
contact=contact,
activity=adh_activity,
acquisition=dp.Acquisition(sb_id=1029394),
)
EXAMPLES.append(dl0_muon)
DL0/Event/Subarray/Trigger (SWAT)#
dl0_swat = dp.Product(
description="Raw data from SWAT",
creation_time=Time.now(),
curation=curation,
data=dp.ProductType(
level=dp.DataLevel.DL0,
division=dp.DataDivision.EVENT,
association=dp.DataAssociation.SUBARRAY,
type=dp.DataType.TRIGGER,
),
instance=dp.InstanceIdentifier(
obs_id=1000012345,
subarray_id=5,
chunk_id=1,
site_id=dm.models.common.SiteID.CTAO_NORTH, # optional, as it is linked to the obs_id
data_source=dp.ACADADataSource(component="SWAT", id=1),
observing_night="2022-05-09",
),
model=model,
contact=contact,
activity=adh_activity,
acquisition=dp.Acquisition(sb_id=1029394),
)
EXAMPLES.append(dl0_swat)
Output in flattened JSON format#
import json
for example in EXAMPLES:
flat = dm.flatten_model_instance(example)
print(f"====== {example.data} =======")
print()
print(json.dumps(dm.flatten_model_instance(example), indent=2))
print()
====== DL0/Event/Telescope/AirShower =======
{
"ctao_metadata_version": "1.0.1.dev4+g2af9abbfb",
"description": "Raw Events from a single telescope, chunked.",
"data.level": "DL0",
"data.division": "Event",
"data.association": "Telescope",
"data.type": "AirShower",
"instance.id": "c48fa2dd-4527-41f5-8a85-9b406912c025",
"instance.obs_id": 1000012345,
"instance.ae_id": 2,
"instance.ae_class": "TEL",
"instance.subarray_id": 5,
"instance.chunk_id": 3,
"instance.observing_night": "2022-05-09",
"instance.facility_name": "CTAO",
"instance.site_id": "CTAO-North",
"instance.data_source": "SDH001",
"curation.release": "CTAO/DL0",
"curation.reference": "https://unknown/",
"curation.license": "cc-by-sa",
"curation.license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
"curation.copyright": "CTAO ERIC",
"curation.rights": "secure",
"model.name": "CTAO",
"model.version": "v1.0",
"model.url": "https://docs.ctao.org/data-model/dl0",
"creation_time": "2026-04-16T10:04:58.322968000",
"contact.name": "CTAO HelpDesk",
"contact.organization": "CTAO",
"contact.email": "help@ctao.org",
"activity.process": "acquisition",
"activity.name": "ACADA",
"activity.description": "Acquisition of observtion data",
"activity.id": "b939bf1e-397b-11f1-b275-12a8ba823d72",
"activity.start": "2025-11-28T13:45:12.620000000",
"activity.software.name": "ACADA-ADH",
"activity.software.version": "v0.1.2",
"activity.software.url": "https://some/ref",
"activity.configuration_id": "ACADA-Something?",
"acquisition.sb_id": 1029394
}
====== DL0/Event/Telescope/MuonCandidate =======
{
"ctao_metadata_version": "1.0.1.dev4+g2af9abbfb",
"description": "Raw Muon candidate events from a single telescope, chunked.",
"data.level": "DL0",
"data.division": "Event",
"data.association": "Telescope",
"data.type": "MuonCandidate",
"instance.id": "8deb4d73-1f0e-48db-a4a3-871332ff84e6",
"instance.obs_id": 1000012345,
"instance.ae_id": 2,
"instance.ae_class": "TEL",
"instance.subarray_id": 5,
"instance.chunk_id": 1,
"instance.observing_night": "2022-05-09",
"instance.facility_name": "CTAO",
"instance.site_id": "CTAO-North",
"instance.data_source": "SDH001",
"curation.release": "CTAO/DL0",
"curation.reference": "https://unknown/",
"curation.license": "cc-by-sa",
"curation.license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
"curation.copyright": "CTAO ERIC",
"curation.rights": "secure",
"model.name": "CTAO",
"model.version": "v1.0",
"model.url": "https://docs.ctao.org/data-model/dl0",
"creation_time": "2026-04-16T10:04:58.329073000",
"contact.name": "CTAO HelpDesk",
"contact.organization": "CTAO",
"contact.email": "help@ctao.org",
"activity.process": "acquisition",
"activity.name": "ACADA",
"activity.description": "Acquisition of observtion data",
"activity.id": "b939bf1e-397b-11f1-b275-12a8ba823d72",
"activity.start": "2025-11-28T13:45:12.620000000",
"activity.software.name": "ACADA-ADH",
"activity.software.version": "v0.1.2",
"activity.software.url": "https://some/ref",
"activity.configuration_id": "ACADA-Something?",
"acquisition.sb_id": 1029394
}
====== DL0/Event/Subarray/Trigger =======
{
"ctao_metadata_version": "1.0.1.dev4+g2af9abbfb",
"description": "Raw data from SWAT",
"data.level": "DL0",
"data.division": "Event",
"data.association": "Subarray",
"data.type": "Trigger",
"instance.id": "a62f222f-b851-4f86-ae94-c42c7edebf5d",
"instance.obs_id": 1000012345,
"instance.subarray_id": 5,
"instance.chunk_id": 1,
"instance.observing_night": "2022-05-09",
"instance.facility_name": "CTAO",
"instance.site_id": "CTAO-North",
"instance.data_source": "SWAT001",
"curation.release": "CTAO/DL0",
"curation.reference": "https://unknown/",
"curation.license": "cc-by-sa",
"curation.license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
"curation.copyright": "CTAO ERIC",
"curation.rights": "secure",
"model.name": "CTAO",
"model.version": "v1.0",
"model.url": "https://docs.ctao.org/data-model/dl0",
"creation_time": "2026-04-16T10:04:58.334417000",
"contact.name": "CTAO HelpDesk",
"contact.organization": "CTAO",
"contact.email": "help@ctao.org",
"activity.process": "acquisition",
"activity.name": "ACADA",
"activity.description": "Acquisition of observtion data",
"activity.id": "b939bf1e-397b-11f1-b275-12a8ba823d72",
"activity.start": "2025-11-28T13:45:12.620000000",
"activity.software.name": "ACADA-ADH",
"activity.software.version": "v0.1.2",
"activity.software.url": "https://some/ref",
"activity.configuration_id": "ACADA-Something?",
"acquisition.sb_id": 1029394
}
Output as FITS Headers#
for ii, example in enumerate(EXAMPLES):
header = dm.instance_to_fits_header(example)
print("=" * 70)
print(f"Example {ii}: {example.data} ({len(header)} keys)")
print(
textwrap.fill(
example.description, initial_indent=" ", subsequent_indent=" "
)
)
print("=" * 70)
print()
print(repr(header))
print()
======================================================================
Example 0: DL0/Event/Telescope/AirShower (39 keys)
Raw Events from a single telescope, chunked.
======================================================================
CTAOMETA= '1.0.1.dev4+g2af9abbfb' / CTAO DataProducts Metadata Version
TITLE = 'Raw Events from a single telescope, chunked.' / Human-readable descri
DATALEVL= 'DL0 ' / CTAO Data Level, see Top-Level Data Model. Opti
DATADIV = 'Event ' / Primary data type. See the CTAO Top-level Data
DATAASSO= 'Telescope' / The main associated instrument or analysis part
DATATYPE= 'AirShower' / The specific type of the product. This is used
DATAID = 'c48fa2dd-4527-41f5-8a85-9b406912c025' / A locally-generated unique ID
OBS_ID = 1000012345 / Unique identifier of the observation block, in
AE_ID = 2 / ID of a CTAO array element. See common data mod
AE_CLASS= 'TEL ' / Classification of Array Element, following ACAD
SUB_ID = 5 / Subarray id, for data products from a subarray.
CHUNK_ID= 3 / For files that are split into multiple pieces (
OBSNIGHT= '2022-05-09' / Date associated with the start of data taking.
TELESCOP= 'CTAO ' / Observatory or facility used to collect the dat
INSTRUME= 'CTAO-North' / CTAO site associated with this instance.
DATASRC = 'SDH001 ' / ACADA Data Source, see ACADA-DPPS ICD.
RELEASE = 'CTAO/DL0' / Name of the data release (data collection) that
REFERENC= 'https://unknown/' / a reference to where the data product is publis
LICENSE = 'cc-by-sa' / License for this data product
LICENURL= 'https://creativecommons.org/licenses/by-sa/4.0/' / URL pointing to th
COPYRIGH= 'CTAO ERIC' / Copyright holder(s) of this data product
RIGHTS = 'secure ' / Availability of the data product, if known at t
MODEL = 'CTAO ' / Name of the overall data model, which may conta
MODELVER= 'v1.0 ' / Version number of the data model
MODELURL= 'https://docs.ctao.org/data-model/dl0' / URL or DOI linking to more de
CREATED = '2026-04-16T10:04:58.322968000' / UTC Date-time the data product was c
AUTHOR = 'CTAO HelpDesk' / Contact name for this data product.
ORIGIN = 'CTAO ' / Contact organization name of this data product.
EMAIL = 'help@ctao.org' / Contact's email address
ACTPROC = 'acquisition' / Observatory operational Process. These are the
ACTIVITY= 'ACADA ' / Name of the activity that produced this data pr
ACTDESC = 'Acquisition of observtion data' / Human-readable details of the activ
ACTID = 'b939bf1e-397b-11f1-b275-12a8ba823d72' / Unique identifier of this pro
ACTSTART= '2025-11-28T13:45:12.620000000' / Start time of the activity
SOFTWARE= 'ACADA-ADH' / Descriptive name of the software.
SOFTVER = 'v0.1.2 ' / Version number
SOFTURL = 'https://some/ref' / URL or DOI linking to more detail.
ANAMODE = 'ACADA-Something?' / Identifier for the configuration for the softwa
SB_ID = 1029394 / Unique identifier of the scheduling block assoc
======================================================================
Example 1: DL0/Event/Telescope/MuonCandidate (39 keys)
Raw Muon candidate events from a single telescope, chunked.
======================================================================
CTAOMETA= '1.0.1.dev4+g2af9abbfb' / CTAO DataProducts Metadata Version
TITLE = 'Raw Muon candidate events from a single telescope, chunked.' / Human-
DATALEVL= 'DL0 ' / CTAO Data Level, see Top-Level Data Model. Opti
DATADIV = 'Event ' / Primary data type. See the CTAO Top-level Data
DATAASSO= 'Telescope' / The main associated instrument or analysis part
DATATYPE= 'MuonCandidate' / The specific type of the product. This is used
DATAID = '8deb4d73-1f0e-48db-a4a3-871332ff84e6' / A locally-generated unique ID
OBS_ID = 1000012345 / Unique identifier of the observation block, in
AE_ID = 2 / ID of a CTAO array element. See common data mod
AE_CLASS= 'TEL ' / Classification of Array Element, following ACAD
SUB_ID = 5 / Subarray id, for data products from a subarray.
CHUNK_ID= 1 / For files that are split into multiple pieces (
OBSNIGHT= '2022-05-09' / Date associated with the start of data taking.
TELESCOP= 'CTAO ' / Observatory or facility used to collect the dat
INSTRUME= 'CTAO-North' / CTAO site associated with this instance.
DATASRC = 'SDH001 ' / ACADA Data Source, see ACADA-DPPS ICD.
RELEASE = 'CTAO/DL0' / Name of the data release (data collection) that
REFERENC= 'https://unknown/' / a reference to where the data product is publis
LICENSE = 'cc-by-sa' / License for this data product
LICENURL= 'https://creativecommons.org/licenses/by-sa/4.0/' / URL pointing to th
COPYRIGH= 'CTAO ERIC' / Copyright holder(s) of this data product
RIGHTS = 'secure ' / Availability of the data product, if known at t
MODEL = 'CTAO ' / Name of the overall data model, which may conta
MODELVER= 'v1.0 ' / Version number of the data model
MODELURL= 'https://docs.ctao.org/data-model/dl0' / URL or DOI linking to more de
CREATED = '2026-04-16T10:04:58.329073000' / UTC Date-time the data product was c
AUTHOR = 'CTAO HelpDesk' / Contact name for this data product.
ORIGIN = 'CTAO ' / Contact organization name of this data product.
EMAIL = 'help@ctao.org' / Contact's email address
ACTPROC = 'acquisition' / Observatory operational Process. These are the
ACTIVITY= 'ACADA ' / Name of the activity that produced this data pr
ACTDESC = 'Acquisition of observtion data' / Human-readable details of the activ
ACTID = 'b939bf1e-397b-11f1-b275-12a8ba823d72' / Unique identifier of this pro
ACTSTART= '2025-11-28T13:45:12.620000000' / Start time of the activity
SOFTWARE= 'ACADA-ADH' / Descriptive name of the software.
SOFTVER = 'v0.1.2 ' / Version number
SOFTURL = 'https://some/ref' / URL or DOI linking to more detail.
ANAMODE = 'ACADA-Something?' / Identifier for the configuration for the softwa
SB_ID = 1029394 / Unique identifier of the scheduling block assoc
======================================================================
Example 2: DL0/Event/Subarray/Trigger (37 keys)
Raw data from SWAT
======================================================================
CTAOMETA= '1.0.1.dev4+g2af9abbfb' / CTAO DataProducts Metadata Version
TITLE = 'Raw data from SWAT' / Human-readable description of this data product
DATALEVL= 'DL0 ' / CTAO Data Level, see Top-Level Data Model. Opti
DATADIV = 'Event ' / Primary data type. See the CTAO Top-level Data
DATAASSO= 'Subarray' / The main associated instrument or analysis part
DATATYPE= 'Trigger ' / The specific type of the product. This is used
DATAID = 'a62f222f-b851-4f86-ae94-c42c7edebf5d' / A locally-generated unique ID
OBS_ID = 1000012345 / Unique identifier of the observation block, in
SUB_ID = 5 / Subarray id, for data products from a subarray.
CHUNK_ID= 1 / For files that are split into multiple pieces (
OBSNIGHT= '2022-05-09' / Date associated with the start of data taking.
TELESCOP= 'CTAO ' / Observatory or facility used to collect the dat
INSTRUME= 'CTAO-North' / CTAO site associated with this instance.
DATASRC = 'SWAT001 ' / ACADA Data Source, see ACADA-DPPS ICD.
RELEASE = 'CTAO/DL0' / Name of the data release (data collection) that
REFERENC= 'https://unknown/' / a reference to where the data product is publis
LICENSE = 'cc-by-sa' / License for this data product
LICENURL= 'https://creativecommons.org/licenses/by-sa/4.0/' / URL pointing to th
COPYRIGH= 'CTAO ERIC' / Copyright holder(s) of this data product
RIGHTS = 'secure ' / Availability of the data product, if known at t
MODEL = 'CTAO ' / Name of the overall data model, which may conta
MODELVER= 'v1.0 ' / Version number of the data model
MODELURL= 'https://docs.ctao.org/data-model/dl0' / URL or DOI linking to more de
CREATED = '2026-04-16T10:04:58.334417000' / UTC Date-time the data product was c
AUTHOR = 'CTAO HelpDesk' / Contact name for this data product.
ORIGIN = 'CTAO ' / Contact organization name of this data product.
EMAIL = 'help@ctao.org' / Contact's email address
ACTPROC = 'acquisition' / Observatory operational Process. These are the
ACTIVITY= 'ACADA ' / Name of the activity that produced this data pr
ACTDESC = 'Acquisition of observtion data' / Human-readable details of the activ
ACTID = 'b939bf1e-397b-11f1-b275-12a8ba823d72' / Unique identifier of this pro
ACTSTART= '2025-11-28T13:45:12.620000000' / Start time of the activity
SOFTWARE= 'ACADA-ADH' / Descriptive name of the software.
SOFTVER = 'v0.1.2 ' / Version number
SOFTURL = 'https://some/ref' / URL or DOI linking to more detail.
ANAMODE = 'ACADA-Something?' / Identifier for the configuration for the softwa
SB_ID = 1029394 / Unique identifier of the scheduling block assoc
WARNING: VerifyWarning: Card is too long, comment will be truncated. [astropy.io.fits.card]
Convert FITS header back to model instance#
p = dm.fits_header_to_instance(header, dp.Product)
p
Product(ctao_metadata_version='1.0.1.dev4+g2af9abbfb', description='Raw data from SWAT', data=ProductType(level=<DataLevel.DL0: 'DL0'>, division=<DataDivision.EVENT: 'Event'>, association=<DataAssociation.SUBARRAY: 'Subarray'>, type=<DataType.TRIGGER: 'Trigger'>), instance=InstanceIdentifier(id=UUID('a62f222f-b851-4f86-ae94-c42c7edebf5d'), obs_id=1000012345, event_type=None, ae_id=None, ae_class=None, subarray_id=5, chunk_id=1, batch_id=None, calibration_service_id=None, observing_night=datetime.date(2022, 5, 9), sublevel_id=None, target_id=None, region_id=None, observing_period_id=None, lunar_cycle_id=None, facility_name=<FacilityName.CTAO: 'CTAO'>, site_id=<SiteID.CTAO_NORTH: 'CTAO-North'>, particle_pdgid=None, category=None, data_source=ACADADataSource(component='SWAT', id=1), assembly_name=None), curation=Curation(release='CTAO/DL0', reference=AnyUrl('https://unknown/'), license='cc-by-sa', license_url='https://creativecommons.org/licenses/by-sa/4.0/', copyright='CTAO ERIC', rights=<DataRights.SECURE: 'secure'>, release_date=None, valid_from=None, valid_to=None), model=DataModel(name='CTAO', version='v1.0', url=AnyUrl('https://docs.ctao.org/data-model/dl0')), disclaimer=None, creation_time=<Time object: scale='utc' format='isot' value=2026-04-16T10:04:58.334>, contact=Contact(name='CTAO HelpDesk', organization='CTAO', email='help@ctao.org'), activity=Activity(process=<ObservatoryProcess.ACQUISITION: 'acquisition'>, name='ACADA', description='Acquisition of observtion data', id=UUID('b939bf1e-397b-11f1-b275-12a8ba823d72'), start=<Time object: scale='utc' format='isot' value=2025-11-28T13:45:12.620>, end=None, software=Software(name='ACADA-ADH', version='v0.1.2', url=AnyUrl('https://some/ref')), configuration_id='ACADA-Something?', inputs=None), observation=None, acquisition=Acquisition(sb_id=1029394), sdc=None)
p.creation_time
<Time object: scale='utc' format='isot' value=2026-04-16T10:04:58.334>
p.instance.observing_night
datetime.date(2022, 5, 9)
Mapping to filename scheme in the ACADA-DPPS ICD#
see page 17 of https://plm-extern.desy.de/#/com.siemens.splm.clientfx.tcui.xrt.showObject?uid=B0tlI15bKPLEjD
Table(
{
"ProductType": [e.data for e in EXAMPLES],
"ACADA Filename": [dp.utils.acada_filename(e) for e in EXAMPLES],
}
)
Table length=3
| ProductType | ACADA Filename |
|---|---|
| object | str88 |
| DL0/Event/Telescope/AirShower | TEL002_SDH001_20260416T100458_SBID0001029394_OBSID1000012345_TEL_SHOWER_CHUNK003.fits.fz |
| DL0/Event/Telescope/MuonCandidate | TEL002_SDH001_20260416T100458_SBID0001029394_OBSID1000012345_TEL_MUON_CHUNK001.fits.fz |
| DL0/Event/Subarray/Trigger | SUB005_SWAT001_20260416T100458_SBID0001029394_OBSID1000012345_SUB_CHUNK001.fits.fz |