{ "cells": [ { "cell_type": "markdown", "id": "e73b14a7-719e-4e57-93bd-a6e0424efc4a", "metadata": { "editable": true, "slideshow": { "slide_type": "" }, "tags": [ "hide-cell" ] }, "source": [ "```{eval-rst}\n", ".. currentmodule:: ctao_datamodel\n", "```" ] }, { "cell_type": "markdown", "id": "ab5b8404-25f8-4af5-a383-51a2b57339c6", "metadata": {}, "source": [ "# DL0 Data Products\n", "\n", "Examples of using {py:class}`~models.dataproducts.Product` for **DL0** (raw archived) data product metadata." ] }, { "cell_type": "code", "execution_count": null, "id": "dcb3ad51-330d-4727-88f3-c86ff55b469a", "metadata": {}, "outputs": [], "source": [ "import json\n", "import textwrap\n", "import uuid\n", "from pathlib import Path\n", "\n", "import ctao_datamodel as dm\n", "import ctao_datamodel.models.dataproducts as dp\n", "import yaml\n", "from astropy import units as u\n", "from astropy.coordinates import EarthLocation, SkyCoord\n", "from astropy.table import Table\n", "from astropy.time import Time\n", "from astropydantic import AstroPydanticICRS, AstroPydanticQuantity, AstroPydanticTime\n", "from IPython.display import HTML" ] }, { "cell_type": "markdown", "id": "aa2d6a52-6ec4-4f1c-896f-e59f166f7fa2", "metadata": {}, "source": [ "## Some common DL0 info\n", "\n", "For these examples, we will make some common metadata" ] }, { "cell_type": "code", "execution_count": null, "id": "79fcc652-eef3-4787-8941-389fb8586da5", "metadata": {}, "outputs": [], "source": [ "EXAMPLES = []\n", "\n", "curation = dp.Curation(\n", " release=\"CTAO/DL0\",\n", " reference=\"https://unknown\",\n", " license=\"cc-by-sa\",\n", " license_url=\"https://creativecommons.org/licenses/by-sa/4.0/\",\n", " copyright=\"CTAO ERIC\",\n", " rights=dp.DataRights.SECURE,\n", ")\n", "model = dp.DataModel(\n", " name=\"CTAO\",\n", " version=\"v1.0\",\n", " url=\"https://docs.ctao.org/data-model/dl0\",\n", ")\n", "contact = dp.Contact(\n", " name=\"CTAO HelpDesk\", # for DL0, do we have another contact?\n", " email=\"help@ctao.org\",\n", " organization=\"CTAO\",\n", ")\n", "adh_activity = dp.Activity(\n", " process=dp.ObservatoryProcess.ACQUISITION,\n", " configuration_id=\"ACADA-Something?\", # the ACADA config ID?\n", " name=\"ACADA\",\n", " description=\"Acquisition of observtion data\",\n", " id=uuid.uuid1(),\n", " start=Time(\"2025-11-28 13:45:12.62\"),\n", " software=dp.Software(\n", " name=\"ACADA-ADH\",\n", " version=\"v0.1.2\",\n", " url=\"https://some/ref\",\n", " ),\n", ")" ] }, { "cell_type": "markdown", "id": "20ff1361-c8a8-4a64-acd9-19c10434936e", "metadata": {}, "source": [ "## Examples:\n", "\n", "### DL0/Event/Telescope/AirShower" ] }, { "cell_type": "code", "execution_count": null, "id": "6ee866c3-0096-4f21-8209-66af3cce2066", "metadata": {}, "outputs": [], "source": [ "TARGET_DIR = SkyCoord.from_name(\"Crab\")\n", "WOBBLE_OFFSET = 0.9 * u.deg\n", "POINTING_DIR = SkyCoord(\n", " ra=TARGET_DIR.ra, dec=TARGET_DIR.dec + WOBBLE_OFFSET, frame=TARGET_DIR.frame\n", ")\n", "EPOCH = Time(0, format=\"mjd\", scale=\"utc\") # MJD epoch\n", "T_START = (Time(\"2022-05-10T03:18:01.00\") - EPOCH).tai.to_value(u.s)\n", "T_STOP = (Time(\"2022-05-10T03:18:01.00\") - EPOCH + 30 * u.min).tai.to_value(u.s)\n", "LOCATION = EarthLocation.of_site(\n", " \"Roque de los Muchachos\"\n", ") # should use real CTAO North coords" ] }, { "cell_type": "code", "execution_count": null, "id": "aef585be-1d79-4cd4-a3e6-e04b1fe4a1f7", "metadata": {}, "outputs": [], "source": [ "dl0_shower = dp.Product(\n", " description=\"Raw Events from a single telescope, chunked.\",\n", " creation_time=Time.now(),\n", " curation=curation,\n", " data=dp.ProductType(\n", " level=dp.DataLevel.DL0,\n", " division=dp.DataDivision.EVENT,\n", " association=dp.DataAssociation.TELESCOPE,\n", " type=dp.DataType.AIRSHOWER,\n", " ),\n", " instance=dp.InstanceIdentifier(\n", " obs_id=1000012345,\n", " ae_id=2,\n", " ae_class=\"TEL\",\n", " subarray_id=5,\n", " chunk_id=3,\n", " site_id=dm.models.common.SiteID.CTAO_NORTH, # optional, as it is linked to the obs_id\n", " data_source=dp.ACADADataSource(component=\"SDH\", id=1),\n", " # observing_period_id=\"2027\", linked to obs_id, so needed?\n", " observing_night=\"2022-05-09\", \n", " # calibration_service_id # should maybe be context here, not identifier?\n", " ),\n", " model=model,\n", " contact=contact,\n", " activity=adh_activity,\n", " acquisition=dp.Acquisition(\n", " sb_id=1029394,\n", " ),\n", ")\n", "\n", "EXAMPLES.append(dl0_shower)" ] }, { "cell_type": "markdown", "id": "a980b1af-a3b6-4274-b97f-eff72eab3090", "metadata": {}, "source": [ "### DL0/Event/Telescope/MuonCandidate" ] }, { "cell_type": "code", "execution_count": null, "id": "14e88bb3-8960-4851-99f4-7e7a99603507", "metadata": {}, "outputs": [], "source": [ "dl0_muon = dp.Product(\n", " description=\"Raw Muon candidate events from a single telescope, chunked.\",\n", " creation_time=Time.now(),\n", " curation=curation,\n", " data=dp.ProductType(\n", " level=dp.DataLevel.DL0,\n", " division=dp.DataDivision.EVENT,\n", " association=dp.DataAssociation.TELESCOPE,\n", " type=dp.DataType.MUON_CANDIDATE,\n", " ),\n", " instance=dp.InstanceIdentifier(\n", " obs_id=1000012345,\n", " observing_night=\"2022-05-09\", \n", " ae_id=2,\n", " ae_class=\"TEL\",\n", " subarray_id=5,\n", " chunk_id=1,\n", " site_id=dm.models.common.SiteID.CTAO_NORTH, # optional, as it is linked to the obs_id\n", " data_source=dp.ACADADataSource(component=\"SDH\", id=1),\n", " ),\n", " model=model,\n", " contact=contact,\n", " activity=adh_activity,\n", " acquisition=dp.Acquisition(sb_id=1029394),\n", ")\n", "\n", "EXAMPLES.append(dl0_muon)" ] }, { "cell_type": "markdown", "id": "ca1101ae-bd28-4b5e-a9f2-0015f3199d9d", "metadata": {}, "source": [ "### DL0/Event/Subarray/Trigger (SWAT)" ] }, { "cell_type": "code", "execution_count": null, "id": "b15eeae9-87e6-4237-885e-9cf305262dac", "metadata": {}, "outputs": [], "source": [ "dl0_swat = dp.Product(\n", " description=\"Raw data from SWAT\",\n", " creation_time=Time.now(),\n", " curation=curation,\n", " data=dp.ProductType(\n", " level=dp.DataLevel.DL0,\n", " division=dp.DataDivision.EVENT,\n", " association=dp.DataAssociation.SUBARRAY,\n", " type=dp.DataType.TRIGGER,\n", " ),\n", " instance=dp.InstanceIdentifier(\n", " obs_id=1000012345,\n", " subarray_id=5,\n", " chunk_id=1,\n", " site_id=dm.models.common.SiteID.CTAO_NORTH, # optional, as it is linked to the obs_id\n", " data_source=dp.ACADADataSource(component=\"SWAT\", id=1),\n", " observing_night=\"2022-05-09\", \n", " ),\n", " model=model,\n", " contact=contact,\n", " activity=adh_activity,\n", " acquisition=dp.Acquisition(sb_id=1029394),\n", ")\n", "\n", "EXAMPLES.append(dl0_swat)" ] }, { "cell_type": "markdown", "id": "25d5f1ea-39e5-4aa7-ae67-a69e307f207e", "metadata": {}, "source": [ "## Output in flattened JSON format " ] }, { "cell_type": "code", "execution_count": null, "id": "01e84ffa-47d7-409d-98ec-79171c4d9c58", "metadata": {}, "outputs": [], "source": [ "import json\n", "\n", "for example in EXAMPLES:\n", " flat = dm.flatten_model_instance(example)\n", " print(f\"====== {example.data} =======\")\n", " print()\n", " print(json.dumps(dm.flatten_model_instance(example), indent=2))\n", " print()" ] }, { "cell_type": "markdown", "id": "0091c8b3-4b11-49dd-8417-8eeec2387f89", "metadata": {}, "source": [ "## Output as FITS Headers" ] }, { "cell_type": "code", "execution_count": null, "id": "8ed6e17a-f4b0-4a47-b1cc-7d285d2c2dad", "metadata": {}, "outputs": [], "source": [ "for ii, example in enumerate(EXAMPLES):\n", " header = dm.instance_to_fits_header(example)\n", " print(\"=\" * 70)\n", " print(f\"Example {ii}: {example.data} ({len(header)} keys)\")\n", " print(\n", " textwrap.fill(\n", " example.description, initial_indent=\" \", subsequent_indent=\" \"\n", " )\n", " )\n", " print(\"=\" * 70)\n", "\n", " print()\n", " print(repr(header))\n", " print()" ] }, { "cell_type": "markdown", "id": "8ae4b2c5-ce9b-4306-911b-334fea4519f8", "metadata": {}, "source": [ "## Convert FITS header back to model instance" ] }, { "cell_type": "code", "execution_count": null, "id": "0c429ba7-2f2c-4ff7-8124-4a18d48b10ec", "metadata": {}, "outputs": [], "source": [ "p = dm.fits_header_to_instance(header, dp.Product)\n", "p" ] }, { "cell_type": "code", "execution_count": null, "id": "99af2abb-af9e-47e4-b66b-6960cd86860d", "metadata": {}, "outputs": [], "source": [ "p.creation_time" ] }, { "cell_type": "code", "execution_count": null, "id": "f00aec48-b804-4025-b113-c5bf0d659e27", "metadata": {}, "outputs": [], "source": [ "p.instance.observing_night" ] }, { "cell_type": "markdown", "id": "1b5fc15a-bc40-4b86-8f6c-4a66a33f649a", "metadata": {}, "source": [ "## Mapping to filename scheme in the ACADA-DPPS ICD\n", "\n", "see page 17 of https://plm-extern.desy.de/#/com.siemens.splm.clientfx.tcui.xrt.showObject?uid=B0tlI15bKPLEjD" ] }, { "cell_type": "code", "execution_count": null, "id": "9ed88b84-b7e8-4272-95ca-c571f8f7ae14", "metadata": {}, "outputs": [], "source": [ "Table(\n", " {\n", " \"ProductType\": [e.data for e in EXAMPLES],\n", " \"ACADA Filename\": [dp.utils.acada_filename(e) for e in EXAMPLES],\n", " }\n", ")" ] } ], "metadata": {}, "nbformat": 4, "nbformat_minor": 5 }