{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "240dfac4-b8a1-4964-be6b-46d8f51096f3",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": [
     "hide-cell"
    ]
   },
   "source": [
    "```{eval-rst}\n",
    ".. currentmodule:: ctao_datamodel\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab5b8404-25f8-4af5-a383-51a2b57339c6",
   "metadata": {},
   "source": [
    "# DL3 Data Products\n",
    "\n",
    "Examples of using {py:class}`~models.dataproducts.Product` for **DL3** (science-ready) data product metadata."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcb3ad51-330d-4727-88f3-c86ff55b469a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import textwrap\n",
    "import uuid\n",
    "from pathlib import Path\n",
    "\n",
    "import ctao_datamodel as dm\n",
    "import ctao_datamodel.models.dataproducts as dp\n",
    "import numpy as np\n",
    "import yaml\n",
    "from astropy import units as u\n",
    "from astropy.coordinates import AltAz, EarthLocation, SkyCoord\n",
    "from astropy.table import Table\n",
    "from astropy.time import Time\n",
    "from astropydantic import AstroPydanticICRS, AstroPydanticQuantity, AstroPydanticTime\n",
    "from IPython.display import HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4bf2c0ff-2efe-47c4-9d09-b10e43903635",
   "metadata": {},
   "outputs": [],
   "source": [
    "EXAMPLES = []"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20ff1361-c8a8-4a64-acd9-19c10434936e",
   "metadata": {},
   "source": [
    "## Setup: some metadata common to all examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ee866c3-0096-4f21-8209-66af3cce2066",
   "metadata": {},
   "outputs": [],
   "source": [
    "EXAMPLE_DISCLAIMER = \"\"\"\n",
    "DISCLAIMER: This data product was generated from simulations. When publishing any result\n",
    "using it, please cite the CTAO Science Data challenge...\n",
    "\"\"\".replace(\n",
    "    \"\\n\", \" \"\n",
    ")\n",
    "TARGET_DIR = SkyCoord.from_name(\"Crab\")\n",
    "WOBBLE_OFFSET = 0.9 * u.deg\n",
    "POINTING_DIR = SkyCoord(\n",
    "    ra=TARGET_DIR.ra, dec=TARGET_DIR.dec + WOBBLE_OFFSET, frame=TARGET_DIR.frame\n",
    ")\n",
    "EPOCH = Time(0, format=\"mjd\", scale=\"utc\")  # MJD epoch\n",
    "T_START = Time(\"2022-05-10T01:18:01.00\")\n",
    "T_STOP = Time(\"2022-05-10T01:28:01.00\")\n",
    "T_START_S = (Time(\"2022-05-10T01:18:01.00\") - EPOCH).tai.to_value(u.s)\n",
    "T_STOP_S = (Time(\"2022-05-10T01:18:01.00\") - EPOCH + 30 * u.min).tai.to_value(u.s)\n",
    "OBSERVING_NIGHT = \"2022-05-09\"  # can also be  datetime\n",
    "\n",
    "# should use real CTAO North coords here, but for this example we can cheat:\n",
    "LOCATION = EarthLocation.of_site(\"Roque de los Muchachos\")\n",
    "\n",
    "# for TrackingCoverage\n",
    "DELTA_T = T_STOP - T_START\n",
    "TIME_COVERAGE = T_START + np.arange(0, DELTA_T.sec, 1.0) * u.s\n",
    "TARGET_ALTAZ = TARGET_DIR.transform_to(AltAz(obstime=TIME_COVERAGE, location=LOCATION))\n",
    "\n",
    "# Some common things for all examples\n",
    "OBSERVATION = dp.Observation(\n",
    "    coverage=dp.Coverage(\n",
    "        time=dp.TimeCoverage(\n",
    "            reference=dp.TimeReference(\n",
    "                position=dp.TimeRefPos.TOPOCENTER,  # if TOPOCENTER need also observation.location\n",
    "                time_mjd=EPOCH.mjd,  # MJD reference at 0\n",
    "                unit=\"s\",\n",
    "                system=dp.TimeSystem.TAI,\n",
    "            ),\n",
    "            # just some dummy times, here represented in seconds since the\n",
    "            t_min=T_START_S,\n",
    "            t_max=T_STOP_S,\n",
    "        ),\n",
    "        energy=dp.EnergyCoverage(energy_unit=\"TeV\", energy_min=0.003, energy_max=270.0),\n",
    "        space=dp.SpaceCoverage(\n",
    "            ra=POINTING_DIR.ra.deg, dec=POINTING_DIR.dec.deg, field_of_view=6.0\n",
    "        ),\n",
    "        tracking=dp.TrackingCoverage(\n",
    "            elevation_min=TARGET_ALTAZ.alt.min().deg,\n",
    "            elevation_max=TARGET_ALTAZ.alt.max().deg,\n",
    "            azimuth_mean=TARGET_ALTAZ.az.mean().deg,\n",
    "            azimuth_range=0.5 * (TARGET_ALTAZ.az.max() - TARGET_ALTAZ.az.min()).deg,\n",
    "            mode=dm.models.sciops.TrackingMode.FIXED_ICRS,\n",
    "        ),\n",
    "    ),\n",
    "    location=dp.GeodeticEarthLocation(\n",
    "        latitude=LOCATION.lat.deg,\n",
    "        longitude=LOCATION.lon.deg,\n",
    "        height=LOCATION.height.to_value(\"m\"),\n",
    "    ),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28cfcc2c-e6c5-41a9-b15f-0242ac7c86db",
   "metadata": {},
   "source": [
    "## Examples of DL3/Event/Subarray/Observation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58e77f72-09d4-4fa7-b10d-8442821578a5",
   "metadata": {},
   "source": [
    "### From SDC\n",
    "\n",
    "This is for a *simulated* DL3 file created for a *Science Data Challenge* (SDC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aef585be-1d79-4cd4-a3e6-e04b1fe4a1f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "dl3_sdc = dp.Product(\n",
    "    description=\"Simulated gamma-like events and IRFs.\",\n",
    "    creation_time=Time.now(),\n",
    "    curation=dp.Curation(\n",
    "        release=\"CTAO-Simulation/DL3-SDC1\",  # note: recommendation\n",
    "        reference=\"doi://reference/to/the/sdc/archive\",\n",
    "        license=\"cc-by-sa\",\n",
    "        license_url=\"https://creativecommons.org/licenses/by-sa/4.0/\",\n",
    "        copyright=\"CTAO ERIC\",\n",
    "        rights=\"public\",\n",
    "    ),\n",
    "    disclaimer=EXAMPLE_DISCLAIMER,\n",
    "    data=dp.ProductType(\n",
    "        level=dp.DataLevel.DL3,\n",
    "        division=dp.DataDivision.EVENT,\n",
    "        association=dp.DataAssociation.SUBARRAY,\n",
    "        type=dp.DataType.OBSERVATION_SIM,\n",
    "    ),\n",
    "    instance=dp.InstanceIdentifier(\n",
    "        obs_id=1000012345,\n",
    "        site_id=dm.models.common.SiteID.CTAO_NORTH,  # optional, as it is linked to the obs_id\n",
    "        facility_name=dp.FacilityName.SIMULATED_CTAO, # this is simulated data, so need to be clear\n",
    "        event_type=\"hires-soft\",\n",
    "    ),\n",
    "    model=dp.DataModel(\n",
    "        name=\"CTAO-GADF\",\n",
    "        version=\"v0.3\",\n",
    "        url=\"https://gamma-astro-data-formats.readthedocs.io/en/v0.3/\",\n",
    "    ),\n",
    "    contact=dp.Contact(\n",
    "        name=\"CTAO Science Data Challenge HelpDesk\",\n",
    "        email=\"sdc@ctao.org\",\n",
    "        organization=\"CTAO\",\n",
    "    ),\n",
    "    activity=dp.Activity(\n",
    "        process=dp.ObservatoryProcess.SCIENCE_DATA_CHALLENGE,\n",
    "        configuration_id=\"eventdisplay-v1\",  # this is the \"data processing configuration\", for SDC, not sure\n",
    "        name=\"sdc-production\",\n",
    "        description=\"production of SDC data products\",\n",
    "        id=\"2accb086-068d-11f1-8a8d-acde48001122\",\n",
    "        start=Time(\"2025-11-28 13:45:12.62\"),\n",
    "        software=dp.Software(\n",
    "            name=\"SDC_Simulator.py\",\n",
    "            version=\"UNRELEASED\",\n",
    "            url=\"https://gitlab.cta-observatory.org/cta-science/sdc/sdc-simulations\",\n",
    "        ),\n",
    "        inputs=[\n",
    "            dp.ExternalDataProduct(\n",
    "                id=\"1fcade22-d04d-11f0-84af-acde48001122\",\n",
    "                uri=\"file:./irf_calibration.fits\",\n",
    "                role=\"IRF calibration coefficients\",\n",
    "            ),\n",
    "            dp.ExternalDataProduct(\n",
    "                id=\"1e6f6306-d050-11f0-8af5-acde48001122\",\n",
    "                uri=\"file:./other_input.fits\",\n",
    "                role=\"Some critical input\",\n",
    "            ),\n",
    "        ],\n",
    "    ),\n",
    "    observation=OBSERVATION,\n",
    "    sdc=dp.ScienceDataChallenge(nsb_conditions=\"bright\"), # moonlight=bright, normal=dark\n",
    ")\n",
    "EXAMPLES.append(dl3_sdc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16cfae68-65c0-40b2-b611-97163c067a95",
   "metadata": {},
   "source": [
    "### From ACADA (Cat-A)\n",
    "\n",
    "This is a Cat-A (real-time) DL3 product from ACADA-SAG (Science Alert Generation pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0a8eed2-a2f9-460d-b09b-6c83af4402cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "dl3_acada_sag = dp.Product(\n",
    "    description=\"DL3 Event Data Product from Realtime analysis (SAG)\",\n",
    "    creation_time=Time.now(),\n",
    "    curation=dp.Curation(\n",
    "        release=\"CTAO/DL3-Realtime-DR1\",  # note: recommendation\n",
    "        reference=\"doi://reference/to/release-info\",\n",
    "        license=\"cc-by-sa\",\n",
    "        license_url=\"https://creativecommons.org/licenses/by-sa/4.0/\",\n",
    "        copyright=\"CTAO ERIC\",\n",
    "        rights=\"public\",\n",
    "    ),\n",
    "    disclaimer=(\n",
    "        \"This data product was produced with the Cat-A analysis, which is optimized \"\n",
    "        \"for semi-realtime results but may have high systematic uncertainties. \"\n",
    "        \"If a CatB or CatC product is available for the same OB, it should be used instead.\"\n",
    "    ),\n",
    "    data=dp.ProductType(\n",
    "        level=dp.DataLevel.DL3,\n",
    "        division=dp.DataDivision.EVENT,\n",
    "        association=dp.DataAssociation.SUBARRAY,\n",
    "        type=dp.DataType.OBSERVATION,\n",
    "    ),\n",
    "    instance=dp.InstanceIdentifier(\n",
    "        obs_id=1000012345,\n",
    "        site_id=dm.models.common.SiteID.CTAO_SOUTH,  # optional, as it is linked to the obs_id\n",
    "        category=dp.DataProcessingCategory.A,\n",
    "    ),\n",
    "    model=dp.DataModel(\n",
    "        name=\"CTAO-GADF\",\n",
    "        version=\"v0.3\",\n",
    "        url=\"https://gamma-astro-data-formats.readthedocs.io/en/v0.3/\",\n",
    "    ),\n",
    "    contact=dp.Contact(\n",
    "        name=\"CTAO HelpDesk\",\n",
    "        email=\"help@ctao.org\",\n",
    "        organization=\"CTAO\",\n",
    "    ),\n",
    "    activity=dp.Activity(\n",
    "        process=dp.ObservatoryProcess.DATA_PROCESSING,\n",
    "        configuration_id=\"sag-standard\",  # this is the \"data processing configuration\"\n",
    "        name=\"sag-reco\",\n",
    "        description=\"realtime reconstruction and IRF generation\",\n",
    "        id=\"225c364c-068d-11f1-8a8d-acde48001122\",\n",
    "        start=Time(\"2025-11-28 13:45:12.62\"),\n",
    "        software=dp.Software(\n",
    "            name=\"acada-sag-reco\",\n",
    "            version=\"v2.0.1\",\n",
    "            url=\"https://gitlab.cta-observatory.org/cta-computing/acada/sag/sag-reconstruction\",\n",
    "        ),\n",
    "        inputs=[\n",
    "            dp.ExternalDataProduct(\n",
    "                id=\"1fcade22-d04d-11f0-84af-acde48001122\",\n",
    "                uri=\"lfn://dpps/datapipe/models/v7/gammaness_model.pkl\",\n",
    "                role=\"gammaness reconstruction model\",\n",
    "            ),\n",
    "            dp.ExternalDataProduct(\n",
    "                id=\"AA214079-1D3E-4BB0-9B30-F2BDB72094B3\",\n",
    "                uri=\"lfn://dpps/datapipe/models/v7/energy_model.pkl\",\n",
    "                role=\"energy reconstruction model\",\n",
    "            ),\n",
    "        ],\n",
    "    ),\n",
    "    observation=OBSERVATION,\n",
    "    acquisition=dp.Acquisition(sb_id=102934192),\n",
    ")\n",
    "EXAMPLES.append(dl3_acada_sag)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afaa0e56-100a-46aa-983d-12f037d5e148",
   "metadata": {},
   "source": [
    "### From DPPS (Cat-B)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24e45ebb-2a80-4ee1-bb0d-01323011445d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dl3_acada_sag = dp.Product(\n",
    "    description=\"DL3 Event Data Product from DPPS DataPipe, Cat-C\",\n",
    "    creation_time=Time.now(),\n",
    "    curation=dp.Curation(\n",
    "        release=\"CTAO/DL3-DR1\",  # note: recommendation\n",
    "        reference=\"doi://reference/to/release-info\",\n",
    "        license=\"cc-by-sa\",\n",
    "        license_url=\"https://creativecommons.org/licenses/by-sa/4.0/\",\n",
    "        copyright=\"CTAO ERIC\",\n",
    "        rights=\"public\",\n",
    "    ),\n",
    "    disclaimer=(\n",
    "        \"This data product was produced with the Cat-A analysis, which is optimized \"\n",
    "        \"for semi-realtime results but may have high systematic uncertainties. \"\n",
    "        \"If a CatB or CatC product is available for the same OB, it should be used instead.\"\n",
    "    ),\n",
    "    data=dp.ProductType(\n",
    "        level=dp.DataLevel.DL3,\n",
    "        division=dp.DataDivision.EVENT,\n",
    "        association=dp.DataAssociation.SUBARRAY,\n",
    "        type=dp.DataType.OBSERVATION,\n",
    "    ),\n",
    "    instance=dp.InstanceIdentifier(\n",
    "        obs_id=1000012345,\n",
    "        site_id=dm.models.common.SiteID.CTAO_SOUTH,  # optional, as it is linked to the obs_id\n",
    "        category=dp.DataProcessingCategory.A,\n",
    "        observing_night=OBSERVING_NIGHT,\n",
    "    ),\n",
    "    model=dp.DataModel(\n",
    "        name=\"CTAO-GADF\",\n",
    "        version=\"v0.3\",\n",
    "        url=\"https://gamma-astro-data-formats.readthedocs.io/en/v0.3/\",\n",
    "    ),\n",
    "    contact=dp.Contact(\n",
    "        name=\"CTAO HelpDesk\",\n",
    "        email=\"help@ctao.org\",\n",
    "        organization=\"CTAO\",\n",
    "    ),\n",
    "    activity=dp.Activity(\n",
    "        process=dp.ObservatoryProcess.DATA_PROCESSING,\n",
    "        configuration_id=\"standard-hillas-v3.1\",  # this is the \"data processing configuration\"\n",
    "        name=\"process-ob.cwl\",\n",
    "        description=\"data processing pipeline\",\n",
    "        id=\"225c364c-068d-11f1-8a8d-acde48001122\",\n",
    "        start=Time(\"2025-11-28 13:45:12.62\"),\n",
    "        software=dp.Software(\n",
    "            name=\"ctao-dpps-datapipe\",\n",
    "            version=\"v1.1.2\",\n",
    "            url=\"https://gitlab.cta-observatory.org/cta-computing/dpps/datapipe\",\n",
    "        ),\n",
    "        inputs=[\n",
    "            dp.ExternalDataProduct(\n",
    "                id=\"1fcade22-d04d-11f0-84af-acde48001122\",\n",
    "                uri=\"lfn://dpps/datapipe/models/v7/gammaness_model.pkl\",\n",
    "                role=\"gammaness reconstruction model\",\n",
    "            ),\n",
    "            dp.ExternalDataProduct(\n",
    "                id=\"AA214079-1D3E-4BB0-9B30-F2BDB72094B3\",\n",
    "                uri=\"lfn://dpps/datapipe/models/v7/energy_model.pkl\",\n",
    "                role=\"energy reconstruction model\",\n",
    "            ),\n",
    "        ],\n",
    "    ),\n",
    "    observation=OBSERVATION,\n",
    "    acquisition=dp.Acquisition(sb_id=102934192),\n",
    ")\n",
    "EXAMPLES.append(dl3_acada_sag)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25d5f1ea-39e5-4aa7-ae67-a69e307f207e",
   "metadata": {},
   "source": [
    "## Output in flattened JSON format "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01e84ffa-47d7-409d-98ec-79171c4d9c58",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "for example in EXAMPLES:\n",
    "    flat = dm.flatten_model_instance(example)\n",
    "    print(f\"====== {example.data} =======\")\n",
    "    print()\n",
    "    print(json.dumps(dm.flatten_model_instance(example), indent=2))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93029f4d-5988-43d0-adfc-9f1990ff16e1",
   "metadata": {},
   "source": [
    "## Output as FITS Headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad0c41c4-f6e0-4cd6-963b-39e9d5f758d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "for ii, example in enumerate(EXAMPLES):\n",
    "    header = dm.instance_to_fits_header(example)\n",
    "    print(\"=\" * 70)\n",
    "    print(f\"Example {ii}: {example.data} ({len(header)} keys)\")\n",
    "    print(\n",
    "        textwrap.fill(\n",
    "            example.description, initial_indent=\"    \", subsequent_indent=\"   \"\n",
    "        )\n",
    "    )\n",
    "    print(\"=\" * 70)\n",
    "\n",
    "    print()\n",
    "    print(repr(header))\n",
    "    print()"
   ]
  }
 ],
 "metadata": {},
 "nbformat": 4,
 "nbformat_minor": 5
}