From 6278aed0e8262b991ca55efd406501e1fb9fb880 Mon Sep 17 00:00:00 2001 From: Ian Nigel Evans Date: Thu, 6 Nov 2025 16:42:16 -0500 Subject: [PATCH 1/2] Near-final updates These updates are mostly those from issue #28, but include some changes from issue #29 and feedback from Pierre. There are additional changes because the TeX file that I downloaded from the repo included more changes than the PDF Preview. --- HighEnergyObsCoreExt.bib | 6 +- HighEnergyObsCoreExt.glg | 10 +- HighEnergyObsCoreExt.gls | 48 ++--- HighEnergyObsCoreExt.tex | 451 +++++++++++++++++++++------------------ Makefile | 2 +- UseCases.tex | 302 +++++++++++++------------- 6 files changed, 421 insertions(+), 398 deletions(-) diff --git a/HighEnergyObsCoreExt.bib b/HighEnergyObsCoreExt.bib index f952157..52db993 100644 --- a/HighEnergyObsCoreExt.bib +++ b/HighEnergyObsCoreExt.bib @@ -1,5 +1,5 @@ @MISC{2024ivoa.note.heig, - author = {{Servillat}, M. and {Boisson}, C. and {Bonnarel}, F. and {Cresitello-Dittmar}, M. and {Cristofari}, P. and {Evans}, I. and {Evans}, J. and {Fuessling}, M. and {Jaffe}, T. and {Khélifi}, B. and {Kosack}, K. and {Louys}, M. and {Michel}, L. and {Nebot}, A. and {Schnabel}, J. and {Schussler}, F. and {the HE discussion group at IVOA}}, + author = {{Servillat}, M. and {Boisson}, C. and {Bonnarel}, F. and {Cresitello-Dittmar}, M. and {Cristofari}, P. and {Evans}, I. and {Evans}, J. and {Fuessling}, M. and {Jaffe}, T. and {Kh\'elifi}, B. and {Kosack}, K. and {Louys}, M. and {Michel}, L. and {Nebot}, A. and {Schnabel}, J. and {Schussler}, F. and {the HE discussion group at IVOA}}, title = "{Virtual Observatory and High Energy Astrophysics Version 1.0}", howpublished = {IVOA Note 12 November 2024}, year = 2024, @@ -51,8 +51,8 @@ @misc{deil_2022_7304668 {Hassan}, Tarek and {Boisson}, Catherine and {Contreras}, Jose Luis and - {Knödlseder}, Jürgen and - {Khelifi}, Bruno and + {Kn\"odlseder}, J\"urgen and + {Kh\'elifi}, Bruno and {King}, Johannes and {Mohrmann}, Lars and {Linhoff}, Maximilian and diff --git a/HighEnergyObsCoreExt.glg b/HighEnergyObsCoreExt.glg index b7605e3..c962b2b 100644 --- a/HighEnergyObsCoreExt.glg +++ b/HighEnergyObsCoreExt.glg @@ -1,7 +1,7 @@ -This is makeindex, version 2.15 [TeX Live 2022/dev] (kpathsea + Thai support). -Scanning style file ./HighEnergyObsCoreExt.ist.............................done (29 attributes redefined, 0 ignored). -Scanning input file HighEnergyObsCoreExt.glo....done (121 entries accepted, 0 rejected). -Sorting entries....done (925 comparisons). -Generating output file HighEnergyObsCoreExt.gls....done (52 lines written, 0 warnings). +This is makeindex, version 2.17 [TeX Live 2025] (kpathsea + Thai support). +Scanning style file ./HighEnergyObsCoreExt.ist...........................done (27 attributes redefined, 0 ignored). +Scanning input file HighEnergyObsCoreExt.glo....done (115 entries accepted, 0 rejected). +Sorting entries....done (844 comparisons). +Generating output file HighEnergyObsCoreExt.gls....done (38 lines written, 0 warnings). Output written in HighEnergyObsCoreExt.gls. Transcript written in HighEnergyObsCoreExt.glg. diff --git a/HighEnergyObsCoreExt.gls b/HighEnergyObsCoreExt.gls index f0279d9..54515bf 100644 --- a/HighEnergyObsCoreExt.gls +++ b/HighEnergyObsCoreExt.gls @@ -2,51 +2,37 @@ \begin{theglossary}\glossaryheader \glsgroupheading{G}\relax \glsresetentrylist % \glossentry{GTI}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{15}}}\glsgroupskip + \setentrycounter[]{page}\glsnumberformat{13}}}\glsgroupskip \glsgroupheading{H}\relax \glsresetentrylist % \glossentry{HEA}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{1}\delimN - \setentrycounter[]{page}\glsnumberformat{5\delimR 11}\delimN - \setentrycounter[]{page}\glsnumberformat{13\delimR 19}\delimN - \setentrycounter[]{page}\glsnumberformat{21}\delimN - \setentrycounter[]{page}\glsnumberformat{23\delimR 25}\delimN - \setentrycounter[]{page}\glsnumberformat{28\delimN 29}\delimN - \setentrycounter[]{page}\glsnumberformat{32}\delimN - \setentrycounter[]{page}\glsnumberformat{41}}}% + \setentrycounter[]{page}\glsnumberformat{1\delimR 9}\delimN + \setentrycounter[]{page}\glsnumberformat{11\delimR 19}\delimN + \setentrycounter[]{page}\glsnumberformat{24\delimR 29}\delimN + \setentrycounter[]{page}\glsnumberformat{31}}}% \glossentry{HEIG}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{5}\delimN - \setentrycounter[]{page}\glsnumberformat{41}}}\glsgroupskip + \setentrycounter[]{page}\glsnumberformat{2}}}\glsgroupskip \glsgroupheading{I}\relax \glsresetentrylist % \glossentry{IACT}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{17\delimN 18}}}% + \setentrycounter[]{page}\glsnumberformat{15\delimN 16}}}% \glossentry{IRF}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{3\delimN 4}\delimN - \setentrycounter[]{page}\glsnumberformat{6\delimR 9}\delimN - \setentrycounter[]{page}\glsnumberformat{11}\delimN - \setentrycounter[]{page}\glsnumberformat{18\delimR 20}\delimN - \setentrycounter[]{page}\glsnumberformat{33}\delimN - \setentrycounter[]{page}\glsnumberformat{35\delimR 37}}}% + \setentrycounter[]{page}\glsnumberformat{4\delimR 7}\delimN + \setentrycounter[]{page}\glsnumberformat{9}\delimN + \setentrycounter[]{page}\glsnumberformat{17\delimR 19}}}% \glossentry{IVOA}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{5}\delimN - \setentrycounter[]{page}\glsnumberformat{7\delimN 8}\delimN - \setentrycounter[]{page}\glsnumberformat{19}\delimN - \setentrycounter[]{page}\glsnumberformat{41}}}\glsgroupskip + \setentrycounter[]{page}\glsnumberformat{2\delimR 4}\delimN + \setentrycounter[]{page}\glsnumberformat{18}}}\glsgroupskip \glsgroupheading{M}\relax \glsresetentrylist % \glossentry{MOC}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{16}}}\glsgroupskip + \setentrycounter[]{page}\glsnumberformat{13}}}\glsgroupskip \glsgroupheading{S}\relax \glsresetentrylist % \glossentry{STI}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{15}}}\glsgroupskip -\glsgroupheading{U}\relax \glsresetentrylist % -\glossentry{UHE}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{25}}}\glsgroupskip + \setentrycounter[]{page}\glsnumberformat{13}}}\glsgroupskip \glsgroupheading{V}\relax \glsresetentrylist % \glossentry{VHE}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{25}}}% + \setentrycounter[]{page}\glsnumberformat{26}}}% \glossentry{VO}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{4\delimN 5}\delimN - \setentrycounter[]{page}\glsnumberformat{7}}}\glsgroupskip + \setentrycounter[]{page}\glsnumberformat{2\delimR 4}}}\glsgroupskip \glsgroupheading{W}\relax \glsresetentrylist % \glossentry{WCD}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{18}}}% + \setentrycounter[]{page}\glsnumberformat{16}}}% \end{theglossary}\glossarypostamble diff --git a/HighEnergyObsCoreExt.tex b/HighEnergyObsCoreExt.tex index c2a99f3..c8f65bb 100644 --- a/HighEnergyObsCoreExt.tex +++ b/HighEnergyObsCoreExt.tex @@ -10,7 +10,7 @@ \author{The IVOA High Energy Interest Group} -\editor{Ian Evans, Mathieu Servillat, Bruno Khélifi, Janet Evans} +\editor{Ian Evans, Mathieu Servillat, Bruno Kh\'elifi, Janet Evans} \previousversion{This is the first public release} @@ -38,10 +38,10 @@ \newacronym{HESS}{H.E.S.S.}{High Energy Stereoscopic System} \newacronym{CTAO}{CTAO}{Cherenkov Telescope Array Observatory} \newacronym{IACT}{IACT}{Imaging Atmospheric Cherenkov Telescope} -\newacronym[plural=IRFs,firstplural=Instrument Response Functions (IRFs)]{IRF}{IRF}{Instrument Response Function} +\newacronym[plural={IRFs},firstplural={Instrument Response Functions (IRFs)}]{IRF}{IRF}{Instrument Response Function} \newacronym{PSF}{PSF}{Point Spread Function} \newacronym{RMF}{RMF}{Redistribution Matrix File} -\newacronym{ARF}{ARF}{Auxiliary Response File} +\newacronym{ARF}{ARF}{Ancillary Response File} \newacronym{ESA}{ESA}{European Space Agency} \newacronym{XMM-Newton}{XMM-Newton}{X-ray Multi-Mirror Mission} \newacronym{SSC}{SSC}{Survey Science Centre} @@ -57,8 +57,8 @@ \newacronym{ANTARES}{ANTARES}{Astronomy with a Neutrino Telescope and Abyss Environmental Research} \newacronym{GW}{GW}{Gravitational wave} \newacronym{WCD}{WCD}{Water Cherenkov Detector} -\newacronym[plural=STIs]{STI}{STI}{Stable Time Interval} -\newacronym[plural=GTIs]{GTI}{GTI}{Good Time Interval} +\newacronym[plural={STIs}]{STI}{STI}{Stable Time Interval} +\newacronym[plural={GTIs}]{GTI}{GTI}{Good Time Interval} \newacronym{FITS}{FITS}{Flexible Image Transport System} \newacronym{ACIS}{ACIS}{Advanced CCD Imaging Spectrometer} \newacronym{HRC}{HRC}{High Resolution Camera} @@ -78,7 +78,7 @@ \newacronym{CAOM}{CAOM}{Common Archive Observation Model} \newacronym{MOC}{MOC}{Multi-Order-Coverage} -\makeglossaries +%\makeglossaries \begin{document} @@ -102,80 +102,78 @@ \section*{Conformance-related definitions} \section{Introduction} -The \gls{IVOA} \gls{HEIG} was formed in the Fall of 2024, and developed an \gls{IVOA} Note \citep{2024ivoa.note.heig} that explores the connections between the \gls{VO} and \gls{HEA}. Here, the \gls{HEA} covers experiments and observatories from the X-ray range up to the PeV range, as well as the astrophysical neutrinos above the TeV range, called here the \gls{HEA} domain. The HEIG Note includes an outline of several important topics that have formed a roadmap for the group. An ObsCore \citep{2017ivoa.spec.0509L} extension for \gls{HEA} data is the first priority in order to meet the needs of HEA, and to coincide with similar work being carried out by the Radio IG, Time Domain IG, and discussions on DM standards, such that current and future \gls{HEA} experiments and observatories are able to release data on the \gls{VO}. +The \gls{IVOA} \gls{HEIG} was formed in the Fall of 2024, and developed an \gls{IVOA} Note \citep{2024ivoa.note.heig} that explores the connections between the \gls{VO} and \gls{HEA}\null. \gls{HEA} covers experiments and observatories from the X-ray range up to the PeV range and beyond, as well as astrophysical neutrinos above the TeV range. These regimes are referred to here as the \gls{HEA} domain. The \gls{HEIG} Note includes an outline of several important topics that have formed a roadmap for the group. An ObsCore \citep{2017ivoa.spec.0509L} extension for \gls{HEA} data is the first priority in order to meet the needs of \gls{HEA}, and to coincide with similar work being carried out by the Radio Interest Group, Time Domain Interest Group, and discussions on DataModel standards, such that current and future \gls{HEA} experiments and observatories are able to release data through the \gls{VO}. -The goal is to explore elements needed to reliably discover and select \gls{HEA} data through \gls{IVOA} interfaces. It implies defining an extension to ObsCore with the possibility to use the DataLink mechanism and to enhance vocabularies of keywords for ObsCore and DataLink. We suggest that, if an attribute is unique to \gls{HEA} data, that element should appear in an \gls{HEA} ObsCore extension. Whereas, if an attribute makes sense for more than one domain and can be shared across those domains, then that element should be added to the base ObsCore model. This note proposes recommendations in both of these categories. We also discuss enhancements to the vocabulary of data products, DataLink semantics, UCDs and MIME-types to correctly represent \gls{HEA} data. Topics related to the Registry are currently outlined in the proposed Radio extension document and are not discussed here. +The goal is to explore elements needed to reliably discover and select \gls{HEA} data through \gls{IVOA} interfaces. This requires defining an extension to ObsCore with the possibility to use the DataLink mechanism and to enhance vocabularies of keywords for ObsCore and DataLink. We suggest that, if an attribute is unique to \gls{HEA} data, that element should appear in an \gls{HEA} ObsCore extension. Whereas, if an attribute makes sense for more than one domain and can be shared across those domains, then that element should be added to the base ObsCore model. This note proposes recommendations in both of these categories. We also discuss enhancements to the vocabulary of data\/ products, DataLink semantics, Unified Content Descriptors (UCDs), and MIME-types to correctly represent \gls{HEA} data. Topics related to the Registry are currently outlined in the proposed Radio extension document and are not discussed here. \section{High Energy Astrophysics Data} -\gls{HEA} data include observations obtained using photon detectors covering X-ray (from $\sim$0.1 eV to $\sim$100 keV) through gamma-ray (from 100 MeV up to $\gtrsim$ PeV) energies, as well as cosmic-ray and astrophysical neutrino ($\gtrsim$ TeV) detectors, or other messenger related to \gls{HEA} phenomena. The domain is now sufficiently mature to provide open data that are science-ready and work with open analysis tools ({\em e.g.\/} CIAO \citep{2006SPIE.6270E..1VF} or Gammapy \citep{gammapy:2023}). The science output of the \gls{HEA} domain already includes high-level products such as images, cubes, spectra, and time series such as light curves and time-resolved spectra. Additional data products include fitted sky models with a spatial, spectral and/or temporal component(s), along with their confidence intervals or confidence limits, and covariance matrices. Finally, multiple \gls{HEA} instruments produce source catalogs and surveys covering up to the full the sky, which include maps of photon or particle flux, exposure, sensitivity, and aperture-photometry likelihood profiles. +\gls{HEA} data include observations obtained using photon detectors covering X-ray (from $\sim$0.1 keV to $\sim$120 keV) through gamma-ray (from 120 keV up to $\gtrsim$ PeV) energies, as well as cosmic-ray and astrophysical neutrino ($\gtrsim$ TeV) detectors, or other messengers related to \gls{HEA} phenomena. The domain is now sufficiently mature to provide open data that are science-ready and work with open analysis tools ({\em e.g.\/}, CIAO \citep{2006SPIE.6270E..1VF} or Gammapy \citep{gammapy:2023}). The science output of the \gls{HEA} domain already includes high-level products such as images, cubes, spectra, and time series such as light curves and time-resolved spectra. Additional data products include fitted sky models with a spatial, spectral, and/or temporal component(s), along with their confidence intervals or confidence limits, and covariance matrices. Finally, multiple \gls{HEA} instruments produce source catalogs and surveys covering up to the full the sky, which include maps of photon or particle flux, exposure, sensitivity, and aperture-photometry likelihood profiles. -Observations of the Universe at the highest energies are based on techniques that are radically different compared to the UV through radio domains. \gls{HEA} observatories\footnote{One can cite Chandra, XMM-Newton, Fermi, H.E.S.S., MAGIC, VERITAS, HAWC, LHAASO, IceCube, Auger and soon CTAO and KM3NeT, SWGO.} are generally designed to detect particles ({\em e.g.\/}, individual photons, cosmic-rays or neutrinos) with the ability to estimate multiple observables for those particles. These detection techniques rely all on \emph{event counting}\footnote{As opposed to signal integrating using a light detector.}, where an event has some probability of being due to the interaction of an astronomical particle with the detectors, but also some probability from being from instrumental or background effects. The data corresponding to an event are first an instrumental signal, which is then calibrated and processed to estimate physical quantities such as a time of arrival, point-of-origin on the sky, and an energy proxy associated with the event. Several other intermediate and qualifying characteristics may be associated with a detected event, depending on the detection technique. The ensemble of events detected over a given time interval and spatial field-of-view is referred to as an \emph{event list}. +Observations of the universe at the highest energies are based on techniques that are radically different compared to the UV through radio domains. \gls{HEA} observatories\footnote{For example, Chandra, XMM-Newton, Fermi, H.E.S.S., MAGIC, VERITAS, HAWC, LHAASO, IceCube, Auger and soon CTAO and KM3NeT, SWGO.} are generally designed to detect particles ({\em e.g.\/}, individual photons, cosmic-rays or neutrinos) with the ability to estimate multiple observables for those particles. These detection techniques all rely on {\em event counting\/}\footnote{As opposed to signal integrating ({\em e.g.\/}, using a detector that accumulates the total photon signal during an exposure).}, where an event has some probability of being due to the interaction of a particle from an astrophysical source with the detectors, but also has some probability from being from instrumental or background effects. The data corresponding to an event are first an instrumental signal, which is then calibrated and processed to estimate physical quantities such as a time of arrival, point-of-origin on the sky, and an energy proxy associated with the event. Several other intermediate and qualifying characteristics may be associated with a detected event, depending on the detection technique. The ensemble of events detected over a given time interval and spatial field-of-view is referred to as an {\em event list\/}, which we designate an {\bf event-list} in this document. -Though \textbf{event-lists} \emph{may} include estimators for calibrated physical values, they typically still have to be corrected for the photometric, spectral, spatial, and/or temporal responses of the telescope and detector combination to yield scientifically interpretable information. The mappings between the observables and physical measurements of the source properties are called \glspl{IRF}\footnote{We try to avoid using the term \gls{IRF} in a normative sense since historical usage across the broad \gls{HEA} community (and from facility to facility) varies. In some cases, \gls{IRF} has been used to mean specifically the X-ray product of the ``ARF'' and ``RMF'', whereas in other cases \gls{IRF} -has been used more generally to mean any instrumental response function regardless of type.}; using techniques like forward-folding, they enable one to fit a model (with any combination of spectral, spatial, and temporal components) of the true flux of particles from a source arriving at the instrument to the measured quantities. The \glspl{IRF} generally evolve over time with the instrument and observation characteristics, and are usually defined for a specific time interval and decomposed into a standard set of independent components (see section 3.1.5 of \citep{2024ivoa.note.heig}), such as the spatial point-spread function or the energy-migration matrix, where each component may be stored or computed separately. Some of the \glspl{IRF} are probabilistic in nature\footnote{The energy matrix is a probability density funtion}, and in addition may depend on the set of events selected for analysis by the end user. They are usually not invertible, so methods such as forward-folding fitting (using source models that are estimated) are needed to estimate physical properties given observables. Since both \glspl{IRF} and \textbf{event-lists} are required to process \gls{HEA} data, some \gls{IVOA} standards must be be modified in order to expose both of them via the \gls{VO}. +Though {\bf event-list}s {\em may\/} include estimators for calibrated physical values, they typically still have to be corrected for the photometric, spectral, spatial, and/or temporal responses of the telescope and detector combination to yield scientifically interpretable information. The mappings between physical measurements of the source properties and the observables are called Instrument Response Functions (\glspl{IRF}\footnote{We try to avoid using the term \gls{IRF} in a normative sense since historical usage across the broad \gls{HEA} community (and from facility to facility) varies. In some cases, \gls{IRF} has been used to mean specifically the X-ray product of the \gls{ARF} and \gls{RMF}, whereas in other cases \gls{IRF} has been used more generally to mean any instrumental response function regardless of type.}). Some \glspl{IRF} are probabilistic in nature\footnote{For example, the energy matrix is a probability density function.}, and in addition may depend on the set of events selected for analysis by the end user. They are usually not invertible, so methods such as forward-folding fitting (using source models with any combination of spectral, spatial, temporal, and/or polarization components that are estimated) are needed to estimate physical properties, such as the true flux of particles from a source arriving at the instrument, given the measured observable quantities. The \glspl{IRF} generally evolve over time with the instrument and observation characteristics, and are usually defined for a specific time interval and decomposed into a standard set of independent components (see \S~3.1.5 of \citep{2024ivoa.note.heig}), such as the spatial point-spread function or the energy-migration matrix, where each component may be stored or computed separately. Since both \glspl{IRF} and {\bf event-list}s are required to analyze \gls{HEA} data, some \gls{IVOA} standards must be modified in order to expose both of them via the \gls{VO}. -In the following, the current ObsCore standard will be discussed in Section \ref{sec:obscore}, focusing on attributes that need to be modified. Then, we propose the creation of a \gls{HEA} extension of ObsCore in Section \ref{sec:obscoreext}, as some attributes are very specific to our domain (see \S\ref{sec:ibscoreext}). In these two sections, the discussion focuses on the attribute definitions rather on the attribute values. In Section \ref{sec:voc}, enhancement of vocabulary is proposed for some ObsCore attributes, DataLink semantics, UCDs and MIME-types. +In the following, the current ObsCore standard will be discussed in \S~\ref{sec:obscore}, focusing on attributes that need to be modified. Then, we propose the creation of a \gls{HEA} extension of ObsCore in \S~\ref{sec:obscoreext}, as some attributes are very specific to our domain (see \S~\ref{sec:ibscoreext}). In these two sections, the discussion focuses on the attribute definitions rather on the attribute values. In \S~\ref{sec:voc}, enhancement of vocabulary is proposed for some ObsCore attributes, DataLink semantics, UCDs and MIME-types. \section{ObsCore Attribute Definitions for High Energy Astrophysics Data} \label{sec:obscore} -The ObsCore representation of any \gls{HEA} \textbf{event-list} data products is described in terms of curation, coverage, and access. However, given the \gls{HEA} data specificities, several properties, including resolutions, observable axis descriptions, and polarization states would be simply set to ``NULL'', and data axis lengths are set to ``$-1$''. Therefore, for these data products and associated \glspl{IRF}, the definitions of some ObsCore attributes should be adjusted so that they better represent the content of the data from the perspective of data discovery. We note that many properties, including spatial and spectral coverage and resolution can vary strongly with energy and off-axis angle. These adjustments will also typically apply to advanced, high-level data products derived from \textbf{event-list} data. +The ObsCore representation of any \gls{HEA} \textbf{event-list} data products is described in terms of curation, coverage, and access. However, given the \gls{HEA} data specificities, several properties, including resolutions, observable axis descriptions, and polarization states would be simply set to ``NULL'', and data axis lengths set to ``$-1$''. Therefore, for these data products and associated \glspl{IRF}, the definitions of some ObsCore attributes should be adjusted so that they better represent the content of the data from the perspective of data discovery. We note that many properties, including spatial and spectral coverage and resolution can vary strongly with energy and off-axis angle. These adjustments will also typically apply to advanced, high-level data products derived from \textbf{event-list} data. -In addition, the hereafter modification proposal faces to the issue that some values of ObsCore attributes ({\em dataproduct\_type} and {\em calib\_level}) are defined both into the ObsCore standard document \citep{2017ivoa.spec.0509L} and in the vocabularies documents \citep{2023ivoa.spec.0206D, 2021ivoa.spec.0525D}, which might create some issues for the users. In this context, we have opted to propose in this document some modifications of both standards, even if we would have prefered that everything is uniquely defined in the \gls{IVOA} Vocabulary. Some harmonization should be taken by the Data Model and Semantics working groups in order to avoid duplications. But until such work is achieved, we require modifications in ObsCore and Vocabulary. +Currently, some ObsCore attributes ({\em dataproduct\_type\/} and {\em calib\_level\/}) are formally defined in the ObsCore Recommendation Version 1.1 \citep{2017ivoa.spec.0509L} and also in the vocabularies documents \citep{2023ivoa.spec.0206D, 2021ivoa.spec.0525D}\footnote{Primarily the Data Product Type Vocabulary, \url{https://www.ivoa.net/rdf/product_type}.}, which may be referenced in future versions of the ObsCore Recommendation. For completeness, we are proposing in this document modifications to both the existing ObsCore Recommendation and IVOA vocabularies. \subsection{{\em dataproduct\_type}} \label{sec:dataproduct_type} The attribute {\em dataproduct\_type\/} provides a scientific classification of the data product and is of primary importance for data discovery, especially when there may be many different types of data product associated with an observation\footnote{We use the term ``observation'' in the broad sense, as is done in the ObsCore Recommendation. We note that in this context an ``observation'' may not correspond to a single pointed observation defined in the traditional sense.} (as is often the case for \gls{HEA} datasets). -The ObsCore v1.1 recommendation \citep{2017ivoa.spec.0509L} only defines an {\bf event} {\em dataproduct\_type} as: +The modifications/additions to the set of ObsCore Recommendation Version 1.1 \citep{2017ivoa.spec.0509L} {\em dataproduct\_type\/} (and, in some cases, {\em dataproduct\_subtype\/}) terms described herein apply equally to the IVOA Data Product Type Vocabulary, and will therefore be repeated in \S~5. + +The ObsCore Recommendation Version 1.1 \citep{2017ivoa.spec.0509L} defines an {\bf event} {\em dataproduct\_type\/} as: \begin{quote} {\bf event}: an event-counting ({\em e.g.\/}, X-ray or other high energy) dataset of some sort. Typically this is instrumental data, {\em i.e.\/}, ``event data''. An event dataset is often a complex object containing multiple files or other substructures. An event dataset may contain data with spatial, spectral, and time information for each measured event, although the spectral resolution (energy) is sometimes limited. Event data may be used to produce higher level data products such as images or spectra. \end{quote} -We propose to add the following {\em dataproduct\_type} term in both the ObsCore standard and into the \gls{IVOA} Vocabulary of Data Product Types\footnote{See \url{https://www.ivoa.net/rdf/product-type}.} to better define an \gls{HEA} \textbf{event-list} and an \textbf{event-bundle} that includes the event-list and its associated data: +We propose to add the following {\em dataproduct\_type\/} terms to ObsCore to better define a \gls{HEA} {\bf event-list} and an {\bf event-bundle} that includes the {\bf event-list} and associated data: \begin{quote} -{\bf event-list}: a collection of observed particle-detection events, such as incoming high-energy particles. The table of event list is typically characterised by a spatial position, a time and an energy proxy. +{\bf event-list}: a dataset that records a collection of observed particle-detection events, such as incoming high-energy particles, where an event is typically characterized by a spatial position, a time, and a spectral value ({\em e.g.\/}, an energy, a channel, a pulse height). -{\bf event-bundle}: compounded dataset containing an {\bf event-list} and multiple files or other substructures that are products necessary to analyze the event-list. Data in an event-bundle may thus be used to produce higher level data products calibrated in physical units when containing \glspl{IRF} or other data products that can be used to construct \glspl{IRF}. +{\bf event-bundle}: a compounded dataset containing an {\bf event-list} and multiple files or other substructures that are products necessary to analyze the event-list. Data in an {\bf event-bundle} may thus be used to produce higher level data products calibrated in physical units when containing \glspl{IRF} or other data products that can be used to construct \glspl{IRF}. \end{quote} -It may be worth mentioning that the term ``event'' caused confusion in the past, as it also is used for astrophysical events like supernova explosions ({\em e.g.\/} VOEvent), and that is not the type of event that is being described here, which are particle detection events. Using "event-list" was meant to help to resolve this ambiguity.\\ +We note that the term {\bf event} has caused confusion in the past, since ``event'' may also used to describe notifications ({\em e.g.\/}, as in ``VOEvent'') of astrophysical events such as supernova explosions. Such ``events'' are quite different from the particle detection events being described herein. Using {\bf event-list} will help to resolve this ambiguity. %An {\bf event-bundle} might for example consist of an {\bf event-list} and the associated {\bf response-functions} (see below) used to calibrate the dataset; alternatively an {\bf event-bundle} may include the {\bf event-list} and associated data products necessary for the user to create the {\bf response-functions} (for those X-ray cases where detailed knowledge of the scientific use case — for example, the user’s selection of events — may be required to compute the responses).\\ %particle-detection -In addition to {\em dataproduct\_type} terms that focus on event data, we note that existing ObsCore definitions do not adequately span the breadth of {\bf advanced data products} (with {\em calib\_level} $\ge$ 2) that may be generated from astronomical observations by users or observatories removing instrumental effects. The computational complexity of analyzing \gls{HEA} data robustly in the extreme Poisson regime ({\em e.g.\/}, Bayesian X-ray aperture photometry applied simultaneously to multiple overlapping detections and observations) means that data providers may choose to provide such analysis products directly to the end user. For example, the Chandra Source Catalog includes 38 types of advanced data products (for a total of $\sim$90 million files) and $\sim$50\% of these data product types are not well represented by a {\em dataproduct\_type} value that allows for meaningful data discovery. Users will certainly want to discover these data products independently from the associated observation data (and many of these data products combine data from multiple observations). We therefore propose the following additional {\em dataproduct\_type} (or {\em dataproduct\_subtype}) terms for these advanced data products, and note that these terms will certainly be useful independent of waveband ({\em i.e.\/}, they can be equally applicable to UV/optical, IR, and radio datasets): +In addition to {\em dataproduct\_type\/} terms that focus on event data, we note that existing ObsCore definitions do not adequately span the breadth of ``advanced data products'' (typically with {\em calib\_level\/} $\ge$ 3) that may be generated from astronomical observations by users or observatories. The computational complexity of analyzing \gls{HEA} data robustly in the extreme Poisson regime ({\em e.g.\/}, Bayesian X-ray aperture photometry applied simultaneously to multiple overlapping detections and observations) means that data providers may choose to provide such analysis products directly to the end user. For example, the Chandra Source Catalog includes 38 types of advanced data products (for a total of $\sim\!90$ million files) and $\sim\!50$\% of these data product types are not well represented by a {\em dataproduct\_type\/} value that allows for meaningful data discovery. Users will certainly want to discover these data products independently from the associated progenitor observation data (and many of these data products combine data from multiple observations). We therefore propose the following additional {\em dataproduct\_type\/} (or {\em dataproduct\_subtype\/}) terms for these advanced data products, and note that these terms will certainly be useful independent of waveband ({\em i.e.\/}, they can be equally applicable to UV/optical, IR, and radio datasets): \begin{quote} -{\bf draws}: a dataset that represents draws computed from a probability distribution, for example the Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable. The draws -can be interpreted to provide a robust estimation of the probability distribution of variable, and correlations between the draws provide information about how well the draws converge to the parent probability distribution. +{\bf draws}: a dataset that records statistical draws computed from a probability distribution, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable. The draws can be interpreted to provide a robust estimation of the probability distribution of variable, and correlations between the draws provide information about how well the draws converge to the parent probability distribution. -{\bf pdf}: a dataset that represents the probability density function of a quantity, for example the Bayesian marginal probability density function for a random variable. The probability density function provides a robust estimation of the variable and allows arbitrary confidence intervals to be computed directly from the distribution. +{\bf pdf}: a dataset that records the probability density function of a quantity, for example the Bayesian marginal probability density function for a random variable. The probability density function provides a robust estimation of the variable and allows arbitrary confidence intervals to be computed directly from the distribution. -{\bf region}: a dataset that includes an encoding of (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary, which does not offer a MOC. +{\bf region}: a dataset that includes an encoding of (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary. -{\bf response-function}: a dataset that represents a mapping from a physical quantity to an observable. For \gls{HEA}, this may be the components of the composite \gls{IRF} such as an Auxiliary Response File ({\bf ARF}), Redistribution Matrix File ({\bf RMF}), Effective Area ({\bf AEFF}), Energy Dispersion ({\bf EDISP}), the Background Rate ({\bf BKGRATE}). The Point Spread Function ({\bf PSF}) is a response function that is generally applicable across multiple wavebands. While these datasets may generally be represented as an N-dimensional data cube, designating them as {\bf response-functions} enhances data discovery for very common types of \gls{HEA} dataset (see the use cases in appendix \ref{sec:uc}). +{\bf response-function}: a dataset that records a mapping from a physical quantity to an observable quantity. For \gls{HEA}, this may be the components of the composite \gls{IRF} such as an Auxiliary Response File ({\bf arf}), Redistribution Matrix File ({\bf rmf}), Effective Area ({\bf aeff}), Energy Dispersion ({\bf edisp}), the Background Rate ({\bf bkgrate}). The Point Spread Function ({\bf psf}) is a response function that is generally applicable across multiple wavebands. While these datasets may generally be represented as an N-dimensional data cube, designating them as {\bf response-functions} enhances data discovery for very common types of \gls{HEA} dataset (see the use cases in Appendix~\ref{sec:uc}). \end{quote} -The {\bf measurements} data product type is quite useful for many different types of advanced data products (that may be derived from multiple observations). But users of those products often may not be interested the progenitor datasets, especially if many advanced data products are extracted from a single or a few progenitors ({\em e.g.\/}, measurements associated with sources detected in a single observation field). We propose to delete the caveat associated with {\bf dataproduct\_type} = ``measurements'' in the ObsCore IVOA Recommendation (\S4.1.1) that requires the derived data products be exposed ``together with the progenitor observation dataset''.\\ - -Note that these terms will be repeated in the section \ref{sec:voc}, as mentioned in the introduction of this sub-section. +The {\bf measurements} {\em dataproduct\_type\/} is quite useful for many different types of advanced data products (that may be derived from multiple observations). But users of those products often may not be interested the progenitor datasets, especially if many advanced data products are extracted from a single or a few progenitors ({\em e.g.\/}, measurements associated with sources detected in a single observation field). We propose to delete the caveat associated with {\em dataproduct\_type\/} = ``measurements'' in the ObsCore IVOA Recommendation (\S~4.1.1) that requires the derived data products be exposed ``{\bf together} with the progenitor observation dataset''. \subsection{{\em dataproduct\_subtype}} -The optional attribute {\em dataproduct\_subtype} may be used by the data provider to specify additional information about the nature of the data product. For some datasets, this attribute aims to be combined with {\em dataproduct\_type\/} to more precisely define the content of the dataset ({\em e.g.\/}, {\em dataproduct\_type\/} = {\bf image}${}+{}${\em dataproduct\_subtype\/} = {\bf exposuremap}). Even if the vocabulary of such data product (sub-)types is free, we recommand for \gls{HEA} data providers to use standardized vocabulary: {\em exposure} for the exposure map in any astrophysical dimension (ie, not with the instrumental ones), {\em psfkernel} for the PSF map in any astrophysical dimension, {\em edispkernel} for the energy dispersion matrix map in any astrophysical dimension, {\em significance} for the signifiance map in any astrophysical dimension, {\em probability} for a probability map in any astrophysical dimension. +The optional attribute {\em dataproduct\_subtype} may be used by the data provider to specify more precisely the scientific nature of a data product. Although no vocabulary is defined for {\em dataproduct\_subtype\/}, we recommend that data providers formulate and use a standardized vocabulary for this attribute for data products that are commonly used in \gls{HEA}\null. We have proposed several terms in \S~5 for commonly used \gls{HEA} {\bf response-function} types ({\em e.g.\/}, {\bf aeff}, {\bf edisp}, {\bf psf}), but additional terms could be standardized for other common data products. For example, standardizing using {\bf exposuremap} for an exposure map would enable queries such as ({\em dataproduct\_type\/} = {\bf image}) AND ({\em dataproduct\_subtype\/} = {\bf exposuremap}) to work across multiple facilities. Other possible terms could include {\bf significancemap} for a significance map, and {\bf probabilitymap} for aprobability map. \subsection{{\em calib\_level}} ObsCore defines calibration {\bf Level 1} as ``Instrumental data in a standard format (FITS, VOTable, SDFITS, ASDM, etc.) which could be manipulated with standard astronomical packages.'' and {\bf Level 2} as ``Calibrated, science ready data with the instrument signature removed.'' -However, some \gls{HEA} {\bf event-list}s include spatial and time axes that are calibrated physical quantities, but the spectral axis is instrumental and requires application of the IRFs to remove this signature. In X-ray, this is typically done because the {\bf response-function}s can depend on the choice of region (spatial/time) from which the events are extracted (especially for telescope/detector combinations where the telescope position dithers on the sky during the exposure), which depends on the specific science case and therefore cannot be determined {\em a priori\/}. Such {\bf event-list}s fall ``between'' {\em calib\_level\/} 1 and 2. +However, some \gls{HEA} {\bf event-list}s include spatial and time axes that are calibrated physical quantities, but the spectral axis is instrumental and requires application of the IRFs to remove this signature. This is typically done because the {\bf response-function}s can depend on the choice of region (spatial/time) from which the events are extracted (especially for telescope/detector combinations where the telescope position dithers on the sky during the exposure), which depends on the specific science case and therefore cannot be determined {\em a priori\/}. Such {\bf event-list}s fall ``between'' {\em calib\_level\/} 1 and 2. On the other hand, other {\bf event-list}s may not have any calibrated axes or may have all axes calibrated, and it is important to be able to differentiate between these for data discovery. While the value for {\em calib\_level\/} for any data product is left for the data provider to determine, we suggest that individual data providers set {\em calib\_level\/} = 1 if an {\bf event-list} is considered to be ``uncalibrated'' according to normal usage for their data products and set {\em calib\_level\/} = 2 if an {\bf event-list} is considered to be ``calibrated'' according to normal usage for their data products. @@ -183,25 +181,25 @@ \subsection{{\em calib\_level}} \subsection{{\em access\_url}} -Given the complexity and number of HE data products, the {\em access\_url} may point either directly to a file ({\em e.g.\/} to the {\bf event-list} or an {\bf event-bundle}), or to a DataLink service that will provide links to the data and to associated data ({\em e.g.\/} {\bf response-function}s). +Given the complexity and number of \gls{HEA} data products, the {\em access\_url\/} may point either directly to a file ({\em e.g.\/}, to the {\bf event-list} or an {\bf event-bundle}), or to a DataLink service that will provide links to the data and to associated data ({\em e.g.\/}, {\bf response-function}s). -If DataLink is provided, it should be indicated that the URL points to a Datalink service via the {\em access\_format} = application/x-votable+xml;content\\=datalink. +If a DataLink is provided, {\em access\_format\/} should be set to ``application/x-votable+xml;content=datalink'' to indicate that the URL points to a Data\-Link service. -If the {\em access\_url} points to a bundle, the detailed content of the bundle is not exposed; therefore using a DataLink service has advantages. +If the {\em access\_url\/} points to a bundle, the detailed content of the bundle is not exposed; therefore using a DataLink service has advantages. \subsection{{\em access\_format}} -The {\em access\_format\/} attribute specifies the format of the data product when downloaded as a file from the {\em access\_url\/}. The analysis of \gls{HEA} data often requires use of multiple, related data products, for example an {\bf event-list} combined with associated \glspl{IRF} or ancillary files that can be employed by the user to create \glspl{IRF}. These associated products are often bundled together with the {\bf event-list} and we proposed in \S~\ref{sec:dataproduct_type} to assign such bundles {\em dataproduct\_type\/} = {\bf event-bundle}. While these bundles are typically not standardized across different projects, knowledge of the bundle content is useful for client applications to properly handle the bundles (for example to send the data to an appropriate visualization tool). This is readily achieved by encoding an appropriate MIME-type using the {\em access\_format\/} attribute. In Section~\ref{sec:mimetypes} we propose additional MIME-types for some common {\bf event-bundle}s. +The {\em access\_format\/} attribute specifies the format of the data product when downloaded as a file from the {\em access\_url\/}. The analysis of \gls{HEA} data often requires use of multiple, related data products, for example an {\bf event-list} combined with associated \glspl{IRF} or ancillary files that can be employed by the user to create \glspl{IRF}\null. These associated products are often bundled together with the {\bf event-list} and we propose in \S~\ref{sec:dataproduct_type} to assign such bundles {\em dataproduct\_type\/} = {\bf event-bundle}. While these bundles are typically not standardized across different projects, knowledge of the bundle content is useful for client applications to properly handle the bundles (for example to send the data to an appropriate visualization tool). This is readily achieved by encoding an appropriate MIME-type using the {\em access\_format\/} attribute. In Section~\ref{sec:mimetypes} we propose additional MIME-types for some common {\bf event-bundle}s. \subsection{{\em s\_ra\/}/{\em s\_dec}} -We propose that the attributes {\em s\_ra\/}/{\em s\_dec} be redefined to be the ICRS right ascension and ICRS declination of ``a reference position (typically the center)'' of an observation on the sky, rather than the ICRS right ascension and ICRS declination of ``the center'' of the observation. The center of an observation often is not useful for advanced data products that may be extracted from a cut-out from the progenitor observation, and many facilities allow an instrument to be displaced from the optical axis of the telescope, which means that the definition of ``the center'' of an observation may be unclear (especially when the tracking is not fixed in the ICRS system). +We propose that the attributes {\em s\_ra\/}/{\em s\_dec\/} be redefined to be the ICRS right ascension and ICRS declination of ``a reference position (typically the center)'' of an observation on the sky, rather than the ICRS right ascension and ICRS declination of ``the center'' of the observation. For some facilities, the center (RA, Dec) may have a specific meaning (such as the location of the optical axis of the telescope), which often is not useful for advanced data products that may be extracted from a cut-out from the progenitor observation. Some facilities also allow an instrument to be displaced from the center of the focal plane, which means that the definition of ``the center'' of an observation may be unclear (especially when not tracking at sidereal rate or for facilities for which the PSF varies strongly across the telescope field of view). Since these cases effectively displace the observation field-of-view, ObsCore attributes such as {\em s\_fov\/} that are implicitly referenced to ({\em s\_ra\/}, {\em s\_dec\/}) will continue to behave as expected using the revised definition. For non-pointing instruments (which may include all-sky instruments such as KM3NeT or HAWC), these fields are poorly defined (as is the case, generally for observations that are drift scans). For the time duration of the observation, one can compute an effective center position of the exposure skymap and the maximum radius of the covered area ({\em i.e.\/}, for an all-sky instrument this would be $2\pi\,\rm Sr$ solid angle in Alt/Az, which can be converted into a rotated area in RA/Dec). However, the utility of such a characterization depends on both the duration of the observation and the use case. \subsection{{\em s\_calib\_status}} -We propose that {\em s\_calib\_status} encode the calibration status of an {\bf event-list} dataset's spatial axes. Where multiple spatial axes are included in a dataset ({\em e.g.\/}, physical detector pixel coordinates, virtual detector coordinates corrected for distortions, world coordinates), then we recommend that the data provider use the coordinate system that is most likely to be preferred by the end user (typically the most fully calibrated spatial axes) to define {\em s\_calib\_status\/}. +We propose that {\em s\_calib\_status\/} encode the calibration status of an {\bf event-list} dataset's spatial axes. Where multiple spatial axes are included in a dataset ({\em e.g.\/}, physical detector pixel coordinates, virtual detector coordinates corrected for distortions, world coordinates), we recommend that the data provider use the coordinate system that is most likely to be preferred by the end user (typically the most fully calibrated spatial axes) to define {\em s\_calib\_status\/}. Under the (reasonable) assumption that an end-user searching for {\bf event-bundle} datasets is typically querying based on the properties of the primary {\bf event-list}, we suggest that those values also be used for the {\bf event-bundle}. However, the data provider should ultimately decide which value best describes their {\bf event-bundle} dataset. @@ -209,15 +207,15 @@ \subsection{{\em s\_calib\_status}} \subsection{{\em t\_calib\_status}} -We propose that {\em t\_calib\_status} encode the calibration status of an {\bf event-list} dataset's time axis. Where multiple time axes are included in a dataset ({\em e.g.\/}, instrument counter, absolute time), then we recommend that the data provider use the coordinate system that is most likely to be preferred by the end user (typically the most fully calibrated time axis) to define {\em t\_calib\_status\/}. +We propose that {\em t\_calib\_status\/} encode the calibration status of an {\bf event-list} dataset's time axis. Where multiple time axes are included in a dataset ({\em e.g.\/}, instrument counter, absolute time), we recommend that the data provider use the coordinate system that is most likely to be preferred by the end user (typically the most fully calibrated time axis) to define {\em t\_calib\_status\/}. Under the (reasonable) assumption that an end-user searching for {\bf event-bundle} datasets is typically querying based on the properties of the primary {\bf event-list}, we suggest that those values also be used for the {\bf event-bundle}. However, the data provider should ultimately decide which value best describes their {\bf event-bundle} dataset. -For dataset types that do not not encode time coordinates, we suggest setting this value to ``NULL''. +For dataset types that do not encode time coordinates, we suggest setting this value to ``NULL''. \subsection{{\em em\_calib\_status}} -We propose that {\em em\_calib\_status} encode the calibration status of an {\bf event-list} dataset's spectral axis. Where multiple spectral axes are included in a dataset ({\em e.g.\/}, PHA, PI, energy), then we recommend that the data provider use the coordinate system that is most likely to be preferred by the end user (typically the most fully calibrated spectral axis) to define {\em t\_calib\_status\/}. +We propose that {\em em\_calib\_status\/} encode the calibration status of an {\bf event-list} dataset's spectral axis. Where multiple spectral axes are included in a dataset ({\em e.g.\/}, PHA, PI, energy), we recommend that the data provider use the coordinate system that is most likely to be preferred by the end user (typically the most fully calibrated spectral axis) to define {\em t\_calib\_status\/}. Under the (reasonable) assumption that an end-user searching for {\bf event-bundle} datasets is typically querying based on the properties of the primary {\bf event-list}, we suggest that those values also be used for the {\bf event-bundle}. However, the data provider should ultimately decide which value best describes their {\bf event-bundle} dataset. @@ -227,24 +225,20 @@ \subsection{{\em o\_ucd}} For an {\bf event-list}, we can consider that all measures stored in column values are observables. This is {\em the\/} fundamental difference between \gls{HEA} {\bf event-list}s and typical pixelated datasets. The current ObsCore Recommendation suggests that {\em o\_ucd\/} be set to ``NULL'' for event lists. However this significantly hampers data discovery for \gls{HEA} datasets. Since the data content of {\bf event-list}s may vary significantly from facility to facility, meaningful discovery of \gls{HEA} datasets {\em requires\/} the user be able to query the UCDs of the set of observables included in an {\bf event-list}. -A natural way of doing this that is consistent with current usage would be to extend {\em o\_ucd\/} to allow specification of {\em multiple\/} observables for {\bf event-list}s (and {\bf event-bundle}s), for example, {\em o\_ucd\/} = {\em pos.eq,time,instr.pulseHeight\/}. - -However, the UCD standard \citep{2005ivoa.spec.0819D} states explicitly that such a combination with commas is better used to improve a column description. So, an other synthax could be use of used, using a semicolon to separate the UCDs of different column. Using the previous example, it will become {\em o\_ucd\/} = {\em pos.eq;time;instr.pulseHeight\/}. - -We note that extending {\em o\_ucd\/} to allow specification of multiple observables would require similar adjustments to the other observable axis attributes {\em o\_unit}, {\em o\_calib\_status}, and {\em o\_stat\_err}.\\ +A natural way of doing this that is consistent with current usage would be to extend {\em o\_ucd\/} to allow specification of {\em multiple\/} observables for {\bf event-list}s (and {\bf event-bundle}s), for example, {\em o\_ucd\/} = {\em 'pos.eq;time;instr.event.pulse\-Height'\/}. -Note that real {\bf event-list}s may include an extensive set of columns ({\em e.g.\/}, a {\em Chandra\/} ACIS Level~1 {\bf event-list} includes $\sim\!20$ columns, depending on observing mode) and several columns may represent similar (but not identical) observables ({\em e.g.\/}, event position in detector pixel coordinates, projected onto the focal surface, corrected for geometric distortions, corrected for spacecraft dither motion, mapped to world coordinates). Currently defined UCDs are not sufficiently fine-grained to be able to differentiate between these various cases. But that is likely not necessary, since for data discovery purposes the user is typically interested in the ``most calibrated'' properties in each of the spatial/spectral/time(/polarization) axes ({\em e.g.\/}, world coordinates in the above example). +We note that extending {\em o\_ucd\/} to allow specification of multiple observables would require similar adjustments to the other observable axis attributes {\em o\_unit\/}, {\em o\_calib\_status\/}, and {\em o\_stat\_err\/}. -In the example {\em o\_ucd\/} above, the example UCD {\em instr.pulseHeight\/} is used to represent the detector Pulse Height Amplitude (PHA). There is currently no UCD defined for a raw measure like PHA, but we propose the addition of {\em instr.pulseHeight\/} to the UCDs list vocabulary, together with other UCDs that are relevant for \gls{HEA} data, in Section~\ref{sec:UCDs}.\\ +Note that real {\bf event-list}s may include an extensive set of columns ({\em e.g.\/}, a Chandra ACIS Level~1 {\bf event-list} includes $\sim\!20$ columns, depending on observing mode) and several columns may represent similar (but not identical) observables ({\em e.g.\/}, event position in detector pixel coordinates, projected onto the focal surface, corrected for geometric distortions, corrected for spacecraft dither motion, mapped to world coordinates). Currently defined UCDs are not sufficiently fine-grained to be able to differentiate between these various cases. But that is very likely not necessary, since for data discovery purposes the user is typically interested in the ``most calibrated'' properties in each of the spatial/spectral/time(/polarization) axes ({\em e.g.\/}, world coordinates in the above example). -Advanced data products may similarly record multiple observables that can only be differentiated through their UCDs. For example, a {\em Chandra\/} Source Catalog {\bf pdf} dataset for a detection may include multiple marginalized probability density functions computed using a Bayesian X-ray aperture photometry algorithm in units of net counts, net count rates, photon fluxes, and energy fluxes in multiple apertures. The observables recorded in the different MPDFs may be distinguished by their UCDs which then become relevant for data discovery when a user is searching for specific aperture photometry datasets. +In the example {\em o\_ucd\/} above, the UCD {\em instr.event.pulseHeight\/} is used to represent the detector Pulse Height Amplitude (PHA)\null. There is currently no UCD defined for a raw measure like PHA, but we propose the addition of {\em instr.event.pulseHeight\/} to the UCDs list vocabulary, together with other UCDs that are relevant for \gls{HEA} data, in \S~\ref{sec:UCDs}. Several additional UCDs, including electromagnetic spectrum, physical quantities, and statistical parameters UCDs, are also proposed in \S~\ref{sec:UCDs} that are relevant for \gls{HEA} data products but could also be of use for other domains such as cosmology. -For the \gls{HEA} products, one should also improve some statistical UCDs, that could be of use for other domains, like the cosmology. For instance, one should be able the confidence level of intervals, ie a probability. A typical example is the location error of a source, that could be defined at 68\%, 90\% of 95\%. Also, one should extend the UCD list with particles, new energy ranges (see \ref{sec:UCDs}), and also refine the definition of flux (see \ref{sec:flux}). +Advanced data products may similarly record multiple observables that can only be differentiated through their UCDs. For example, a Chandra Source Catalog {\bf pdf} dataset for a detection may include multiple marginalized probability density functions computed using a Bayesian X-ray aperture photometry algorithm in units of net counts, net count rates, photon fluxes, and energy fluxes in multiple apertures. The observables recorded in the different MPDFs may be distinguished by their UCDs which then become relevant for data discovery when a user is searching for specific aperture photometry datasets. \subsection{{\em proposal\_id}} -To support advanced data products that may be constructed using data from multiple progenitor observations, we propose to modify the ObsCore Recommendation for {\em proposal\_id\/} to allow multiple values, similar to {\em facility\_name\/} and {\em instrument\_name}. +To support advanced data products that may be constructed using data from multiple progenitor observations, we propose to modify the ObsCore Recommendation for {\em proposal\_id\/} to allow multiple values, similar to {\em facility\_name\/} and {\em instrument\_name\/}. \section{Extensions to ObsCore Specific to High Energy Astrophysics Data} @@ -252,17 +246,17 @@ \section{Extensions to ObsCore Specific to High Energy Astrophysics Data} \subsection{{\em ev\_xel}} -The lengths of each data axis (spatial, spectral, time, polarization) captured in attributes {\em s\_xel1\/}, {\em s\_xel2\/}, {\em em\_xel\/}, {\em t\_xel\/}, {\em pol\_xel\/} do not apply non-pixelated data including {\bf event-list}s, and ObsCore recommends that these attributes be set to $-1$. However, the dimensionality of an event list is an important property for data discovery, as the number of events often scales with signal-to-noise (and also data volume scales with number of events). We propose to add a new, optional attribute {\em ev\_xel\/} that records the number of events in an {\bf event-list} (effectively, the length of the ``events'' axis in the {\bf event-list}s table). +The lengths of each data axis (spatial, spectral, time, polarization) captured in attributes {\em s\_xel1\/}, {\em s\_xel2\/}, {\em em\_xel\/}, {\em t\_xel\/}, {\em pol\_xel\/} do not apply non-pixelated data including {\bf event-list}s, and ObsCore recommends that these attributes be set to $-1$. However, the dimensionality of an event list is an important property for data discovery, as the number of events often scales with signal-to-noise (and also data volume scales with number of events). We propose to add a new, optional attribute {\em ev\_xel\/} that records the number of events in an {\bf event-list} (effectively, the length of the ``events'' axis in the {\bf event-list}'s table). \subsection{{\em s\_ref\_energy\/}/{\em em\_ref\_energy\/}/{\em s\_ref\_oaa\/}/{\em em\_ref\_oaa}} -For \gls{HEA} datasets that typically span decades of energy, spatial resolution and sky coverage, and spectral resolution, can be strongly dependent on particle energy. The ObsCore Recommendation suggests that in such circumstances a {\em characteristic\/} value be specified for the spatial and spectral characterization attributes ({\em e.g.\/}, {\em s\_fov\/}, {\em s\_region\/}, {\em s\_resolution\/}, {\em em\_res\_power\/}, {\em em\_resolution\/}). We propose adding optional attributes ({\em s\_ref\_energy\/} for spatial characterization attributes and {\em em\_ref\_energy\/} for spectral characterization attributes) that define the energy (in units of eV) at which these characteristic values are specified. +For \gls{HEA} datasets that typically span decades of energy, spatial resolution, sky coverage, and spectral resolution can be strongly dependent on particle energy. The ObsCore Recommendation suggests that in such circumstances a {\em characteristic\/} value be specified for the spatial and spectral characterization attributes ({\em e.g.\/}, {\em s\_fov\/}, {\em s\_region\/}, {\em s\_resolution\/}, {\em em\_res\_power\/}, {\em em\_resolution\/}). We propose adding optional attributes ({\em s\_ref\_energy\/} for spatial characterization attributes and {\em em\_ref\_energy\/} for spectral characterization attributes) that define the energy (in units of eV) at which these characteristic values are specified. For some \gls{HEA} datasets, these attributes vary strongly with position in the field of view, typically as a function of off-axis angle ({\em i.e.\/}, the angular separation of the target or source from the telescope optical axis). We similarly propose adding optional attributes ({\em s\_ref\_oaa\/} for spatial characterization attributes and {\em em\_ref\_oaa\/} for spectral characterization attributes) that define the off-axis angle (in units of degrees) at which these characteristic values are specified. \subsection{{\em t\_intervals}} -The global time bounds described by {\em t\_min\/}/{\em t\_max} in general are not sufficiently flexible when representing \gls{HEA} datasets and advanced data products from any waveband. The former are typically composed of many \glspl{STI}/\glspl{GTI}, where data are only valid during the stable or good intervals, while advanced data products may be constructed from multiple progenitor observations that can span decades from the start time of the first observations to the stop time of the last observation (albeit very sparsely). For both cases, data queries using only {\em t\_min\/}/{\em t\_max} may not be adequate to determine whether useful scientific data coincide with a transient cosmic phenomenon. In such cases, a more detailed knowledge of the observation time coverage is necessary. We propose to add a new optional attribute {\em t\_intervals} that would contain the list of observation intervals or STIs/GTIs as a TMOC description following the \gls{MOC} IVOA standard \citep{2022ivoa.spec.0727F}. This element could then be compared across data collections to make the data set selection via simple intersection or union operations in TMOC representation. +The global time bounds described by {\em t\_min\/}/{\em t\_max\/} in general are not sufficiently flexible when representing \gls{HEA} datasets or advanced data products from any waveband. The former are typically composed of many \glspl{STI}/\glspl{GTI}, where data are only valid during the stable or good intervals, while advanced data products may be constructed from multiple progenitor observations that can span decades from the start time of the first observations to the stop time of the last observation (albeit very sparsely). For both cases, data queries using only {\em t\_min\/}/{\em t\_max\/} will not be adequate to determine whether useful scientific data coincide with a transient cosmic phenomenon. In such cases, a more detailed knowledge of the observation time coverage is necessary. We propose to add a new optional attribute {\em t\_intervals\/} that would contain the list of observation intervals or \glspl{STI}/\glspl{GTI} as a TMOC description following the \gls{MOC} IVOA standard \citep{2022ivoa.spec.0727F}. This element could then be compared across data collections to make the data set selection via simple intersection or union operations in TMOC representation. We recognize that performing such queries will require enhancements to ADQL, but this capability is sufficiently important for some \gls{HEA} data discovery scenarios that we have chosen to add {\em t\_intervals\/}, in anticipation that ADQL will eventually provide this functionality. @@ -270,29 +264,51 @@ \subsection{{\em energy\_min\/}/{\em energy\_max\/}} The existing attributes {\em em\_min\/} and {\em em\_max\/} that define the coverage of the spectral axis (defined as wavelength expressed in units of m) are not user friendly for \gls{HEA} where datasets are generally selected according to an energy range ({\em i.e.\/}, inverse wavelength) in units of eV (or scaled units of eV, for example keV, MeV, GeV, TeV, PeV). Unlike the radio domain where $\lambda = c/\nu$, where $c$ is an almost universally remembered physical constant, the conversion $\lambda = hc/E$ is not simple for the user to express. As the spectral range covered by \gls{HEA} data is many decades larger than for other wavebands, the accurate numerical representations of typical \gls{HEA} spectral ranges as {\em em\_min\/}/{\em em\_max\/} requires quantities with many digits of precision and exponents ranging from $\sim\!10^{-5}$--$10^{-22}$. Since specification of the spectral range is largely fundamental to data discovery in the \gls{HEA} regime, we propose to add attributes {\em energy\_min\/} and {\em energy\_max\/} that specify the minimum and maximum spectral range values in units of eV\null. Note that the sense of these attributes is {\em opposite\/} that of {\em em\_min\/} and {\em em\_max\/} because of the inverse wavelength relationship between energy and wavelength, so numerical comparisons must be transposed ({\em e.g.\/}, $E>E_{\rm thresh}$ becomes $\lambda100$ TeV) \cr +Q & {\em instr.event\/} & Particle event detection \cr +Q & {\em instr.event.grade\/} & Particle event grade \cr +Q & {\em instr.pulseHeight\/} & Pulse height amplitude measure \cr +Q & {\em instr.event.type\/} & Particle event type \cr +E & {\em phot.count.density\/} & Count or particle flux density (dimensionality: $\rm [L^{-2}\,T^{-1}\,E^{-1}]$) \cr +E & {\em phot.count.density.sb\/} & Count or particle flux density surface brightness (dimensionality: $\rm [L^{-2}\,T^{-1}\,E^{-1}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.count.radiance\/} & Count or particle flux radiance (dimensionality: $\rm [L^{-2}\,T^{-1}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.count.sb\/} & Count or particle flux surface brightness (dimensionality: $\rm [L^{-2}\,T^{-1}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.flux.energy\/} & Energy flux or irradiance (dimensionality: $\rm [M\,T^{-3}]$) \cr +E & {\em phot.flux.energy.density\/} & Energy flux density (dimensionality: $\rm [M\,T^{-3}\,E^{-1}]$) \cr +E & {\em phot.flux.energy.density.sb\/} & Energy flux density surface brightness (dimensionality: $\rm [M\,T^{-3}\,E^{-1}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.flux.energy.sb\/} & Energy flux syrface brightness (dimensionality: $\rm [M\,T^{-3}\,\hbox{sr}^{-1}]$) \cr +S & {\em phys.particle.antiprotron\/} & Related to anti-proton \cr +S & {\em phys.particle.cosmicray\/} & Related to cosmic rays particles \cr +S & {\em phys.particle.electron\/} & Related to electron \cr +S & {\em phys.particle.positron\/} & Related to positron \cr +P & {\em stat.distribution\/} & Type or shape of statistical distribution \cr +P & {\em stat.error.negative\/} & Negative statistical error \cr +P & {\em stat.error.positive\/} & Positive statistical error \cr +P & {\em stat.lowerlimit\/} & Lower limit \cr +P & {\em stat.upperlimit\/} & Upper limit \cr \sptablerule -\caption{UCD words proposed extension} +\caption{Proposed New UCD Entries} \label{tab:he_ucds} \end{longtable} -And here is the proposal to update some existing terms: -\begin{longtable}{p{0.1\linewidth}p{0.3\linewidth}p{0.6\linewidth}} +\begin{longtable}{p{0.05\linewidth}p{0.35\linewidth}p{0.6\linewidth}} \sptablerule -\textbf{Label} & \textbf{UCD word} & \textbf{Description}\cr + \textbf{} & \textbf{UCD word} & \textbf{Description}\cr \sptablerule -S & phys.electron & Electron (not recommended) \cr -S & stat.min & Minimum of a parameter \cr -S & stat.max & Maximum or a parameter \cr -E & phot.count & Flux expressed in counts (dimension: [L$^{-2}$ T$^{-1}$]) \cr -Q & phys.flux & Flux or flow of particle, energy, etc. (dimension: [L$^{-2}$ T$^{-1}$]) \cr -Q & phot.flux.bol & Bolometric flux or irradiance or energy flux (dimension: [W L$^{-2}$] or [E T$^{-1}$ L$^{-2}$]) \cr -E & phot.flux.density & Flux density or differential flux (dimension: [L$^{-2}$ T$^{-1}$ E$^{-1}$]) \cr -E & phot.flux.density.sb & Flux density surface brightness (dimension: [L$^{-2}$ T$^{-1}$ E$^{-1}$ sr$^{-1}$])\cr -E & phot.flux.sb & Flux surface brightness or flux per steradian (dimension: [L$^{-2}$ T$^{-1}$ sr$^{-1}$]) \cr -E & phot.fluence & Radiant photon energy received by a surface per unit area or irradiance of a surface integrated over time of irradiation (dimension: [L$^{-2}$]) \cr -Q & phys.flux.energy & Energy flux, heat flux (dimension: [W L$^{-2}$] or [E T$^{-1}$ L$^{-2}$]) \cr -E & phys.luminosity & Luminosity or energy par unit of time (dimension: [W] or [E T$^{-1}$]) \cr -S & em.gamma.hard & 500keV-100MeV \cr +S & {\em em.gamma.hard\/} & Hard gamma ray (500 keV -- 100 MeV) \cr +P & {\em stat.confidenceLevel\/} & Level of confidence for a statistical measure, detection, or classification process \cr +E & {\em phot.count\/} & Count or particle flux (dimensionality: $\rm [L^{-2}\,T^{-1}]$) \cr +E & {\em phot.fluence\/} & Radiant photon energy received by a surface per unit area or irradiance of a surface integrated over time of irradiation (dimensionality: $\rm [L^{-2}]$) \cr +Q & {\em phot.flux.bol\/} & Bolometric flux (dimensionality: $\rm [M\,T^{-3}]$) \cr +E & {\em phot.radiance\/} & Radiance as energy flux per solid angle (dimensionality: $\rm [L^{-2}\,T^{-1}\,\hbox{sr}^{-1}]$) \cr +S & {\em phys.electron\/} & Electron (not recommended/deprecate) \cr +S & {\em stat.min\/} & Minimum value \cr +S & {\em stat.max\/} & Maximum value \cr \sptablerule -\caption{UCD words proposed upgrade} +\caption{Proposed Revised UCD Entries} \label{tab:upgrade_he_ucds} \end{longtable} -\subsection{MIME-types Enhancements}\label{sec:mimetypes} +\subsection{MIME-type Enhancements}\label{sec:mimetypes} -Data files used in the \gls{HEA} domain should have appropriate MIME-types, so that they can be shown in ObsCore tables or elsewhere. +Data files used in the \gls{HEA} domain should have appropriate MIME-types, so that they can be included in ObsCore tables or elsewhere. -Formats based on FITS could thus be declared as: +Many \gls{HEA} FITS format data products will comply with existing MIME-types discussed in the ObsCore Recommendation, such as {\bf application/fits} or {\bf application/x-fits-bintable}. However, the gamma-ray community has developed two additional data format standards and we propose to add the following MIME-types: \begin{itemize} -\item {\bf x-fits-gadf}: for FITS files following the GADF specification \citep{deil_2022_7304668} -\item {\bf x-fits-vodf}: for FITS files following the VODF specification \citep{2023arXiv230813385K} +\item {\bf x-fits-gadf}: for FITS files following the Gamma-Astro-Data-Format (GADF) specification \citep{deil_2022_7304668}, +\item {\bf x-fits-vodf}: for FITS files following the Very-high-energy Open Data Format (VODF) specification \citep{2023arXiv230813385K}. \end{itemize} -\section{Proposal of the {\it ivoa.obscore\_hea} attributes}\label{sec:ibscoreext} +\section{Proposed ivoa.obscore\_hea Table Attributes}\label{sec:ibscoreext} -This section summarizes the proposal for the \gls{HEA} extension of ObsCore. Note that the name of the extension could be {\it ivoa.obscore\_hea}, {\it ivoa.obscore-hea}, {\it ivoa.obscore\_he} or {\it ivoa.obscore-he}. We use here the first term in the Appendix \ref{sec:uc}. +This section summarizes the proposal for the \gls{HEA} extension of ObsCore. We use the term {\em ivoa.obscore\_hea\/} to described the extension here and in Appendix~\ref{sec:uc}. +\begin{landscape} \begin{center} -%\begin{tabular}{ | m{2.5cm} | m{3em} | m{3em} | m{3em} | m{6cm} | m{2.3em} |} -\begin{longtable}{ | m{2.5cm} | m{3em} | m{3em} | m{3em} | m{6cm} | m{2.3em} |} +%\begin{longtable}{ | m{2.5cm} | m{3em} | m{3em} | m{3em} | m{6cm} | m{2.3em} |} +\begin{longtable}{ | p{0.125\linewidth} | p{0.075\linewidth} | p{0.075\linewidth} | p{0.075\linewidth} | p{0.6\linewidth} | p{0.05\linewidth} |} \hline {\centering \bf Column Name} &{\centering \bf UType} &{\centering \bf Unit} &{\centering \bf Type} &{\centering \bf Description} &{\centering \bf MAN}\\ \hline - ev\_xel & TBD & unitless & int & Number of events in an event\_list & NO \\ +{\em ev\_xel\/} & TBD & unitless & integer & Number of events in an event list & NO \\ +\hline +{\em s\_ref\_energy\/} & TBD & eV & double & Energy at which the ObsCore spatial characterization attributes {\em s\_fov\/}, {\em s\_region\/}, {\em s\_resolution\/} are defined & NO \\ \hline - s\_ref\_energy & TBD & eV & float & Energy at which the ObsCore spatial characterisation attributes s\_fov , s\_region, s\_resolution are defined & NO \\ +{\em em\_ref\_energy\/} & TBD & eV & double & Energy at which the ObsCore spectral characterization attributes {\em em\_res\_power\/}, {\em em\_resolution} are defined & NO \\ \hline - em\_ref\_energy & TBD & eV & float & Energy at which the ObsCore spatial characterisation attributes em\_res\_power, em\_resolution are defined & NO \\ +{\em s\_ref\_oaa\/} & TBD & deg & double & Off-axis angle ({\em i.e.\/}, the angular separation of the target or source from the telescope optical axis) at which the ObsCore spatial characterization attributes {\em s\_fov\/}, {\em s\_region\/}, {\em s\_resolution\/} are defined & NO \\ \hline - s\_ref\_oaa & TBD & deg & float & Off-axis angle ({\em i.e.\/}, the angular separation of the target or source from the telescope optical axis) at which the ObsCore spatial characterisation attributes s\_fov , s\_region, s\_resolution are defined & NO \\ +{\em em\_ref\_oaa\/} & TBD & deg & double & Off-axis angle ({\em i.e.\/}, the angular separation of the target or source from the telescope optical axis) at which the ObsCore spectral characterization attributes {\em em\_res\_power\/}, {\em em\_resolution\/} are defined & NO \\ \hline - em\_ref\_oaa & TBD & deg & float & Off-axis angle ({\em i.e.\/}, the angular separation of the target or source from the telescope optical axis) at which the ObsCore spectral characterisation attributes em\_res\_power, em\_resolution are defined & NO \\ +{\em t\_intervals\/} & TBD & unitless & string & List of observation intervals or stable/good time intervals describing the exact observation time coverage as a TMOC & NO \\ \hline - t\_intervals & TBD & unitless & TMOC &List of observation intervals or stable/good time intervals describing the exact observation time coverage & NO \\ +{\em energy\_min\/} & TBD & eV & double & Energy associated to the ObsCore attribute {\em em\_max\/}, describing the minimum energy of the dataset & NO \\ \hline - energy\_min & TBD & float & eV & Energy associated to the ObsCore attribute em\_max, describing the minimal energy of the dataset & NO \\ +{\em energy\_max\/} & TBD & eV & double & Energy associated to the ObsCore attribute {\em em\_min\/}, describing the maximum energy of the dataset & NO \\ \hline - energy\_max & TBD & float & eV & Energy associated to the ObsCore attribute em\_min, describing the maximal energy of the dataset & NO \\ +{\em obs\_mode\/} & TBD & unitless & string & Observation mode of an observation & NO \\ \hline - obs\_mode & TBD & unitless & string & Observation mode of the observation (e.g. TBU) & NO \\ +{\em tracking\_type\/} & TBD & unitless & string & Tracking type of an observation & NO \\ \hline - tracking\_mode & TBD & unitless & string & Tracking mode of an observation (e.g., sidereal rate, moving target [solar system] tracking, drift scans) & NO \\ +{\em scan\_mode\/} & TBD & unitless & string & Scan mode of an observation & NO \\ \hline - analysis\_mode & TBD & unitless & string & Data reduction/analysis mode& NO \\ +{\em pointing\_mode\/} & TBD & unitless & string & Pointing mode of an observation & NO \\ \hline - event\_type & TBD & unitless & string & Data quality flag of the events (e.g. ``good psf'', ``good rejection'', ``Nhit (100,200)'' & NO \\ +{\em analysis\_mode\/} & TBD & unitless & string & Data reduction/analysis mode & NO \\ \hline -\caption{Attributes for the \gls{HEA} extension of ObsCore} +{\em event\_type\/} & TBD & unitless & string & Event subset indicator ({\em e.g.\/}, data quality flag for the events) & NO \\ +\hline +\caption{Attributes for the \gls{HEA} Extension of ObsCore} \label{tab:hea_ext_attr} \end{longtable} \end{center} +\end{landscape} -\section{Summary}\label{sec:summary} - -\todo[inline]{\bf BKH: Ian, could you start writing the final section?} - \pagebreak -\printglossaries +%\printglossaries \bibliography{ivoatex/docrepo, ivoatex/ivoabib, HighEnergyObsCoreExt} @@ -600,17 +629,17 @@ \section{Changes from Previous Versions} \section{Contributions to the Note} -The authors of this Note contributed to write and structure the text. However, the note was initiated and elaborated in several dedicated workshops, Interop meetings, and in specific \gls{IVOA} \gls{HEIG} group meetings, involving more people. The \gls{IVOA} \gls{HEA} group keeps track of its activities on an \gls{IVOA} web page: \url{https://wiki.ivoa.net/twiki/bin/view/IVOA/HEGroup}. +The authors of this Note contributed to the content and structure the text. However, the note was initiated and elaborated in several dedicated workshops, Interop meetings, and in specific \gls{IVOA} \gls{HEIG} group meetings, involving more people. The \gls{IVOA} \gls{HEIG} group keeps track of its activities on an \gls{IVOA} web page: \url{https://wiki.ivoa.net/twiki/bin/view/IVOA/HEGroup}. Further material can be found with those links: \begin{itemize} - \item 2025-04-29: French ASOV workship ``High Energy in the VO'', \url{https://indico.obspm.fr/event/2674/} - \item 2024-11-16: IVOA Malta meeting, DM session with 2 High Energy presentations (B. Khelifi/I. Evans), \url{https://wiki.ivoa.net/twiki/bin/view/IVOA/InterOpNov2024DM} - \item 2024-11-15: IVOA Malta Plenary, CSP Plenary session, \url{https://wiki.ivoa.net/twiki/bin/view/IVOA/InterOpNov2024CSPPlenary} - \item 2024-05-21: IVOA Sydney meeting, DM Session High Energy focus, \url{https://wiki.ivoa.net/twiki/bin/view/IVOA/InterOpMay2024DM} - \item 2023-06-28: IVOA standards for High Energy Astrophysics (French VO Workshop), \url{https://indico.obspm.fr/event/1963/} - \item 2023-05-11: IVOA Bologna meeting: presentation ("DM for High Energy astrophysics", M. Servillat) and first IVOA HE group meeting, \url{https://wiki.ivoa.net/internal/IVOA/IntropMay3023DM/2023-05-11_IVOA_meeting_-_VOHE.pdf} - \item 2022-10-11: Virtual Observatory and High Energy Astrophysics (French VO Workshop), \url{https://indico.obspm.fr/event/1489/} + \item 2025-04-29: French ASOV workship ``High Energy in the VO'', \url{https://indico.obspm.fr/event/2674/}, + \item 2024-11-16: IVOA Malta meeting, DM session with two High Energy presentations (B. Kh\'elifi/I. Evans), \url{https://wiki.ivoa.net/twiki/bin/view/IVOA/InterOpNov2024DM}, + \item 2024-11-15: IVOA Malta Plenary, CSP Plenary session, \url{https://wiki.ivoa.net/twiki/bin/view/IVOA/InterOpNov2024CSPPlenary}, + \item 2024-05-21: IVOA Sydney meeting, DM Session High Energy focus, \url{https://wiki.ivoa.net/twiki/bin/view/IVOA/InterOpMay2024DM}, + \item 2023-06-28: IVOA standards for High Energy Astrophysics (French VO Workshop), \url{https://indico.obspm.fr/event/1963/}, + \item 2023-05-11: IVOA Bologna meeting: presentation (``DM for High Energy astrophysics'', M. Servillat) and first IVOA HE group meeting, \url{https://wiki.ivoa.net/internal/IVOA/IntropMay3023DM/2023-05-11_IVOA_meeting_-_VOHE.pdf}, + \item 2022-10-11: Virtual Observatory and High Energy Astrophysics (French VO Workshop), \url{https://indico.obspm.fr/event/1489/}. \end{itemize} % NOTE: IVOA recommendations must be cited from docrepo rather than ivoabib diff --git a/Makefile b/Makefile index c554bca..e99386c 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ DOCNAME = HighEnergyObsCoreExt DOCVERSION = 1.0 # Publication date, ISO format; update manually for "releases" -DOCDATE = 2025-10-26 +DOCDATE = 2025-11-06 # What is it you're writing: NOTE, WD, PR, REC, PEN, or EN DOCTYPE = PEN diff --git a/UseCases.tex b/UseCases.tex index 9729e39..0d361d9 100644 --- a/UseCases.tex +++ b/UseCases.tex @@ -8,8 +8,8 @@ \subsubsection{Use Case --- Search for event lists surrounding Sgr A*, for examp \medskip \noindent Find all datasets satisfying: \begin{enumerate}[(i)] - \item Target name = ``Sgr A*'' or position inside 30 arcmin from (266.4168, $-29.0078$) - \item dataproduct\_type = ``event-list'' + \item Target name = ``Sgr A*'' or position inside 30 arcmin from (266.4168, $-29.0078$), + \item dataproduct\_type = ``event-list''. \end{enumerate} \begin{verbatim} @@ -22,141 +22,179 @@ \subsubsection{Use Case --- Search for event lists surrounding Sgr A*, for examp \end{verbatim} -\subsubsection{Use Case --- Search for event bundles that include Cas A for X-ray spectrophotometric evolution studies} +\subsubsection{Use Case --- Search for event lists that include a fully calibrated spectral axis for BL Lac for rapid X-ray spectrophotometric evaluation} -{\em Identify all event bundles that include the Cas A SNR and have at least 1 million events for subsequent spectrophotometric studies of the SNR expansion. Since only a few observations are expected to match this request and because the focus is on X-ray spectrophotometric studies, the event bundles that include the responses or the ancillary products used to make the responses are required.\/} +{\em Identify all event lists that include the BL Lac, have a fully calibrated spectral axis (i.e., spectral responses have already been applied), and have at least 10,000 events. These data will be used to prepare slides for a presentation. Note that since calib\_status = 2 may not specify that the spectral axis is fully calibrated in physical units (HEA event lists are often considered ``calibrated'' even if the spectral axis is in pulse height units) the calibration status of the spectral axis must be checked explicitly.\/} \medskip \noindent Find all datasets satisfying: \begin{enumerate}[(i)] - \item Target name = ``Cas A'' or position inside 10 arcmin from (350.8584, $+58.8113$) - \item dataproduct\_type = ``event-bundle'' - \item ev\_xel >= 1,000,000 + \item Target name = ``BL Lac'' or position inside 5 arcmin from (330.680338, $+42.27777$), + \item dataproduct\_type = ``event-list'', + \item calib\_level $\geq 2$, + \item em\_calib\_status = ``calibrated'', + \item ev\_xel $\geq 10000$. \end{enumerate} \begin{verbatim} SELECT * FROM ivoa.obscore NATURAL JOIN ivoa.obscore_hea WHERE -(target_name = 'Cas A' OR -CONTAINS(POINT(s_ra, s_dec), CIRCLE, 350.8584, +58.8113, 0.16667) = 1) -AND (dataproduct_type = 'event-bundle') -AND (ev_xel >= 1000000) +(target_name = 'BL Lac' OR +CONTAINS(POINT(s_ra, s_dec), CIRCLE, 330.680338, +42.27777, 0.083333) = 1) +AND (dataproduct_type = 'event-list') +AND (calib_level >= 2) +AND (em_calib_status = 'calibrated') +AND (ev_xel >= 10000) \end{verbatim} -\subsubsection{Use Case --- Search for event bundles via DataLink that include Cas A for a TeV spectromorphology study} -{\em Identify all event lists and their associated \glspl{IRF} that include the Cas A SNR for subsequent TeV spectromorphology studies from a VERITAS data release. Since the instrumental responses are mandatory to remove instrumetal effects, the event bundles that include the \glspl{IRF} are required.\/} +\subsubsection{Use Case --- Search for SWGO event lists and their \glspl{IRF} for the event type `very-good' in the region of Cygnus loop for a TeV spectromorphology study} + +{\em Identify all event lists and their associated \glspl{IRF} of the region of Cygnus loop ($3^{\circ}$ diameter) taken with SWGO. Only data of the event type `very-good' are selected, in order to limit the amount of downloaded data.\/} \medskip -\noindent Find all VERITAS datasets satisfying: +\noindent Find all SWGO datasets satisfying: \begin{enumerate}[(i)] - \item Target name = ``Cas A'' or position inside 4 deg from (350.8584, $+58.8113$) - \item dataproduct\_type = ``event-bundle'' - \item obs\_collection = ``VERITAS-DR1'' - \item access\_format =``datalink'' + \item s\_region position intersects with the source modeled by a circle of 1.5 deg around (312.775, 30.683), + \item dataproduct\_type = ``event-list'' or ``aeff'' or ``edisp'' or ``psf'' or ``bkgrate'', + \item obs\_collection = ``SWGO-DR1'', + \item event\_type = ``very-good''. \end{enumerate} +First, run the ObCore query: \begin{verbatim} SELECT * FROM ivoa.obscore NATURAL JOIN ivoa.obscore_hea -WHERE -(target_name = 'Cas A' OR -CONTAINS(POINT(s_ra, s_dec), CIRCLE, 350.8584, +58.8113, 4.0) = 1) -AND (dataproduct_type = 'event-bundle') -AND (obs_collection = 'VERITAS-DR1') -AND (access_format = ’application/x-votable+xml;content=datalink’) +WHERE (INTERSECTS(s_region, CIRCLE(312.775, 30.683, 1.5)) = 1) +AND (dataproduct_type = 'event-list' OR dataproduct_type = 'aeff' + OR dataproduct_type = 'edisp' OR dataproduct_type = 'psf' + OR dataproduct_type = 'bkgrate') +AND (obs_collection = 'SWGO-DR1') +AND (event_type = 'very-good') \end{verbatim} -Then, for each row of the output, we get the ``access\_url'' of the DataLink and access to the data. +Then, for each row of the output, we identify the nature of the data product, and retrieve them using the ``access\_url''. -\subsubsection{Use Case --- Identify PSF response-functions for further analysis of previously downloaded data products} +\subsubsection{Use Case --- Search for event lists and their \glspl{IRF} of CTAO South observations at energies above 10 TeV for blind search of PeVatrons from a data release using DataLink} -{\em Identify all Chandra Source Catalog point spread functions for source detections that fall within 2 arcmin radius of (83.84358, $-5.43639$) in the Orion star-forming complex for Chandra observation 4374. These PSFs will be used to analyze previously downloaded catalog data products for the same field.\/ } +{\em Identify all event lists and their associated \glspl{IRF} taken by CTAO South that contains events above 10 TeV. Data taken with the Small Size Telescopes or Medium Size Telescopes can be then selected. \/} \medskip -\noindent Find all datasets satisfying: +\noindent Find all CTAO datasets satisfying: \begin{enumerate}[(i)] - \item Position inside 3 arcmin from (83.84358, $-5.43639$) - \item dataproduct\_type = ``response-function'' - \item dataproduct\_subtype = ``psf'' - \item obs\_id = ``4374'' - \item obs\_collection = ``CSC2'' + \item dataproduct\_type = ``event-list'', + \item obs\_collection = ``CTAO-DR1'', + \item access\_format = ``datalink'', + \item instrument\_name contains ``CTAO-S'', + \item energy\_max >= $10^{12}$. \end{enumerate} +First, run the ObCore query: \begin{verbatim} SELECT * FROM ivoa.obscore NATURAL JOIN ivoa.obscore_hea -WHERE -(CONTAINS(POINT(s_ra, s_dec), CIRCLE, 83.84358, -5.43639, 0.033333) = 1) -AND (dataproduct_type = 'response-function') -AND (dataproduct_subtype = 'psf') -AND (obs_id = '4374') -AND (obs_collection = 'CSC2') +WHERE (dataproduct_type = 'event-list') +AND (obs_collection = 'CTAO-DR1') +AND (access_format = 'application/x-votable+xml;content=datalink') +AND (instrument_name LIKE 'CTAO-S') +AND (energy\_max >= 1.0e+12) \end{verbatim} +The query output is a VOTable that follows the DALI specification. -\subsubsection{Use Case --- Get all the \glspl{IRF} for a given CTAO observation, for simulation purposes} +We process this VOTABLE to access to the data: -{\em Simulations are frequently used to estimate the science performance for a given astrophysical use case. To realise such simulations, \glspl{IRF} are required. \/ } +\begin{enumerate}[(i)] + \item for each row of the query output, get the ``obs\_id'' and the ``access\_url'' of the DataLink, + \item get the VOTABLE associated with the ``access\_url'', + \item for each row of the VOTABLE, get the ``content\_qualifier'' and the ``accessURL'', + \item download the data associated to each ``accessURL''. +\end{enumerate} + +\begin{verbatim} +FOR EACH ROW OF OUTPUT_VOTABLE: + OBS_ID = OUTPUT_VOTABLE['ID'] + DATALINK_TABLE = GET OUTPUT_VOTABLE['access_url'] + FOR EACH ROW OF DATALINK_TABLE: + - IF ROW['content_qualifier'] = 'aeff' + AEFF_FILE['OBS_ID'] = GET RAW['accessURL'] + - IF ROW['content_qualifier'] = 'edisp' + EDISP_FILE['OBS_ID'] = GET RAW['accessURL'] + - IF ROW['content_qualifier'] = 'psf' + PSF_FILE['OBS_ID'] = GET RAW['accessURL'] + - IF ROW['content_qualifier'] = 'bkgrate' + BKGRATE_FILE['OBS_ID'] = GET RAW['accessURL'] + - IF ROW['content_qualifier'] = 'event-list' + EVENT_FILE['OBS_ID'] = GET RAW['accessURL'] +\end{verbatim} + + +\subsubsection{Use Case --- Search for event bundles that include Cas A for X-ray spectrophotometric evolution studies} + +{\em Identify all event bundles that include the Cas A SNR and have at least 1 million events for subsequent spectrophotometric studies of the SNR expansion. Since only a few observations are expected to match this request and because the focus is on X-ray spectrophotometric studies, the event bundles that include the responses or the ancillary products used to make the responses are required.\/} \medskip -\noindent Find the CTAO datasets satisfying: +\noindent Find all datasets satisfying: \begin{enumerate}[(i)] - \item dataproduct\_type = ``aeff'' or dataproduct\_type = ``psf'' or dataproduct\_type = ``edisp'' or dataproduct\_type = ``bkgrate'' - \item obs\_id = ``4374'' - \item obs\_collection = ``CTAO-DR1'' + \item Target name = ``Cas A'' or position inside 10 arcmin from (350.8584, $+58.8113$), + \item dataproduct\_type = ``event-bundle'', + \item ev\_xel $\geq 1000000$. \end{enumerate} \begin{verbatim} SELECT * FROM ivoa.obscore NATURAL JOIN ivoa.obscore_hea WHERE -(dataproduct_type = 'event-list' OR dataproduct_type = 'aeff' - OR dataproduct_type = 'edisp' - OR dataproduct_type = 'psf' OR dataproduct_type = 'bkgrate') -AND (obs_id = '4374') -AND (obs_collection = 'CTAO-DR1') +(target_name = 'Cas A' OR +CONTAINS(POINT(s_ra, s_dec), CIRCLE, 350.8584, +58.8113, 0.16667) = 1) +AND (dataproduct_type = 'event-bundle') +AND (ev_xel >= 1000000) \end{verbatim} -\subsubsection{Use Case --- Search for event lists that include a fully calibrated spectral axis for BL Lac for rapid X-ray spectrophotometric evaluation} +\subsubsection{Use Case --- Search for event bundles via DataLink that include Cas A for a TeV spectromorphology study} -{\em Identify all event lists that include the BL Lac, have a fully calibrated spectral axis (i.e., spectral responses have already been applied), and have at least 10,000 events. These data will be used to prepare slides for a presentation. Note that since calib\_status = 2 may not specify that the spectral axis is fully calibrated in physical units (HEA event lists are often considered ``calibrated'' even if the spectral axis is in pulse height units ) the calibration status of the spectral axis must be checked explicitly.\/} +{\em Identify all event lists and their associated \glspl{IRF} that include the Cas A SNR for subsequent TeV spectromorphology studies from a VERITAS data release. Since the instrumental responses are mandatory to remove instrumental effects, the event bundles that include the \glspl{IRF} are required.\/} \medskip -\noindent Find all datasets satisfying: +\noindent Find all VERITAS datasets satisfying: \begin{enumerate}[(i)] - \item Target name = ``BL Lac'' or position inside 5 arcmin from (330.680338, $+42.27777$) - \item dataproduct\_type = ``event-list'' - \item calib\_level $\geq 2$ - \item em\_calib\_status = ``calibrated'' - \item ev\_xel $\geq 10000$ + \item Target name = ``Cas A'' or position inside 4 deg from (350.8584, $+58.8113$), + \item dataproduct\_type = ``event-bundle'', + \item obs\_collection = ``VERITAS-DR1'', + \item access\_format = ``datalink''. \end{enumerate} \begin{verbatim} SELECT * FROM ivoa.obscore NATURAL JOIN ivoa.obscore_hea WHERE -(target_name = 'BL Lac' OR -CONTAINS(POINT(s_ra, s_dec), CIRCLE, 330.680338, +42.27777, 0.083333) = 1) -AND (dataproduct_type = 'event-list') -AND (calib_level >= 2) -AND (em_calib_status = 'calibrated') -AND (ev_xel >= 10000) +(target_name = 'Cas A' OR +CONTAINS(POINT(s_ra, s_dec), CIRCLE, 350.8584, +58.8113, 4.0) = 1) +AND (dataproduct_type = 'event-bundle') +AND (obs_collection = 'VERITAS-DR1') +AND (access_format = ’application/x-votable+xml;content=datalink’) \end{verbatim} +Then, for each row of the output, we get the ``access\_url'' of the DataLink to provide access to the data. + -\subsubsection{Use Case --- Search for spatially resolved spectropolarimetric observation of the Crab with spectral resolution R > 100} +\subsubsection{Use Case --- Search for spatially resolved spectropolarimetric observations of the Crab with spectral resolution R > 100} {\em Identify all event bundles for observations of the Crab that intersect the 1.0--100.0 keV energy range, have calibrated spatial and time axes, are spatially resolved in 2 dimensions in equatorial coordinates, have spectral resolution $R>100$, and include polarimetry measurements. Note that ObsCore specifies that the axes lengths --- s\_xel1, s\_xel2, em\_xel, t\_xel, pol\_xel --- should be set to $-1$ for non-pixelated data like event lists, so these quantities are not useful for this query.\/} \medskip \noindent Find all datasets satisfying: \begin{enumerate}[(i)] - \item Target name = ``Crab'' or position inside 5 arcmin from (83.6324, $+22.0174$) - \item dataproduct\_type = ``event-bundle'' - \item calib\_level $\geq 2$ + \item Target name = ``Crab'' or position inside 5 arcmin from (83.6324, $+22.0174$), + \item dataproduct\_type = ``event-bundle'', + \item calib\_level $\geq 2$, + \item s\_resolution $> 100$, + \item energy\_min $< 100000$, + \item energy\_max $> 1000$, + \item o\_ucd contains ``phys.polarization'', + \item o\_ucd contains ``pos.eq''. \end{enumerate} \begin{verbatim} @@ -173,87 +211,58 @@ \subsubsection{Use Case --- Search for spatially resolved spectropolarimetric ob AND (o.ucd LIKE '%pos.eq%') \end{verbatim} -\subsubsection{Use Case --- Search for SWGO event lists and their \glspl{IRF} for the event type `very-good' in the region of Cygnus loop for a TeV spectromorphology study} -{\em Identify all event lists and their associated \glspl{IRF} of the region of Cygnus loop (3$^{\circ}$ diameter) taken with SWGO. Only data of the event type `very-good' are selected, in order to limit the amount of downloaded data.\/} +\subsubsection{Use Case --- Identify PSF response-functions for further analysis of previously downloaded data products} + +{\em Identify all Chandra Source Catalog point spread functions for source detections that fall within 2 arcmin radius of (83.84358, $-5.43639$) in the Orion star-forming complex for Chandra observation 4374. These PSFs will be used to analyze previously downloaded catalog data products for the same field.\/ } \medskip -\noindent Find all SWGO datasets satisfying: +\noindent Find all datasets satisfying: \begin{enumerate}[(i)] - \item s\_region position intersects with the source modeled by a circle of 1.5 deg around (312.775, 30.683) - \item dataproduct\_type = ``event-list'' or ``aeff'' or ``edisp'' or ``psf'' or ``bkgrate'' - \item obs\_collection = ``SWGO-DR1'' - \item event\_type = ``very-good'' + \item Position inside 3 arcmin from (83.84358, $-5.43639$), + \item dataproduct\_type = ``response-function'', + \item dataproduct\_subtype = ``psf'', + \item obs\_id = ``4374'', + \item obs\_collection = ``CSC2''. \end{enumerate} -First, run the ObCore query: \begin{verbatim} SELECT * FROM ivoa.obscore NATURAL JOIN ivoa.obscore_hea -WHERE (INTERSECTS(s_region, CIRCLE(312.775, 30.683, 1.5)) = 1) -AND (dataproduct_type = 'event-list' OR dataproduct_type = 'aeff' - OR dataproduct_type = 'edisp' OR dataproduct_type = 'psf' OR - dataproduct_type = 'bkgrate') -AND (obs_collection = 'SWGO-DR1') -AND (event_type = 'very-good') +WHERE +(CONTAINS(POINT(s_ra, s_dec), CIRCLE, 83.84358, -5.43639, 0.033333) = 1) +AND (dataproduct_type = 'response-function') +AND (dataproduct_subtype = 'psf') +AND (obs_id = '4374') +AND (obs_collection = 'CSC2') \end{verbatim} -Then, for each row of the output, we identify the nature of the data and get it from its ``access\_url''. -\subsubsection{Use Case --- Search for event lists and their \glspl{IRF} of CTAO South observations at energies above 10 TeV for blind search of PeVatrons from a data release using DataLink} +\subsubsection{Use Case --- Get all the \glspl{IRF} for a given CTAO observation, for simulation purposes} -{\em Identify all event lists and their associated \glspl{IRF} taken by CTAO South that contains events above 10 TeV. Data taken with the Small Size Telescopes or Medium Size Telescopes can be then selected. \/} +{\em Simulations are frequently used to estimate the science performance for a given astrophysical use case. To realise such simulations, \glspl{IRF} are required.\/ } \medskip -\noindent Find all CTAO datasets satisfying: +\noindent Find the CTAO datasets satisfying: \begin{enumerate}[(i)] - \item dataproduct\_type = ``event-list'' - \item obs\_collection = ``CTAO-DR1'' - \item access\_format = ``datalink'' - \item ``CTA-S'' in instrument\_name - \item energy\_max >= 10e+12 + \item dataproduct\_type = ``event-list'' or dataproduct\_type = ``aeff'' or dataproduct\_type = ``psf'' or dataproduct\_type = ``edisp'' or dataproduct\_type = ``bkgrate'', + \item obs\_id = ``4374'', + \item obs\_collection = ``CTAO-DR1''. \end{enumerate} -First, run the ObCore query: \begin{verbatim} SELECT * FROM ivoa.obscore NATURAL JOIN ivoa.obscore_hea -WHERE (dataproduct_type = 'event-list') +WHERE +(dataproduct_type = 'event-list' OR dataproduct_type = 'aeff' + OR dataproduct_type = 'edisp' OR dataproduct_type = 'psf' + OR dataproduct_type = 'bkgrate') +AND (obs_id = '4374') AND (obs_collection = 'CTAO-DR1') -AND (access_format = 'application/x-votable+xml;content=datalink') -AND (instrument_name LIKE 'CTAO-S') -AND (ENERGYMAX GE 10e+12) \end{verbatim} -The query output is a VOTable that follows the DALI specification. -\medskip -\noindent Then, process this VOTABLE to access to the data: -\begin{enumerate}[(i)] - \item for each row of the query output, get the ``obs\_id'' and the ``access\_url'' of the DataLink - \item get the VOTABLE associated with the ``access\_url'' - \item for each row of the VOTABLE, get the ``content\_qualifier'' and the ``accessURL'' - \item download the data associated to each ``accessURL'' -\end{enumerate} - -\begin{verbatim} -FOR EACH ROW OF OUTPUT_VOTABLE: - OBS_ID = OUTPUT_VOTABLE['ID'] - DATALINK_TABLE = GET OUTPUT_VOTABLE['access_url'] - FOR EACH ROW OF DATALINK_TABLE: - - IF ROW['content_qualifier'] = 'aeff' - AEFF_FILE['OBS_ID'] = GET RAW['accessURL'] - - IF ROW['content_qualifier'] = 'edisp' - EDISP_FILE['OBS_ID'] = GET RAW['accessURL'] - - IF ROW['content_qualifier'] = 'psf' - PSF_FILE['OBS_ID'] = GET RAW['accessURL'] - - IF ROW['content_qualifier'] = 'bkgrate' - BKGRATE_FILE['OBS_ID'] = GET RAW['accessURL'] - - IF ROW['content_qualifier'] = 'event-list' - EVENT_FILE['OBS_ID'] = GET RAW['accessURL'] -\end{verbatim} - -\subsection{Very-High-Level Data Products} +\subsection{Advanced Data Products} \subsubsection{Use Case --- Search for Chandra Source Catalog position error MCMC draws for X-ray detections in the vicinity of Gaia DR3 486718823701242368} @@ -262,10 +271,10 @@ \subsubsection{Use Case --- Search for Chandra Source Catalog position error MCM \medskip \noindent Find all datasets satisfying: \begin{enumerate}[(i)] - \item Position within 5.0 arcsec from (54.036061, $+61.907633$) - \item dataproduct\_type = ``draws'' - \item dataproduct\_subtype = ``poserr'' - \item obs\_collection = ``CSC2'' + \item Position within 5.0 arcsec from (54.036061, $+61.907633$), + \item dataproduct\_type = ``draws'', + \item dataproduct\_subtype = ``poserr'', + \item obs\_collection = ``CSC2''. \end{enumerate} \begin{verbatim} @@ -286,12 +295,12 @@ \subsubsection{Use Case --- Search for flux maps for CTAO-North observations bet \medskip \noindent Find all datasets satisfying: \begin{enumerate}[(i)] - \item dataproduct\_type = ``image'' - \item dataproduct\_subtype = ``flux-map'' - \item obs\_collection = ``CTAO-DR1'' - \item int(obs\_id) >= 4374 - \item int(obs\_id) <= 4379 - \item ``CTAO-N'' in instrument\_name + \item dataproduct\_type = ``image'', + \item dataproduct\_subtype = ``fluxmap'', + \item obs\_collection = ``CTAO-DR1'', + \item int(obs\_id) $> 4374$, + \item int(obs\_id) $\leq 4379$, + \item instrument\_name contains ``CTAO-N''. \end{enumerate} \begin{verbatim} @@ -299,7 +308,7 @@ \subsubsection{Use Case --- Search for flux maps for CTAO-North observations bet NATURAL JOIN ivoa.obscore_hea WHERE (dataproduct_type = 'image') -AND (dataproduct_subtype = 'flux-map') +AND (dataproduct_subtype = 'fluxmap') AND (obs_collection = 'CTAO-DR1') AND (instrument_name LIKE 'CTAO-N') AND (CAST(obs_id AS INTEGER) > 4374) @@ -314,11 +323,11 @@ \subsubsection{Use Case --- Search for M31 source light curves and aperture phot \medskip \noindent Find all datasets satisfying: \begin{enumerate}[(i)] - \item Position within 1.5 degrees from (10.6847, $+41.2688$) - \item dataproduct\_type = ``timeseries'' {\em or\/} ``pdf'' - \item calib\_level = 4 - \item energy\_min $\leq 0.3$ {\em and\/} energy\_max $\geq 7.0$ - \item t\_intervals TMOC intersects\footnote{We note that this functiondoes not yet exist in ADQL} MJD 56320--56325 TT + \item Position within 1.5 degrees from (10.6847, $+41.2688$), + \item dataproduct\_type = ``timeseries'' {\em or\/} ``pdf'', + \item calib\_level = 4, + \item energy\_min $\leq 0.3$ {\em and\/} energy\_max $\geq 7.0$, + \item t\_intervals TMOC intersects\footnote{We note that this functiondoes not yet exist in ADQL} MJD 56320--56325 TT. \end{enumerate} @@ -332,7 +341,6 @@ \subsubsection{Use Case --- Search for M31 source light curves and aperture phot AND (energy_min <= 300.0) AND (energy_max >= 7000.0) AND (INTERSECTS(TMOC(17, t_intervals), ...) = 1) \end{verbatim} -% Can someone please update the last line? \subsubsection{Use Case --- Search for the CTAO flux light curves of PKS 2155-304 in 2030} @@ -341,12 +349,12 @@ \subsubsection{Use Case --- Search for the CTAO flux light curves of PKS 2155-30 \medskip \noindent Find all datasets satisfying: \begin{enumerate}[(i)] - \item dataproduct\_type = ``timeseries'' - \item dataproduct\_subtype = ``flux''. - \item obs\_collection = ``CTAO-DR1''. - \item tmin >= 62502 ({\em e.g.\/} 2030-01-01) - \item tmax >= 62866 ({\em e.g.\/} 2030-12-31) - \item target\_name = ``PKS 2155-304'' + \item dataproduct\_type = ``timeseries'', + \item dataproduct\_subtype = ``flux'', + \item obs\_collection = ``CTAO-DR1'', + \item tmin $\geq 62502$ ({\em i.e.\/}, 2030-01-01), + \item tmax $\leq 62866$ ({\em i.e.\/}, 2030-12-31), + \item target\_name = ``PKS 2155-304''. \end{enumerate} \begin{verbatim} From 4a997d13a780ac83a2037f2b6048b76e9f4a267d Mon Sep 17 00:00:00 2001 From: Ian Nigel Evans Date: Wed, 12 Nov 2025 13:45:22 -0500 Subject: [PATCH 2/2] Update to Near-final updates A few fixes in response to feedback from Bruno --- HighEnergyObsCoreExt.bib | 12 +++++ HighEnergyObsCoreExt.glg | 6 +-- HighEnergyObsCoreExt.gls | 52 ++++++++++++------ HighEnergyObsCoreExt.tex | 67 ++++++++++++----------- Makefile | 2 +- UseCases.tex | 112 +++++++++++++++++++-------------------- 6 files changed, 144 insertions(+), 107 deletions(-) diff --git a/HighEnergyObsCoreExt.bib b/HighEnergyObsCoreExt.bib index 52db993..673d980 100644 --- a/HighEnergyObsCoreExt.bib +++ b/HighEnergyObsCoreExt.bib @@ -98,3 +98,15 @@ @misc{2023arXiv230813385K adsnote = {Provided by the SAO/NASA Astrophysics Data System} } +@INCOLLECTION{2019adds.book..283C, + author = {{Czerny}, Bo{\.z}ena and {Beaton}, Rachael and {Bejger}, Micha{\l} and {Cackett}, Edward and {Dall'Ora}, Massimo and {Holanda}, R.~F.~L. and {Jensen}, Joseph B. and {Jha}, Saurabh W. and {Lusso}, Elisabeta and {Minezaki}, Takeo and {Risaliti}, Guido and {Salaris}, Maurizio and {Toonen}, Silvia and {Yoshii}, Yuzuru}, + title = "{Astronomical Distance Determination in the Space Age}", + booktitle = {Astronomical Distance Determination in the Space Age}, + year = 2019, + editor = {{de Grijs}, Richard and {Falanga}, Maurizio}, + volume = {66}, + pages = {283-351}, + doi = {10.1007/978-94-024-1631-2_7}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2019adds.book..283C}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} \ No newline at end of file diff --git a/HighEnergyObsCoreExt.glg b/HighEnergyObsCoreExt.glg index c962b2b..8e234b7 100644 --- a/HighEnergyObsCoreExt.glg +++ b/HighEnergyObsCoreExt.glg @@ -1,7 +1,7 @@ This is makeindex, version 2.17 [TeX Live 2025] (kpathsea + Thai support). Scanning style file ./HighEnergyObsCoreExt.ist...........................done (27 attributes redefined, 0 ignored). -Scanning input file HighEnergyObsCoreExt.glo....done (115 entries accepted, 0 rejected). -Sorting entries....done (844 comparisons). -Generating output file HighEnergyObsCoreExt.gls....done (38 lines written, 0 warnings). +Scanning input file HighEnergyObsCoreExt.glo....done (138 entries accepted, 0 rejected). +Sorting entries....done (1103 comparisons). +Generating output file HighEnergyObsCoreExt.gls....done (56 lines written, 0 warnings). Output written in HighEnergyObsCoreExt.gls. Transcript written in HighEnergyObsCoreExt.glg. diff --git a/HighEnergyObsCoreExt.gls b/HighEnergyObsCoreExt.gls index 54515bf..5013c23 100644 --- a/HighEnergyObsCoreExt.gls +++ b/HighEnergyObsCoreExt.gls @@ -1,38 +1,56 @@ \glossarysection[\glossarytoctitle]{\glossarytitle}\glossarypreamble \begin{theglossary}\glossaryheader +\glsgroupheading{A}\relax \glsresetentrylist % +\glossentry{ARF}{\glossaryentrynumbers{\relax + \setentrycounter[]{page}\glsnumberformat{6}}}\glsgroupskip \glsgroupheading{G}\relax \glsresetentrylist % \glossentry{GTI}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{13}}}\glsgroupskip + \setentrycounter[]{page}\glsnumberformat{16}}}\glsgroupskip \glsgroupheading{H}\relax \glsresetentrylist % \glossentry{HEA}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{1\delimR 9}\delimN - \setentrycounter[]{page}\glsnumberformat{11\delimR 19}\delimN - \setentrycounter[]{page}\glsnumberformat{24\delimR 29}\delimN - \setentrycounter[]{page}\glsnumberformat{31}}}% + \setentrycounter[]{page}\glsnumberformat{1}\delimN + \setentrycounter[]{page}\glsnumberformat{5\delimR 11}\delimN + \setentrycounter[]{page}\glsnumberformat{14\delimR 22}\delimN + \setentrycounter[]{page}\glsnumberformat{26\delimR 29}\delimN + \setentrycounter[]{page}\glsnumberformat{31}\delimN + \setentrycounter[]{page}\glsnumberformat{33}\delimN + \setentrycounter[]{page}\glsnumberformat{36}}}% \glossentry{HEIG}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{2}}}\glsgroupskip + \setentrycounter[]{page}\glsnumberformat{5}\delimN + \setentrycounter[]{page}\glsnumberformat{45}}}\glsgroupskip \glsgroupheading{I}\relax \glsresetentrylist % \glossentry{IACT}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{15\delimN 16}}}% + \setentrycounter[]{page}\glsnumberformat{18\delimN 19}}}% \glossentry{IRF}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{4\delimR 7}\delimN - \setentrycounter[]{page}\glsnumberformat{9}\delimN - \setentrycounter[]{page}\glsnumberformat{17\delimR 19}}}% + \setentrycounter[]{page}\glsnumberformat{3\delimN 4}\delimN + \setentrycounter[]{page}\glsnumberformat{6\delimR 8}\delimN + \setentrycounter[]{page}\glsnumberformat{10}\delimN + \setentrycounter[]{page}\glsnumberformat{12}\delimN + \setentrycounter[]{page}\glsnumberformat{20\delimR 22}\delimN + \setentrycounter[]{page}\glsnumberformat{37\delimR 39}\delimN + \setentrycounter[]{page}\glsnumberformat{41}}}% \glossentry{IVOA}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{2\delimR 4}\delimN - \setentrycounter[]{page}\glsnumberformat{18}}}\glsgroupskip + \setentrycounter[]{page}\glsnumberformat{5}\delimN + \setentrycounter[]{page}\glsnumberformat{7}\delimN + \setentrycounter[]{page}\glsnumberformat{21}\delimN + \setentrycounter[]{page}\glsnumberformat{45}}}\glsgroupskip \glsgroupheading{M}\relax \glsresetentrylist % \glossentry{MOC}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{13}}}\glsgroupskip + \setentrycounter[]{page}\glsnumberformat{9}\delimN + \setentrycounter[]{page}\glsnumberformat{16}}}\glsgroupskip +\glsgroupheading{R}\relax \glsresetentrylist % +\glossentry{RMF}{\glossaryentrynumbers{\relax + \setentrycounter[]{page}\glsnumberformat{6}}}\glsgroupskip \glsgroupheading{S}\relax \glsresetentrylist % \glossentry{STI}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{13}}}\glsgroupskip + \setentrycounter[]{page}\glsnumberformat{16}}}\glsgroupskip \glsgroupheading{V}\relax \glsresetentrylist % \glossentry{VHE}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{26}}}% + \setentrycounter[]{page}\glsnumberformat{28}}}% \glossentry{VO}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{2\delimR 4}}}\glsgroupskip + \setentrycounter[]{page}\glsnumberformat{4\delimN 5}\delimN + \setentrycounter[]{page}\glsnumberformat{7}}}\glsgroupskip \glsgroupheading{W}\relax \glsresetentrylist % \glossentry{WCD}{\glossaryentrynumbers{\relax - \setentrycounter[]{page}\glsnumberformat{16}}}% + \setentrycounter[]{page}\glsnumberformat{19}}}% \end{theglossary}\glossarypostamble diff --git a/HighEnergyObsCoreExt.tex b/HighEnergyObsCoreExt.tex index c8f65bb..da3ec24 100644 --- a/HighEnergyObsCoreExt.tex +++ b/HighEnergyObsCoreExt.tex @@ -104,18 +104,18 @@ \section{Introduction} The \gls{IVOA} \gls{HEIG} was formed in the Fall of 2024, and developed an \gls{IVOA} Note \citep{2024ivoa.note.heig} that explores the connections between the \gls{VO} and \gls{HEA}\null. \gls{HEA} covers experiments and observatories from the X-ray range up to the PeV range and beyond, as well as astrophysical neutrinos above the TeV range. These regimes are referred to here as the \gls{HEA} domain. The \gls{HEIG} Note includes an outline of several important topics that have formed a roadmap for the group. An ObsCore \citep{2017ivoa.spec.0509L} extension for \gls{HEA} data is the first priority in order to meet the needs of \gls{HEA}, and to coincide with similar work being carried out by the Radio Interest Group, Time Domain Interest Group, and discussions on DataModel standards, such that current and future \gls{HEA} experiments and observatories are able to release data through the \gls{VO}. -The goal is to explore elements needed to reliably discover and select \gls{HEA} data through \gls{IVOA} interfaces. This requires defining an extension to ObsCore with the possibility to use the DataLink mechanism and to enhance vocabularies of keywords for ObsCore and DataLink. We suggest that, if an attribute is unique to \gls{HEA} data, that element should appear in an \gls{HEA} ObsCore extension. Whereas, if an attribute makes sense for more than one domain and can be shared across those domains, then that element should be added to the base ObsCore model. This note proposes recommendations in both of these categories. We also discuss enhancements to the vocabulary of data\/ products, DataLink semantics, Unified Content Descriptors (UCDs), and MIME-types to correctly represent \gls{HEA} data. Topics related to the Registry are currently outlined in the proposed Radio extension document and are not discussed here. +The goal is to explore elements needed to reliably discover and select \gls{HEA} data through \gls{IVOA} interfaces. This requires defining an extension to ObsCore with the possibility to use the DataLink mechanism and to enhance vocabularies of keywords for ObsCore and DataLink. We suggest that if an attribute is unique to \gls{HEA} data, then that element should appear in an \gls{HEA} ObsCore extension. Whereas, if an attribute makes sense for more than one domain and can be shared across those domains, then that element should be added to the base ObsCore model. This note proposes recommendations in both of these categories. We also discuss enhancements to the vocabulary of data\/ products, DataLink semantics, Unified Content Descriptors (UCDs), and MIME-types to correctly represent \gls{HEA} data. Topics related to the Registry are currently outlined in the proposed Radio extension document and are not discussed here. \section{High Energy Astrophysics Data} -\gls{HEA} data include observations obtained using photon detectors covering X-ray (from $\sim$0.1 keV to $\sim$120 keV) through gamma-ray (from 120 keV up to $\gtrsim$ PeV) energies, as well as cosmic-ray and astrophysical neutrino ($\gtrsim$ TeV) detectors, or other messengers related to \gls{HEA} phenomena. The domain is now sufficiently mature to provide open data that are science-ready and work with open analysis tools ({\em e.g.\/}, CIAO \citep{2006SPIE.6270E..1VF} or Gammapy \citep{gammapy:2023}). The science output of the \gls{HEA} domain already includes high-level products such as images, cubes, spectra, and time series such as light curves and time-resolved spectra. Additional data products include fitted sky models with a spatial, spectral, and/or temporal component(s), along with their confidence intervals or confidence limits, and covariance matrices. Finally, multiple \gls{HEA} instruments produce source catalogs and surveys covering up to the full the sky, which include maps of photon or particle flux, exposure, sensitivity, and aperture-photometry likelihood profiles. +\gls{HEA} data include observations obtained using photon detectors covering X-ray (from $\sim$0.1 keV to $\sim$120 keV) through gamma-ray (from 120 keV up to $\gtrsim$ PeV) energies, as well as cosmic-ray and astrophysical neutrino ($\gtrsim$ TeV) detectors, or other messengers related to \gls{HEA} phenomena. The domain is now sufficiently mature to provide open data that are science-ready and work with open analysis tools ({\em e.g.\/}, CIAO \citep{2006SPIE.6270E..1VF} or Gammapy \citep{gammapy:2023}). The science output of the \gls{HEA} domain already includes advanced products such as images, cubes, spectra, and time series such as light curves and time-resolved spectra. Additional data products include fitted sky models with spatial, spectral, and/or temporal component(s), along with their confidence intervals or confidence limits, and covariance matrices. Finally, multiple \gls{HEA} instruments produce source catalogs and surveys covering up to the full the sky, which include maps of photon or particle flux, exposure, sensitivity, and aperture-photometry likelihood profiles. -Observations of the universe at the highest energies are based on techniques that are radically different compared to the UV through radio domains. \gls{HEA} observatories\footnote{For example, Chandra, XMM-Newton, Fermi, H.E.S.S., MAGIC, VERITAS, HAWC, LHAASO, IceCube, Auger and soon CTAO and KM3NeT, SWGO.} are generally designed to detect particles ({\em e.g.\/}, individual photons, cosmic-rays or neutrinos) with the ability to estimate multiple observables for those particles. These detection techniques all rely on {\em event counting\/}\footnote{As opposed to signal integrating ({\em e.g.\/}, using a detector that accumulates the total photon signal during an exposure).}, where an event has some probability of being due to the interaction of a particle from an astrophysical source with the detectors, but also has some probability from being from instrumental or background effects. The data corresponding to an event are first an instrumental signal, which is then calibrated and processed to estimate physical quantities such as a time of arrival, point-of-origin on the sky, and an energy proxy associated with the event. Several other intermediate and qualifying characteristics may be associated with a detected event, depending on the detection technique. The ensemble of events detected over a given time interval and spatial field-of-view is referred to as an {\em event list\/}, which we designate an {\bf event-list} in this document. +Observations of the universe at the highest energies are based on techniques that are radically different compared to the UV through radio domains. \gls{HEA} observatories\footnote{For example, Chandra, XMM-Newton, Fermi, H.E.S.S., MAGIC, VERITAS, HAWC, LHAASO, IceCube, Auger and soon CTAO and KM3NeT, SWGO.} are generally designed to detect particles ({\em e.g.\/}, individual photons, cosmic-rays, or neutrinos) with the ability to estimate multiple observables for those particles. These detection techniques all rely on {\em event counting\/}\footnote{As opposed to signal integrating ({\em e.g.\/}, using a detector that accumulates the total photon signal during an exposure).}, where an event has some probability of being due to the interaction of a particle from an astrophysical source with the detectors, but also has some probability of being from instrumental or background effects. The data corresponding to an event are first an instrumental signal, which is then calibrated and processed to estimate physical quantities such as a time of arrival, point-of-origin on the sky, and an energy proxy associated with the event. Several other intermediate and qualifying characteristics may be associated with a detected event, depending on the detection technique. The ensemble of events detected over a given time interval and spatial field-of-view is referred to as an {\em event list\/}, which we designate an {\bf event-list} in this document. -Though {\bf event-list}s {\em may\/} include estimators for calibrated physical values, they typically still have to be corrected for the photometric, spectral, spatial, and/or temporal responses of the telescope and detector combination to yield scientifically interpretable information. The mappings between physical measurements of the source properties and the observables are called Instrument Response Functions (\glspl{IRF}\footnote{We try to avoid using the term \gls{IRF} in a normative sense since historical usage across the broad \gls{HEA} community (and from facility to facility) varies. In some cases, \gls{IRF} has been used to mean specifically the X-ray product of the \gls{ARF} and \gls{RMF}, whereas in other cases \gls{IRF} has been used more generally to mean any instrumental response function regardless of type.}). Some \glspl{IRF} are probabilistic in nature\footnote{For example, the energy matrix is a probability density function.}, and in addition may depend on the set of events selected for analysis by the end user. They are usually not invertible, so methods such as forward-folding fitting (using source models with any combination of spectral, spatial, temporal, and/or polarization components that are estimated) are needed to estimate physical properties, such as the true flux of particles from a source arriving at the instrument, given the measured observable quantities. The \glspl{IRF} generally evolve over time with the instrument and observation characteristics, and are usually defined for a specific time interval and decomposed into a standard set of independent components (see \S~3.1.5 of \citep{2024ivoa.note.heig}), such as the spatial point-spread function or the energy-migration matrix, where each component may be stored or computed separately. Since both \glspl{IRF} and {\bf event-list}s are required to analyze \gls{HEA} data, some \gls{IVOA} standards must be modified in order to expose both of them via the \gls{VO}. +Though {\bf event-list}s {\em may\/} include estimators for calibrated physical values, they typically still have to be corrected for the photometric, spectral, spatial, and/or temporal responses of the telescope and detector combination to yield scientifically interpretable information. The mappings between physical measurements of the source properties and the observables are called Instrument Response Functions (\glspl{IRF}\footnote{We try to avoid using the term \gls{IRF} in a normative sense since historical usage across the broad \gls{HEA} community (and from facility to facility) varies. In some cases, \gls{IRF} has been used to mean specifically the X-ray product of the \gls{ARF} and \gls{RMF}, whereas in other cases \gls{IRF} has been used more generally to mean any instrumental response function regardless of type.}). Some \glspl{IRF} are probabilistic in nature\footnote{For example, the energy matrix is a probability density function.}, and in addition may depend on the set of events selected for analysis by the end user. They are usually not invertible, so methods such as forward-folding fitting (using source models with any combination of spectral, spatial, temporal, and/or polarization components that are estimated) are needed to estimate physical properties, such as the true flux of particles from a source arriving at the instrument, given the measured observable quantities. The \glspl{IRF} generally evolve over time with the instrument and observation characteristics, and are usually defined for a specific time interval and may be decomposed into a standard set of independent components (see \S~3.1.5 of \citep{2024ivoa.note.heig}), such as the spatial point-spread function or the energy-migration matrix, where each component may be stored or computed separately. Since both \glspl{IRF} and {\bf event-list}s are required to analyze \gls{HEA} data, some \gls{IVOA} standards must be modified in order to expose both of them via the \gls{VO}. -In the following, the current ObsCore standard will be discussed in \S~\ref{sec:obscore}, focusing on attributes that need to be modified. Then, we propose the creation of a \gls{HEA} extension of ObsCore in \S~\ref{sec:obscoreext}, as some attributes are very specific to our domain (see \S~\ref{sec:ibscoreext}). In these two sections, the discussion focuses on the attribute definitions rather on the attribute values. In \S~\ref{sec:voc}, enhancement of vocabulary is proposed for some ObsCore attributes, DataLink semantics, UCDs and MIME-types. +In the following, the current ObsCore standard will be discussed in \S~\ref{sec:obscore}, focusing on attributes that need to be modified. Then, we propose the creation of a \gls{HEA} extension of ObsCore in \S~\ref{sec:obscoreext}, as some attributes are very specific to our domain. In these two sections, the discussion focuses on the attribute definitions rather on the attribute values. In \S~\ref{sec:voc}, enhancement of vocabulary is proposed for some ObsCore attributes, DataLink semantics, UCDs, and MIME-types. \section{ObsCore Attribute Definitions for High Energy Astrophysics Data} \label{sec:obscore} @@ -150,24 +150,23 @@ \subsection{{\em dataproduct\_type}} %An {\bf event-bundle} might for example consist of an {\bf event-list} and the associated {\bf response-functions} (see below) used to calibrate the dataset; alternatively an {\bf event-bundle} may include the {\bf event-list} and associated data products necessary for the user to create the {\bf response-functions} (for those X-ray cases where detailed knowledge of the scientific use case — for example, the user’s selection of events — may be required to compute the responses).\\ %particle-detection -In addition to {\em dataproduct\_type\/} terms that focus on event data, we note that existing ObsCore definitions do not adequately span the breadth of ``advanced data products'' (typically with {\em calib\_level\/} $\ge$ 3) that may be generated from astronomical observations by users or observatories. The computational complexity of analyzing \gls{HEA} data robustly in the extreme Poisson regime ({\em e.g.\/}, Bayesian X-ray aperture photometry applied simultaneously to multiple overlapping detections and observations) means that data providers may choose to provide such analysis products directly to the end user. For example, the Chandra Source Catalog includes 38 types of advanced data products (for a total of $\sim\!90$ million files) and $\sim\!50$\% of these data product types are not well represented by a {\em dataproduct\_type\/} value that allows for meaningful data discovery. Users will certainly want to discover these data products independently from the associated progenitor observation data (and many of these data products combine data from multiple observations). We therefore propose the following additional {\em dataproduct\_type\/} (or {\em dataproduct\_subtype\/}) terms for these advanced data products, and note that these terms will certainly be useful independent of waveband ({\em i.e.\/}, they can be equally applicable to UV/optical, IR, and radio datasets): +In addition to {\em dataproduct\_type\/} terms that focus on event data, we note that existing ObsCore definitions do not adequately span the breadth of ``advanced data products'' (typically with {\em calib\_level\/} $\ge$ 3) that may be generated from astronomical observations by users or observatories. The computational complexity of analyzing \gls{HEA} data robustly in the extreme Poisson regime ({\em e.g.\/}, Bayesian X-ray aperture photometry applied simultaneously to multiple overlapping detections and observations, or Frequentist adjustment of models of electron populations for multi-wavelength data spanning from X-rays to PeV gamma rays) means that data providers may choose to provide such analysis products directly to the end user. For example, the Chandra Source Catalog includes 38 types of advanced data products (for a total of $\sim\!90$ million files) and $\sim\!50$\% of these data product types are not well represented by a {\em dataproduct\_type\/} value that allows for meaningful data discovery. Users will certainly want to discover these data products independently from the associated progenitor observation data (and many of these data products combine data from multiple observations). We therefore propose the following additional {\em dataproduct\_type\/} (or {\em dataproduct\_subtype\/}) terms for these advanced data products, and note that these terms will certainly be useful independent of waveband ({\em i.e.\/}, they can be equally applicable to UV/optical, IR, and radio datasets): \begin{quote} -{\bf draws}: a dataset that records statistical draws computed from a probability distribution, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable. The draws can be interpreted to provide a robust estimation of the probability distribution of variable, and correlations between the draws provide information about how well the draws converge to the parent probability distribution. +{\bf draws}: a dataset that records statistical draws computed from a probability distribution, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable. The draws can be interpreted to provide a robust estimation of the probability distribution of variable, and correlations between the draws provide information about how well the draws converge to the parent probability distribution.\footnote{As an example, within the standard $\Lambda CDM$ cosmological model, estimates of the cosmological density parameters $\Omega M$ and $\Omega\Lambda$ can be derived from the intersection of confidence contours from Hubble diagram of quasars with those from the Type Ia supernovae \citep{2019adds.book..283C}. These contours are {\bf draws}.} -{\bf pdf}: a dataset that records the probability density function of a quantity, for example the Bayesian marginal probability density function for a random variable. The probability density function provides a robust estimation of the variable and allows arbitrary confidence intervals to be computed directly from the distribution. +{\bf pdf}: a dataset that records the probability density function of a quantity, for example the Bayesian marginal probability density function for a random variable, or the DeltaTS associated with a quantity from a Frequentist analysis. The probability density function provides a robust estimation of the variable and allows arbitrary confidence intervals to be computed directly from the distribution. -{\bf region}: a dataset that includes an encoding of (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary. +{\bf region}: a dataset that includes an encoding of (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary.\footnote{One possible encoding is a \gls{MOC}; however the vast majority of pre-existing region data products in \gls{HEA} data archives currently use other encodings.} {\bf response-function}: a dataset that records a mapping from a physical quantity to an observable quantity. For \gls{HEA}, this may be the components of the composite \gls{IRF} such as an Auxiliary Response File ({\bf arf}), Redistribution Matrix File ({\bf rmf}), Effective Area ({\bf aeff}), Energy Dispersion ({\bf edisp}), the Background Rate ({\bf bkgrate}). The Point Spread Function ({\bf psf}) is a response function that is generally applicable across multiple wavebands. While these datasets may generally be represented as an N-dimensional data cube, designating them as {\bf response-functions} enhances data discovery for very common types of \gls{HEA} dataset (see the use cases in Appendix~\ref{sec:uc}). - \end{quote} -The {\bf measurements} {\em dataproduct\_type\/} is quite useful for many different types of advanced data products (that may be derived from multiple observations). But users of those products often may not be interested the progenitor datasets, especially if many advanced data products are extracted from a single or a few progenitors ({\em e.g.\/}, measurements associated with sources detected in a single observation field). We propose to delete the caveat associated with {\em dataproduct\_type\/} = ``measurements'' in the ObsCore IVOA Recommendation (\S~4.1.1) that requires the derived data products be exposed ``{\bf together} with the progenitor observation dataset''. +The {\bf measurements} {\em dataproduct\_type\/} is quite useful for many different types of advanced data products (which may be derived from multiple observations). But users of those products often may not be interested in the progenitor datasets, especially as multiple advanced data products may be extracted from the same single progenitor or a few progenitors ({\em e.g.\/}, measurements associated with multiple sources detected in a single observation field). We propose to delete the caveat associated with {\em dataproduct\_type\/} = ``measurements'' in the ObsCore IVOA Recommendation (\S~4.1.1) that requires the derived data products be exposed ``{\bf together} with the progenitor observation dataset''. The recovery of progenitor observation datasets may be achieved using provenance information, if desired. \subsection{{\em dataproduct\_subtype}} -The optional attribute {\em dataproduct\_subtype} may be used by the data provider to specify more precisely the scientific nature of a data product. Although no vocabulary is defined for {\em dataproduct\_subtype\/}, we recommend that data providers formulate and use a standardized vocabulary for this attribute for data products that are commonly used in \gls{HEA}\null. We have proposed several terms in \S~5 for commonly used \gls{HEA} {\bf response-function} types ({\em e.g.\/}, {\bf aeff}, {\bf edisp}, {\bf psf}), but additional terms could be standardized for other common data products. For example, standardizing using {\bf exposuremap} for an exposure map would enable queries such as ({\em dataproduct\_type\/} = {\bf image}) AND ({\em dataproduct\_subtype\/} = {\bf exposuremap}) to work across multiple facilities. Other possible terms could include {\bf significancemap} for a significance map, and {\bf probabilitymap} for aprobability map. +The optional attribute {\em dataproduct\_subtype} may be used by the data provider to specify more precisely the scientific nature of a data product. Although no vocabulary is defined for {\em dataproduct\_subtype\/}, we recommend that data providers formulate and use a standardized vocabulary for this attribute for data products that are commonly used in \gls{HEA}\null. We have proposed several terms in \S~5 for commonly used \gls{HEA} {\bf response-function} types ({\em e.g.\/}, {\bf aeff}, {\bf edisp}, {\bf psf}), but additional terms could be standardized for other common data products. For example, standardizing using {\bf exposuremap} for an exposure map would enable queries such as ({\em dataproduct\_type\/} = {\bf image}) AND ({\em dataproduct\_subtype\/} = {\bf exposuremap}) to work across multiple facilities. Other possible terms could include (but are not limited to) {\bf significancemap} for a significance map, {\bf probabilitymap} for aprobability map, and {\bf exclusionmap} for an exclusion map ({\em e.g.\/}, as used to adjust TeV background models). \subsection{{\em calib\_level}} @@ -175,7 +174,7 @@ \subsection{{\em calib\_level}} However, some \gls{HEA} {\bf event-list}s include spatial and time axes that are calibrated physical quantities, but the spectral axis is instrumental and requires application of the IRFs to remove this signature. This is typically done because the {\bf response-function}s can depend on the choice of region (spatial/time) from which the events are extracted (especially for telescope/detector combinations where the telescope position dithers on the sky during the exposure), which depends on the specific science case and therefore cannot be determined {\em a priori\/}. Such {\bf event-list}s fall ``between'' {\em calib\_level\/} 1 and 2. -On the other hand, other {\bf event-list}s may not have any calibrated axes or may have all axes calibrated, and it is important to be able to differentiate between these for data discovery. While the value for {\em calib\_level\/} for any data product is left for the data provider to determine, we suggest that individual data providers set {\em calib\_level\/} = 1 if an {\bf event-list} is considered to be ``uncalibrated'' according to normal usage for their data products and set {\em calib\_level\/} = 2 if an {\bf event-list} is considered to be ``calibrated'' according to normal usage for their data products. +On the other hand, other {\bf event-list}s may not have any calibrated axes or may have all axes calibrated, and it is important to be able to differentiate between these for data discovery. While the value for {\em calib\_level\/} for any data product is left for the data provider to determine, we suggest that individual data providers set {\em calib\_level\/} = 1 if an {\bf event-list} is considered to be ``uncalibrated'' according to normal usage for their data products, and set {\em calib\_level\/} = 2 if an {\bf event-list} is considered to be ``calibrated'' according to normal usage for their data products. Also, we propose that the calibration status of the spatial/spectral/time data axes be identified using the appropriate axis ObsCore {\em calib\_status\/} keyword ({\em s\_calib\_status\/} for the spatial axes, {\em em\_calib\_status\/} for the spectral axis, and {\em t\_calib\_status\/} for the time axis). @@ -207,7 +206,7 @@ \subsection{{\em s\_calib\_status}} \subsection{{\em t\_calib\_status}} -We propose that {\em t\_calib\_status\/} encode the calibration status of an {\bf event-list} dataset's time axis. Where multiple time axes are included in a dataset ({\em e.g.\/}, instrument counter, absolute time), we recommend that the data provider use the coordinate system that is most likely to be preferred by the end user (typically the most fully calibrated time axis) to define {\em t\_calib\_status\/}. +We propose that {\em t\_calib\_status\/} encode the calibration status of an {\bf event-list} dataset's time axis. Where multiple time axes are included in a dataset ({\em e.g.\/}, instrument counter, absolute time), we recommend that the data provider use the coordinate system that is most likely to be preferred by the end user (typically the most fully calibrated time axis) to define {\em t\_calib\_sta\-tus\/}. Under the (reasonable) assumption that an end-user searching for {\bf event-bundle} datasets is typically querying based on the properties of the primary {\bf event-list}, we suggest that those values also be used for the {\bf event-bundle}. However, the data provider should ultimately decide which value best describes their {\bf event-bundle} dataset. @@ -215,7 +214,7 @@ \subsection{{\em t\_calib\_status}} \subsection{{\em em\_calib\_status}} -We propose that {\em em\_calib\_status\/} encode the calibration status of an {\bf event-list} dataset's spectral axis. Where multiple spectral axes are included in a dataset ({\em e.g.\/}, PHA, PI, energy), we recommend that the data provider use the coordinate system that is most likely to be preferred by the end user (typically the most fully calibrated spectral axis) to define {\em t\_calib\_status\/}. +We propose that {\em em\_calib\_status\/} encode the calibration status of an {\bf event-list} dataset's spectral axis. Where multiple spectral axes are included in a dataset ({\em e.g.\/}, PHA, PI, energy), we recommend that the data provider use the coordinate system that is most likely to be preferred by the end user (typically the most fully calibrated spectral axis) to define {\em em\_calib\_status\/}. Under the (reasonable) assumption that an end-user searching for {\bf event-bundle} datasets is typically querying based on the properties of the primary {\bf event-list}, we suggest that those values also be used for the {\bf event-bundle}. However, the data provider should ultimately decide which value best describes their {\bf event-bundle} dataset. @@ -442,14 +441,14 @@ \subsubsection{Clarification of ``Flux'' in Data Product Type Vocabulary Definit \item Particle flux, with units $\rm\hbox{particles}\,m^{-2}\,s^{-1}$, \item Particle radiance (flux per steradian), with units $\rm\hbox{particles}\,m^{-2}\,s^{-1}\,\hbox{sr}^{-1}$, \item Particle flux density (differential flux), with units $\rm\hbox{particles}\,m^{-2}\,s^{-1}\,\hbox{eV}^{-1}$, -\item Energy flux, with units $\rm W\,m^{-2}\,s^{-1}$, -\item Energy radiance (flux per steradian), with units $\rm W\,m^{-2}\,s^{-1}\,\hbox{sr}^{-1}$, -\item Energy flux density (differential flux), with units $\rm W\,m^{-2}\,s^{-1}\,\hbox{eV}^{-1}$. +\item Energy flux, with units $\rm J\,m^{-2}\,s^{-1}$, +\item Energy radiance (flux per steradian), with units $\rm J\,m^{-2}\,s^{-1}\,\hbox{sr}^{-1}$, +\item Energy flux density (differential flux), with units $\rm J\,m^{-2}\,s^{-1}\,\hbox{eV}^{-1}$. \end{itemize} -The common \gls{HEA} flux, radiance, and flux density definitions listed above are not all covered by the UCD vocabulary. We propose to add new UCDs to the UCD list in \S~\ref{sec:UCDs} below to address this, and where appropriate refine the definitions of existing UCDs based on dimensionality. Such refinement would be particularly useful for cases where a \gls{HEA} quantity should have the same UCD as ({\em e.g.\/}) a radio measurement but where very different units are used. For example, $\rm Jy$ ( Jansky) and $\rm erg\,cm^{-2}\,s^{-1}\,TeV$ are both spectral flux densities but the unit analysis doesn’t agree due to the frequency-energy equivalency. +The common \gls{HEA} flux, radiance, and flux density definitions listed above are not all covered by the UCD vocabulary. We propose to add new UCDs in \S~\ref{sec:UCDs} below to address this, and where appropriate refine the definitions of existing UCDs based on dimensionality. With the proposed additions and refinements, we can, for example, easily distinguish between a counts flux with UCD {\em phot.counts\/}, a photon flux with UCDs {\em phot.flux.particle;phys.particle.photon\/}, and an energy flux with UCD {\em phot.\-flux.energy\/}. These refinement will also be particularly useful for cases where a \gls{HEA} quantity should have the same UCD as ({\em e.g.\/}) a radio measurement but where very different units are used. For example, $\rm Jy$ ( Jansky) and $\rm erg\,cm^{-2}\,s^{-1}\,TeV$ are both spectral flux densities but the unit analysis doesn’t agree due to the frequency-energy equivalency. -Restating the existing IVOA Data Product Type Vocabulary descriptions that use the term ``flux'' to explicitly state ``count/particle/energy flux or magnitude'' where appropriate would resolve the concern with the current use of the term ``flux''. +Restating the existing IVOA Data Product Type Vocabulary descriptions that use the term ``flux'' to explicitly state ``count/particle/energy flux or magnitude'' where appropriate would resolve the concern raised above with the current use of the term ``flux''. \subsection{DataLink Vocabularies}\label{sec:DLs} @@ -488,13 +487,15 @@ \subsubsection{Physical Quantities} The messengers for \gls{HEA} observations may include particles other than the ones currently described in the UCD list. Because some instruments can now distinguish electrons from positrons\footnote{For example, the Fermi-LAT instrument.}, as well antiprotons from protons\footnote{For example, the AMS-2 experiment.}, we also propose to add {\em phys.particle.positron\/} and {\em phys.particle.antiproton\/}, as well as {\em phys.particle.cosmicray\/} and unify them all under the {\em phys.particle\/} UCD hierarchy. -One should note that electrons are denoted by the UCD {\em phys.electron\/} in the current version of the UCD list \citep{2024ivoa.spec.1218C} and are inappropriately not grouped under the {\em phys.particle\/} hierarchy. This causes some inconsistencies that could be solved by marking {\em phys.electron\/} (and {\em phys.electron.degen\/}) as obsolete or not recommended, and adding the term {\em phys.particle.electron\/} to the UCD list. +One should note that electrons are denoted by the UCD {\em phys.electron\/} in the current version of the UCD list \citep{2024ivoa.spec.1218C} and are inappropriately not grouped under the {\em phys.particle\/} hierarchy. This causes some inconsistencies that could be solved by marking {\em phys.electron\/} (and {\em phys.electron.degen\/}) as obsolete or not recommended, and adding the term {\em phys.particle.electron\/} to the UCD list. + +Finally, we propose to add the most commonly used astronomical messenger to the UCD list as {\em phys.particle.photon\/}. \subsubsection{Statistical Parameters} Since statistical lower and upper limits play a significant role in \gls{HEA} data analysis, we propose adding new UCDs {\em stat.lowerlimit\/} and {\em stat.upperlimit\/} to explicitly identify data quantities as lower or upper limits. We suggest that the existing UCDs {\em stat.min\/} and {\em stat.max\/} be restricted to meaning the minimum and maximum statistic, rather than the current definitions ``Minimum or lowest limit'' and ``Maximum or upper limit'', which blend statistical confidence intervals and limits into a single UCD\null. A specification of a confidence level is necessary for the user to interpret both confidence intervals and lower/upper limits meaningfully, and this level can be described by the existing UCD {\em stat.confidenceLevel\/}. -The shape of any statistical distribution is an essential quantity for interpreting the meaning of any statistical properties. Too often a Gaussian distribution or a distribution that can be simply characterized by a simple set of moments ({\em e.g.\/}, mean, variance, skewness, kurtosis) are assumed, but in the extreme Poisson regime common in \gls{HEA} these assumptions are often invalid. We propose adding a UCD {\em stat.distribution\/} to identify a quantity that defines the distribution of a statistical variable such as a likelihood profile. +The shape of any statistical distribution is an essential quantity for interpreting the meaning of any statistical properties. Too often a Gaussian distribution or a distribution that can be characterized by a simple set of moments ({\em e.g.\/}, mean, variance, skewness, kurtosis) are assumed, but in the extreme Poisson regime common in \gls{HEA} these assumptions are often invalid. We propose adding a UCD {\em stat.distribution\/} to identify a quantity that defines the distribution of a statistical variable such as a likelihood profile. \subsubsection{Evolution of UCD list} @@ -511,17 +512,23 @@ \subsubsection{Evolution of UCD list} Q & {\em instr.event.grade\/} & Particle event grade \cr Q & {\em instr.pulseHeight\/} & Pulse height amplitude measure \cr Q & {\em instr.event.type\/} & Particle event type \cr -E & {\em phot.count.density\/} & Count or particle flux density (dimensionality: $\rm [L^{-2}\,T^{-1}\,E^{-1}]$) \cr -E & {\em phot.count.density.sb\/} & Count or particle flux density surface brightness (dimensionality: $\rm [L^{-2}\,T^{-1}\,E^{-1}\,\hbox{sr}^{-1}]$) \cr -E & {\em phot.count.radiance\/} & Count or particle flux radiance (dimensionality: $\rm [L^{-2}\,T^{-1}\,\hbox{sr}^{-1}]$) \cr -E & {\em phot.count.sb\/} & Count or particle flux surface brightness (dimensionality: $\rm [L^{-2}\,T^{-1}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.count.density\/} & Count flux density (dimensionality: $\rm [L^{-2}\,T^{-1}\,E^{-1}]$) \cr +E & {\em phot.count.density.sb\/} & Count flux density surface brightness (dimensionality: $\rm [L^{-2}\,T^{-1}\,E^{-1}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.count.radiance\/} & Count flux radiance (dimensionality: $\rm [L^{-2}\,T^{-1}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.count.sb\/} & Count flux surface brightness (dimensionality: $\rm [L^{-2}\,T^{-1}\,\hbox{sr}^{-1}]$) \cr E & {\em phot.flux.energy\/} & Energy flux or irradiance (dimensionality: $\rm [M\,T^{-3}]$) \cr E & {\em phot.flux.energy.density\/} & Energy flux density (dimensionality: $\rm [M\,T^{-3}\,E^{-1}]$) \cr -E & {\em phot.flux.energy.density.sb\/} & Energy flux density surface brightness (dimensionality: $\rm [M\,T^{-3}\,E^{-1}\,\hbox{sr}^{-1}]$) \cr -E & {\em phot.flux.energy.sb\/} & Energy flux syrface brightness (dimensionality: $\rm [M\,T^{-3}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.flux.energy.density.sb\/} & Energy flux density surface brightness (dimensionality: $\rm [M\,T^{-3}\,E^{-1}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.flux.energy.sb\/} & Energy flux surface brightness (dimensionality: $\rm [M\,T^{-3}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.flux.particle\/} & Particle flux or irradiance (dimensionality: $\rm [L^{-2}\,T^{-1}]$) \cr +E & {\em phot.flux.particle.density\/} & Particle flux density (dimensionality: $\rm [L^{-2}\,T^{-1}\,E^{-1}]$) \cr +E & {\em phot.flux.particle.density.sb\/} & Particle flux density surface brightness (dimensionality: $\rm [L^{-2}\,T^{-1}\,E^{-1}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.flux.particle.radiance\/} & Particle flux radiance (dimensionality: $\rm [L^{-2}\,T^{-1}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.flux.particle.sb\/} & Particle flux surface brightness (dimensionality: $\rm [L^{-2}\,T^{-1}\,\hbox{sr}^{-1}]$) \cr S & {\em phys.particle.antiprotron\/} & Related to anti-proton \cr S & {\em phys.particle.cosmicray\/} & Related to cosmic rays particles \cr S & {\em phys.particle.electron\/} & Related to electron \cr +S & {\em phys.particle.photon\/} & Related to photon \cr S & {\em phys.particle.positron\/} & Related to positron \cr P & {\em stat.distribution\/} & Type or shape of statistical distribution \cr P & {\em stat.error.negative\/} & Negative statistical error \cr @@ -539,10 +546,10 @@ \subsubsection{Evolution of UCD list} \sptablerule S & {\em em.gamma.hard\/} & Hard gamma ray (500 keV -- 100 MeV) \cr P & {\em stat.confidenceLevel\/} & Level of confidence for a statistical measure, detection, or classification process \cr -E & {\em phot.count\/} & Count or particle flux (dimensionality: $\rm [L^{-2}\,T^{-1}]$) \cr +E & {\em phot.count\/} & Count flux (dimensionality: $\rm [L^{-2}\,T^{-1}]$) \cr E & {\em phot.fluence\/} & Radiant photon energy received by a surface per unit area or irradiance of a surface integrated over time of irradiation (dimensionality: $\rm [L^{-2}]$) \cr Q & {\em phot.flux.bol\/} & Bolometric flux (dimensionality: $\rm [M\,T^{-3}]$) \cr -E & {\em phot.radiance\/} & Radiance as energy flux per solid angle (dimensionality: $\rm [L^{-2}\,T^{-1}\,\hbox{sr}^{-1}]$) \cr +E & {\em phot.radiance\/} & Radiance as energy flux per solid angle (dimensionality: $\rm [M\,T^{-3}\,\hbox{sr}^{-1}]$) \cr S & {\em phys.electron\/} & Electron (not recommended/deprecate) \cr S & {\em stat.min\/} & Minimum value \cr S & {\em stat.max\/} & Maximum value \cr diff --git a/Makefile b/Makefile index e99386c..397f0cd 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ DOCNAME = HighEnergyObsCoreExt DOCVERSION = 1.0 # Publication date, ISO format; update manually for "releases" -DOCDATE = 2025-11-06 +DOCDATE = 2025-11-12 # What is it you're writing: NOTE, WD, PR, REC, PEN, or EN DOCTYPE = PEN diff --git a/UseCases.tex b/UseCases.tex index 0d361d9..e68e72f 100644 --- a/UseCases.tex +++ b/UseCases.tex @@ -66,7 +66,7 @@ \subsubsection{Use Case --- Search for SWGO event lists and their \glspl{IRF} f \begin{verbatim} SELECT * FROM ivoa.obscore NATURAL JOIN ivoa.obscore_hea -WHERE (INTERSECTS(s_region, CIRCLE(312.775, 30.683, 1.5)) = 1) +WHERE (INTERSECTS(s_region, CIRCLE(312.775, 30.683, 1.5)) = 6) AND (dataproduct_type = 'event-list' OR dataproduct_type = 'aeff' OR dataproduct_type = 'edisp' OR dataproduct_type = 'psf' OR dataproduct_type = 'bkgrate') @@ -76,14 +76,64 @@ \subsubsection{Use Case --- Search for SWGO event lists and their \glspl{IRF} f Then, for each row of the output, we identify the nature of the data product, and retrieve them using the ``access\_url''. +\subsubsection{Use Case --- Search for event bundles via DataLink that include Cas A for a TeV spectromorphology study} + +{\em Identify all event bundles (event lists and their associated \glspl{IRF}) that include the Cas A SNR for subsequent TeV spectromorphology studies from a VERITAS data release. Since the instrumental responses are mandatory to remove instrumental effects, the event bundles that include the \glspl{IRF} are required.\/} + +\medskip +\noindent Find all VERITAS datasets satisfying: +\begin{enumerate}[(i)] + \item Target name = ``Cas A'' or position inside 2.5 arcmin from (350.8584, $+58.8113$), + \item dataproduct\_type = ``event-bundle'', + \item obs\_collection = ``VERITAS-DR1'', + \item access\_format = ``datalink''. +\end{enumerate} + +\begin{verbatim} +SELECT * FROM ivoa.obscore +NATURAL JOIN ivoa.obscore_hea +WHERE +(target_name = 'Cas A' OR +CONTAINS(POINT(s_ra, s_dec), CIRCLE, 350.8584, +58.8113, 0.042) = 2) +AND (dataproduct_type = 'event-bundle') +AND (obs_collection = 'VERITAS-DR1') +AND (access_format = ’application/x-votable+xml;content=datalink’) +\end{verbatim} + +Then, for each row of the output, we get the ``access\_url'' of the DataLink to provide access to the data. + + +\subsubsection{Use Case --- Search for event bundles that include Cas A for X-ray spectrophotometric evolution studies} + +{\em Identify all event bundles that include the Cas A SNR and have at least 1 million events for subsequent spectrophotometric studies of the SNR expansion. Since only a few observations are expected to match this request and because the focus is on X-ray spectrophotometric studies, the event bundles that include the responses or the ancillary products used to make the responses are required.\/} + +\medskip +\noindent Find all datasets satisfying: +\begin{enumerate}[(i)] + \item Target name = ``Cas A'' or position inside 10 arcmin from (350.8584, $+58.8113$), + \item dataproduct\_type = ``event-bundle'', + \item ev\_xel $\geq 1000000$. +\end{enumerate} + +\begin{verbatim} +SELECT * FROM ivoa.obscore +NATURAL JOIN ivoa.obscore_hea +WHERE +(target_name = 'Cas A' OR +CONTAINS(POINT(s_ra, s_dec), CIRCLE, 350.8584, +58.8113, 0.16667) = 1) +AND (dataproduct_type = 'event-bundle') +AND (ev_xel >= 1000000) +\end{verbatim} + + \subsubsection{Use Case --- Search for event lists and their \glspl{IRF} of CTAO South observations at energies above 10 TeV for blind search of PeVatrons from a data release using DataLink} -{\em Identify all event lists and their associated \glspl{IRF} taken by CTAO South that contains events above 10 TeV. Data taken with the Small Size Telescopes or Medium Size Telescopes can be then selected. \/} +{\em Identify all event bundles (event lists and their associated \glspl{IRF}) taken by CTAO South that contains events above 10 TeV. Data taken with the Small Size Telescopes or Medium Size Telescopes can be then selected. \/} \medskip \noindent Find all CTAO datasets satisfying: \begin{enumerate}[(i)] - \item dataproduct\_type = ``event-list'', + \item dataproduct\_type = ``event-bundle'', \item obs\_collection = ``CTAO-DR1'', \item access\_format = ``datalink'', \item instrument\_name contains ``CTAO-S'', @@ -94,7 +144,7 @@ \subsubsection{Use Case --- Search for event lists and their \glspl{IRF} of CTAO \begin{verbatim} SELECT * FROM ivoa.obscore NATURAL JOIN ivoa.obscore_hea -WHERE (dataproduct_type = 'event-list') +WHERE (dataproduct_type = 'event-bundle') AND (obs_collection = 'CTAO-DR1') AND (access_format = 'application/x-votable+xml;content=datalink') AND (instrument_name LIKE 'CTAO-S') @@ -130,56 +180,6 @@ \subsubsection{Use Case --- Search for event lists and their \glspl{IRF} of CTAO \end{verbatim} -\subsubsection{Use Case --- Search for event bundles that include Cas A for X-ray spectrophotometric evolution studies} - -{\em Identify all event bundles that include the Cas A SNR and have at least 1 million events for subsequent spectrophotometric studies of the SNR expansion. Since only a few observations are expected to match this request and because the focus is on X-ray spectrophotometric studies, the event bundles that include the responses or the ancillary products used to make the responses are required.\/} - -\medskip -\noindent Find all datasets satisfying: -\begin{enumerate}[(i)] - \item Target name = ``Cas A'' or position inside 10 arcmin from (350.8584, $+58.8113$), - \item dataproduct\_type = ``event-bundle'', - \item ev\_xel $\geq 1000000$. -\end{enumerate} - -\begin{verbatim} -SELECT * FROM ivoa.obscore -NATURAL JOIN ivoa.obscore_hea -WHERE -(target_name = 'Cas A' OR -CONTAINS(POINT(s_ra, s_dec), CIRCLE, 350.8584, +58.8113, 0.16667) = 1) -AND (dataproduct_type = 'event-bundle') -AND (ev_xel >= 1000000) -\end{verbatim} - - -\subsubsection{Use Case --- Search for event bundles via DataLink that include Cas A for a TeV spectromorphology study} - -{\em Identify all event lists and their associated \glspl{IRF} that include the Cas A SNR for subsequent TeV spectromorphology studies from a VERITAS data release. Since the instrumental responses are mandatory to remove instrumental effects, the event bundles that include the \glspl{IRF} are required.\/} - -\medskip -\noindent Find all VERITAS datasets satisfying: -\begin{enumerate}[(i)] - \item Target name = ``Cas A'' or position inside 4 deg from (350.8584, $+58.8113$), - \item dataproduct\_type = ``event-bundle'', - \item obs\_collection = ``VERITAS-DR1'', - \item access\_format = ``datalink''. -\end{enumerate} - -\begin{verbatim} -SELECT * FROM ivoa.obscore -NATURAL JOIN ivoa.obscore_hea -WHERE -(target_name = 'Cas A' OR -CONTAINS(POINT(s_ra, s_dec), CIRCLE, 350.8584, +58.8113, 4.0) = 1) -AND (dataproduct_type = 'event-bundle') -AND (obs_collection = 'VERITAS-DR1') -AND (access_format = ’application/x-votable+xml;content=datalink’) -\end{verbatim} - -Then, for each row of the output, we get the ``access\_url'' of the DataLink to provide access to the data. - - \subsubsection{Use Case --- Search for spatially resolved spectropolarimetric observations of the Crab with spectral resolution R > 100} {\em Identify all event bundles for observations of the Crab that intersect the 1.0--100.0 keV energy range, have calibrated spatial and time axes, are spatially resolved in 2 dimensions in equatorial coordinates, have spectral resolution $R>100$, and include polarimetry measurements. Note that ObsCore specifies that the axes lengths --- s\_xel1, s\_xel2, em\_xel, t\_xel, pol\_xel --- should be set to $-1$ for non-pixelated data like event lists, so these quantities are not useful for this query.\/} @@ -245,7 +245,7 @@ \subsubsection{Use Case --- Get all the \glspl{IRF} for a given CTAO observation \medskip \noindent Find the CTAO datasets satisfying: \begin{enumerate}[(i)] - \item dataproduct\_type = ``event-list'' or dataproduct\_type = ``aeff'' or dataproduct\_type = ``psf'' or dataproduct\_type = ``edisp'' or dataproduct\_type = ``bkgrate'', + \item dataproduct\_type = ``event-bundle'' or dataproduct\_type = ``aeff'' or dataproduct\_type = ``psf'' or dataproduct\_type = ``edisp'' or dataproduct\_type = ``bkgrate'', \item obs\_id = ``4374'', \item obs\_collection = ``CTAO-DR1''. \end{enumerate} @@ -254,7 +254,7 @@ \subsubsection{Use Case --- Get all the \glspl{IRF} for a given CTAO observation SELECT * FROM ivoa.obscore NATURAL JOIN ivoa.obscore_hea WHERE -(dataproduct_type = 'event-list' OR dataproduct_type = 'aeff' +(dataproduct_type = 'event-bundle' OR dataproduct_type = 'aeff' OR dataproduct_type = 'edisp' OR dataproduct_type = 'psf' OR dataproduct_type = 'bkgrate') AND (obs_id = '4374')