Skip to content

SST1RSoXSDB metadata change: make loadMonitors handle scans without monitors #215

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pbeaucage wants to merge 3 commits into
base: main
Choose a base branch
from
Open

SST1RSoXSDB metadata change: make loadMonitors handle scans without monitors #215

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6d17573
Fix loadMonitors to handle scans without monitors
pbeaucage Jun 21, 2025
5400040
Dramatic speedup by not double-downloading images
pbeaucage Jun 21, 2025
b992882
Merge branch 'main' into sst1-non-monitor-loading
pbeaucage Jun 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 25 additions & 80 deletions src/PyHyperScattering/SST1RSoXSDB.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ def __init__(
use_precise_positions=False,
use_chunked_loading=False,
suppress_time_dimension=True,
double_diode_norm_scanid=None,
):
"""
Args:
Expand Down Expand Up @@ -138,6 +139,7 @@ def __init__(
self.exposure_offset = exposure_offset
self.use_precise_positions = use_precise_positions
self.suppress_time_dimension = suppress_time_dimension
self.double_diode_norm_scanid = double_diode_norm_scanid

self.catalog_df = None
self.catalog_df_kwargs = None
Expand Down Expand Up @@ -883,9 +885,7 @@ def loadRun(
dims = pyhyper_axes_to_use

data = run["primary"]["data"][md["detector"] + "_image"]
if isinstance(data,tiled.client.array.ArrayClient):
data = run["primary"]["data"].read()[md["detector"] + "_image"]
elif isinstance(data,tiled.client.array.DaskArrayClient):
if isinstance(data,tiled.client.array.ArrayClient) or isinstance(data,tiled.client.array.DaskArrayClient):
data = run["primary"]["data"].read()[md["detector"] + "_image"]
# Handle extra dimensions (non-pixel and non-intended dimensions from repeat exposures) by averaging them along the dim_0 axis
if len(data.shape) > 3:
Expand All @@ -896,8 +896,8 @@ def loadRun(
if self.dark_subtract:
dark = run["dark"]["data"][md["detector"] + "_image"]
if (
type(dark) == tiled.client.array.ArrayClient
or type(dark) == tiled.client.array.DaskArrayClient
isinstance(dark,tiled.client.array.ArrayClient)
or isinstance(dark,tiled.client.array.DaskArrayClient)
):
dark = run["dark"]["data"].read()[md["detector"] + "_image"]
darkframe = np.copy(data.time)
Expand All @@ -916,7 +916,7 @@ def subtract_dark(img, pedestal=100, darks=None):
for dim in dims:
try:
test = len(md[dim]) # this will throw a typeerror if single value
if type(md[dim]) == dask.array.core.Array:
if isinstance(md[dim], dask.array.core.Array):
dims_to_join.append(md[dim].compute())
else:
dims_to_join.append(md[dim])
Expand Down Expand Up @@ -979,6 +979,18 @@ def subtract_dark(img, pedestal=100, darks=None):
frozen_attrs = retxr.attrs
if self.corr_mode == "i0":
retxr = retxr / monitors["RSoXS Au Mesh Current"]
if self.corr_mode == "double_diode_i0":
diode_reference = self.loadMonitors(self.c[self.double_diode_norm_scanid])
assert diode_reference.len('time') == retxr.len('system') # must be the exact same scan
diode_reference = (
diode_reference.rename({"time": "system"})
.reset_index("system")
.assign_coords(mindex_coords)
)
# in double-diode normalization, we correct for the mesh sensitivity by using a diode sensitivity scan to normalize the mesh current
# this assumes the diode is clean/true/correct, and the mesh sensitivity didn't change between the runs
retxr = retxr / monitors['RSoXS Au Mesh Current'] * (diode_reference['RSoXS Au Mesh Current'] / diode_reference['WAXS Beamstop'])

elif self.corr_mode != "none":
warnings.warn(
"corrections other than none or i0 are not supported at the moment",
Expand Down Expand Up @@ -1067,9 +1079,13 @@ def loadMonitors(
# the time dimension inherited all time values from all streams
# the data variables (Mesh current, sample current etc.) are all sparse, with lots of nans

# if there are no monitors, return an empty xarray Dataset
# if there are no monitors, return the dataset as it is, assume it's already been treated upstream
if raw_monitors is None:
return xr.Dataset()
if isinstance(entry['primary']['data'],tiled.client.xarray.DatasetClient) or isinstance(entry['primary']['data'],tiled.client.xarray.DaskDatasetClient):
keys_to_load = [x for x in list(entry['primary']['data'].keys()) if 'image' not in x]
return entry['primary']['data'].read(keys_to_load)
else:
return entry['primary']['data']

# For each nan value, replace with the closest value ahead of it in time
# For remaining nans, replace with closest value behind it in time
Expand Down Expand Up @@ -1350,75 +1366,4 @@ def loadMd(self, run):
md["rot3"] = 0

md.update(run.metadata)
return md

def loadSingleImage(self, filepath, coords=None, return_q=False, **kwargs):
"""
DO NOT USE

This function is preserved as reference for the qx/qy loading conventions.

NOT FOR ACTIVE USE. DOES NOT WORK.
"""
if len(kwargs.keys()) > 0:
warnings.warn(
f"Loader does not support features for args: {kwargs.keys()}",
stacklevel=2,
)
img = Image.open(filepath)

headerdict = self.loadMd(filepath)
# two steps in this pre-processing stage:
# (1) get and apply the right scalar correction term to the image
# (2) find and subtract the right dark
if coords != None:
headerdict.update(coords)

# step 1: correction term

if self.corr_mode == "expt":
corr = headerdict["exposure"] # (headerdict['AI 3 Izero']*expt)
elif self.corr_mode == "i0":
corr = headerdict["AI 3 Izero"]
elif self.corr_mode == "expt+i0":
corr = headerdict["exposure"] * headerdict["AI 3 Izero"]
elif self.corr_mode == "user_func":
corr = self.user_corr_func(headerdict)
elif self.corr_mode == "old":
corr = (
headerdict["AI 6 BeamStop"]
* 2.4e10
/ headerdict["Beamline Energy"]
/ headerdict["AI 3 Izero"]
)
# this term is a mess... @TODO check where it comes from
else:
corr = 1

if corr < 0:
warnings.warn(
f"Correction value is negative: {corr} with headers {headerdict}.",
stacklevel=2,
)
corr = abs(corr)

# # step 2: dark subtraction
# try:
# darkimg = self.darks[headerdict['EXPOSURE']]
# except KeyError:
# warnings.warn(f"Could not find a dark image with exposure time {headerdict['EXPOSURE']}. Using zeros.",stacklevel=2)
# darkimg = np.zeros_like(img)

# img = (img-darkimg+self.dark_pedestal)/corr
if return_q:
qpx = (
2 * np.pi * 60e-6 / (headerdict["sdd"] / 1000) / (headerdict["wavelength"] * 1e10)
)
qx = (np.arange(1, img.size[0] + 1) - headerdict["beamcenter_x"]) * qpx
qy = (np.arange(1, img.size[1] + 1) - headerdict["beamcenter_y"]) * qpx
# now, match up the dims and coords
return xr.DataArray(
img, dims=["qy", "qx"], coords={"qy": qy, "qx": qx}, attrs=headerdict
)
else:
return xr.DataArray(img, dims=["pix_x", "pix_y"], attrs=headerdict)
return md
Loading