From 08beb39e64fa1f6c2fdf804eeac5ae9d90aa1a94 Mon Sep 17 00:00:00 2001 From: nisar Date: Wed, 2 Jul 2025 21:36:11 -0400 Subject: [PATCH 01/11] Replace Black and Flake8 with Ruff --- .flake8 | 12 ------------ requirements.txt | 1 + 2 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index f9b52a9..0000000 --- a/.flake8 +++ /dev/null @@ -1,12 +0,0 @@ -[flake8] -exclude = - .git, - __pycache__, - build, - dist, - versioneer.py, - csxtools/doc/conf.py - *.ipynb_checkpoints, - -max-line-length = 140 -ignore = E203, W503 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 24ce15a..a6b1da2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ numpy +ruff>=0.4.0 From db3735f8ccf4e660a62beceba3e2b91367660dea Mon Sep 17 00:00:00 2001 From: nisar Date: Wed, 2 Jul 2025 22:02:53 -0400 Subject: [PATCH 02/11] replace .github/workflows/flake8.yml with .github/workflows/ruff.yml --- .github/workflows/{flake8.yml => ruff.yml} | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) rename .github/workflows/{flake8.yml => ruff.yml} (51%) diff --git a/.github/workflows/flake8.yml b/.github/workflows/ruff.yml similarity index 51% rename from .github/workflows/flake8.yml rename to .github/workflows/ruff.yml index 528bc64..1bcdd96 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/ruff.yml @@ -1,4 +1,4 @@ -name: Check Code Style - FLAKE8 +name: Check Code Style - RUFF on: [push, pull_request] @@ -12,12 +12,8 @@ jobs: python-version: "3.10" - name: Install Dependencies run: | - # These packages are installed in the base environment but may be older - # versions. Explicitly upgrade them because they often create - # installation problems if out of date. python -m pip install --upgrade pip setuptools numpy - - pip install flake8 - - name: Run flake8 + pip install ruff + - name: Run ruff run: | - flake8 + ruff . From 95f6ed7c9f8ae46d9649a40255e15f2ea808793c Mon Sep 17 00:00:00 2001 From: nisar Date: Wed, 2 Jul 2025 22:08:36 -0400 Subject: [PATCH 03/11] Bug fix in .github/workflows/ruff.yml --- .github/workflows/ruff.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 1bcdd96..599d4cf 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -15,5 +15,4 @@ jobs: python -m pip install --upgrade pip setuptools numpy pip install ruff - name: Run ruff - run: | - ruff . + run: ruff check . From 7e9ab6bc18a2b5c54e7d008b410264022b1f1f3a Mon Sep 17 00:00:00 2001 From: nisar Date: Thu, 3 Jul 2025 21:14:27 -0400 Subject: [PATCH 04/11] Update pyproject.toml to exclude 'examples' and other diectories like 'src' from Ruff checks --- pyproject.toml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 145d9bf..c058f1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,3 +6,12 @@ requires = [ "versioneer[toml]>=0.28" ] build-backend = "setuptools.build_meta" +[tool.ruff] +exclude = [ + "examples", + "doc", + "build", + "dist", + "csxtools.egg-info", + "__pycache__" +] From f228d2321b749c1eae2f876d906ed7fdfed2bd3f Mon Sep 17 00:00:00 2001 From: nisar Date: Fri, 4 Jul 2025 09:41:37 -0400 Subject: [PATCH 05/11] Detector 'tag' handling inside _get_axis1_images is improved and fixed some potential issues --- csxtools/utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/csxtools/utils.py b/csxtools/utils.py index 38795a8..edf172a 100644 --- a/csxtools/utils.py +++ b/csxtools/utils.py @@ -173,7 +173,13 @@ def _get_axis1_images(light_header, dark_header=None, flat=None, tag=None, roi=N if tag is None: logger.error("Must pass 'tag' argument to get_axis_images()") raise ValueError("Must pass 'tag' argument") + + if tag not in detectors: + raise ValueError(f"Unknown detector tag '{tag}'. Valid options: {list(detectors)}") + tag_mapped = detectors[tag] + logger.info(f"Using detector tag '{tag}' mapped to internal tag '{tag_mapped}'") + # Now lets sort out the ROI if roi is not None: roi = list(roi) @@ -191,7 +197,7 @@ def _get_axis1_images(light_header, dark_header=None, flat=None, tag=None, roi=N t = ttime.time() d = dark_header - bgnd_events = _get_images(d, tag, roi) + bgnd_events = _get_images(d, tag_mapped, roi) tt = ttime.time() b = bgnd_events.astype(dtype=np.uint16) @@ -204,7 +210,7 @@ def _get_axis1_images(light_header, dark_header=None, flat=None, tag=None, roi=N logger.info("Computed dark images in %.3f seconds", ttime.time() - t) - events = _get_images(light_header, tag, roi) + events = _get_images(light_header, tag_mapped, roi) # Ok, so lets return a pims pipeline which does the image conversion From 3233b5335deac9c6c0d0ab5a55e8145a110dac10 Mon Sep 17 00:00:00 2001 From: nisar Date: Thu, 10 Jul 2025 11:26:43 -0400 Subject: [PATCH 06/11] fixed varible names --- csxtools/utils.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/csxtools/utils.py b/csxtools/utils.py index edf172a..22bf030 100644 --- a/csxtools/utils.py +++ b/csxtools/utils.py @@ -323,6 +323,34 @@ def get_fastccd_timestamps(header, tag="fccd_image"): return timestamps +def get_axis_timestamps(header, tag= None): + """Return the AXIS timestamps from the Areadetector Data File + + Return a list of numpy arrays of the timestamps for the images as + recorded in the datafile. + + Parameters + ---------- + header : databroker header + This header defines the run + tag : string + User-level tag (e.g., 'axis1'). Internally mapped to the correct timestamp key. + + Returns + ------- + list of arrays of the timestamps + """ + + if tag is None: + raise ValueError("Must pass a detector tag (e.g., 'axis1', 'axis_standard', etc.)") + + tag_key = f"{tag}_hdf5_time_stamp" + logger.info(f"Using detector tag '{tag}' mapped to timestamp key '{tag_key}'") + + timestamps = list(header.data(tag_key)) + + return timestamps + def get_axis_timestamps(header, tag="axis1_hdf5_time_stamp"): """Return the AXIS timestamps from the Areadetector Data File From 753d44b33a028ebcb2a4879a1f9d8b15445fce9d Mon Sep 17 00:00:00 2001 From: nisar Date: Thu, 10 Jul 2025 11:39:47 -0400 Subject: [PATCH 07/11] fixed issue with method browse_3Darray in csxtools/helpers/fastccd.py --- csxtools/helpers/fastccd.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/csxtools/helpers/fastccd.py b/csxtools/helpers/fastccd.py index 30277ca..9e0df48 100644 --- a/csxtools/helpers/fastccd.py +++ b/csxtools/helpers/fastccd.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -def browse_3Darray(res, title="Frame"): # , extra_scalar_dict=None): +def browse_3Darray(res, fig, ax, im, title="Frame"): # , extra_scalar_dict=None): """Widget for notebooks. Sliding bar to browse 3D python array. Must plot using subplots method with 1 axes. res : 3D array with the first element being interated @@ -21,24 +21,15 @@ def browse_3Darray(res, title="Frame"): # , extra_scalar_dict=None): """ N = len(res) - def view_image(i=0): + def view_image(i=0, fig = fig, ax = ax, im = im): im.set_data(res[i]) - # if extra_scalar_dict is not None: - # key = extra_scalr_dict.keys()[0] - # values = extra_scalar_dict.values() - - # if extra_scalar_dict is None: - # ax.set_title(f'{title} {i} {key} {values[i]}') - # else: ax.set_title(f"{title} {i}") fig.canvas.draw_idle() interact(view_image, i=(0, N - 1)) - # FCCD specific stuff starts here - def find_possible_darks( header, dark_gain, From 3799914447a147a0ffcaba16d3ee951aeeab7bd9 Mon Sep 17 00:00:00 2001 From: nisar Date: Thu, 10 Jul 2025 12:35:42 -0400 Subject: [PATCH 08/11] modified: csxtools/utils.py to address the issue with detector tag --- csxtools/settings.py | 4 ---- csxtools/utils.py | 37 ++++++------------------------------- 2 files changed, 6 insertions(+), 35 deletions(-) diff --git a/csxtools/settings.py b/csxtools/settings.py index cdd9c41..462c331 100644 --- a/csxtools/settings.py +++ b/csxtools/settings.py @@ -1,8 +1,4 @@ detectors = {} detectors["fccd"] = "fccd_image" -detectors["axis1"] = "axis1_image" -detectors["axis"] = "axis_image" -detectors["axis_standard"] = "axis_standard_image" -detectors["axis_cont"] = "axis_cont_image" diff_angles = ["delta", "theta", "gamma", None, None, None] diff --git a/csxtools/utils.py b/csxtools/utils.py index 22bf030..8c75b84 100644 --- a/csxtools/utils.py +++ b/csxtools/utils.py @@ -177,8 +177,8 @@ def _get_axis1_images(light_header, dark_header=None, flat=None, tag=None, roi=N if tag not in detectors: raise ValueError(f"Unknown detector tag '{tag}'. Valid options: {list(detectors)}") - tag_mapped = detectors[tag] - logger.info(f"Using detector tag '{tag}' mapped to internal tag '{tag_mapped}'") + tag_key = f"{tag}_image" + logger.info(f"Using detector tag '{tag}' converted to internal tag '{tag_key}'") # Now lets sort out the ROI if roi is not None: @@ -197,7 +197,7 @@ def _get_axis1_images(light_header, dark_header=None, flat=None, tag=None, roi=N t = ttime.time() d = dark_header - bgnd_events = _get_images(d, tag_mapped, roi) + bgnd_events = _get_images(d, tag_key, roi) tt = ttime.time() b = bgnd_events.astype(dtype=np.uint16) @@ -210,7 +210,7 @@ def _get_axis1_images(light_header, dark_header=None, flat=None, tag=None, roi=N logger.info("Computed dark images in %.3f seconds", ttime.time() - t) - events = _get_images(light_header, tag_mapped, roi) + events = _get_images(light_header, tag_key, roi) # Ok, so lets return a pims pipeline which does the image conversion @@ -334,7 +334,7 @@ def get_axis_timestamps(header, tag= None): header : databroker header This header defines the run tag : string - User-level tag (e.g., 'axis1'). Internally mapped to the correct timestamp key. + User-level tag (e.g., 'axis1'). Internally converted to the correct timestamp key. Returns ------- @@ -345,37 +345,12 @@ def get_axis_timestamps(header, tag= None): raise ValueError("Must pass a detector tag (e.g., 'axis1', 'axis_standard', etc.)") tag_key = f"{tag}_hdf5_time_stamp" - logger.info(f"Using detector tag '{tag}' mapped to timestamp key '{tag_key}'") + logger.info(f"Using detector tag '{tag}' converted to timestamp key '{tag_key}'") timestamps = list(header.data(tag_key)) return timestamps - -def get_axis_timestamps(header, tag="axis1_hdf5_time_stamp"): - """Return the AXIS timestamps from the Areadetector Data File - - Return a list of numpy arrays of the timestamps for the images as - recorded in the datafile. - - Parameters - ---------- - header : databorker header - This header defines the run - tag : string - This is the tag or name of the fastccd. - - Returns - ------- - list of arrays of the timestamps - - """ - - timestamps = list(header.data(tag)) - - return timestamps - - def calculate_flatfield(image, limits=(0.6, 1.4)): """Calculate a flatfield from fluo data From 2ce04c2c9d4ec230694925eba5c7db178852c60c Mon Sep 17 00:00:00 2001 From: nisar Date: Fri, 11 Jul 2025 16:12:00 -0400 Subject: [PATCH 09/11] Small bugs related to detector tag is fixed --- csxtools/utils.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/csxtools/utils.py b/csxtools/utils.py index 8c75b84..9813062 100644 --- a/csxtools/utils.py +++ b/csxtools/utils.py @@ -152,8 +152,7 @@ def get_axis_images(light_header, dark_header=None, flat=None, tag=None, roi=Non tag : string Data tag used to retrieve images. Used in the call to - ``databroker.get_images()``. If `None`, use the defualt from - the settings. + ``databroker.get_images()``. roi : tuple coordinates of the upper-left corner and width and height of @@ -171,14 +170,9 @@ def get_axis_images(light_header, dark_header=None, flat=None, tag=None, roi=Non def _get_axis1_images(light_header, dark_header=None, flat=None, tag=None, roi=None): if tag is None: - logger.error("Must pass 'tag' argument to get_axis_images()") - raise ValueError("Must pass 'tag' argument") - - if tag not in detectors: - raise ValueError(f"Unknown detector tag '{tag}'. Valid options: {list(detectors)}") + raise ValueError("Must pass a detector tag (e.g., 'axis1', 'axis_standard', etc.)") tag_key = f"{tag}_image" - logger.info(f"Using detector tag '{tag}' converted to internal tag '{tag_key}'") # Now lets sort out the ROI if roi is not None: @@ -345,8 +339,6 @@ def get_axis_timestamps(header, tag= None): raise ValueError("Must pass a detector tag (e.g., 'axis1', 'axis_standard', etc.)") tag_key = f"{tag}_hdf5_time_stamp" - logger.info(f"Using detector tag '{tag}' converted to timestamp key '{tag_key}'") - timestamps = list(header.data(tag_key)) return timestamps From caec32748a74127b272856c4c925e91bf9e2196d Mon Sep 17 00:00:00 2001 From: nisar Date: Mon, 14 Jul 2025 15:06:04 -0400 Subject: [PATCH 10/11] modified utils.py/get_axis_flatfield() to add 'tag', an essential paramaeter now --- csxtools/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/csxtools/utils.py b/csxtools/utils.py index 9813062..6829bd4 100644 --- a/csxtools/utils.py +++ b/csxtools/utils.py @@ -424,8 +424,7 @@ def get_fastccd_flatfield( ) return flat - -def get_axis_flatfield(light, dark, flat=None, limits=(0.6, 1.4), half_interval=False): +def get_axis_flatfield(light, dark, flat=None, tag=None, limits=(0.6, 1.4), half_interval=False): """Calculate a flatfield from two headers This routine calculates the flatfield using the @@ -440,6 +439,8 @@ def get_axis_flatfield(light, dark, flat=None, limits=(0.6, 1.4), half_interval= The header(s) from the run containin the dark images. flat : flatfield image (optional) The array to be used for the initial flatfield + tag : string + Data tag used to retrieve images. Used in the call to ``databroker.get_images()``. limits : tuple limits used for returning corrected pixel flatfield The tuple setting lower and upper bound. np.nan returned value is outside bounds half_interval : boolean or tuple to perform calculation for only half of the FastCCD @@ -452,7 +453,7 @@ def get_axis_flatfield(light, dark, flat=None, limits=(0.6, 1.4), half_interval= array_like Flatfield correction. The correction is orientated as "raw data" not final data generated by get_fastccd_images(). """ - images = get_images_to_3D(_get_axis1_images(light, dark, flat)) + images = get_images_to_3D(_get_axis1_images(light, dark, flat, tag)) images = stackmean(images) if half_interval: if isinstance(half_interval, bool): From cd296f609a64641aae4da39d9a41a26949887379 Mon Sep 17 00:00:00 2001 From: nisar Date: Mon, 14 Jul 2025 16:43:50 -0400 Subject: [PATCH 11/11] some doc strings in utils.py are improved --- csxtools/utils.py | 104 ++++++++++++++++++++++++---------------------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/csxtools/utils.py b/csxtools/utils.py index 6829bd4..35dfb1b 100644 --- a/csxtools/utils.py +++ b/csxtools/utils.py @@ -128,41 +128,35 @@ def get_fastccd_images( def get_axis_images(light_header, dark_header=None, flat=None, tag=None, roi=None): - """Retreive and correct AXIS Images from associated headers - - Retrieve AXIS Images from databroker and correct for: - - - Bad Pixels (converted to ``np.nan``) - - Backgorund. - - Flatfield correction. - - Rotation (returned images are rotated 90 deg cw) - + """Retrieve and correct AXIS Images from associated headers. + + This function retrieves AXIS images from a databroker header and applies several corrections: + - **Bad Pixels**: Converts bad pixels to ``np.nan``. + - **Background Subtraction**: Subtracts a background from the images. + - **Flatfield Correction**: Applies a flatfield correction. + - **Rotation**: Rotates the returned images by 90 degrees clockwise. + Parameters ---------- - light_header : databorker header - This header defines the images to convert - - dark_header : databroker header , optional - The header is the dark images. - - flat : array_like - Array to use for the flatfield correction. This should be a 2D - array sized as the last two dimensions of the image stack. - - - tag : string - Data tag used to retrieve images. Used in the call to - ``databroker.get_images()``. - - roi : tuple - coordinates of the upper-left corner and width and height of - the ROI: e.g., (x, y, w, h) + light_header : databroker header + This header defines the images to convert. + dark_header : databroker header, optional + The header containing dark images for background subtraction. + flat : array_like, optional, default=None + 2D array to use for the flatfield correction, sized as the last two + dimensions of the image stack. If None, no flatfield correction is applied. + tag : str + Data tag used to retrieve images (e.g., obtained by ``header.start["detectors"]``). + roi : tuple, optional, default=None + Coordinates of the upper-left corner and width and height of the ROI: + e.g., ``(x, y, w, h)``. If None, the entire image is used. Returns ------- - dask.array : corrected images - + dask.array + The corrected image stack. """ + flipped_image = _get_axis1_images(light_header, dark_header, flat, tag, roi) return flipped_image[..., ::-1] @@ -328,7 +322,8 @@ def get_axis_timestamps(header, tag= None): header : databroker header This header defines the run tag : string - User-level tag (e.g., 'axis1'). Internally converted to the correct timestamp key. + User-level tag (e.g., 'axis1', 'axis_standard', etc.). + Internally converted to the correct timestamp key. Returns ------- @@ -425,34 +420,46 @@ def get_fastccd_flatfield( return flat def get_axis_flatfield(light, dark, flat=None, tag=None, limits=(0.6, 1.4), half_interval=False): - """Calculate a flatfield from two headers + """Calculate a flatfield from two databroker headers. - This routine calculates the flatfield using the - :func:calculate_flatfield() function after obtaining the images from - the headers. + This routine calculates the flatfield using the :func:`calculate_flatfield()` + function after obtaining the images from the provided light and dark headers. Parameters ---------- light : databroker header - The header containing the light images - dark : databroker header(s) - The header(s) from the run containin the dark images. - flat : flatfield image (optional) - The array to be used for the initial flatfield - tag : string - Data tag used to retrieve images. Used in the call to ``databroker.get_images()``. - limits : tuple limits used for returning corrected pixel flatfield - The tuple setting lower and upper bound. np.nan returned value is outside bounds - half_interval : boolean or tuple to perform calculation for only half of the FastCCD - Default is False. If True, then the hard-code portion is retained. Customize image - manipulation using a tuple of length 2 for (row_start, row_stop). - + The header containing the light images. + dark : databroker header + The header from the run containing the dark images. This can be a + single header or a list/tuple of headers if dark images span multiple + runs. + flat : array_like, optional, default=None + An array to be used as the initial flatfield. If provided, the calculation + will refine this initial flatfield. If None, a flatfield is calculated + from scratch. + tag : str + Data tag used to retrieve images (e.g., obtained by ``header.start["detectors"]``). + limits : tuple, optional, default=None + A tuple ``(lower_bound, upper_bound)`` defining the limits for the + corrected pixel flatfield values. Any calculated flatfield values + outside these bounds will be converted to ``np.nan``. If None, no + clipping is performed. + half_interval : bool or tuple, optional, default=False + Controls the image manipulation for FastCCD. + - If ``False`` (default), calculations are performed on the full image. + - If ``True``, a hard-coded portion (e.g., half) of the FastCCD image + is retained for the calculation. + - If a ``tuple`` of length 2, ``(row_start, row_stop)``, it specifies + a custom row interval for image manipulation. Returns ------- array_like - Flatfield correction. The correction is orientated as "raw data" not final data generated by get_fastccd_images(). + The calculated flatfield correction array. The orientation of this + correction array corresponds to the "raw data" orientation, not the + final data generated by ``get_axis_images()``. """ + images = get_images_to_3D(_get_axis1_images(light, dark, flat, tag)) images = stackmean(images) if half_interval: @@ -471,7 +478,6 @@ def get_axis_flatfield(light, dark, flat=None, tag=None, limits=(0.6, 1.4), half ) return flat - def fccd_mask(): """Return the initial flatfield mask for the FastCCD