From a4ecaa078e4262c54ac5fdcf0df42489552f632f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dennis=20G=C3=B6ries?= Date: Sun, 17 Nov 2024 18:40:15 +0100 Subject: [PATCH 1/3] feat: Nogil for LTOB --- src/downsample/_ltob.c | 49 ++++++++++++++---------- src/downsample/tests/test_lttb_memory.py | 2 + 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/src/downsample/_ltob.c b/src/downsample/_ltob.c index 3cb9c6c..4f840e9 100644 --- a/src/downsample/_ltob.c +++ b/src/downsample/_ltob.c @@ -1,7 +1,6 @@ #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include #include -#define PY_SSIZE_T_CLEAN #include #include @@ -57,25 +56,27 @@ static PyObject *largest_triangle_one_bucket(PyObject *self, PyObject *args) { double *x = (double *)PyArray_DATA(x_array); double *y = (double *)PyArray_DATA(y_array); - npy_intp dims[1] = {threshold}; - PyArrayObject *x_result = (PyArrayObject *)PyArray_Empty( - 1, dims, PyArray_DescrFromType(NPY_DOUBLE), 0); - PyArrayObject *y_result = (PyArrayObject *)PyArray_Empty( - 1, dims, PyArray_DescrFromType(NPY_DOUBLE), 0); - - double *x_result_data = (double *)PyArray_DATA((PyArrayObject *)x_result); - double *y_result_data = (double *)PyArray_DATA((PyArrayObject *)y_result); + double *result_x = (double *)malloc(threshold * sizeof(double)); + double *result_y = (double *)malloc(threshold * sizeof(double)); + if (!result_x || !result_y) { + PyErr_SetString(PyExc_MemoryError, + "Failed to allocate memory for result arrays."); + free(result_x); + free(result_y); + goto fail; + } // Add the first point and last - x_result_data[0] = npy_isfinite(x[0]) ? x[0] : 0.0; - y_result_data[0] = npy_isfinite(y[0]) ? y[0] : 0.0; - x_result_data[threshold - 1] = + result_x[0] = npy_isfinite(x[0]) ? x[0] : 0.0; + result_y[0] = npy_isfinite(y[0]) ? y[0] : 0.0; + result_x[threshold - 1] = npy_isfinite(x[len_points - 1]) ? x[len_points - 1] : 0.0; - y_result_data[threshold - 1] = + result_y[threshold - 1] = npy_isfinite(y[len_points - 1]) ? y[len_points - 1] : 0.0; double bucket_size = (double)(len_points - 2) / (double)(threshold - 2); + Py_BEGIN_ALLOW_THREADS; // Main loop for (npy_intp i = 1; i < threshold - 1; i++) { npy_intp start_index = (npy_intp)floor((double)i * bucket_size); @@ -95,18 +96,24 @@ static PyObject *largest_triangle_one_bucket(PyObject *self, PyObject *args) { max_area_index = j; } } - x_result_data[i] = (double)x[max_area_index]; - y_result_data[i] = (double)y[max_area_index]; + result_x[i] = (double)x[max_area_index]; + result_y[i] = (double)y[max_area_index]; } + Py_END_ALLOW_THREADS; - // Return x and y arrays as a tuple - PyObject *result = PyTuple_Pack(2, x_result, y_result); - // Clean up references + npy_intp dims[1] = {threshold}; + PyObject *npx = + PyArray_SimpleNewFromData(1, dims, NPY_DOUBLE, (void *)result_x); + PyObject *npy = + PyArray_SimpleNewFromData(1, dims, NPY_DOUBLE, (void *)result_y); + PyArray_ENABLEFLAGS((PyArrayObject *)npx, NPY_ARRAY_OWNDATA); + PyArray_ENABLEFLAGS((PyArrayObject *)npy, NPY_ARRAY_OWNDATA); + + PyObject *result = PyTuple_Pack(2, npx, npy); Py_DECREF(x_array); Py_DECREF(y_array); - Py_DECREF(x_result); - Py_DECREF(y_result); - + Py_DECREF(npx); + Py_DECREF(npy); return result; fail: diff --git a/src/downsample/tests/test_lttb_memory.py b/src/downsample/tests/test_lttb_memory.py index 1a453bd..849bc23 100644 --- a/src/downsample/tests/test_lttb_memory.py +++ b/src/downsample/tests/test_lttb_memory.py @@ -1,5 +1,7 @@ import tracemalloc + import numpy as np + from downsample import largest_triangle_three_buckets From 8a38270d36c9ff82ea32848cb16bcd1ce6ab482f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dennis=20G=C3=B6ries?= Date: Sun, 17 Nov 2024 18:44:25 +0100 Subject: [PATCH 2/3] Memory leak --- ...est_lttb_memory.py => test_memory_leak.py} | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) rename src/downsample/tests/{test_lttb_memory.py => test_memory_leak.py} (57%) diff --git a/src/downsample/tests/test_lttb_memory.py b/src/downsample/tests/test_memory_leak.py similarity index 57% rename from src/downsample/tests/test_lttb_memory.py rename to src/downsample/tests/test_memory_leak.py index 849bc23..f2c11f5 100644 --- a/src/downsample/tests/test_lttb_memory.py +++ b/src/downsample/tests/test_memory_leak.py @@ -2,28 +2,44 @@ import numpy as np -from downsample import largest_triangle_three_buckets +from downsample import lttb, ltob, ltd +import pytest -def test_memory_leak(): + +@pytest.mark.parametrize("func", [lttb, ltob]) +def test_memory_leak(func): + """ + Test memory leak for different LTTB functions. + + Args: + func (callable): The function to test. + """ tracemalloc.start() + # Test parameters (shared for all functions) size = 1_000_000 + threshold = 100 + iterations = 1_000 + + # Generate test data x = np.linspace(0, 10, size) y = np.sin(x) - threshold = 100 + # Snapshot before function execution before_snapshot = tracemalloc.take_snapshot() - for _ in range(1_000): - result = largest_triangle_three_buckets(x, y, threshold) + for _ in range(iterations): + result = func(x, y, threshold) del result import gc gc.collect() + # Snapshot after function execution after_snapshot = tracemalloc.take_snapshot() tracemalloc.stop() + # Calculate memory usage before_size = sum( stat.size for stat in before_snapshot.statistics("filename")) after_size = sum( From 1080951ddf435adeb5b1370de2d2cc0cd235ec8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dennis=20G=C3=B6ries?= Date: Sun, 17 Nov 2024 18:44:43 +0100 Subject: [PATCH 3/3] Memory leak --- src/downsample/tests/test_memory_leak.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/downsample/tests/test_memory_leak.py b/src/downsample/tests/test_memory_leak.py index f2c11f5..bf2a6cf 100644 --- a/src/downsample/tests/test_memory_leak.py +++ b/src/downsample/tests/test_memory_leak.py @@ -1,11 +1,10 @@ import tracemalloc import numpy as np - -from downsample import lttb, ltob, ltd - import pytest +from downsample import ltob, lttb + @pytest.mark.parametrize("func", [lttb, ltob]) def test_memory_leak(func):