From e40ac402cfb562a3588e637496b2a723e6695fd5 Mon Sep 17 00:00:00 2001 From: Jack Cliff Date: Fri, 24 Oct 2025 19:52:32 +0100 Subject: [PATCH 1/6] cache initial commit --- cache/cache.md | 0 cache/cache.q | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ cache/init.q | 5 +++++ cache/test.csv | 0 4 files changed, 57 insertions(+) create mode 100644 cache/cache.md create mode 100644 cache/cache.q create mode 100644 cache/init.q create mode 100644 cache/test.csv diff --git a/cache/cache.md b/cache/cache.md new file mode 100644 index 0000000..e69de29 diff --git a/cache/cache.q b/cache/cache.q new file mode 100644 index 0000000..ec751af --- /dev/null +++ b/cache/cache.q @@ -0,0 +1,52 @@ +/ Library to provide a mechanism for storing function results in a cache and returning them from the cache if they are available and non stale. + +/ return timestamp function +cp:{.z.p}; + +/ the maximum size of the cache in MB +maxsize:10; + +/ the maximum size of any individual result set in MB +maxindividual:50; + +/ make sure the maxindividual isn't bigger than maxsize +maxindividual:maxsize&maxindividual; + +MB:2 xexp 20 + +/ a table to store the cache values in memory +cache:([id:`u#`long$()] lastrun:`timestamp$();lastaccess:`timestamp$();size:`long$()) + +/ a dictionary of the functions +funcs:(`u#`long$())!() +/ the results of the functions +results:(`u#`long$())!() + +/ table to track the cache performance +perf:([]time:`timestamp$();id:`long$();status:`symbol$()) + +id:0j +getid:{:id+::1} + +/ add to cache +add:{[function;id;status] + / Don't trap the error here - if it throws an error, we want it to be propagated out + res:value function; + $[(.z.m.maxindividual*MB)>size:-22!res; + / check if we need more space to store this item + [now:.z.m.cp[]; + if[0>requiredsize:(.z.m.maxsize*MB) - size+sum exec size from cache; evict[neg requiredsize;now]]; + / Insert to the cache table + .z.M.cache upsert (id;now;now;size); + / and insert to the function and results dictionary + .z.m.funcs[id]:enlist function; + .z.m.results[id]:enlist res; + / Update the performance + trackperf[id;status;now]]; + / Otherwise just log it as an addfail - the result set is too big + trackperf[id;`fail;.z.m.cp[]]]; + / Return the result + res}; + +trackperf:{[id;status;currenttime] .z.M.perf insert ((count id)#currenttime;id;(count id)#status)}; + diff --git a/cache/init.q b/cache/init.q new file mode 100644 index 0000000..111b605 --- /dev/null +++ b/cache/init.q @@ -0,0 +1,5 @@ +/ Load core functionality into root module namespace +\l ::cache.q + +export:([add:add]); + diff --git a/cache/test.csv b/cache/test.csv new file mode 100644 index 0000000..e69de29 From ae8129f85289b7f754442f3613e3b0d0bb5d920c Mon Sep 17 00:00:00 2001 From: Joseph Muldoon Date: Tue, 11 Nov 2025 17:42:36 +0000 Subject: [PATCH 2/6] placed module in di directory --- {cache => di/cache}/cache.md | 0 {cache => di/cache}/cache.q | 0 {cache => di/cache}/init.q | 0 {cache => di/cache}/test.csv | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {cache => di/cache}/cache.md (100%) rename {cache => di/cache}/cache.q (100%) rename {cache => di/cache}/init.q (100%) rename {cache => di/cache}/test.csv (100%) diff --git a/cache/cache.md b/di/cache/cache.md similarity index 100% rename from cache/cache.md rename to di/cache/cache.md diff --git a/cache/cache.q b/di/cache/cache.q similarity index 100% rename from cache/cache.q rename to di/cache/cache.q diff --git a/cache/init.q b/di/cache/init.q similarity index 100% rename from cache/init.q rename to di/cache/init.q diff --git a/cache/test.csv b/di/cache/test.csv similarity index 100% rename from cache/test.csv rename to di/cache/test.csv From 92f5805350a1b829cf6fcea91f5f16bc9278a345 Mon Sep 17 00:00:00 2001 From: ryanCheale Date: Wed, 12 Nov 2025 10:20:36 +0000 Subject: [PATCH 3/6] Added ; to line ends --- di/cache/cache.q | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/di/cache/cache.q b/di/cache/cache.q index ec751af..8b73fe1 100644 --- a/di/cache/cache.q +++ b/di/cache/cache.q @@ -12,21 +12,21 @@ maxindividual:50; / make sure the maxindividual isn't bigger than maxsize maxindividual:maxsize&maxindividual; -MB:2 xexp 20 +MB:2 xexp 20; / a table to store the cache values in memory -cache:([id:`u#`long$()] lastrun:`timestamp$();lastaccess:`timestamp$();size:`long$()) +cache:([id:`u#`long$()] lastrun:`timestamp$();lastaccess:`timestamp$();size:`long$()); / a dictionary of the functions -funcs:(`u#`long$())!() +funcs:(`u#`long$())!(); / the results of the functions -results:(`u#`long$())!() +results:(`u#`long$())!(); / table to track the cache performance -perf:([]time:`timestamp$();id:`long$();status:`symbol$()) +perf:([]time:`timestamp$();id:`long$();status:`symbol$()); -id:0j -getid:{:id+::1} +id:0j; +getid:{:id+::1}; / add to cache add:{[function;id;status] From e87f652d137618ad16a6d09a4e94388b73e4ff12 Mon Sep 17 00:00:00 2001 From: ryanCheale Date: Wed, 12 Nov 2025 11:32:10 +0000 Subject: [PATCH 4/6] - Removed instances of .z.m - Removed instances of .z.M and replaced ` --- di/cache/cache.q | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/di/cache/cache.q b/di/cache/cache.q index 8b73fe1..2be1246 100644 --- a/di/cache/cache.q +++ b/di/cache/cache.q @@ -32,21 +32,21 @@ getid:{:id+::1}; add:{[function;id;status] / Don't trap the error here - if it throws an error, we want it to be propagated out res:value function; - $[(.z.m.maxindividual*MB)>size:-22!res; + $[(maxindividual*MB)>size:-22!res; / check if we need more space to store this item - [now:.z.m.cp[]; - if[0>requiredsize:(.z.m.maxsize*MB) - size+sum exec size from cache; evict[neg requiredsize;now]]; + [now:cp[]; + if[0>requiredsize:(maxsize*MB) - size+sum exec size from cache; evict[neg requiredsize;now]]; / Insert to the cache table - .z.M.cache upsert (id;now;now;size); + `cache upsert (id;now;now;size); / and insert to the function and results dictionary - .z.m.funcs[id]:enlist function; - .z.m.results[id]:enlist res; + funcs[id]:enlist function; + results[id]:enlist res; / Update the performance - trackperf[id;status;now]]; + trackperf[0N!id;0N!status;0N!now]]; / Otherwise just log it as an addfail - the result set is too big - trackperf[id;`fail;.z.m.cp[]]]; + trackperf[id;`fail;cp[]]]; / Return the result res}; -trackperf:{[id;status;currenttime] .z.M.perf insert ((count id)#currenttime;id;(count id)#status)}; +trackperf:{[id;status;currenttime] `perf insert ((count id)#currenttime;id;(count id)#status)}; From 1e03bd58e0089b5fd8281afdab53fc8506d27704 Mon Sep 17 00:00:00 2001 From: ryanCheale Date: Thu, 13 Nov 2025 15:37:13 +0000 Subject: [PATCH 5/6] removal of 0N\!, tidied up init.q --- di/cache/cache.q | 2 +- di/cache/init.q | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/di/cache/cache.q b/di/cache/cache.q index 2be1246..b2ec527 100644 --- a/di/cache/cache.q +++ b/di/cache/cache.q @@ -42,7 +42,7 @@ add:{[function;id;status] funcs[id]:enlist function; results[id]:enlist res; / Update the performance - trackperf[0N!id;0N!status;0N!now]]; + trackperf[id;status;now]]; / Otherwise just log it as an addfail - the result set is too big trackperf[id;`fail;cp[]]]; / Return the result diff --git a/di/cache/init.q b/di/cache/init.q index 111b605..7ea6d4d 100644 --- a/di/cache/init.q +++ b/di/cache/init.q @@ -1,5 +1,5 @@ / Load core functionality into root module namespace \l ::cache.q -export:([add:add]); +export:([add]); From cb82ec51712305b8e7046f9a354d6ec93a97db9e Mon Sep 17 00:00:00 2001 From: ryanCheale Date: Fri, 21 Nov 2025 14:55:40 +0000 Subject: [PATCH 6/6] Complete cache code transfer, runs but local namespacing for variables needs checking --- di/cache/.cache.q.swo | Bin 0 -> 12288 bytes di/cache/cache.q | 45 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 di/cache/.cache.q.swo diff --git a/di/cache/.cache.q.swo b/di/cache/.cache.q.swo new file mode 100644 index 0000000000000000000000000000000000000000..7ccb9820f935b78def2719568ecee5493ae019f2 GIT binary patch literal 12288 zcmeHNO>7)R7A_$CO!x^bpuIkm2y9}{I7UdQ2LhW#f=EP2{7O3vnR0i{OvOFjld5iy zO=KXK<-l?QB#=13X>Y5QmJ1gSAZ{RWK?s)BA`m~~N1RzL68pWX>Y1@g_P}XZYAc^T zQ`PnA)%U*lUXA1R)klt>pxc^j0$*1Pam5bl=Po=e4*cbIVd9~Zsp6Uc`Zdf)$~5Ka zQl4(cd9&O&&(lu^gRPBZeQRZuCuOQ^E77f3VkiH-dHOty>*1P#nt@->Kw0!xZo5+4 zvUX&ZjU2pTfv)|{!@u5My|QMYW}s%EW}s%EW}s%EW}s%EX5jyhfnt1tcnV%$7(722 zeqONW=g;A*7sjvKGaug8KWYYQ25JUs25JUs25JUs25JUs25JUs25JUs2L1;bkckim zAO836gLC-*|LXJq_<#^O@E~w6a1XEmTn}6a{PQXy-UHqRUH~#+1e^pO2UdZPuN2}V z;0@q4;4Clz`oNvQKHwtYrz?c`4)_-M1b7QL1v~((06$(X#5ce*zyLS~><7NvFU0%6 z-+&i@6F?LAEpPz13izQR#OJ_A!0W(Ez>C21!0&*6T?YSvSAb`MBfvGlKH$?!h4=^X z3GjE|V_+ND0w%x`a6RzZB|`iO7y^F){(Z3!UjuIfe*}&Lhk*A};vz^4`;||aG_b|FC(-Mcs%Pd)@^%BnIIpt|WD`eCP7Q--MQW>|Wl_Nzuaa;AZ zu?3#8Wm;Ho)tjZl$asU#=0@ASp)T54K5?_u=mh3cX$u+0%GwB%o~~Zt%hNL%D#P2| z;SR%_!sD=dMk1^fb{UQorI}+U39bpEM&nRN4ip`wMeGzc$0m=P@wn4n;$!yM5uB7M zf_;@*MTtrkg6l}d*rSRvr4o=s`16{0!*RhDS4j6gF=G|$zJ@&fzgi@EPT`ZLFP}@n)8G+I-b$paxI32~|yP>bTgo z4Pvva!ut9(Hfpm|JCz(et{iIaJKjO2zw9e2+=V%&*mb}ar1GAwZZ z&uab3tPW!%Q^~=OWz1d7H>pfb+-lKqaqEQ^MG=y~(n+cYI9$HYyIp$oxuD@OdS7Dk z5WX)*i%V_6!F3w5j&n{N!jxvbh{7c;+%$Q|7Tuj^H{rHsq)#Jd z4Ak$Mg1eJICdJU?6DJinxy+tTB=VT&pbvx_0+s`GpoH?W5KvU8(;N^khrR|v?Cy$* zMUEeQbm04GbSBct;V3$?(-wHQdJ2`21r#lIx?FXm9)E|4(fAOu2aoj9JR9(9Z#U}M z?a_LkVgzw*QNi(J*PLXXVtiodWhsItqs3l%urj!`!08DK(@e0G`-@qnIV+rC&$&#_ ziSU=}gmVI?;uFmF>{95bd(Js4&3h(XY^NTxyK;8qN*~oH0~c~=C8LpolD6@Y>{zr$ zTWV`eYpY;-mv*9{?i+WF@)oYt+R|xH;oY3zP$*bJs1t+g3KGohwSFr$^yIRE5z@wG z3>cQ#HhiokysHXjbmEvZykW~v7{jI)I(zJ#MOx~y0 z4eAC<22w^ExwfTZ+@`c0xqiUBYTO$t83Y#{voh@Ks}b+RPr@H8<size:-22!res; / check if we need more space to store this item [now:cp[]; @@ -48,5 +48,48 @@ add:{[function;id;status] / Return the result res}; +// Drop some ids from the cache +drop:{[ids] + ids,:(); + delete from `cache where id in ids; + `results : ids _ `results; + } + +// evict some items from the cache - need to clear enough space for the new item +// evict the least recently accessed items which make up the total size +// feel free to write a more intelligent cache eviction policy ! +evict:{[reqsize;currenttime] + r:select from + (update totalsize:sums size from `lastaccess xasc select lastaccess,id,size from cache) + where prev[totalsize] (now:.proc.cp[]) - r`lastrun; + // update the cache stats, return the cached result + [update lastaccess:now from `.cache.cache where id=r`id; + trackperf[r`id;`hit;now]; + first results[r`id]]; + // value found, but too old - re-run it under the same id + [drop[r`id]; + add[func;r`id;`rerun]]]]; + // it's not in the cache, so add it + add[func;getid[];`add]]} + +// get the cache performance +getperf:{update function:.cache.funcs[id] from .cache.perf} +