From 7cf5f643f2217e2e5efac0086e75204b1a8d351c Mon Sep 17 00:00:00 2001 From: Benedictus Alvian Sofjan Date: Fri, 11 Sep 2026 07:11:07 +0100 Subject: [PATCH 1/3] Anycache --- code/common/anymaputils.q | 5 ++ code/common/getter.q | 28 ++++++++++ code/processes/anycachereadme.md | 1 + code/processes/gateway.q | 3 ++ code/processes/setter.q | 92 ++++++++++++++++++++++++++++++++ config/cacheconfig.json | 23 ++++++++ 6 files changed, 152 insertions(+) create mode 100644 code/common/anymaputils.q create mode 100644 code/common/getter.q create mode 100644 code/processes/anycachereadme.md create mode 100644 code/processes/setter.q create mode 100644 config/cacheconfig.json diff --git a/code/common/anymaputils.q b/code/common/anymaputils.q new file mode 100644 index 000000000..ee7e88189 --- /dev/null +++ b/code/common/anymaputils.q @@ -0,0 +1,5 @@ +.anymap.writetoanymap:{[filepath;data] (hsym filepath) 1: data; :(::)}; +.anymap.deriveanymapfiles:{[filepath] `$string[filepath],/:("";"#";"##")}; +.anymap.util.copyanymap:{[fromfilepath; destfilepath] fromfilepaths:.anymap.deriveanymapfiles[fromfilepath]; .os.cpy[;destfilepath] each fromfilepaths}; +.anymap.util.removeanymap:{[filepath] filepaths:.anymap.deriveanymapfiles[filepath]; .os.del each filepaths}; +.anymap.util.moveanymap:{[fromfilepath; destfilepath] fromfilepaths:.anymap.deriveanymapfiles[fromfilepath]; .os.ren[;destfilepath] each fromfilepaths}; \ No newline at end of file diff --git a/code/common/getter.q b/code/common/getter.q new file mode 100644 index 000000000..152b516e8 --- /dev/null +++ b/code/common/getter.q @@ -0,0 +1,28 @@ +// Bootstrap script that will enable an existing process to load in cache getter functionalities. + +\d .anycache.getter + +// Get cache from disk +getcachefromdisk:{[filepath] get hsym filepath}; + +// Get location of cache config and load it in. +cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"]; +cacheconfig:.j.k raze read0 hsym first cacheconfiglocation; +cachename: cacheconfig`cachename; +asyncprocessname: cacheconfig`asyncprocessname; + +loadcaches:{ + caches:key cacheconfig`componentcaches; + cachefilepaths: ` sv' ((hsym `$cacheconfig`cacherootdir),2#`$cachename),/:`$(string caches),\:"/data"; + cachesdata:getcachefromdisk each cachefilepaths; + cachevarnames:` sv' `.anycache.cache,/:caches; + cachevarnames set' cachesdata; + }; + +// Example of args: `cache1`cache2!(`a`b`c! 1 2 3;`d`e`f!4 5 6) +requestnewcache:{[args] + maincache:` sv (hsym `$cacheconfig`cacherootdir),(`$cachename),`$asyncprocessname, "_", string .z.P; + (` sv maincache,`args) set args + }; + +\d . \ No newline at end of file diff --git a/code/processes/anycachereadme.md b/code/processes/anycachereadme.md new file mode 100644 index 000000000..7700bbf32 --- /dev/null +++ b/code/processes/anycachereadme.md @@ -0,0 +1 @@ +AnyCache README \ No newline at end of file diff --git a/code/processes/gateway.q b/code/processes/gateway.q index 32d076bc7..6dbb5c27f 100644 --- a/code/processes/gateway.q +++ b/code/processes/gateway.q @@ -592,6 +592,9 @@ if[@[value;`.timer.enabled;0b]; .timer.repeat[.proc.cp[];0Wp;0D00:00:05;(`.gw.checktimeout;`);"Timeout queries which have been waiting too long"]; .timer.repeat[.proc.cp[];0Wp;0D00:05;(`.gw.removeinactive;.gw.clearinactivetime);"Remove data for inactive handles"]]; +//AnyCache sample analytic +.anycache.sampleanalytic:{ .gw.syncexec["select by sym from trade";`rdb] }; + // add in some api details .api.add[`.gw.asyncexecjpt;1b;"Execute a function asynchronously. The result is posted back to the client either directly down the socket (in which case the client must block and wait for the result - deferred synchronous) or wrapped in the postback function";"[(string | mixed list): the query to execute; symbol(list): the list of servers to query against; lambda: the function used to join the resulting data; symbol or lambda: postback;timespan: query timeout]";"The result of the query either directly or through the postback function"] .api.add[`.gw.asyncexec;1b;"Execute a function asynchronously. The result is posted back to the client directly down the socket. The client must block and wait for the result - deferred synchronous. Equivalent to .gw.asyncexecjpt with join function of raze, no postback and no timeout";"[(string | mixed list): the query to execute; symbol(list): the list of servers to query against]";"The result of the query"] diff --git a/code/processes/setter.q b/code/processes/setter.q new file mode 100644 index 000000000..f057a89c6 --- /dev/null +++ b/code/processes/setter.q @@ -0,0 +1,92 @@ +// Setter process to set cache to disk + +\d .anycache.setter + +// Save cache down to disk +savecachedowntodisk:{[data;filepath] (hsym filepath) set data }; + +// Get location of cache config and load it in. +cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"]; +cacheconfig:.j.k raze read0 hsym first cacheconfiglocation; + +// Mode of cache +main:{[args] + isRequest:: first `$(.Q.opt args)[`isRequest] + }; + +if[count .z.x; main[.z.x]]; + +detectandwritecache:{ + cacheinfo:detectcachetobuild[]; + if[not count cacheinfo; + :(::) + ]; + writetoken[cachepath:cacheinfo`cachepath;`start]; + //Included some basic error trapping here + success:@[`generateandwritecache[cachepath];cacheinfo`args;0b]; + if[not success; + cleanupcache cachepath; + :(::) + ]; + writetoken[cachepath;`end]; + //Can eject now if there are still caches to be built in the main cache + if[not count remainingcaches:cacheinfo`maincachepath; + :(::) + ]; + completecache cacheinfo`maincachepath + }; + + writetoken:{[dir;stage] + //accepts start or end and saves the current time as a timestamp to a flat file + //in dir as `:startTime or `:endTime + (` sv (dir;stage)) set .z.P + }; + +detectcachetobuild:{ + latestcache:{cachename:"MyFirstCache"; + maincachepath:` sv (hsym `$.setter.cacheconfig.cacheRootDir),`$cachename; + caches:` sv' maincachepath,'(key maincachepath) where (key maincachepath) like cachename,"_*"; + if[0 = count caches; :`cachename`newcache!(cachename,"_",string .z.P;1b)]; + cachewithmaxstarttime:starts ? max starts:cands!{get ` sv x,`start} each cands:key[d] where not `end in/: value d:caches!key each caches; + if[(not isRequest) and ("N"$.setter.cacheconfig.setter.interval) < .z.P - "P"$@[last "_" vs string cachewithmaxstarttime;13 16 19;:;"::."];:`cachename`newcache!(cachename,"_",string .z.P;1b)]; + :`cachename`newcache!(cachewithmaxstarttime;0b)}[]; + + if[latestcache[`newcache];writetoken[latestcache[`cachename];`start]]; + if[latestcache[`newcache]; cachewithmaxstarttime:` sv maincachepath,`$latestcache[`cachename]]; + + componentcaches:` sv' cachewithmaxstarttime,/:key .setter.cacheconfig.componentCaches; + incompletecomponentcaches:key[d2] where not `end in/: value d2:componentcaches!key each componentcaches; + writetoken[;`start] each incompletecomponentcaches; + writetoken[;`setter1] each incompletecomponentcaches; + args:enlist`; + if[isRequest; + argpaths:` sv' maincachepath,'(key maincachepath) where (key maincachepath) like "AsyncCache*"; + maxstarttime:string first max "P"$-1#' "_" vs' string argpaths; + argwithmaxstarttime:first argpaths where argpaths like "*",maxstarttime; + args:: get ` sv argwithmaxstarttime,`args; + ] + `maincachepath`cachepath`args!(maincachepath;incompletecomponentcaches;args) + }; + +generateandwritecache:{[cachepath; args] + cachename:last ` vs cachepath; + connectiondetails: .anycache.config.componentCaches[cachename].dataSource; + cache: connectiondetails".anycache.sampleanalytic[(::)]"; + .anymap.writetoanymap[cache;cachepath] + }; + +cleanupcache:{[cachepath] + //Want to just remove the component cache (cacheName) from the main cache directory in event of a failure + hdel cachepath + }; + +completecache:{[maincachepath] + cachename: string last ` vs maincachepath; + latestcache:first system"ls -lt ",(1_string maincachepath), " | grep ", cachename, " | grep -vE '(^l|total)' | head -n 1 | awk '{print $NF}'"; + latestcachefilepath: ` sv maincachepath,`$latestcache; + writetoken[latestcachefilepath;`end]; + if[not isRequest; system"ln -sfn ", latestcache, " ", (1_string maincachepath), "/", cachename]; + if[not isRequest; hdel each ` sv' maincachepath,'(key maincachepath) except (`$cachename;`$latestcache)] + }; + +\d . \ No newline at end of file diff --git a/config/cacheconfig.json b/config/cacheconfig.json new file mode 100644 index 000000000..efb7f1565 --- /dev/null +++ b/config/cacheconfig.json @@ -0,0 +1,23 @@ +{ + "cacherootdir": "opt/anycache", + "cachename": "myfirstcache", + "asyncprocessname": "asynccache", + "componentcaches":{ + "cache1": { + "analytic": "analytic1", + "dataSource": "connection1" + }, + "cache2": { + "analytic": "analytic2", + "dataSource": "connection2" + } + }, + "getter":{ + "interval": "0D00:05" + }, + "setter":{ + "mode": "timer", + "interval": "0D00:30", + "compression": "16 3 0" + } +} \ No newline at end of file From 3152fa2f6ad93791b626c2710a329d4cdc4ed11a Mon Sep 17 00:00:00 2001 From: Benedictus Alvian Sofjan Date: Fri, 11 Sep 2026 08:22:13 +0100 Subject: [PATCH 2/3] Update setter.q --- code/processes/setter.q | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/code/processes/setter.q b/code/processes/setter.q index f057a89c6..f203a8e90 100644 --- a/code/processes/setter.q +++ b/code/processes/setter.q @@ -10,11 +10,7 @@ cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"]; cacheconfig:.j.k raze read0 hsym first cacheconfiglocation; // Mode of cache -main:{[args] - isRequest:: first `$(.Q.opt args)[`isRequest] - }; - -if[count .z.x; main[.z.x]]; +isrequest: "request" ~ cacheconfig.setter.mode; detectandwritecache:{ cacheinfo:detectcachetobuild[]; @@ -42,19 +38,21 @@ detectandwritecache:{ (` sv (dir;stage)) set .z.P }; -detectcachetobuild:{ - latestcache:{cachename:"MyFirstCache"; - maincachepath:` sv (hsym `$.setter.cacheconfig.cacheRootDir),`$cachename; +detectcachetobuild:{ + cachename: cacheconfig`cachename; + maincachepath:` sv (hsym `$cacheconfig.cacheRootDir),`$cachename; caches:` sv' maincachepath,'(key maincachepath) where (key maincachepath) like cachename,"_*"; + if[not 0 = count caches; cachewithmaxstarttime:starts ? max starts:cands!{get ` sv x,`start} each cands:key[d] where not `end in/: value d:caches!key each caches]; + + latestcache:{ if[0 = count caches; :`cachename`newcache!(cachename,"_",string .z.P;1b)]; - cachewithmaxstarttime:starts ? max starts:cands!{get ` sv x,`start} each cands:key[d] where not `end in/: value d:caches!key each caches; - if[(not isRequest) and ("N"$.setter.cacheconfig.setter.interval) < .z.P - "P"$@[last "_" vs string cachewithmaxstarttime;13 16 19;:;"::."];:`cachename`newcache!(cachename,"_",string .z.P;1b)]; + if[(not isRequest) and ("N"$cacheconfig.setter.interval) < .z.P - "P"$@[last "_" vs string cachewithmaxstarttime;13 16 19;:;"::."];:`cachename`newcache!(cachename,"_",string .z.P;1b)]; :`cachename`newcache!(cachewithmaxstarttime;0b)}[]; if[latestcache[`newcache];writetoken[latestcache[`cachename];`start]]; if[latestcache[`newcache]; cachewithmaxstarttime:` sv maincachepath,`$latestcache[`cachename]]; - componentcaches:` sv' cachewithmaxstarttime,/:key .setter.cacheconfig.componentCaches; + componentcaches:` sv' cachewithmaxstarttime,/:key cacheconfig.componentCaches; incompletecomponentcaches:key[d2] where not `end in/: value d2:componentcaches!key each componentcaches; writetoken[;`start] each incompletecomponentcaches; writetoken[;`setter1] each incompletecomponentcaches; @@ -63,16 +61,16 @@ detectcachetobuild:{ argpaths:` sv' maincachepath,'(key maincachepath) where (key maincachepath) like "AsyncCache*"; maxstarttime:string first max "P"$-1#' "_" vs' string argpaths; argwithmaxstarttime:first argpaths where argpaths like "*",maxstarttime; - args:: get ` sv argwithmaxstarttime,`args; - ] + args: get ` sv argwithmaxstarttime,`args + ]; `maincachepath`cachepath`args!(maincachepath;incompletecomponentcaches;args) }; generateandwritecache:{[cachepath; args] cachename:last ` vs cachepath; - connectiondetails: .anycache.config.componentCaches[cachename].dataSource; + connectiondetails: cacheconfig.componentCaches[cachename].dataSource; cache: connectiondetails".anycache.sampleanalytic[(::)]"; - .anymap.writetoanymap[cache;cachepath] + .anymap.writetoanymap[cachepath;cache] }; cleanupcache:{[cachepath] From 6b27c69d50f11687dda591b2fe523044f9945d5c Mon Sep 17 00:00:00 2001 From: Benedictus Alvian Sofjan Date: Wed, 16 Sep 2026 10:12:05 +0100 Subject: [PATCH 3/3] Update setter.q, getter.q, cacheconfig.json --- code/common/getter.q | 24 +++++++-------- code/processes/setter.q | 65 +++++++++++++++++++++-------------------- config/cacheconfig.json | 6 ++-- 3 files changed, 48 insertions(+), 47 deletions(-) diff --git a/code/common/getter.q b/code/common/getter.q index 152b516e8..20dc2b8c1 100644 --- a/code/common/getter.q +++ b/code/common/getter.q @@ -1,27 +1,27 @@ // Bootstrap script that will enable an existing process to load in cache getter functionalities. -\d .anycache.getter +\d .anycache // Get cache from disk -getcachefromdisk:{[filepath] get hsym filepath}; +getter.getcachefromdisk:{[filepath] get hsym filepath}; // Get location of cache config and load it in. -cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"]; -cacheconfig:.j.k raze read0 hsym first cacheconfiglocation; -cachename: cacheconfig`cachename; -asyncprocessname: cacheconfig`asyncprocessname; +getter.cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"]; +getter.cacheconfig:.j.k raze read0 hsym first getter.cacheconfiglocation; +getter.cachename: getter.cacheconfig`cachename; +getter.asyncprocessname: getter.cacheconfig`asyncprocessname; -loadcaches:{ - caches:key cacheconfig`componentcaches; - cachefilepaths: ` sv' ((hsym `$cacheconfig`cacherootdir),2#`$cachename),/:`$(string caches),\:"/data"; - cachesdata:getcachefromdisk each cachefilepaths; +getter.loadcaches:{ + caches:key getter.cacheconfig`componentcaches; + cachefilepaths: ` sv' ((hsym `$getter.cacheconfig`cacherootdir),2#`$getter.cachename),/:`$(string caches),\:"/data"; + cachesdata:getter.getcachefromdisk each cachefilepaths; cachevarnames:` sv' `.anycache.cache,/:caches; cachevarnames set' cachesdata; }; // Example of args: `cache1`cache2!(`a`b`c! 1 2 3;`d`e`f!4 5 6) -requestnewcache:{[args] - maincache:` sv (hsym `$cacheconfig`cacherootdir),(`$cachename),`$asyncprocessname, "_", string .z.P; +getter.requestnewcache:{[args] + maincache:` sv (hsym `$getter.cacheconfig`cacherootdir),(`$getter.cachename),`$getter.asyncprocessname, "_", string .z.P; (` sv maincache,`args) set args }; diff --git a/code/processes/setter.q b/code/processes/setter.q index f203a8e90..384139ead 100644 --- a/code/processes/setter.q +++ b/code/processes/setter.q @@ -1,64 +1,65 @@ // Setter process to set cache to disk -\d .anycache.setter +\d .anycache // Save cache down to disk -savecachedowntodisk:{[data;filepath] (hsym filepath) set data }; +setter.savecachedowntodisk:{[data;filepath] (hsym filepath) set data }; // Get location of cache config and load it in. -cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"]; -cacheconfig:.j.k raze read0 hsym first cacheconfiglocation; +setter.cacheconfiglocation:.proc.getconfigfile["cacheconfig.json"]; +setter.cacheconfig:.j.k raze read0 hsym first setter.cacheconfiglocation; // Mode of cache -isrequest: "request" ~ cacheconfig.setter.mode; +setter.isrequest: "request" ~ setter.cacheconfig.setter.mode; -detectandwritecache:{ - cacheinfo:detectcachetobuild[]; +setter.detectandwritecache:{ + cacheinfo:setter.detectcachetobuild[]; if[not count cacheinfo; :(::) ]; - writetoken[cachepath:cacheinfo`cachepath;`start]; + setter.writetoken[cachepath:cacheinfo`cachepath;`start]; //Included some basic error trapping here - success:@[`generateandwritecache[cachepath];cacheinfo`args;0b]; + success:@[setter.generateandwritecache[cachepath;];cacheinfo`args;0b]; if[not success; - cleanupcache cachepath; + setter.cleanupcache cachepath; :(::) ]; - writetoken[cachepath;`end]; + setter.writetoken[cachepath;`end]; //Can eject now if there are still caches to be built in the main cache if[not count remainingcaches:cacheinfo`maincachepath; :(::) ]; - completecache cacheinfo`maincachepath + setter.completecache cacheinfo`maincachepath }; - writetoken:{[dir;stage] +setter.writetoken:{[dir;stage] //accepts start or end and saves the current time as a timestamp to a flat file //in dir as `:startTime or `:endTime (` sv (dir;stage)) set .z.P }; -detectcachetobuild:{ - cachename: cacheconfig`cachename; - maincachepath:` sv (hsym `$cacheconfig.cacheRootDir),`$cachename; +setter.detectcachetobuild:{ + cachename: setter.cacheconfig`cachename; + asyncprocessname: setter.cacheconfig`asyncprocessname; + maincachepath:` sv (hsym `$setter.cacheconfig.cacherootdir),`$cachename; caches:` sv' maincachepath,'(key maincachepath) where (key maincachepath) like cachename,"_*"; - if[not 0 = count caches; cachewithmaxstarttime:starts ? max starts:cands!{get ` sv x,`start} each cands:key[d] where not `end in/: value d:caches!key each caches]; + if[not 0 = count caches; cachewithmaxstarttime:starts ? max starts:cands!{get ` sv x,`start} each cands:key[d1] where not `end in/: value d1:caches!key each caches]; latestcache:{ if[0 = count caches; :`cachename`newcache!(cachename,"_",string .z.P;1b)]; - if[(not isRequest) and ("N"$cacheconfig.setter.interval) < .z.P - "P"$@[last "_" vs string cachewithmaxstarttime;13 16 19;:;"::."];:`cachename`newcache!(cachename,"_",string .z.P;1b)]; + if[(not setter.isrequest) and ("N"$setter.cacheconfig.setter.interval) < .z.P - "P"$@[last "_" vs string cachewithmaxstarttime;13 16 19;:;"::."];:`cachename`newcache!(cachename,"_",string .z.P;1b)]; :`cachename`newcache!(cachewithmaxstarttime;0b)}[]; - if[latestcache[`newcache];writetoken[latestcache[`cachename];`start]]; + if[latestcache[`newcache]; setter.writetoken[latestcache[`cachename];`start]]; if[latestcache[`newcache]; cachewithmaxstarttime:` sv maincachepath,`$latestcache[`cachename]]; - componentcaches:` sv' cachewithmaxstarttime,/:key cacheconfig.componentCaches; + componentcaches:` sv' cachewithmaxstarttime,/:key setter.cacheconfig.componentcaches; incompletecomponentcaches:key[d2] where not `end in/: value d2:componentcaches!key each componentcaches; - writetoken[;`start] each incompletecomponentcaches; - writetoken[;`setter1] each incompletecomponentcaches; + setter.writetoken[;`start] each incompletecomponentcaches; + setter.writetoken[;`setter1] each incompletecomponentcaches; args:enlist`; - if[isRequest; - argpaths:` sv' maincachepath,'(key maincachepath) where (key maincachepath) like "AsyncCache*"; + if[setter.isrequest; + argpaths:` sv' maincachepath,'(key maincachepath) where (key maincachepath) like asyncprocessname,"*"; maxstarttime:string first max "P"$-1#' "_" vs' string argpaths; argwithmaxstarttime:first argpaths where argpaths like "*",maxstarttime; args: get ` sv argwithmaxstarttime,`args @@ -66,25 +67,25 @@ detectcachetobuild:{ `maincachepath`cachepath`args!(maincachepath;incompletecomponentcaches;args) }; -generateandwritecache:{[cachepath; args] +setter.generateandwritecache:{[cachepath; args] cachename:last ` vs cachepath; - connectiondetails: cacheconfig.componentCaches[cachename].dataSource; + connectiondetails: setter.cacheconfig.componentcaches[cachename].datasource; cache: connectiondetails".anycache.sampleanalytic[(::)]"; .anymap.writetoanymap[cachepath;cache] }; -cleanupcache:{[cachepath] - //Want to just remove the component cache (cacheName) from the main cache directory in event of a failure +setter.cleanupcache:{[cachepath] + //Want to just remove the component cache (cachename) from the main cache directory in event of a failure hdel cachepath }; -completecache:{[maincachepath] +setter.completecache:{[maincachepath] cachename: string last ` vs maincachepath; latestcache:first system"ls -lt ",(1_string maincachepath), " | grep ", cachename, " | grep -vE '(^l|total)' | head -n 1 | awk '{print $NF}'"; latestcachefilepath: ` sv maincachepath,`$latestcache; - writetoken[latestcachefilepath;`end]; - if[not isRequest; system"ln -sfn ", latestcache, " ", (1_string maincachepath), "/", cachename]; - if[not isRequest; hdel each ` sv' maincachepath,'(key maincachepath) except (`$cachename;`$latestcache)] + setter.writetoken[latestcachefilepath;`end]; + if[not setter.isrequest; system"ln -sfn ", latestcache, " ", (1_string maincachepath), "/", cachename]; + if[not setter.isrequest; hdel each ` sv' maincachepath,'(key maincachepath) except (`$cachename;`$latestcache)] }; \d . \ No newline at end of file diff --git a/config/cacheconfig.json b/config/cacheconfig.json index efb7f1565..8d445e8cc 100644 --- a/config/cacheconfig.json +++ b/config/cacheconfig.json @@ -1,15 +1,15 @@ { - "cacherootdir": "opt/anycache", + "cacherootdir": "/home/user/torq/opt/anycache", "cachename": "myfirstcache", "asyncprocessname": "asynccache", "componentcaches":{ "cache1": { "analytic": "analytic1", - "dataSource": "connection1" + "datasource": "connection1" }, "cache2": { "analytic": "analytic2", - "dataSource": "connection2" + "datasource": "connection2" } }, "getter":{