Skip to content

Add mockDataGenerator Module #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
NikhilBiradar2000 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add mockDataGenerator Module #75

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions di/mockDataGenerator/mockDataGen.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
initschema:{[]
trades::([] time:`timestamp$(); sym:`g#`$(); src:`g#`$(); price:`float$(); size:`int$());
quotes::([] time:`timestamp$(); sym:`g#`$(); src:`g#`$(); bid:`float$(); ask:`float$(); bsize:`int$(); asize:`int$());
depth::([] time:`timestamp$(); sym:`g#`$(); bid1:`float$(); bsize1:`int$(); bid2:`float$(); bsize2:`int$(); bid3:`float$(); bsize3:`int$(); bid4:`float$(); bsize4:`int$(); bid5:`float$(); bsize5:`int$(); ask1:`float$(); asize1:`int$(); ask2:`float$(); asize2:`int$(); ask3:`float$(); asize3:`int$(); ask4:`float$(); asize4:`int$(); ask5:`float$(); asize5:`int$());
};

// Utility Functions
rnd:{0.01*floor 100*x};

/ Generate realistic trades, quotes, and level-2 depth data for a single symbol/instrument
mockDataOne:{[sym;date;startTime;endTime;tradeCnts;startPxs;level]
tradeCnt:$[99h=type tradeCnts; tradeCnts[sym]; tradeCnts];
startPx:$[99h=type startPxs; startPxs[sym]; startPxs];
quoteCnt:5*tradeCnt;
depthCnt:25*tradeCnt;
hoursinday:endTime-startTime;
t0:date+startTime;
t1:date+endTime;
ttimes:date+ `#asc startTime+tradeCnt?hoursinday;
qtimes:date+ `#asc startTime+quoteCnt?hoursinday;
dtimes:date+ `#asc startTime+depthCnt?hoursinday;
/mids:startPx+sums 0.002*startPx*((quoteCnt?1f)-0.5);
mids:startPx* exp sums 0.0005*-1+quoteCnt?2f;
mids:0.01*floor 100*mids;
spread:0.02+0.01*quoteCnt?1f;
half:spread%2;
bid:rnd mids-quoteCnt?0.03;
ask:rnd mids+quoteCnt?0.03;
bsize:`int$(600*1+quoteCnt?20);
asize:`int$(600*1+quoteCnt?20);
tradeIdx:til tradeCnt;
quoteIdx:5*tradeIdx;
side:tradeCnt?`buy`sell;
price:0.01*floor 100*?[side=`buy; ask[quoteIdx]; bid[quoteIdx]];
tsize:`int$((tradeCnt?1f)*?[side=`buy; asize[quoteIdx]; bsize[quoteIdx]]);
if[0=count tables[];initschema[]];
trades,:flip `time`sym`src`price`size!(ttimes;tradeCnt#sym;tradeCnt?`N`O`L;price;tsize);
quotes,:flip `time`sym`src`bid`ask`bsize`asize!(qtimes;quoteCnt#sym;quoteCnt?`N`O`L;bid;ask;bsize;asize);
if[level=2;
dIdx:(til depthCnt) mod quoteCnt;
dBid:bid[dIdx];dAsk:ask[dIdx];
b1:`int$(600*1+depthCnt?20);b2:b1+`int$(600*1+depthCnt?5);b3:b1+`int$(600*1+depthCnt?10);b4:b1+`int$(600*1+depthCnt?15);b5:b1+`int$(600*1+depthCnt?20);
a1:`int$(600*1+depthCnt?20);a2:a1+`int$(600*1+depthCnt?5);a3:a1+`int$(600*1+depthCnt?5);a4:a1+`int$(600*1+depthCnt?5);a5:a1+`int$(600*1+depthCnt?5);
depth,:flip `time`sym`bid1`bsize1`bid2`bsize2`bid3`bsize3`bid4`bsize4`bid5`bsize5`ask1`asize1`ask2`asize2`ask3`asize3`ask4`asize4`ask5`asize5!(dtimes;depthCnt#sym;dBid;b1;dBid-0.01;b2;dBid-0.02;b3;dBid-0.03;b4;dBid-0.04;b5;dAsk;a1;dAsk+0.01;a2;dAsk+0.02;a3;dAsk+0.03;a4;dAsk+0.04;a5);
];
};

/ Wrapper function to generate the realistic data for multiple syms
mockData:{[syms;date;startTime;endTime;rowCnts;startPxs;level]
syms:$[11h=type syms; syms; enlist syms];
rc:$[99h=type rowCnts; rowCnts; (enlist syms)!enlist rowCnts];
sp:$[99h=type startPxs; startPxs; (enlist syms)!enlist startPxs];
{[s;rc;sp;date;startTime;endTime;level]
mockDataOne[s;date;startTime;endTime;rc[s];sp[s];level]}[;rc;sp;date;startTime;endTime;level] each syms;
};

/to generate mock data for multiple syms in a given date range
mockDataR:{[syms;datelist;startTime;endTime;rowCnts;startPxs;level]
mockData[syms;datelist[0];startTime;endTime;rowCnts;startPxs;level];
sp::exec last price by sym from trades;
{[syms;x;startTime;endTime;rowCnts;sp;level]
sp::exec last price by sym from trades;
mockData[syms;x;startTime;endTime;rowCnts;sp;2]}[syms;;startTime;endTime;rowCnts;sp;2]each 1_datelist;
};

/to write the data down to HDB
mockHdb:{[dir;syms;dates;startTime;endTime;rowCnts;startPxs;level]
{[dir;syms;d;startTime;endTime;rowCnts;startPxs;level]
mockData[syms;d;startTime;endTime;rowCnts;startPxs;level];
.Q.hdpf[`:;dir;d;`sym]}[dir;syms;;startTime;endTime;rowCnts;startPxs;level] each dates;
};



-1"USAGE: mockDataOne[sym;date;startTime;endTime;tradeCnts;startPxs;level] eg mockDataOne[`AAPL;2025.01.10;09:30;16:00;300;22.35;2] / tradeCnts is equal to rowCounts\n\nmockData[syms;date;startTime;endTime;rowCnts;startPxs;level] eg mockData[`AAPL`MSFT`META;2025.01.10;09:30:00;16:00:00;`AAPL`MSFT`META!300 500 200;`AAPL`MSFT`META!22.33 38.34 29.43;2];\n\nmakeDataR[syms;datelist;startTime;endTime;rowCnts;startPxs;level] eg mockDataR[`AAPL`MSFT`META;2025.01.10 2025.01.11 2025.01.12;09:30:00;16:00:00;`AAPL`MSFT`META!7000 5000 9000;`AAPL`MSFT`META!27.33 38.34 29.43;2];\n\nmockHdb[dir;syms;dates;startTime;endTime;rowCnts;startPxs;level] eg mockHdb[`:hdb;`AAPL`MSFT;2014.12.15 2014.12.16 2014.12.17 2014.12.18 2025.12.19; 09:30:00;17:30:00;`AAPL`MSFT!20000 300000;`AAPL`MSFT!23.45 45.89;2]\n";