diff --git a/di/datadog/datadog.md b/di/datadog/datadog.md new file mode 100644 index 0000000..9a2f065 --- /dev/null +++ b/di/datadog/datadog.md @@ -0,0 +1,133 @@ +# `datadog.q` – Metric and event publishing to DataDog for kdb+ + +A library used to publish metrics and events to the DataDog application through DataDog agents or https, dynamically adapting the delivery +mechanism depending on host operating system. + +> **Note:** Using the functions `sendmetric` and `sendevent` on a Windows OS relies on Poweshell being installed. +> If Powershell is not installed please initialise the package using `init[1b]` to send data via https. +> +> **Note:** To send metrics and events to DataDog via https you must either have TLS certificates set up on your machine or set the environment variable `SSL_VERIFY_SERVER=NO`. + +--- + +## :sparkles: Features + +- Send custom metrics and events to DataDog platform. +- Allows posts to be pushed via DataDog agent or https +- Log all posts and delivery status to in memory tables. + +--- + +## :gear: Configuration + +Config variables used to connect to DataDog and change the mode of delivery can be set **before initialising** the package: + +```q +agentport : 8125 // (int) Port that the DataDog agent is listening on, should be passed in through the environment variable "DOGSTATSD_PORT". The default DataDog agent port is 8125. +apikey : "your api key" // (str) API key used to connect with your DataDog account, should be passed in through the environment variable "DOGSTATSD_APIKEY". +baseurl : "DataDog web address" // (str) Web address to base DataDog api. (default: ":https://.api.datadoghq.eu/api/v1"). +``` + +--- + +## :memo: Initialisation + +The package is initialised by calling the monadic function `init` with a boolean argument, `1b: use https delivery; 0b: use DataDog agent`. +The init function will then set the appropriate variables and call `setfunctions` in order to define the `sendmetric` and `sendevent` functions. +--- + +## :wrench: Functions + + +### :rocket: Data Delivery Functions + +Primary functions used to push data to DataDog. These are the only functions required to send data as they are overridden depending on os/https. + +| Function | Params | Description | +|------------------|---------------------------------------------------------------------------------------------|----------------------------------| +| `sendmetric` | (`metricname`: string; `metricvalue`: float; `tags`:string) | Primary metric delivery function | +| `sendevent` | (`eventtitle`: string; `eventtext`: string; `priority`: string; `tags`: string; `alerttype`: string ) | Primary event delivery function | + +#### :mag_right:Parameters in depth + +`sendmetric` +```q +metricname : "string" // The name of the timeseries. +metricvalue : "short/real/int/long/float" // Point relating to a metric. A scalar value (cannot be a string). +tags : "string" // A list of tags associated with the metric. +``` +`sendevent` +```q +eventtitle : "string" // The event title. +eventtext : "string" // The body of the event. Limited to 4000 characters. The text supports markdown. To use markdown in the event text, start the text block with %%% \n and end the text block with \n %%%. +priority : "string" // The priority of the event. For example, normal or low. Allowed values: normal,low. +tags : "string" // A list of tags associated with the metric. +alerttype : "string" // Allowed values: error,warning,info,success,user_update,recommendation,snapshot. +``` + +--- + +## :label: Log Tables Schema + +The metric Log is used to record all metrics delivered to the DataDog application. It allows the user to determine if packages are being delivered successfully, +analyse the package sent along with the metric names and values and determine if a package was delivered via the DataDog agent or via https. +The log can be retrived using `getmerticlog` and includes the following columns: + +| Column | Type | Description | +|-------------|-------------|-------------------------------------------| +| time | `timestamp` | Time of the event | +| host | `symbol` | host of request origin | +| message | `char` | package delivered | +| metricname | `char` | metric name | +| metricvalue | `float` | metric value | +| https | `boolean` | 1b if https was used, 0b if DataDog agent | +| status | `char` | Repsonse from DataDog confirming delivery | + +The event Log is used to record all events delivered to the DataDog application. It allows the user to determine if packages are being delivered successfully, +analyse the package sent along with the event title and text and determine if a package was delivered via the DataDog agent or via https. +The log can be retrived using `geteventlog` includes the following columns: + +| Column | Type | Description | +|------------|-------------|-------------------------------------------| +| time | `timestamp` | Time of the event | +| host | `symbol` | host of request origin | +| message | `char` | package delivered | +| eventtitle | `char` | Name for event | +| eventtext | `char` | Message sent with event | +| https | `boolean` | 1b if https was used, 0b if DataDog agent | +| status | `char` | Repsonse from DataDog confirming delivery | + + +--- + +## :test_tube: Example +Set your environment variables. +agentport = 8125 +apikey = "yourapikey" +baseurl = ":https://api.datadoghq.eu/api/v1/" + +```q +// Import datadog package into a session +datadog:use`di.datadog + +// Initialise the package and send data via https +datadog.init[1b] + +datadog.sendmetric["custom.metric";123;"shell"]; +datadog.sendevent["Test_Event";"This is a test";"normal";"test";"info"] + +// Check log tables for delivery success +select from datadog.getmetriclog[]; + +time host message name metric https status +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +2025.07.16D08:53:51.158486300 hostname "{\"series\":[{\"metric\":\"custom.metric\",\"points\":[[1752656031,123]],\"host\":\"hotname\",\"tags\":\"shell\"}]}" "custom.metric" 123 1 "{\"status\": \"ok\"}" + + +select from datadog.geteventlog[]; + +time host message title text https status .. +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------.. +2025.07.16D08:54:36.543890200 hostname "{\"title\":\"Test_Event\",\"text\":\"This is a test\",\"priority\":\"normal\",\"tags\":\"test\",\"alert_type\":\"info\"}" "Test_Event" "This is a test" 1 "{\"status\":\"ok\", + +``` \ No newline at end of file diff --git a/di/datadog/datadog.q b/di/datadog/datadog.q new file mode 100644 index 0000000..e14c23f --- /dev/null +++ b/di/datadog/datadog.q @@ -0,0 +1,192 @@ +/ -----datadog.q ----- +/ package used to push metrics and events from a q process to datadog. +/ default delivery method is through the datadog agent installed on the host. use useweb "1b" to switch delivery to https in the init function + +/ define tables to capture events and metrics +metriclog:([]time:`timestamp$();host:`$();message:();name:();metric:`float$();https:`boolean$();status:()); +eventlog:([]time:`timestamp$();host:`$();message:();title:();text:();https:`boolean$();status:()); +/ pre-define operating system to help with testing +opsys:.z.o; + +/ Filter for sending events +eventfilter:{[dict] + if[not ((`$"alert")=`$dict`category) or (`$"change")=`$dict`category; '"category only change or alert"]; + + requiredpars:`eventtitle`eventtext; + optionalpars: `eventdate`hostname`priority`alerttype`tags; + + $[useweb;$[baseurlversion=`v1; + (optionalpars,:`aggregation_key`source_type_name`related_event_id`device); + baseurlversion=`v2;$[(`$dict`category)=`$"alert"; + (requiredpars,:`category`eventstatus;optionalpars:`aggregation_key`integration_id`message`tags`timestamp`priority`custom; + if[any (string key dict) like "*link*";(requiredpars,:`linkcategory`linkurl;optionalpars,:`linktitle)]); + (`$dict`category)=`$"change";(requiredpars,:`category`cr_name`cr_type;optionalpars:`aggregation_key`integration_id`message`tags`timestamp`change_metadata`new_value`prev_value; + if[any (string key dict) like "*author*"; (requiredpars,:`authorname`authortype)]; + if[any (string key dict) like "*impacted_resource*"; (requiredpars,:`impacted_resource_name`impacted_resource_type)])] + ]; + (requiredpars:`eventtitle`eventtext; + optionalpars: `eventdate`hostname`priority`alerttype`tags) + ]; + + validpars: requiredpars,optionalpars; + / Checks + if[not 99h=type dict; '"input must be a dictionary"]; + pars: key dict; + if[(count validpars) /dev/udp/127.0.0.1/%s\"";ddmsg;string agentport); + response:system cmd; + eventlog,:(.z.p;.z.h;cmd;eventtitle;eventtext;0b;response); + }; + +lin.sendmetric:{[(pars!args):metfilter] + leaders:([metricname:"";metricvalue:":";metrictype:"|";samplerate:"|@";tags:"|#"]); + (metricname;metricvalue):args 0 1; + ddmsg:raze (leaders[pars]),'(args); + cmd:printf("bash -c \"echo -n '%s' > /dev/udp/127.0.0.1/%s\"";ddmsg;string agentport); + response:system cmd; + metriclog,:(.z.p;.z.h;cmd;metricname;"F"$metricvalue;0b;response) + }; + +/ following three functions are used to push metrics and events to datadog through udp and powershell on windows os +/ windows os unable to be tested currently so following three functions have not been unit tested. + +pushtodogagent:{[message] + / shell command to push data + cmd:"powershell -Command \""; + cmd,:" $udpClient = New-Object System.Net.Sockets.UdpClient;"; + cmd,:" $udpClient.Connect('127.0.0.1','",string[agentport], "');"; + cmd,:" $bytes = [System.Text.Encoding]::ASCII.GetBytes('",raze message, "');"; + cmd,:" $udpClient.Send($bytes, $bytes.Length );"; + cmd,:" $udpClient.Close();\""; + response:system cmd; + response + }; + +win.sendmetric:{[(pars!args):metfilter] + leaders:([metricname:"";metricvalue:":";metrictype:"|";samplerate:"|@";tags:"|#"]); + (metricname;metricvalue):args 0 1; + ddmsg:raze (leaders[pars]),'(args); + response:raze@[pushtodogagent;ddmsg;{'"Error pushing data to agent: ",x}]; + metriclog,:(.z.p;.z.h;ddmsg;metricname;`float$metricvalue;0b;response); + }; + +win.sendevent:{[(pars!args):eventfilter] + (eventtitle;eventtext):args 0 1; + leaders:([eventtitle:printf("_e{%d,%d}:";count eventtitle;count eventtext); + eventtext:"|";eventdate:"|d:";hostname:"|h:";priority:"|p:";alerttype:"|t:";tags:"|#"]); + ddmsg:raze (leaders[pars]),'(args); + response:raze@[pushtodogagent;ddmsg;{'"Error pushing data to agent: ",x}]; + eventlog,:(.z.p;.z.h;ddmsg;eventtitle;eventtext;0b;response); + }; + +/ the following two functions are used to push data to datadog through https post using .Q.hp + +/ TODO: Handle variety of web requests (different post requests, different versions) +web.sendevent:{[dict:eventfilter] + (eventtitle;eventtext;priority;tags;alerttype):dict[`eventtitle`eventtext`priority`tags`alerttype]; + if[first baseurlversion=`v2; + (eventtitle;eventtext;priority;tags;alerttype;category):dict[`eventtitle`eventtext`priority`tags`alerttype`category] + ]; + / sends events via https post to datadog api + url:baseurl,"events?api_key=",apikey; + json:.j.j `title`text`priority`tags`alert_type!(eventtitle;eventtext;priority;tags;alerttype); + if[first baseurlversion=`v2; + json:.j.j `title`text`priority`tags`alert_type`category!(eventtitle;eventtext;priority;tags;alerttype;category) + ]; + response:.[.Q.hp;(url;.h.ty`json;json);{'"error with https request: ",x}]; + eventlog,:(.z.p;.z.h;json;eventtitle;eventtext;1b; response); + }; + +web.sendmetric:{[dict:metfilter] + (metricname;metricvalue;metrictype;samplerate;tags):dict[`metricname`metricvalue`metrictype`samplerate`tags]; + / sends metrics via https post to datadog api + url:baseurl,"series?api_key=",apikey; + unixtime:floor((`long$.z.p)-1970.01.01D00:00)%1e9; + json:.j.j(enlist`series)!enlist(enlist(`metric`points`host`tags!(metricname;enlist(unixtime;"F"$metricvalue);upper string .z.h;tags))); + response:.[.Q.hp;(url;.h.ty`json;json);{'"error with https request: ",x}]; + metriclog,:(.z.p;.z.h;json;metricname;"F"$metricvalue;1b;response); + }; + +/ utility functions used to manage send functions + +setfunctions:{[useweb] + / determine if web request is used or datadog agent, then assign appropriate functions + if[null method:$[useweb;`web;("lw"!`lin`win)first string opsys]; + '"Currently only linux and windows operating systems are supported to send metrics and events. Please use ""setfunctions 1b"" to attempt a web request"]; + .z.m.sendmetric:value ` sv .z.M,method,`sendmetric; + .z.m.sendevent:value ` sv .z.M,method,`sendevent; + }; + +init:{[configs] + / Default values + envcheck: {$[count x; x; y]}; + / define datadog agent port + .z.m.agentport:"J"$envcheck[getenv`DOGSTATSD_PORT;string 8125]; + / define datadog api key - default value is empty string, so no need to check + .z.m.apikey:getenv`DOGSTATSD_APIKEY; + / define base api url + .z.m.baseurl:envcheck[getenv `DOGSTATSD_URL;":https://api.datadoghq.eu/api/v1/"]; + / default - don't use web + .z.m.useweb:0b; + + / Values from config dictionary take priority + if[not (configs~(::)) or ((0=count configs) and 99h~type configs); + vars:`agentport`apikey`baseurl`useweb inter key configs; + (.Q.dd[.z.M] each key[vars#configs]) set' value[vars#configs] + ]; + + .z.m.baseurlversion:`$l where (l:"/" vs baseurl) like "v*"; + + / initialisation function + if[not`printf in key .z.m;([.z.m.printf]):@[use;`kx.printf;{'"printf module not found, please install"}]]; + / sets delivery method + setfunctions useweb; + }; \ No newline at end of file diff --git a/di/datadog/init.q b/di/datadog/init.q new file mode 100644 index 0000000..5f94ce7 --- /dev/null +++ b/di/datadog/init.q @@ -0,0 +1,10 @@ +\l ::datadog.q + +/ export functions to be accessible outside private +export:([ + init:init; + getmetriclog:{.z.m.metriclog}; / metric table + geteventlog:{.z.m.eventlog}; / event table + sendmetric:{[dict].z.m.sendmetric[dict]}; + sendevent:{[dict].z.m.sendevent[dict]} + ]) diff --git a/di/datadog/test.csv b/di/datadog/test.csv new file mode 100644 index 0000000..fd0dc20 --- /dev/null +++ b/di/datadog/test.csv @@ -0,0 +1,48 @@ +action,ms,bytes,lang,code,repeat,minver,comment +before,0,0,q,datadog:use`di.datadog,1,,Initialize module +run,0,0,q,resetmetriclog:{.m.di.0datadog.metriclog:0#.m.di.0datadog.metriclog},1,,Set a function to clear metric log table +run,0,0,q,reseteventlog:{.m.di.0datadog.eventlog:0#.m.di.0datadog.eventlog},1,,Set a function to clear event log table +run,0,0,q,resetmetriclog[],1,,Reset metric log table for next test +run,0,0,q,reseteventlog[],1,,Reset event log table for next test + + +comment,,,,,,,Set expected tables +run,0,0,q,sm_lin:enlist`host`message`name`metric`https`status!(.z.h;"bash -c \"echo -n 'test:123' > /dev/udp/127.0.0.1/8125\"";"test";123f;0b;()),1,,Create an expected send metric log when using linux +run,0,0,q,"se_lin:enlist`host`message`title`text`https`status!(.z.h;""bash -c \""echo -n '_e{10,14}:Test_Event|This is a test' > /dev/udp/127.0.0.1/8125\"""";""Test_Event"";""This is a test"";0b;())",1,,Create an expected send event log when using linux +run,0,0,q,"sm_web:enlist`host`message`name`metric`https`status!(.z.h;""{\""series\"":[{\""metric\"":\""test\"",\""points\"":[[0000000000,123]],\""host\"":\""HOMER.AQUAQ.CO.UK\"",\""tags\"":\""tag test\""}]}"";""test"";123f;1b;""No api key specified"")",1,,Create an expected send metric log when using web +run,0,0,q,"se_web:enlist`host`message`title`text`https`status!(.z.h;""{\""title\"":\""Test_Event\"",\""text\"":\""This is a test\"",\""priority\"":\""high\"",\""tags\"":\""testtag\"",\""alert_type\"":\""test_type\""}"";""Test_Event"";""This is a test"";1b;""No api key specified"")",1,,Create an expected send event log when using web + +comment,,,,,,,Test linux +run,0,0,q,k4unit.mock[`.m.di.0datadog.opsys;`l64],1,,Mock linux operating system +run,0,0,q,datadog.init[(::)],1,,Init the package with linux os +run,0,0,q,datadog.sendmetric[([metricname:"test";metricvalue:123])],1,,Send a dummy metric +run,0,0,q,datadog.sendevent[([eventtitle:"Test_Event";eventtext:"This is a test"])],1,,Send a dummy event +true,0,0,q,sm_lin~``time _datadog.getmetriclog[],1,,Check it is as expected +true,0,0,q,se_lin~``time _datadog.geteventlog[],1,,Check it is as expected +run,0,0,q,resetmetriclog[],1,,Reset metric log table for next test +run,0,0,q,reseteventlog[],1,,Reset event log table for next test + +comment,,,,,,,Test windows (NOTE:there is currently no way to test windows functionality until any windows builds of kdbx are available) +run,0,0,q,k4unit.mock[`.m.di.0datadog.opsys;`win],1,,Mock windows operating system +run,0,0,q,datadog.init[(::)],1,,Init the package with windows os +run,0,0,q,resetmetriclog[],1,,Reset metric log table for next test +run,0,0,q,reseteventlog[],1,,Reset event log table for next test + + +comment,,,,,,,Test web +run,0,0,q,k4unit.mock[`.m.di.0datadog.opsys;`l64],1,,Mock linux operating system +run,0,0,q,datadog.init([useweb:1b]),1,,Init the package with linux os with useweb +run,0,0,q,datadog.sendmetric[([metricname:"test";metricvalue:123;metrictype:"test_type";samplerate:0.5;tags:"tag test"])],1,,Send a dummy metric +run,0,0,q,datadog.sendevent[([eventtitle:"Test_Event";eventtext:"This is a test";priority:"high";tags:"testtag";alerttype:"test_type"])],1,,Send a dummy event +run,0,0,q,"@[`.m.di.0datadog.metriclog;`message;{raze@[""[[""vs x;1;""[["",(10#""0""),10_]}']",1,,Update the unixtime in the table to be 0s +true,0,0,q,sm_web~``time _datadog.getmetriclog[],1,,Check it is as expected +true,0,0,q,se_web~``time _datadog.geteventlog[],1,,Check it is as expected +run,0,0,q,resetmetriclog[],1,,Reset metric log table for next test +run,0,0,q,reseteventlog[],1,,Reset event log table for next test + +comment,,,,,,,Tidy up +run,0,0,q,k4unit.unmock(::),1,,unmock all variables +run,0,0,q,resetmetriclog[],1,,Reset metric log table for next test +run,0,0,q,reseteventlog[],1,,Reset event log table for next test + +comment,,,,,,,TODO: Currently tests cannot be repeated (variables might not be cleared correctly) diff --git a/eventv1.json b/eventv1.json new file mode 100644 index 0000000..18682b8 --- /dev/null +++ b/eventv1.json @@ -0,0 +1,15 @@ +{ + "title": "Example-Event", + "text": "A text message.", + "aggregation_key": "aggregate_by_this", + "alert_type": "error", + "date_happened": 120391209, + "device_name": "device name", + "host": "HOST", + "priority": "normal", + "related_event_id": 413, + "source_type_name": "type_of_event", + "tags": [ + "test:ExampleEvent" + ] +} \ No newline at end of file diff --git a/eventv2.json b/eventv2.json new file mode 100644 index 0000000..0724882 --- /dev/null +++ b/eventv2.json @@ -0,0 +1,54 @@ +{ + "data": { + "attributes": { + "aggregation_key": "aggregation_key_123", + "attributes": { + "author": { + "name": "example@datadog.com", + "type": "user" + }, + "change_metadata": { + "dd": { + "team": "datadog_team", + "user_email": "datadog@datadog.com", + "user_id": "datadog_user_id", + "user_name": "datadog_username" + }, + "resource_link": "datadog.com/feature/fallback_payments_test" + }, + "changed_resource": { + "name": "fallback_payments_test", + "type": "one of feature_flag,configuration" + }, + "impacted_resources": [ + { + "name": "payments_api", + "type": "service" + } + ], + "new_value": { + "enabled": true, + "percentage": "50%", + "rule": { + "datacenter": "devcycle.us1.prod" + } + }, + "prev_value": { + "enabled": true, + "percentage": "10%", + "rule": { + "datacenter": "devcycle.us1.prod" + } + } + }, + "category": "one of change,alert", + "integration_id": "custom-events or nothing", + "message": "payment_processed feature flag has been enabled", + "tags": [ + "env:api_client_test" + ], + "title": "payment_processed feature flag updated" + }, + "type": "event" + } +} \ No newline at end of file diff --git a/metricv1.json b/metricv1.json new file mode 100644 index 0000000..f9a0985 --- /dev/null +++ b/metricv1.json @@ -0,0 +1,19 @@ +{ + "series": [ + { + "metric": "system.load.1", + "points": [ + [ + 1636629071, + 1.1 + ] + ], + "tags": [ + "test:ExampleMetric" + ], + "type": "one of , count, gauge, rate", + "host": "string", + "interval": 300 + } + ] +} \ No newline at end of file diff --git a/metricv2.json b/metricv2.json new file mode 100644 index 0000000..215fc62 --- /dev/null +++ b/metricv2.json @@ -0,0 +1,25 @@ +{ + "series": [ + { + "metric": "system.load.1", + "type": 0, + "points": [ + { + "timestamp": 1636629071, + "value": 0.7 + } + ], + "resources": [ + { + "name": "dummyhost", + "type": "host" + } + ], + "tags": [ + "test:examplemetric", + "test:examplemetric2" + ], + "unit": "£" + } + ] +} \ No newline at end of file