diff --git a/.env.example b/.env.example index 6c249675..2a0b5b04 100644 --- a/.env.example +++ b/.env.example @@ -10,11 +10,13 @@ DIFFBOT_API_KEY= UPSTASH_REDIS_REST_URL= UPSTASH_REDIS_REST_TOKEN= -# Analytics (required) -CLICKHOUSE_URL=http://localhost:8123 -CLICKHOUSE_USER=default -CLICKHOUSE_PASSWORD=clickhouse -CLICKHOUSE_DATABASE=smry_analytics +# Analytics - PostHog (required) +POSTHOG_API_KEY=phc_xxx +POSTHOG_HOST=https://us.i.posthog.com +POSTHOG_PROJECT_ID=12345 +POSTHOG_PERSONAL_API_KEY=phx_xxx +NEXT_PUBLIC_POSTHOG_KEY=phc_xxx +NEXT_PUBLIC_POSTHOG_HOST=https://us.i.posthog.com # Alerting ALERT_EMAIL= diff --git a/app/layout.tsx b/app/layout.tsx index 0bf857bd..5ede1568 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -19,6 +19,7 @@ import { ThemeProvider } from "@/components/theme-provider"; import { ClerkProvider } from "@clerk/nextjs"; import { getLocale } from 'next-intl/server'; import { JsonLd, organizationSchema, websiteSchema } from "@/components/seo/json-ld"; +import { PostHogProvider } from "@/components/providers/posthog-provider"; export const metadata: Metadata = { metadataBase: new URL("https://smry.ai"), @@ -84,6 +85,7 @@ export default async function RootLayout({ + + diff --git a/bun.lock b/bun.lock index 10143ef1..2276a04f 100644 --- a/bun.lock +++ b/bun.lock @@ -9,7 +9,6 @@ "@base-ui/react": "^1.1.0", "@clerk/backend": "^2.29.0", "@clerk/nextjs": "^6.36.5", - "@clickhouse/client": "^1.15.0", "@databuddy/sdk": "^2.3.29", "@elysiajs/cors": "^1.4.1", "@elysiajs/cron": "^1.4.1", @@ -49,6 +48,8 @@ "next-themes": "^0.4.6", "nuqs": "^2.8.0", "pino": "^8.19.0", + "posthog-js": "^1.341.1", + "posthog-node": "^5.24.10", "react": "19.2.1", "react-dom": "19.2.1", "react-markdown": "^9.0.1", @@ -170,10 +171,6 @@ "@clerk/types": ["@clerk/types@4.101.13", "", { "dependencies": { "@clerk/shared": "^3.43.2" } }, "sha512-PKv85uHjNXu8KO/Vc4m4e1GByItfuib/T3wNINDrq1k+QuzKwohC+n07ENlzOzr67tfbnfa6CQSgg2HUb4RohQ=="], - "@clickhouse/client": ["@clickhouse/client@1.16.0", "", { "dependencies": { "@clickhouse/client-common": "1.16.0" } }, "sha512-ThPhoRMsKsf/hmBEgWlUsGxFecsr3i+k3JI8JV0Od7UpH2BSmk9VKMGJoyPCrTL0vPUs5rJH+7o4iCqBF09Xvg=="], - - "@clickhouse/client-common": ["@clickhouse/client-common@1.16.0", "", {}, "sha512-qMzkI1NmV29ZjFkNpVSvGNfA0c7sCExlufAQMv+V+5xtNeYXnRfdqzmBLIQoq6Pf1ij0kw/wGLD3HQrl7pTFLA=="], - "@cloudflare/kv-asset-handler": ["@cloudflare/kv-asset-handler@0.4.2", "", {}, "sha512-SIOD2DxrRRwQ+jgzlXCqoEFiKOFqaPjhnNTGKXSRLvp1HiOvapLaFG2kEr9dYQTYe8rKrd9uvDUzmAITeNyaHQ=="], "@cloudflare/next-on-pages": ["@cloudflare/next-on-pages@1.13.16", "", { "dependencies": { "acorn": "^8.8.0", "ast-types": "^0.14.2", "chalk": "^5.2.0", "chokidar": "^3.5.3", "commander": "^11.1.0", "cookie": "^0.5.0", "esbuild": "^0.15.3", "js-yaml": "^4.1.0", "miniflare": "^3.20231218.1", "package-manager-manager": "^0.2.0", "pcre-to-regexp": "^1.1.0", "semver": "^7.5.2" }, "peerDependencies": { "@cloudflare/workers-types": "^4.20240208.0", "next": ">=14.3.0 && <=15.5.2", "vercel": ">=30.0.0 && <=47.0.4", "wrangler": "^3.28.2 || ^4.0.0" }, "optionalPeers": ["@cloudflare/workers-types"], "bin": { "next-on-pages": "bin/index.js" } }, "sha512-52h51WNcfmx3szTdTd+n/xgz4qNxFtjOGG0zwnUAhTg8cjPwSUYmZp0OPRNw2jYG9xHwRS2ttSPAS8tcGkQGsw=="], @@ -422,6 +419,26 @@ "@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="], + "@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.208.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg=="], + + "@opentelemetry/core": ["@opentelemetry/core@2.2.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw=="], + + "@opentelemetry/exporter-logs-otlp-http": ["@opentelemetry/exporter-logs-otlp-http@0.208.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.208.0", "@opentelemetry/core": "2.2.0", "@opentelemetry/otlp-exporter-base": "0.208.0", "@opentelemetry/otlp-transformer": "0.208.0", "@opentelemetry/sdk-logs": "0.208.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-jOv40Bs9jy9bZVLo/i8FwUiuCvbjWDI+ZW13wimJm4LjnlwJxGgB+N/VWOZUTpM+ah/awXeQqKdNlpLf2EjvYg=="], + + "@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.208.0", "", { "dependencies": { "@opentelemetry/core": "2.2.0", "@opentelemetry/otlp-transformer": "0.208.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-gMd39gIfVb2OgxldxUtOwGJYSH8P1kVFFlJLuut32L6KgUC4gl1dMhn+YC2mGn0bDOiQYSk/uHOdSjuKp58vvA=="], + + "@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.208.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.208.0", "@opentelemetry/core": "2.2.0", "@opentelemetry/resources": "2.2.0", "@opentelemetry/sdk-logs": "0.208.0", "@opentelemetry/sdk-metrics": "2.2.0", "@opentelemetry/sdk-trace-base": "2.2.0", "protobufjs": "^7.3.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-DCFPY8C6lAQHUNkzcNT9R+qYExvsk6C5Bto2pbNxgicpcSWbe2WHShLxkOxIdNcBiYPdVHv/e7vH7K6TI+C+fQ=="], + + "@opentelemetry/resources": ["@opentelemetry/resources@2.5.0", "", { "dependencies": { "@opentelemetry/core": "2.5.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-F8W52ApePshpoSrfsSk1H2yJn9aKjCrbpQF1M9Qii0GHzbfVeFUB+rc3X4aggyZD8x9Gu3Slua+s6krmq6Dt8g=="], + + "@opentelemetry/sdk-logs": ["@opentelemetry/sdk-logs@0.208.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.208.0", "@opentelemetry/core": "2.2.0", "@opentelemetry/resources": "2.2.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-QlAyL1jRpOeaqx7/leG1vJMp84g0xKP6gJmfELBpnI4O/9xPX+Hu5m1POk9Kl+veNkyth5t19hRlN6tNY1sjbA=="], + + "@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@2.2.0", "", { "dependencies": { "@opentelemetry/core": "2.2.0", "@opentelemetry/resources": "2.2.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-G5KYP6+VJMZzpGipQw7Giif48h6SGQ2PFKEYCybeXJsOCB4fp8azqMAAzE5lnnHK3ZVwYQrgmFbsUJO/zOnwGw=="], + + "@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.2.0", "", { "dependencies": { "@opentelemetry/core": "2.2.0", "@opentelemetry/resources": "2.2.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-xWQgL0Bmctsalg6PaXExmzdedSp3gyKV8mQBwK/j9VGdCDu2fmXIb2gAehBKbkXCpJ4HPkgv3QfoJWRT4dHWbw=="], + + "@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.39.0", "", {}, "sha512-R5R9tb2AXs2IRLNKLBJDynhkfmx7mX0vi8NkhZb3gUkPWHn6HXk5J8iQ/dql0U3ApfWym4kXXmBDRGO+oeOfjg=="], + "@oxc-resolver/binding-android-arm-eabi": ["@oxc-resolver/binding-android-arm-eabi@11.17.0", "", { "os": "android", "cpu": "arm" }, "sha512-kVnY21v0GyZ/+LG6EIO48wK3mE79BUuakHUYLIqobO/Qqq4mJsjuYXMSn3JtLcKZpN1HDVit4UHpGJHef1lrlw=="], "@oxc-resolver/binding-android-arm64": ["@oxc-resolver/binding-android-arm64@11.17.0", "", { "os": "android", "cpu": "arm64" }, "sha512-Pf8e3XcsK9a8RHInoAtEcrwf2vp7V9bSturyUUYxw9syW6E7cGi7z9+6ADXxm+8KAevVfLA7pfBg8NXTvz/HOw=="], @@ -500,10 +517,34 @@ "@poppinss/exception": ["@poppinss/exception@1.2.3", "", {}, "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw=="], + "@posthog/core": ["@posthog/core@1.20.0", "", { "dependencies": { "cross-spawn": "^7.0.6" } }, "sha512-e/F20we0t6bPMuDOVOe53f908s23vuKEpFKNXmZcx4bSYsPkjRN49akIIHU621HBJdcsFx537vhJYKZxu8uS9w=="], + + "@posthog/types": ["@posthog/types@1.341.1", "", {}, "sha512-Ufuo+6X7BbbhKeqmO/WHyWZZPBSESLO6sPKJEaHpOJLTBXmCmgIQvwXq2SA9QY56htJqLHIluvTXvr3UKoCi8w=="], + "@preact/signals": ["@preact/signals@1.3.2", "", { "dependencies": { "@preact/signals-core": "^1.7.0" }, "peerDependencies": { "preact": "10.x" } }, "sha512-naxcJgUJ6BTOROJ7C3QML7KvwKwCXQJYTc5L/b0eEsdYgPB6SxwoQ1vDGcS0Q7GVjAenVq/tXrybVdFShHYZWg=="], "@preact/signals-core": ["@preact/signals-core@1.12.2", "", {}, "sha512-5Yf8h1Ke3SMHr15xl630KtwPTW4sYDFkkxS0vQ8UiQLWwZQnrF9IKaVG1mN5VcJz52EcWs2acsc/Npjha/7ysA=="], + "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], + + "@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="], + + "@protobufjs/codegen": ["@protobufjs/codegen@2.0.4", "", {}, "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="], + + "@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.0", "", {}, "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="], + + "@protobufjs/fetch": ["@protobufjs/fetch@1.1.0", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" } }, "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ=="], + + "@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="], + + "@protobufjs/inquire": ["@protobufjs/inquire@1.1.0", "", {}, "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="], + + "@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="], + + "@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="], + + "@protobufjs/utf8": ["@protobufjs/utf8@1.1.0", "", {}, "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="], + "@radix-ui/number": ["@radix-ui/number@1.1.1", "", {}, "sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g=="], "@radix-ui/primitive": ["@radix-ui/primitive@1.1.3", "", {}, "sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg=="], @@ -1000,6 +1041,8 @@ "cookie": ["cookie@0.5.0", "", {}, "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw=="], + "core-js": ["core-js@3.48.0", "", {}, "sha512-zpEHTy1fjTMZCKLHUZoVeylt9XrzaIN2rbPXEt0k+q7JE5CkCZdo6bNq55bn24a69CH7ErAVLKijxJja4fw+UQ=="], + "create-require": ["create-require@1.1.1", "", {}, "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ=="], "croner": ["croner@6.0.7", "", {}, "sha512-k3Xx3Rcclfr60Yx4TmvsF3Yscuiql8LSvYLaphTsaq5Hk8La4Z/udmUANMOTKpgGGroI2F6/XOr9cU9OFkYluQ=="], @@ -1268,6 +1311,8 @@ "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], + "fflate": ["fflate@0.4.8", "", {}, "sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA=="], + "file-entry-cache": ["file-entry-cache@8.0.0", "", { "dependencies": { "flat-cache": "^4.0.0" } }, "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ=="], "file-type": ["file-type@21.3.0", "", { "dependencies": { "@tokenizer/inflate": "^0.4.1", "strtok3": "^10.3.4", "token-types": "^6.1.1", "uint8array-extras": "^1.4.0" } }, "sha512-8kPJMIGz1Yt/aPEwOsrR97ZyZaD1Iqm8PClb1nYFclUCkBi0Ma5IsYNQzvSFS9ib51lWyIw5mIT9rWzI/xjpzA=="], @@ -1580,6 +1625,8 @@ "lodash.merge": ["lodash.merge@4.6.2", "", {}, "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ=="], + "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], + "longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="], "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], @@ -1850,6 +1897,10 @@ "postcss-selector-parser": ["postcss-selector-parser@6.0.10", "", { "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" } }, "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w=="], + "posthog-js": ["posthog-js@1.341.1", "", { "dependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/api-logs": "^0.208.0", "@opentelemetry/exporter-logs-otlp-http": "^0.208.0", "@opentelemetry/resources": "^2.2.0", "@opentelemetry/sdk-logs": "^0.208.0", "@posthog/core": "1.20.0", "@posthog/types": "1.341.1", "core-js": "^3.38.1", "dompurify": "^3.3.1", "fflate": "^0.4.8", "preact": "^10.28.2", "query-selector-shadow-dom": "^1.0.1", "web-vitals": "^5.1.0" } }, "sha512-e5IIcZTt9OOUTqQTt5ouYWoLze9gmokYTRBDsDfLhMPrwF2cJ9S7K1lmPz4AcmyOCrfSFRONDM/yMH1zACBfig=="], + + "posthog-node": ["posthog-node@5.24.10", "", { "dependencies": { "@posthog/core": "1.20.0" } }, "sha512-C4ueZUrifTJMDFngybSWQ+GthcqCqPiCcGg5qnjoh+f6ie3+tdhFROqqshjttpQ6Q4DPM40USPTmU/UBYqgsbA=="], + "preact": ["preact@10.28.2", "", {}, "sha512-lbteaWGzGHdlIuiJ0l2Jq454m6kcpI1zNje6d8MlGAFlYvP2GO4ibnat7P74Esfz4sPTdM6UxtTwh/d3pwM9JA=="], "prelude-ls": ["prelude-ls@1.2.1", "", {}, "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g=="], @@ -1870,10 +1921,14 @@ "property-information": ["property-information@7.1.0", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], + "protobufjs": ["protobufjs@7.5.4", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.4", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.0", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.0", "@types/node": ">=13.7.0", "long": "^5.0.0" } }, "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg=="], + "pump": ["pump@3.0.3", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA=="], "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], + "query-selector-shadow-dom": ["query-selector-shadow-dom@1.0.1", "", {}, "sha512-lT5yCqEBgfoMYpf3F2xQRK7zEr1rhIIZuceDK6+xRkJQ4NMbHTwXqk4NkwDwQMNqXgG9r9fyHnzwNVs6zV5KRw=="], + "queue-microtask": ["queue-microtask@1.2.3", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="], "quick-format-unescaped": ["quick-format-unescaped@4.0.4", "", {}, "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg=="], @@ -2240,7 +2295,7 @@ "web-namespaces": ["web-namespaces@2.0.1", "", {}, "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ=="], - "web-vitals": ["web-vitals@0.2.4", "", {}, "sha512-6BjspCO9VriYy12z356nL6JBS0GYeEcA457YyRzD+dD6XYCQ75NKhcOHUMHentOE7OcVCIXXDvOm0jKFfQG2Gg=="], + "web-vitals": ["web-vitals@5.1.0", "", {}, "sha512-ArI3kx5jI0atlTtmV0fWU3fjpLmq/nD3Zr1iFFlJLaqa5wLBkUSzINwBPySCX/8jRyjlmy1Volw1kz1g9XE4Jg=="], "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], @@ -2326,6 +2381,16 @@ "@openrouter/sdk/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], + "@opentelemetry/otlp-transformer/@opentelemetry/resources": ["@opentelemetry/resources@2.2.0", "", { "dependencies": { "@opentelemetry/core": "2.2.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A=="], + + "@opentelemetry/resources/@opentelemetry/core": ["@opentelemetry/core@2.5.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-ka4H8OM6+DlUhSAZpONu0cPBtPPTQKxbxVzC4CzVx5+K4JnroJVBtDzLAMx4/3CDTJXRvVFhpFjtl4SaiTNoyQ=="], + + "@opentelemetry/sdk-logs/@opentelemetry/resources": ["@opentelemetry/resources@2.2.0", "", { "dependencies": { "@opentelemetry/core": "2.2.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A=="], + + "@opentelemetry/sdk-metrics/@opentelemetry/resources": ["@opentelemetry/resources@2.2.0", "", { "dependencies": { "@opentelemetry/core": "2.2.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A=="], + + "@opentelemetry/sdk-trace-base/@opentelemetry/resources": ["@opentelemetry/resources@2.2.0", "", { "dependencies": { "@opentelemetry/core": "2.2.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A=="], + "@poppinss/dumper/supports-color": ["supports-color@10.2.2", "", {}, "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g=="], "@radix-ui/react-dialog/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], @@ -2366,6 +2431,8 @@ "@vercel/fun/semver": ["semver@7.5.4", "", { "dependencies": { "lru-cache": "^6.0.0" }, "bin": { "semver": "bin/semver.js" } }, "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA=="], + "@vercel/gatsby-plugin-vercel-analytics/web-vitals": ["web-vitals@0.2.4", "", {}, "sha512-6BjspCO9VriYy12z356nL6JBS0GYeEcA457YyRzD+dD6XYCQ75NKhcOHUMHentOE7OcVCIXXDvOm0jKFfQG2Gg=="], + "@vercel/gatsby-plugin-vercel-builder/esbuild": ["esbuild@0.14.47", "", { "optionalDependencies": { "esbuild-android-64": "0.14.47", "esbuild-android-arm64": "0.14.47", "esbuild-darwin-64": "0.14.47", "esbuild-darwin-arm64": "0.14.47", "esbuild-freebsd-64": "0.14.47", "esbuild-freebsd-arm64": "0.14.47", "esbuild-linux-32": "0.14.47", "esbuild-linux-64": "0.14.47", "esbuild-linux-arm": "0.14.47", "esbuild-linux-arm64": "0.14.47", "esbuild-linux-mips64le": "0.14.47", "esbuild-linux-ppc64le": "0.14.47", "esbuild-linux-riscv64": "0.14.47", "esbuild-linux-s390x": "0.14.47", "esbuild-netbsd-64": "0.14.47", "esbuild-openbsd-64": "0.14.47", "esbuild-sunos-64": "0.14.47", "esbuild-windows-32": "0.14.47", "esbuild-windows-64": "0.14.47", "esbuild-windows-arm64": "0.14.47" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-wI4ZiIfFxpkuxB8ju4MHrGwGLyp1+awEHAHVpx6w7a+1pmYIq8T9FGEVVwFo0iFierDoMj++Xq69GXWYn2EiwA=="], "@vercel/nft/estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="], diff --git a/components/providers/posthog-provider.tsx b/components/providers/posthog-provider.tsx new file mode 100644 index 00000000..440c7b15 --- /dev/null +++ b/components/providers/posthog-provider.tsx @@ -0,0 +1,46 @@ +"use client"; + +import posthog from "posthog-js"; +import { PostHogProvider as PHProvider, usePostHog } from "posthog-js/react"; +import { useEffect, Suspense } from "react"; +import { usePathname, useSearchParams } from "next/navigation"; + +function PostHogPageView() { + const pathname = usePathname(); + const searchParams = useSearchParams(); + const ph = usePostHog(); + + useEffect(() => { + if (pathname && ph) { + let url = window.origin + pathname; + const search = searchParams?.toString(); + if (search) url += `?${search}`; + ph.capture("$pageview", { $current_url: url }); + } + }, [pathname, searchParams, ph]); + + return null; +} + +export function PostHogProvider({ children }: { children: React.ReactNode }) { + useEffect(() => { + const key = process.env.NEXT_PUBLIC_POSTHOG_KEY; + const host = process.env.NEXT_PUBLIC_POSTHOG_HOST; + if (!key || !host) return; + + posthog.init(key, { + api_host: host, + capture_pageview: false, // we handle it manually above + capture_pageleave: true, + }); + }, []); + + return ( + + + + + {children} + + ); +} diff --git a/docker-compose.yml b/docker-compose.yml index ac3d8d4e..f5849b7a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,33 +1,7 @@ -# Local dev services - ClickHouse starts automatically with: bun run dev +# Local dev services # For full Docker stack: bun run dev:docker services: - # Clickhouse analytics database (memory-optimized) - clickhouse: - build: - context: ./docker/clickhouse - dockerfile: Dockerfile - container_name: smry-clickhouse - ports: - - "8123:8123" - - "9000:9000" - volumes: - - clickhouse-data:/var/lib/clickhouse - - ./docs/clickhouse-schema.sql:/docker-entrypoint-initdb.d/init.sql:ro - environment: - CLICKHOUSE_DB: smry_analytics - CLICKHOUSE_USER: default - CLICKHOUSE_PASSWORD: clickhouse - healthcheck: - test: ["CMD", "clickhouse-client", "--query", "SELECT 1"] - interval: 10s - timeout: 5s - retries: 3 - ulimits: - nofile: - soft: 262144 - hard: 262144 - # Next.js app app: build: @@ -42,22 +16,16 @@ services: - /app/node_modules - /app/.next environment: - # Clickhouse (internal Docker network) - CLICKHOUSE_URL: http://clickhouse:8123 - CLICKHOUSE_USER: default - CLICKHOUSE_PASSWORD: clickhouse - CLICKHOUSE_DATABASE: smry_analytics - ANALYTICS_SECRET_KEY: dev_secret_key # Pass through from host .env + POSTHOG_API_KEY: ${POSTHOG_API_KEY} + POSTHOG_HOST: ${POSTHOG_HOST} + POSTHOG_PROJECT_ID: ${POSTHOG_PROJECT_ID} + POSTHOG_PERSONAL_API_KEY: ${POSTHOG_PERSONAL_API_KEY} + NEXT_PUBLIC_POSTHOG_KEY: ${NEXT_PUBLIC_POSTHOG_KEY} + NEXT_PUBLIC_POSTHOG_HOST: ${NEXT_PUBLIC_POSTHOG_HOST} UPSTASH_REDIS_REST_URL: ${UPSTASH_REDIS_REST_URL} UPSTASH_REDIS_REST_TOKEN: ${UPSTASH_REDIS_REST_TOKEN} DIFFBOT_API_KEY: ${DIFFBOT_API_KEY} OPENROUTER_API_KEY: ${OPENROUTER_API_KEY} CLERK_SECRET_KEY: ${CLERK_SECRET_KEY} NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: ${NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY} - depends_on: - clickhouse: - condition: service_healthy - -volumes: - clickhouse-data: diff --git a/docker/clickhouse/Dockerfile b/docker/clickhouse/Dockerfile deleted file mode 100644 index cfd45c59..00000000 --- a/docker/clickhouse/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM clickhouse/clickhouse-server:24.8 - -# Copy memory-optimized config -COPY memory.xml /etc/clickhouse-server/config.d/memory.xml diff --git a/docker/clickhouse/memory.xml b/docker/clickhouse/memory.xml deleted file mode 100644 index 68de5d6a..00000000 --- a/docker/clickhouse/memory.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - 157286400 - - 0.15 - - - 8388608 - - - 8388608 - - - 4194304 - - - - 0 - - - - 1 - 1 - 1 - - - - - - 67108864 - - 16777216 - 16777216 - - - diff --git a/docs/clickhouse-schema.sql b/docs/clickhouse-schema.sql deleted file mode 100644 index 211ba0bb..00000000 --- a/docs/clickhouse-schema.sql +++ /dev/null @@ -1,237 +0,0 @@ --- SMRY.ai Clickhouse Analytics Schema --- Run this SQL in your Clickhouse instance to set up the analytics tables - --- Create database if not exists -CREATE DATABASE IF NOT EXISTS smry_analytics; - --- Switch to the database -USE smry_analytics; - --- Main events table with MergeTree engine optimized for time-series queries -CREATE TABLE IF NOT EXISTS request_events -( - -- Identifiers - request_id String, - timestamp DateTime64(3) DEFAULT now64(3), - - -- Request metadata - method LowCardinality(String), - endpoint LowCardinality(String), -- /api/article, /api/summary - path String, - - -- Article/content context - url String, - hostname LowCardinality(String), -- nytimes.com, wsj.com, etc. - source LowCardinality(String), -- smry-fast, smry-slow, wayback - - -- Outcome metrics - outcome LowCardinality(String), -- success, error - status_code UInt16, - error_type LowCardinality(String) DEFAULT '', - error_message String DEFAULT '', - - -- Performance metrics - duration_ms UInt32, - fetch_ms UInt32 DEFAULT 0, - cache_lookup_ms UInt32 DEFAULT 0, - cache_save_ms UInt32 DEFAULT 0, - - -- Cache behavior - cache_hit UInt8 DEFAULT 0, -- 0 = miss, 1 = hit - cache_status LowCardinality(String) DEFAULT '', -- hit, miss, invalid, error - - -- Content metrics - article_length UInt32 DEFAULT 0, - article_title String DEFAULT '', - - -- AI Summary specific (for /api/summary) - summary_length UInt32 DEFAULT 0, - input_tokens UInt32 DEFAULT 0, - output_tokens UInt32 DEFAULT 0, - - -- User context - is_premium UInt8 DEFAULT 0, - client_ip String DEFAULT '', - user_agent String DEFAULT '', - - -- System health - heap_used_mb UInt16 DEFAULT 0, - heap_total_mb UInt16 DEFAULT 0, - rss_mb UInt16 DEFAULT 0, - - -- Environment - env LowCardinality(String) DEFAULT 'production', - version String DEFAULT '' -) -ENGINE = MergeTree() -PARTITION BY toYYYYMM(timestamp) -ORDER BY (hostname, source, timestamp, request_id) -TTL toDateTime(timestamp) + INTERVAL 30 DAY -- Auto-delete data older than 30 days -SETTINGS index_granularity = 8192; - --- Index for faster hostname lookups -ALTER TABLE request_events ADD INDEX idx_hostname hostname TYPE bloom_filter GRANULARITY 1; - --- Index for error filtering -ALTER TABLE request_events ADD INDEX idx_outcome outcome TYPE set(2) GRANULARITY 1; - - --- Materialized view for hourly aggregates (pre-computed for dashboard performance) -CREATE MATERIALIZED VIEW IF NOT EXISTS hourly_stats -ENGINE = SummingMergeTree() -PARTITION BY toYYYYMM(hour) -ORDER BY (hostname, source, hour) -TTL hour + INTERVAL 30 DAY -- Match raw data TTL -AS SELECT - toStartOfHour(timestamp) AS hour, - hostname, - source, - count() AS request_count, - countIf(outcome = 'success') AS success_count, - countIf(outcome = 'error') AS error_count, - countIf(cache_hit = 1) AS cache_hits, - sum(duration_ms) AS total_duration_ms, - sum(article_length) AS total_article_length -FROM request_events -GROUP BY hour, hostname, source; - - --- Materialized view for error tracking by hostname -CREATE MATERIALIZED VIEW IF NOT EXISTS error_rates -ENGINE = SummingMergeTree() -PARTITION BY toYYYYMM(hour) -ORDER BY (hostname, source, error_type, hour) -TTL hour + INTERVAL 30 DAY -- Match raw data TTL -AS SELECT - toStartOfHour(timestamp) AS hour, - hostname, - source, - error_type, - count() AS error_count -FROM request_events -WHERE outcome = 'error' -GROUP BY hour, hostname, source, error_type; - - --- ============================================================================ --- AD EVENTS TABLE - Tracks ad requests, fill rates, and performance --- ============================================================================ - -CREATE TABLE IF NOT EXISTS ad_events -( - event_id String, - timestamp DateTime64(3) DEFAULT now64(3), - - -- Request context - url String, - hostname LowCardinality(String), - article_title String DEFAULT '', - article_content_length UInt32 DEFAULT 0, - session_id String, - - -- User context - user_id String DEFAULT '', - is_premium UInt8 DEFAULT 0, - - -- Device context - device_type LowCardinality(String) DEFAULT '', -- desktop, mobile, tablet - os LowCardinality(String) DEFAULT '', -- windows, macos, ios, android - browser LowCardinality(String) DEFAULT '', -- chrome, safari, firefox - - -- Response - status LowCardinality(String), -- filled, no_fill, premium_user, gravity_error, timeout, error - gravity_status_code UInt16 DEFAULT 0, - error_message String DEFAULT '', - - -- Ad data (when filled) - brand_name LowCardinality(String) DEFAULT '', - ad_title String DEFAULT '', - - -- Performance - duration_ms UInt32 DEFAULT 0, - - -- Environment - env LowCardinality(String) DEFAULT 'production' -) -ENGINE = MergeTree() -PARTITION BY toYYYYMM(timestamp) -ORDER BY (hostname, status, timestamp, event_id) -TTL toDateTime(timestamp) + INTERVAL 90 DAY -- Keep ad data longer for analysis -SETTINGS index_granularity = 8192; - --- Index for faster status filtering -ALTER TABLE ad_events ADD INDEX idx_status status TYPE set(10) GRANULARITY 1; - --- Index for brand lookups -ALTER TABLE ad_events ADD INDEX idx_brand brand_name TYPE bloom_filter GRANULARITY 1; - - --- ============================================================================ --- USEFUL QUERIES FOR DEBUGGING AND MONITORING --- ============================================================================ - --- Check data is flowing in --- SELECT count(), max(timestamp), min(timestamp) FROM request_events; - --- Top 10 sites by error count (last 24h) --- SELECT hostname, count() as errors --- FROM request_events --- WHERE timestamp > now() - INTERVAL 24 HOUR AND outcome = 'error' --- GROUP BY hostname --- ORDER BY errors DESC --- LIMIT 10; - --- Source success rates by hostname (last 24h) --- SELECT hostname, source, --- round(countIf(outcome = 'success') / count() * 100, 2) as success_rate, --- count() as total --- FROM request_events --- WHERE timestamp > now() - INTERVAL 24 HOUR --- GROUP BY hostname, source --- HAVING total >= 5 --- ORDER BY hostname, success_rate DESC; - --- Memory usage over time (for leak detection) --- SELECT toStartOfMinute(timestamp) as minute, --- avg(heap_used_mb) as avg_heap, --- max(heap_used_mb) as max_heap, --- avg(rss_mb) as avg_rss --- FROM request_events --- WHERE timestamp > now() - INTERVAL 1 HOUR --- GROUP BY minute --- ORDER BY minute; - --- Cache hit rate by endpoint --- SELECT endpoint, --- round(countIf(cache_hit = 1) / count() * 100, 2) as cache_hit_rate, --- count() as total --- FROM request_events --- WHERE timestamp > now() - INTERVAL 24 HOUR --- GROUP BY endpoint; - - --- ============================================================================ --- MEMORY MANAGEMENT --- ============================================================================ --- --- Built-in safeguards: --- 1. TTL (30 days) - auto-deletes old data via background merges --- 2. LowCardinality columns - reduces memory for repeated strings (hostname, source, etc) --- 3. Monthly partitioning - enables efficient partition drops --- 4. Compression enabled client-side --- --- Monitor disk usage: --- SELECT database, table, formatReadableSize(sum(bytes)) as size --- FROM system.parts --- WHERE active --- GROUP BY database, table; --- --- Manual partition cleanup (if needed): --- ALTER TABLE request_events DROP PARTITION '202501'; --- --- Check TTL progress: --- SELECT table, formatReadableSize(sum(bytes)) as size, --- min(min_date), max(max_date) --- FROM system.parts --- WHERE database = 'smry_analytics' AND active --- GROUP BY table; diff --git a/lib/alerting.ts b/lib/alerting.ts index 7b9ede4b..a511b955 100644 --- a/lib/alerting.ts +++ b/lib/alerting.ts @@ -1,11 +1,11 @@ /** * Error Rate Alerting * - * Monitors ClickHouse for error rate spikes and sends alerts via inbound.new. + * Monitors PostHog for error rate spikes and sends alerts via inbound.new. * Runs on a cron schedule from the Elysia server. */ -import { queryClickhouse } from "./clickhouse"; +import { queryPostHog } from "./posthog"; import { sendAlertEmail } from "./emails"; import { env } from "../server/env"; @@ -38,15 +38,16 @@ interface TopError { async function getErrorRateStats(): Promise { const query = ` SELECT - countIf(outcome = 'error' AND timestamp > now() - INTERVAL 5 MINUTE) as recent_errors, + countIf(properties.outcome = 'error' AND timestamp > now() - INTERVAL 5 MINUTE) as recent_errors, countIf(timestamp > now() - INTERVAL 5 MINUTE) as recent_total, - countIf(outcome = 'error' AND timestamp <= now() - INTERVAL 5 MINUTE AND timestamp > now() - INTERVAL 1 HOUR) as baseline_errors, + countIf(properties.outcome = 'error' AND timestamp <= now() - INTERVAL 5 MINUTE AND timestamp > now() - INTERVAL 1 HOUR) as baseline_errors, countIf(timestamp <= now() - INTERVAL 5 MINUTE AND timestamp > now() - INTERVAL 1 HOUR) as baseline_total - FROM request_events - WHERE timestamp > now() - INTERVAL 1 HOUR + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL 1 HOUR `; - const results = await queryClickhouse<{ + const results = await queryPostHog<{ recent_errors: number; recent_total: number; baseline_errors: number; @@ -72,18 +73,19 @@ async function getErrorRateStats(): Promise { async function getTopRecentErrors(): Promise { const query = ` SELECT - error_type, - error_message, + properties.error_type as error_type, + properties.error_message as error_message, count() as count - FROM request_events - WHERE timestamp > now() - INTERVAL 5 MINUTE - AND outcome = 'error' + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL 5 MINUTE + AND properties.outcome = 'error' GROUP BY error_type, error_message ORDER BY count DESC LIMIT 5 `; - return queryClickhouse(query); + return queryPostHog(query); } /** @@ -93,7 +95,7 @@ export async function checkErrorRateAndAlert(): Promise { try { const stats = await getErrorRateStats(); if (!stats) { - console.log("[alerting] No data from ClickHouse"); + console.log("[alerting] No data from PostHog"); return; } diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts deleted file mode 100644 index 4fb1a63d..00000000 --- a/lib/clickhouse.ts +++ /dev/null @@ -1,767 +0,0 @@ -import { createClient, ClickHouseClient } from "@clickhouse/client"; -import { env } from "../server/env"; - -/** - * Clickhouse Analytics Client - * - * Memory-safe implementation following the same patterns as: - * - redis.ts (module-level singleton) - * - summary/route.ts rate limiters (singleton to prevent memory leaks) - * - * Key memory safeguards: - * 1. Module-level singleton client (not per-request) - * 2. Bounded event buffer (max 500 events) - * 3. Automatic flush every 5 seconds - * 4. Fire-and-forget writes (non-blocking) - * 5. Graceful degradation when Clickhouse not configured - */ - -// Module-level singleton - created once at module load -let client: ClickHouseClient | null = null; -// Track if ClickHouse is unavailable (connection failed) to prevent repeated attempts -let clickhouseDisabled = false; -let lastConnectionAttempt = 0; -const CONNECTION_RETRY_INTERVAL_MS = 60_000; // Retry connection check every 60 seconds - -function getClient(): ClickHouseClient | null { - // Skip if we've determined ClickHouse is unavailable - // Allow retry after CONNECTION_RETRY_INTERVAL_MS - if (clickhouseDisabled) { - const now = Date.now(); - if (now - lastConnectionAttempt < CONNECTION_RETRY_INTERVAL_MS) { - return null; - } - // Reset to allow retry - clickhouseDisabled = false; - } - - if (!client) { - client = createClient({ - url: env.CLICKHOUSE_URL, - username: env.CLICKHOUSE_USER, - password: env.CLICKHOUSE_PASSWORD, - database: env.CLICKHOUSE_DATABASE, - request_timeout: 30_000, - compression: { - request: true, - response: true, - }, - // Keep-alive to reduce connection overhead - keep_alive: { - enabled: true, - }, - }); - } - return client; -} - -/** - * Mark ClickHouse as temporarily disabled due to connection failure - */ -function disableClickhouse(reason: string): void { - if (!clickhouseDisabled) { - console.warn(`[clickhouse] Disabled due to connection failure: ${reason}. Will retry in ${CONNECTION_RETRY_INTERVAL_MS / 1000}s`); - clickhouseDisabled = true; - lastConnectionAttempt = Date.now(); - } -} - -// Analytics event type matching our Clickhouse schema -// Error severity levels for distinguishing expected vs unexpected errors -export type ErrorSeverity = "expected" | "degraded" | "unexpected" | ""; - -export interface AnalyticsEvent { - request_id: string; - timestamp: string; - method: string; - endpoint: string; - path: string; - url: string; - hostname: string; - source: string; - outcome: string; - status_code: number; - error_type: string; - error_message: string; - error_severity: ErrorSeverity; - // Upstream error info - which host/service actually caused the error - upstream_hostname: string; - upstream_status_code: number; - upstream_error_code: string; - upstream_message: string; - duration_ms: number; - fetch_ms: number; - cache_lookup_ms: number; - cache_save_ms: number; - cache_hit: number; - cache_status: string; - article_length: number; - article_title: string; - summary_length: number; - input_tokens: number; - output_tokens: number; - is_premium: number; - client_ip: string; - user_agent: string; - heap_used_mb: number; - heap_total_mb: number; - rss_mb: number; - env: string; - version: string; -} - -// ============================================================================= -// Ad Event Tracking -// ============================================================================= - -// Ad event status - matches ContextResponseStatus in types/api.ts -export type AdEventStatus = "filled" | "no_fill" | "premium_user" | "gravity_error" | "timeout" | "error"; - -// Event type for tracking funnel: request -> impression -> click/dismiss -export type AdEventType = "request" | "impression" | "click" | "dismiss"; - -export interface AdEvent { - event_id: string; - timestamp: string; - // Event type (request, impression, click, dismiss) - event_type: AdEventType; - // Request context - url: string; - hostname: string; - article_title: string; - article_content_length: number; - session_id: string; - // User context - user_id: string; - is_premium: number; - // Device context - device_type: string; - os: string; - browser: string; - // Response - status: AdEventStatus; - gravity_status_code: number; - error_message: string; - // Gravity forwarding status (for impressions) - // 1 = successfully forwarded to Gravity, 0 = failed or not applicable - gravity_forwarded: number; - // Ad data (when filled) - brand_name: string; - ad_title: string; - ad_text: string; - click_url: string; - imp_url: string; - cta: string; - favicon: string; - ad_count: number; // Number of ads returned in this request - // Performance - duration_ms: number; - // Environment - env: string; -} - -// Separate buffer for ad events -const adEventBuffer: AdEvent[] = []; -let adFlushTimer: NodeJS.Timeout | null = null; -let adSchemaMigrated = false; - -/** - * Ensure ad_events table exists - */ -async function ensureAdSchema(): Promise { - if (adSchemaMigrated) return; - - const clickhouse = getClient(); - if (!clickhouse) return; - - try { - await clickhouse.command({ - query: ` - CREATE TABLE IF NOT EXISTS ad_events - ( - event_id String, - timestamp DateTime64(3) DEFAULT now64(3), - -- Event type (request, impression, click, dismiss) - event_type LowCardinality(String) DEFAULT 'request', - -- Request context - url String, - hostname LowCardinality(String), - article_title String DEFAULT '', - article_content_length UInt32 DEFAULT 0, - session_id String, - -- User context - user_id String DEFAULT '', - is_premium UInt8 DEFAULT 0, - -- Device context - device_type LowCardinality(String) DEFAULT '', - os LowCardinality(String) DEFAULT '', - browser LowCardinality(String) DEFAULT '', - -- Response - status LowCardinality(String), - gravity_status_code UInt16 DEFAULT 0, - error_message String DEFAULT '', - -- Gravity forwarding status (for impressions) - -- 1 = successfully forwarded to Gravity, 0 = failed or not applicable - gravity_forwarded UInt8 DEFAULT 0, - -- Ad data (when filled) - brand_name LowCardinality(String) DEFAULT '', - ad_title String DEFAULT '', - ad_text String DEFAULT '', - click_url String DEFAULT '', - imp_url String DEFAULT '', - cta LowCardinality(String) DEFAULT '', - favicon String DEFAULT '', - ad_count UInt8 DEFAULT 0, - -- Performance - duration_ms UInt32 DEFAULT 0, - -- Environment - env LowCardinality(String) DEFAULT 'production' - ) - ENGINE = MergeTree() - PARTITION BY toYYYYMM(timestamp) - ORDER BY (hostname, event_type, status, timestamp, event_id) - TTL toDateTime(timestamp) + INTERVAL 90 DAY - SETTINGS index_granularity = 8192 - `, - }); - - // Add new columns for existing tables (safe migration) - try { - await clickhouse.command({ - query: `ALTER TABLE ad_events ADD COLUMN IF NOT EXISTS event_type LowCardinality(String) DEFAULT 'request'`, - }); - await clickhouse.command({ - query: `ALTER TABLE ad_events ADD COLUMN IF NOT EXISTS ad_text String DEFAULT ''`, - }); - await clickhouse.command({ - query: `ALTER TABLE ad_events ADD COLUMN IF NOT EXISTS click_url String DEFAULT ''`, - }); - await clickhouse.command({ - query: `ALTER TABLE ad_events ADD COLUMN IF NOT EXISTS imp_url String DEFAULT ''`, - }); - await clickhouse.command({ - query: `ALTER TABLE ad_events ADD COLUMN IF NOT EXISTS cta LowCardinality(String) DEFAULT ''`, - }); - await clickhouse.command({ - query: `ALTER TABLE ad_events ADD COLUMN IF NOT EXISTS favicon String DEFAULT ''`, - }); - await clickhouse.command({ - query: `ALTER TABLE ad_events ADD COLUMN IF NOT EXISTS ad_count UInt8 DEFAULT 0`, - }); - // Track whether impression was successfully forwarded to Gravity (for billing) - await clickhouse.command({ - query: `ALTER TABLE ad_events ADD COLUMN IF NOT EXISTS gravity_forwarded UInt8 DEFAULT 0`, - }); - } catch { - // Ignore errors - columns may already exist - } - - // Create materialized views for ad analytics performance - try { - // Hourly ad metrics materialized view - await clickhouse.command({ - query: ` - CREATE MATERIALIZED VIEW IF NOT EXISTS ad_hourly_metrics_mv - ENGINE = SummingMergeTree() - PARTITION BY toYYYYMM(hour) - ORDER BY (hour, device_type, browser) - AS SELECT - toStartOfHour(timestamp) AS hour, - if(device_type = '', 'unknown', device_type) AS device_type, - if(browser = '', 'unknown', browser) AS browser, - countIf(event_type = 'request' AND status = 'filled') AS filled_count, - countIf(event_type = 'impression') AS impression_count, - countIf(event_type = 'click') AS click_count, - countIf(event_type = 'dismiss') AS dismiss_count, - uniqState(session_id) AS unique_sessions_state - FROM ad_events - GROUP BY hour, device_type, browser - `, - }); - - // CTR by hour of day materialized view - await clickhouse.command({ - query: ` - CREATE MATERIALIZED VIEW IF NOT EXISTS ad_ctr_by_hour_mv - ENGINE = SummingMergeTree() - PARTITION BY toYYYYMM(date) - ORDER BY (date, hour_of_day, device_type) - AS SELECT - toDate(timestamp) AS date, - toHour(timestamp) AS hour_of_day, - if(device_type = '', 'unknown', device_type) AS device_type, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - countIf(event_type = 'request' AND status = 'filled') AS filled, - countIf(event_type = 'request' AND status != 'premium_user') AS requests - FROM ad_events - GROUP BY date, hour_of_day, device_type - `, - }); - - console.log("[clickhouse] Ad materialized views created"); - } catch { - // Ignore errors - views may already exist - } - - adSchemaMigrated = true; - console.log("[clickhouse] Ad events schema migration complete"); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - if (message.includes("ECONNREFUSED") || message.includes("ENOTFOUND") || message.includes("ETIMEDOUT") || message.includes("Authentication failed")) { - disableClickhouse(message); - } else { - console.error("[clickhouse] Ad schema migration failed:", message); - } - } -} - -/** - * Flush ad events to ClickHouse - */ -async function flushAdEvents(): Promise { - if (adEventBuffer.length === 0) return; - - const clickhouse = getClient(); - if (!clickhouse) return; - - await ensureAdSchema(); - - const events = adEventBuffer.splice(0, adEventBuffer.length); - - try { - await clickhouse.insert({ - table: "ad_events", - values: events, - format: "JSONEachRow", - }); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - if (message.includes("ECONNREFUSED") || message.includes("ENOTFOUND") || message.includes("ETIMEDOUT") || message.includes("Authentication failed")) { - disableClickhouse(message); - } else { - console.error("[clickhouse] Ad events flush failed:", message); - } - } -} - -/** - * Schedule ad events flush - */ -function scheduleAdFlush(): void { - if (adFlushTimer) return; - adFlushTimer = setTimeout(async () => { - adFlushTimer = null; - await flushAdEvents(); - }, FLUSH_INTERVAL_MS); - adFlushTimer.unref(); -} - -/** - * Track an ad event - */ -export function trackAdEvent(event: Partial): void { - const rawTimestamp = event.timestamp || new Date().toISOString(); - const clickhouseTimestamp = rawTimestamp.replace("T", " ").replace("Z", ""); - - const fullEvent: AdEvent = { - event_id: event.event_id || crypto.randomUUID(), - timestamp: clickhouseTimestamp, - event_type: event.event_type || "request", - url: event.url || "", - hostname: event.hostname || "", - article_title: (event.article_title || "").slice(0, 500), - article_content_length: event.article_content_length || 0, - session_id: event.session_id || "", - user_id: event.user_id || "", - is_premium: event.is_premium || 0, - device_type: event.device_type || "", - os: event.os || "", - browser: event.browser || "", - status: event.status || "error", - gravity_status_code: event.gravity_status_code || 0, - error_message: (event.error_message || "").slice(0, 500), - gravity_forwarded: event.gravity_forwarded || 0, - brand_name: event.brand_name || "", - ad_title: (event.ad_title || "").slice(0, 500), - ad_text: (event.ad_text || "").slice(0, 1000), - click_url: (event.click_url || "").slice(0, 2000), - imp_url: (event.imp_url || "").slice(0, 2000), - cta: (event.cta || "").slice(0, 100), - favicon: (event.favicon || "").slice(0, 500), - ad_count: event.ad_count || 0, - duration_ms: event.duration_ms || 0, - env: event.env || env.NODE_ENV, - }; - - if (adEventBuffer.length >= MAX_BUFFER_SIZE) { - adEventBuffer.shift(); - } - - adEventBuffer.push(fullEvent); - - if (adEventBuffer.length >= BATCH_SIZE) { - flushAdEvents().catch(() => {}); - } else { - scheduleAdFlush(); - } -} - -// ============================================================================= -// Request Event Tracking -// ============================================================================= - -// MEMORY SAFETY: Bounded buffer with strict max size -const MAX_BUFFER_SIZE = 500; -const BATCH_SIZE = 50; -const FLUSH_INTERVAL_MS = 5000; - -// CONCURRENCY CONTROL: Limit concurrent queries to prevent thread exhaustion -// ClickHouse has limited threads (typically 28), so we limit concurrent queries -// Admin dashboard runs 13 queries in parallel, so we need at least ~7 slots -const MAX_CONCURRENT_QUERIES = 8; -const QUERY_SLOT_TIMEOUT_MS = 60_000; // 60s timeout waiting for slot -let activeQueries = 0; -const queryQueue: Array<{ - resolve: () => void; - reject: (err: Error) => void; -}> = []; - -async function acquireQuerySlot(): Promise { - if (activeQueries < MAX_CONCURRENT_QUERIES) { - activeQueries++; - return; - } - // Wait for a slot to become available (with timeout) - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - const idx = queryQueue.findIndex((q) => q.resolve === wrappedResolve); - if (idx !== -1) queryQueue.splice(idx, 1); - reject(new Error("Query slot timeout - too many concurrent queries")); - }, QUERY_SLOT_TIMEOUT_MS); - - const wrappedResolve = () => { - clearTimeout(timeout); - resolve(); - }; - - queryQueue.push({ resolve: wrappedResolve, reject }); - }); -} - -function releaseQuerySlot(): void { - activeQueries--; - const next = queryQueue.shift(); - if (next) { - activeQueries++; - next.resolve(); - } -} - -const eventBuffer: AnalyticsEvent[] = []; -let flushTimer: NodeJS.Timeout | null = null; -let isInitialized = false; -let schemaMigrated = false; - -/** - * Auto-migrate schema on first use - * Creates database and table if they don't exist - */ -async function ensureSchema(): Promise { - if (schemaMigrated) return; - - const clickhouse = getClient(); - if (!clickhouse) return; - - try { - // Create database if not exists - await clickhouse.command({ - query: `CREATE DATABASE IF NOT EXISTS ${env.CLICKHOUSE_DATABASE}`, - }); - - // Create main events table - await clickhouse.command({ - query: ` - CREATE TABLE IF NOT EXISTS request_events - ( - request_id String, - timestamp DateTime64(3) DEFAULT now64(3), - method LowCardinality(String), - endpoint LowCardinality(String), - path String, - url String, - hostname LowCardinality(String), - source LowCardinality(String), - outcome LowCardinality(String), - status_code UInt16, - error_type LowCardinality(String) DEFAULT '', - error_message String DEFAULT '', - error_severity LowCardinality(String) DEFAULT '', - upstream_hostname LowCardinality(String) DEFAULT '', - upstream_status_code UInt16 DEFAULT 0, - upstream_error_code LowCardinality(String) DEFAULT '', - upstream_message String DEFAULT '', - duration_ms UInt32, - fetch_ms UInt32 DEFAULT 0, - cache_lookup_ms UInt32 DEFAULT 0, - cache_save_ms UInt32 DEFAULT 0, - cache_hit UInt8 DEFAULT 0, - cache_status LowCardinality(String) DEFAULT '', - article_length UInt32 DEFAULT 0, - article_title String DEFAULT '', - summary_length UInt32 DEFAULT 0, - input_tokens UInt32 DEFAULT 0, - output_tokens UInt32 DEFAULT 0, - is_premium UInt8 DEFAULT 0, - client_ip String DEFAULT '', - user_agent String DEFAULT '', - heap_used_mb UInt16 DEFAULT 0, - heap_total_mb UInt16 DEFAULT 0, - rss_mb UInt16 DEFAULT 0, - env LowCardinality(String) DEFAULT 'production', - version String DEFAULT '' - ) - ENGINE = MergeTree() - PARTITION BY toYYYYMM(timestamp) - ORDER BY (hostname, source, timestamp, request_id) - TTL toDateTime(timestamp) + INTERVAL 30 DAY - SETTINGS index_granularity = 8192 - `, - }); - - // Add new upstream columns to existing tables (safe for already-existing tables) - try { - await clickhouse.command({ - query: `ALTER TABLE request_events ADD COLUMN IF NOT EXISTS upstream_hostname LowCardinality(String) DEFAULT ''`, - }); - await clickhouse.command({ - query: `ALTER TABLE request_events ADD COLUMN IF NOT EXISTS upstream_status_code UInt16 DEFAULT 0`, - }); - await clickhouse.command({ - query: `ALTER TABLE request_events ADD COLUMN IF NOT EXISTS upstream_error_code LowCardinality(String) DEFAULT ''`, - }); - await clickhouse.command({ - query: `ALTER TABLE request_events ADD COLUMN IF NOT EXISTS upstream_message String DEFAULT ''`, - }); - } catch { - // Ignore errors - columns may already exist - } - - schemaMigrated = true; - console.log("[clickhouse] Schema migration complete"); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - // Check for connection errors and disable to prevent spam - if (message.includes("ECONNREFUSED") || message.includes("ENOTFOUND") || message.includes("ETIMEDOUT") || message.includes("Authentication failed")) { - disableClickhouse(message); - } else { - // Log other errors but don't disable - might be transient - console.error("[clickhouse] Schema migration failed:", message); - } - } -} - -/** - * Flush events to Clickhouse - * Non-blocking, errors are logged but never thrown - */ -async function flushEvents(): Promise { - if (eventBuffer.length === 0) return; - - const clickhouse = getClient(); - if (!clickhouse) return; - - // Ensure schema exists before first insert - await ensureSchema(); - - // Splice out events atomically to prevent duplicates - const events = eventBuffer.splice(0, eventBuffer.length); - - try { - await clickhouse.insert({ - table: "request_events", - values: events, - format: "JSONEachRow", - }); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - // Check for connection errors and disable to prevent spam - if (message.includes("ECONNREFUSED") || message.includes("ENOTFOUND") || message.includes("ETIMEDOUT") || message.includes("Authentication failed")) { - disableClickhouse(message); - } else { - // Log other errors but don't disable - might be transient - console.error("[clickhouse] Flush failed:", message); - } - // Don't push events back - prevents infinite memory growth on persistent errors - } -} - -/** - * Schedule a flush if not already scheduled - */ -function scheduleFlush(): void { - if (flushTimer) return; - flushTimer = setTimeout(async () => { - flushTimer = null; - await flushEvents(); - }, FLUSH_INTERVAL_MS); - // Unref the timer so it doesn't keep the process alive - flushTimer.unref(); -} - -/** - * Track an analytics event - * - * Memory-safe guarantees: - * - Non-blocking (fire-and-forget) - * - Bounded buffer (drops oldest events if full) - * - No promise rejection - */ -export function trackEvent(event: Partial): void { - // Build full event with defaults - // Convert ISO timestamp to Clickhouse-compatible format (remove 'T' and 'Z') - const rawTimestamp = event.timestamp || new Date().toISOString(); - const clickhouseTimestamp = rawTimestamp.replace("T", " ").replace("Z", ""); - - const fullEvent: AnalyticsEvent = { - request_id: event.request_id || "", - timestamp: clickhouseTimestamp, - method: event.method || "", - endpoint: event.endpoint || "", - path: event.path || "", - url: event.url || "", - hostname: event.hostname || "", - source: event.source || "", - outcome: event.outcome || "", - status_code: event.status_code || 0, - error_type: event.error_type || "", - error_message: event.error_message || "", - error_severity: event.error_severity || "", - upstream_hostname: event.upstream_hostname || "", - upstream_status_code: event.upstream_status_code || 0, - upstream_error_code: event.upstream_error_code || "", - upstream_message: (event.upstream_message || "").slice(0, 500), // Truncate - duration_ms: event.duration_ms || 0, - fetch_ms: event.fetch_ms || 0, - cache_lookup_ms: event.cache_lookup_ms || 0, - cache_save_ms: event.cache_save_ms || 0, - cache_hit: event.cache_hit || 0, - cache_status: event.cache_status || "", - article_length: event.article_length || 0, - article_title: (event.article_title || "").slice(0, 500), // Truncate to prevent large strings - summary_length: event.summary_length || 0, - input_tokens: event.input_tokens || 0, - output_tokens: event.output_tokens || 0, - is_premium: event.is_premium || 0, - client_ip: event.client_ip || "", - user_agent: (event.user_agent || "").slice(0, 500), // Truncate - heap_used_mb: event.heap_used_mb || 0, - heap_total_mb: event.heap_total_mb || 0, - rss_mb: event.rss_mb || 0, - env: event.env || env.NODE_ENV, - version: event.version || process.env.npm_package_version || "unknown", - }; - - // MEMORY SAFETY: Drop oldest events if buffer is at capacity - if (eventBuffer.length >= MAX_BUFFER_SIZE) { - eventBuffer.shift(); - } - - eventBuffer.push(fullEvent); - - // Flush immediately if buffer hits batch size - if (eventBuffer.length >= BATCH_SIZE) { - // Fire-and-forget flush - flushEvents().catch(() => {}); - } else { - scheduleFlush(); - } -} - -/** - * Query helper for dashboard - * Returns empty array on error (graceful degradation) - * Uses semaphore to limit concurrent queries and prevent thread exhaustion - */ -export async function queryClickhouse(query: string): Promise { - const clickhouse = getClient(); - if (!clickhouse) return []; - - let slotAcquired = false; - - try { - // Acquire a query slot (may wait if at capacity) - await acquireQuerySlot(); - slotAcquired = true; - - const result = await clickhouse.query({ - query, - format: "JSONEachRow", - }); - return result.json(); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - // Check for connection errors and disable to prevent spam - if (message.includes("ECONNREFUSED") || message.includes("ENOTFOUND") || message.includes("ETIMEDOUT") || message.includes("Authentication failed")) { - disableClickhouse(message); - } else if (message.includes("Query slot timeout")) { - // Log slot timeouts but don't disable - indicates too many concurrent queries - console.warn("[clickhouse] Query slot timeout - too many concurrent queries"); - } else { - console.error("[clickhouse] Query failed:", message); - } - return []; - } finally { - // Only release the slot if we actually acquired one - if (slotAcquired) { - releaseQuerySlot(); - } - } -} - -/** - * Get buffer and query stats for monitoring - */ -export function getBufferStats(): { - size: number; - maxSize: number; - activeQueries: number; - queuedQueries: number; - maxConcurrentQueries: number; -} { - return { - size: eventBuffer.length, - maxSize: MAX_BUFFER_SIZE, - activeQueries, - queuedQueries: queryQueue.length, - maxConcurrentQueries: MAX_CONCURRENT_QUERIES, - }; -} - -/** - * Graceful shutdown - flush remaining events - * Called on process exit - */ -export async function closeClickhouse(): Promise { - if (flushTimer) { - clearTimeout(flushTimer); - flushTimer = null; - } - if (adFlushTimer) { - clearTimeout(adFlushTimer); - adFlushTimer = null; - } - await Promise.all([flushEvents(), flushAdEvents()]); - if (client) { - await client.close(); - client = null; - } -} - -// Register shutdown handler (only once) -if (!isInitialized && typeof process !== "undefined") { - isInitialized = true; - process.on("beforeExit", async () => { - await closeClickhouse(); - }); -} diff --git a/lib/env.ts b/lib/env.ts index 8e69caf5..6e3daf56 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -14,11 +14,11 @@ export const env = createEnv({ UPSTASH_REDIS_REST_URL: z.string().url(), UPSTASH_REDIS_REST_TOKEN: z.string().min(1), - // Analytics - CLICKHOUSE_URL: z.string().url(), - CLICKHOUSE_USER: z.string().min(1), - CLICKHOUSE_PASSWORD: z.string().min(1), - CLICKHOUSE_DATABASE: z.string().min(1), + // Analytics (PostHog) - optional, gracefully degrades when not set + POSTHOG_API_KEY: z.string().optional(), + POSTHOG_HOST: z.string().url().optional(), + POSTHOG_PROJECT_ID: z.string().optional(), + POSTHOG_PERSONAL_API_KEY: z.string().optional(), // Alerting RESEND_API_KEY: z.string().min(1), @@ -35,6 +35,8 @@ export const env = createEnv({ NEXT_PUBLIC_URL: z.string().url(), NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: z.string().min(1), NEXT_PUBLIC_CLERK_PATRON_PLAN_ID: z.string().min(1), + NEXT_PUBLIC_POSTHOG_KEY: z.string().optional(), + NEXT_PUBLIC_POSTHOG_HOST: z.string().url().optional(), }, runtimeEnv: { @@ -43,10 +45,10 @@ export const env = createEnv({ DIFFBOT_API_KEY: process.env.DIFFBOT_API_KEY, UPSTASH_REDIS_REST_URL: process.env.UPSTASH_REDIS_REST_URL, UPSTASH_REDIS_REST_TOKEN: process.env.UPSTASH_REDIS_REST_TOKEN, - CLICKHOUSE_URL: process.env.CLICKHOUSE_URL, - CLICKHOUSE_USER: process.env.CLICKHOUSE_USER, - CLICKHOUSE_PASSWORD: process.env.CLICKHOUSE_PASSWORD, - CLICKHOUSE_DATABASE: process.env.CLICKHOUSE_DATABASE, + POSTHOG_API_KEY: process.env.POSTHOG_API_KEY, + POSTHOG_HOST: process.env.POSTHOG_HOST, + POSTHOG_PROJECT_ID: process.env.POSTHOG_PROJECT_ID, + POSTHOG_PERSONAL_API_KEY: process.env.POSTHOG_PERSONAL_API_KEY, RESEND_API_KEY: process.env.RESEND_API_KEY, ALERT_EMAIL: process.env.ALERT_EMAIL, CORS_ORIGIN: process.env.CORS_ORIGIN, @@ -56,6 +58,8 @@ export const env = createEnv({ NEXT_PUBLIC_URL: process.env.NEXT_PUBLIC_URL, NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: process.env.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY, NEXT_PUBLIC_CLERK_PATRON_PLAN_ID: process.env.NEXT_PUBLIC_CLERK_PATRON_PLAN_ID, + NEXT_PUBLIC_POSTHOG_KEY: process.env.NEXT_PUBLIC_POSTHOG_KEY, + NEXT_PUBLIC_POSTHOG_HOST: process.env.NEXT_PUBLIC_POSTHOG_HOST, }, emptyStringAsUndefined: true, diff --git a/lib/memory-monitor.ts b/lib/memory-monitor.ts index 5b9fe1d7..7c8a69bd 100644 --- a/lib/memory-monitor.ts +++ b/lib/memory-monitor.ts @@ -4,11 +4,11 @@ * Logs memory stats every 30 seconds to help identify memory leaks. * Triggers garbage collection (Node.js --expose-gc) when memory grows. * - * When RSS exceeds threshold, logs to ClickHouse for post-mortem analysis. + * When RSS exceeds threshold, logs to PostHog for post-mortem analysis. * Railway's healthcheck on /health will detect the unhealthy status and restart. */ -import { trackEvent } from "./clickhouse"; +import { trackEvent } from "./posthog"; const INTERVAL_MS = 30_000; // 30 seconds const CRITICAL_RSS_MB = 1500; // 1.5GB - force restart above this @@ -117,7 +117,7 @@ function logMemory(): void { }) ); - // Log to ClickHouse for analysis + // Log to PostHog for analysis trackEvent({ request_id: `gc_${Date.now()}`, endpoint: "/internal/gc", @@ -171,7 +171,7 @@ function logMemory(): void { }) ); - // Log to ClickHouse for post-mortem analysis + // Log to PostHog for post-mortem analysis trackEvent({ request_id: `memory_spike_${Date.now()}`, endpoint: "/internal/memory", @@ -203,7 +203,7 @@ function logMemory(): void { }) ); - // Log to ClickHouse for post-mortem analysis + // Log to PostHog for post-mortem analysis trackEvent({ request_id: `memory_critical_${Date.now()}`, endpoint: "/internal/memory", diff --git a/lib/posthog.ts b/lib/posthog.ts new file mode 100644 index 00000000..709dbcf0 --- /dev/null +++ b/lib/posthog.ts @@ -0,0 +1,239 @@ +import { PostHog } from "posthog-node"; + +/** + * PostHog Analytics Client + * + * Replaces the custom ClickHouse setup. PostHog handles batching, + * retries, and connection management internally via its SDK. + * + * Env vars: + * POSTHOG_API_KEY – project API key (server-side) + * POSTHOG_HOST – PostHog instance URL + * POSTHOG_PROJECT_ID – numeric project ID (for HogQL queries) + * POSTHOG_PERSONAL_API_KEY – personal API key (for HogQL query API) + */ + +let client: PostHog | null = null; + +function getClient(): PostHog | null { + if (client) return client; + + const apiKey = process.env.POSTHOG_API_KEY; + const host = process.env.POSTHOG_HOST; + if (!apiKey || !host) return null; + + client = new PostHog(apiKey, { + host, + flushAt: 50, + flushInterval: 5000, + }); + return client; +} + +// --------------------------------------------------------------------------- +// Type exports (unchanged from clickhouse.ts) +// --------------------------------------------------------------------------- + +export type ErrorSeverity = "expected" | "degraded" | "unexpected" | ""; + +export interface AnalyticsEvent { + request_id: string; + timestamp: string; + method: string; + endpoint: string; + path: string; + url: string; + hostname: string; + source: string; + outcome: string; + status_code: number; + error_type: string; + error_message: string; + error_severity: ErrorSeverity; + upstream_hostname: string; + upstream_status_code: number; + upstream_error_code: string; + upstream_message: string; + duration_ms: number; + fetch_ms: number; + cache_lookup_ms: number; + cache_save_ms: number; + cache_hit: number; + cache_status: string; + article_length: number; + article_title: string; + summary_length: number; + input_tokens: number; + output_tokens: number; + is_premium: number; + client_ip: string; + user_agent: string; + heap_used_mb: number; + heap_total_mb: number; + rss_mb: number; + env: string; + version: string; +} + +// --------------------------------------------------------------------------- +// Ad Event Types +// --------------------------------------------------------------------------- + +export type AdEventStatus = "filled" | "no_fill" | "premium_user" | "gravity_error" | "timeout" | "error"; +export type AdEventType = "request" | "impression" | "click" | "dismiss"; + +export interface AdEvent { + event_id: string; + timestamp: string; + event_type: AdEventType; + url: string; + hostname: string; + article_title: string; + article_content_length: number; + session_id: string; + user_id: string; + is_premium: number; + device_type: string; + os: string; + browser: string; + status: AdEventStatus; + gravity_status_code: number; + error_message: string; + gravity_forwarded: number; + brand_name: string; + ad_title: string; + ad_text: string; + click_url: string; + imp_url: string; + cta: string; + favicon: string; + ad_count: number; + duration_ms: number; + env: string; +} + +// --------------------------------------------------------------------------- +// trackEvent – captures request analytics +// --------------------------------------------------------------------------- + +export function trackEvent(event: Partial): void { + const posthog = getClient(); + if (!posthog) return; + + const distinctId = event.request_id || `req_${crypto.randomUUID().slice(0, 8)}`; + + posthog.capture({ + distinctId, + event: "request_event", + properties: { + ...event, + timestamp: event.timestamp || new Date().toISOString(), + }, + }); +} + +// --------------------------------------------------------------------------- +// trackAdEvent – captures ad funnel analytics +// --------------------------------------------------------------------------- + +export function trackAdEvent(event: Partial): void { + const posthog = getClient(); + if (!posthog) return; + + const eventId = event.event_id || crypto.randomUUID(); + const distinctId = event.session_id || eventId; + + posthog.capture({ + distinctId, + event: "ad_event", + properties: { + ...event, + event_id: eventId, + timestamp: event.timestamp || new Date().toISOString(), + }, + }); +} + +// --------------------------------------------------------------------------- +// queryPostHog – HogQL query API (replaces queryClickhouse) +// --------------------------------------------------------------------------- + +export async function queryPostHog(query: string): Promise { + const host = process.env.POSTHOG_HOST; + const projectId = process.env.POSTHOG_PROJECT_ID; + const personalApiKey = process.env.POSTHOG_PERSONAL_API_KEY; + + if (!host || !projectId || !personalApiKey) return []; + + try { + const response = await fetch(`${host}/api/projects/${projectId}/query/`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${personalApiKey}`, + }, + body: JSON.stringify({ query: { kind: "HogQLQuery", query } }), + }); + + if (!response.ok) { + console.error(`[posthog] HogQL query failed (${response.status}):`, await response.text().catch(() => "")); + return []; + } + + const data = await response.json(); + // HogQL returns { columns: string[], results: any[][] } + const columns: string[] = data.columns ?? []; + const rows: unknown[][] = data.results ?? []; + + return rows.map((row) => { + const obj: Record = {}; + columns.forEach((col, i) => { + obj[col] = row[i]; + }); + return obj as T; + }); + } catch (error) { + console.error("[posthog] HogQL query error:", error instanceof Error ? error.message : String(error)); + return []; + } +} + +// --------------------------------------------------------------------------- +// getBufferStats – simplified (PostHog SDK manages its own buffer) +// --------------------------------------------------------------------------- + +export function getBufferStats(): { + size: number; + maxSize: number; + activeQueries: number; + queuedQueries: number; + maxConcurrentQueries: number; +} { + return { + size: 0, + maxSize: 0, + activeQueries: 0, + queuedQueries: 0, + maxConcurrentQueries: 0, + }; +} + +// --------------------------------------------------------------------------- +// closePostHog – graceful shutdown +// --------------------------------------------------------------------------- + +export async function closePostHog(): Promise { + if (client) { + await client.shutdown(); + client = null; + } +} + +// Register shutdown handler +let isInitialized = false; +if (!isInitialized && typeof process !== "undefined") { + isInitialized = true; + process.on("beforeExit", async () => { + await closePostHog(); + }); +} diff --git a/lib/request-context.ts b/lib/request-context.ts index ce1171e1..be6d3f1e 100644 --- a/lib/request-context.ts +++ b/lib/request-context.ts @@ -1,6 +1,6 @@ import { createLogger } from "./logger"; import { randomUUID } from "crypto"; -import { trackEvent, ErrorSeverity } from "./clickhouse"; +import { trackEvent, ErrorSeverity } from "./posthog"; import { env } from "../server/env"; /** @@ -146,8 +146,7 @@ export function createRequestContext(initial?: InitialContext): RequestContext { logger.error(event, "request completed"); } - // Send to Clickhouse analytics (fire-and-forget, non-blocking) - // trackEvent is memory-safe: bounded buffer, auto-flush, no errors thrown + // Send to PostHog analytics (fire-and-forget, non-blocking) trackEvent({ request_id: event.request_id as string, timestamp: event.timestamp as string, diff --git a/package.json b/package.json index b9a1fc1d..607b8c86 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "node": ">=24" }, "scripts": { - "dev": "docker-compose up -d clickhouse && bun run --watch server/index.ts & next dev", + "dev": "bun run --watch server/index.ts & next dev", "dev:server": "bun run --watch server/index.ts", "dev:next": "next dev", "dev:app-only": "bun run --watch server/index.ts & next dev", @@ -34,7 +34,6 @@ "@base-ui/react": "^1.1.0", "@clerk/backend": "^2.29.0", "@clerk/nextjs": "^6.36.5", - "@clickhouse/client": "^1.15.0", "@databuddy/sdk": "^2.3.29", "@elysiajs/cors": "^1.4.1", "@elysiajs/cron": "^1.4.1", @@ -74,6 +73,8 @@ "next-themes": "^0.4.6", "nuqs": "^2.8.0", "pino": "^8.19.0", + "posthog-js": "^1.341.1", + "posthog-node": "^5.24.10", "react": "19.2.1", "react-dom": "19.2.1", "react-markdown": "^9.0.1", diff --git a/railway.template.json b/railway.template.json index e985afc0..2ee87a38 100644 --- a/railway.template.json +++ b/railway.template.json @@ -1,7 +1,7 @@ { "$schema": "https://railway.app/railway.schema.json", "name": "SMRY.ai", - "description": "Paywall bypass and article summarization with Clickhouse analytics", + "description": "Paywall bypass and article summarization with PostHog analytics", "buttons": [ { "name": "Deploy to Railway", @@ -30,22 +30,29 @@ "description": "Public URL of your deployment (e.g., https://smry.ai)", "required": true }, - "CLICKHOUSE_URL": { - "value": "http://${{clickhouse.RAILWAY_PRIVATE_DOMAIN}}:8123" + "POSTHOG_API_KEY": { + "description": "PostHog project API key (server-side)", + "required": true + }, + "POSTHOG_HOST": { + "description": "PostHog instance URL (e.g., https://us.i.posthog.com)", + "required": true }, - "CLICKHOUSE_USER": { - "value": "default" + "POSTHOG_PROJECT_ID": { + "description": "PostHog project ID (for HogQL queries)", + "required": true }, - "CLICKHOUSE_PASSWORD": { - "value": "${{clickhouse.CLICKHOUSE_PASSWORD}}" + "POSTHOG_PERSONAL_API_KEY": { + "description": "PostHog personal API key (for HogQL query API)", + "required": true }, - "CLICKHOUSE_DATABASE": { - "value": "smry_analytics" + "NEXT_PUBLIC_POSTHOG_KEY": { + "description": "PostHog project API key (client-side)", + "required": true }, - "ANALYTICS_SECRET_KEY": { - "description": "Secret key for accessing /admin/analytics", - "required": true, - "generate": true + "NEXT_PUBLIC_POSTHOG_HOST": { + "description": "PostHog instance URL (client-side)", + "required": true }, "UPSTASH_REDIS_REST_URL": { "description": "Upstash Redis REST URL", @@ -77,43 +84,6 @@ "enabled": true } } - }, - "clickhouse": { - "name": "clickhouse", - "description": "Clickhouse analytics database (memory-optimized)", - "source": { - "repo": "https://github.com/mrmps/SMRY" - }, - "build": { - "builder": "DOCKERFILE", - "dockerfilePath": "docker/clickhouse/Dockerfile", - "buildContext": "docker/clickhouse" - }, - "variables": { - "CLICKHOUSE_DB": { - "value": "smry_analytics" - }, - "CLICKHOUSE_USER": { - "value": "default" - }, - "CLICKHOUSE_PASSWORD": { - "generate": true - }, - "CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT": { - "value": "1" - } - }, - "volumes": [ - { - "mount": "/var/lib/clickhouse", - "name": "clickhouse-data" - } - ], - "networking": { - "public": { - "enabled": false - } - } } } } diff --git a/scripts/analyze-sources.ts b/scripts/analyze-sources.ts deleted file mode 100644 index 0bfff2bb..00000000 --- a/scripts/analyze-sources.ts +++ /dev/null @@ -1,380 +0,0 @@ -/** - * Analyze source effectiveness from ClickHouse logs - * Run with: bun run scripts/analyze-sources.ts - * Make sure .env.local is loaded or env vars are set - */ - -import { createClient } from "@clickhouse/client"; - -// Load .env.local manually for bun -const projectRoot = import.meta.dir.replace("/scripts", ""); -const envFile = Bun.file(`${projectRoot}/.env`); -const envContent = await envFile.text(); -for (const line of envContent.split("\n")) { - const trimmed = line.trim(); - if (trimmed && !trimmed.startsWith("#")) { - const [key, ...valueParts] = trimmed.split("="); - if (key && valueParts.length > 0) { - const value = valueParts.join("=").replace(/^["']|["']$/g, ""); - process.env[key] = value; - } - } -} - -const client = createClient({ - url: process.env.CLICKHOUSE_URL!, - username: process.env.CLICKHOUSE_USER!, - password: process.env.CLICKHOUSE_PASSWORD!, - database: process.env.CLICKHOUSE_DATABASE!, - request_timeout: 60_000, -}); - -async function query(sql: string): Promise { - const result = await client.query({ query: sql, format: "JSONEachRow" }); - return result.json(); -} - -async function main() { - console.log("=== Source Effectiveness Analysis ===\n"); - - // 1. Overall source success rates - console.log("1. OVERALL SOURCE SUCCESS RATES (last 7 days)"); - console.log("-".repeat(60)); - const sourceRates = await query<{ - source: string; - total: string; - successes: string; - success_rate: string; - }>(` - SELECT - source, - count() as total, - countIf(outcome = 'success') as successes, - round(countIf(outcome = 'success') / count() * 100, 2) as success_rate - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source != '' - AND endpoint = '/api/article' - GROUP BY source - ORDER BY total DESC - `); - console.table(sourceRates); - - // 2. For URLs where multiple sources were tried, how many had only one success? - console.log("\n2. URLs WHERE ONLY ONE SOURCE SUCCEEDED (last 7 days)"); - console.log("-".repeat(60)); - const onlyOneWorked = await query<{ - url: string; - sources_tried: string; - sources_succeeded: string; - successful_source: string; - }>(` - SELECT - url, - uniq(source) as sources_tried, - uniqIf(source, outcome = 'success') as sources_succeeded, - groupArrayIf(source, outcome = 'success')[1] as successful_source - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source != '' - AND endpoint = '/api/article' - GROUP BY url - HAVING sources_tried >= 2 AND sources_succeeded = 1 - ORDER BY sources_tried DESC - LIMIT 50 - `); - console.log(`Found ${onlyOneWorked.length} URLs where only 1 source worked`); - if (onlyOneWorked.length > 0) { - console.table(onlyOneWorked.slice(0, 20)); - } - - // 3. Which source is the "only one that works" most often? - console.log("\n3. WHEN ONLY ONE SOURCE WORKS, WHICH ONE? (last 7 days)"); - console.log("-".repeat(60)); - const singleSourceWinner = await query<{ - successful_source: string; - count: string; - percentage: string; - }>(` - WITH single_success_urls AS ( - SELECT - url, - groupArrayIf(source, outcome = 'success')[1] as successful_source - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source != '' - AND endpoint = '/api/article' - GROUP BY url - HAVING uniq(source) >= 2 AND uniqIf(source, outcome = 'success') = 1 - ) - SELECT - successful_source, - count() as count, - round(count() / (SELECT count() FROM single_success_urls) * 100, 2) as percentage - FROM single_success_urls - GROUP BY successful_source - ORDER BY count DESC - `); - console.table(singleSourceWinner); - - // 4. What about when ALL sources fail vs when at least one works? - console.log("\n4. URL OUTCOME DISTRIBUTION (last 7 days)"); - console.log("-".repeat(60)); - const urlOutcomes = await query<{ - outcome_type: string; - url_count: string; - percentage: string; - }>(` - WITH url_stats AS ( - SELECT - url, - uniq(source) as sources_tried, - uniqIf(source, outcome = 'success') as sources_succeeded - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source != '' - AND endpoint = '/api/article' - GROUP BY url - HAVING sources_tried >= 2 - ) - SELECT - CASE - WHEN sources_succeeded = 0 THEN 'all_failed' - WHEN sources_succeeded = 1 THEN 'only_one_worked' - WHEN sources_succeeded = 2 THEN 'two_worked' - WHEN sources_succeeded = 3 THEN 'three_worked' - ELSE 'all_worked' - END as outcome_type, - count() as url_count, - round(count() / (SELECT count() FROM url_stats) * 100, 2) as percentage - FROM url_stats - GROUP BY outcome_type - ORDER BY url_count DESC - `); - console.table(urlOutcomes); - - // 5. Correlation: when smry-fast fails, how often does smry-slow/wayback save the day? - console.log("\n5. FALLBACK EFFECTIVENESS: When smry-fast fails... (last 7 days)"); - console.log("-".repeat(60)); - const fallbackStats = await query<{ - scenario: string; - count: string; - percentage: string; - }>(` - WITH url_outcomes AS ( - SELECT - url, - maxIf(1, source = 'smry-fast' AND outcome = 'success') as fast_success, - maxIf(1, source = 'smry-slow' AND outcome = 'success') as slow_success, - maxIf(1, source = 'wayback' AND outcome = 'success') as wayback_success, - maxIf(1, source = 'smry-fast') as fast_tried, - maxIf(1, source = 'smry-slow') as slow_tried, - maxIf(1, source = 'wayback') as wayback_tried - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow', 'wayback') - AND endpoint = '/api/article' - GROUP BY url - HAVING fast_tried = 1 AND fast_success = 0 -- smry-fast was tried and failed - ) - SELECT - CASE - WHEN slow_success = 1 AND wayback_success = 1 THEN 'both smry-slow and wayback worked' - WHEN slow_success = 1 THEN 'only smry-slow worked' - WHEN wayback_success = 1 THEN 'only wayback worked' - ELSE 'nothing worked' - END as scenario, - count() as count, - round(count() / (SELECT count() FROM url_outcomes) * 100, 2) as percentage - FROM url_outcomes - GROUP BY scenario - ORDER BY count DESC - `); - console.table(fallbackStats); - - // 6. Average latency by source - console.log("\n6. LATENCY BY SOURCE (last 7 days)"); - console.log("-".repeat(60)); - const latencyStats = await query<{ - source: string; - avg_ms: string; - p50_ms: string; - p95_ms: string; - p99_ms: string; - }>(` - SELECT - source, - round(avg(fetch_ms)) as avg_ms, - round(quantile(0.5)(fetch_ms)) as p50_ms, - round(quantile(0.95)(fetch_ms)) as p95_ms, - round(quantile(0.99)(fetch_ms)) as p99_ms - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source != '' - AND endpoint = '/api/article' - AND outcome = 'success' - AND fetch_ms > 0 - GROUP BY source - ORDER BY avg_ms - `); - console.table(latencyStats); - - // 7. If we called sources SEQUENTIALLY (fast -> slow -> wayback), what would be the impact? - console.log("\n7. SEQUENTIAL STRATEGY SIMULATION (last 7 days)"); - console.log("-".repeat(60)); - const sequentialSim = await query<{ - strategy: string; - urls_resolved: string; - avg_api_calls_per_url: string; - total_api_calls: string; - }>(` - WITH url_outcomes AS ( - SELECT - url, - maxIf(1, source = 'smry-fast' AND outcome = 'success') as fast_success, - maxIf(1, source = 'smry-slow' AND outcome = 'success') as slow_success, - maxIf(1, source = 'wayback' AND outcome = 'success') as wayback_success - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow', 'wayback') - AND endpoint = '/api/article' - GROUP BY url - ), - sequential_analysis AS ( - SELECT - url, - fast_success, - slow_success, - wayback_success, - CASE - WHEN fast_success = 1 THEN 1 -- just fast - WHEN slow_success = 1 THEN 2 -- fast failed, then slow - WHEN wayback_success = 1 THEN 3 -- fast+slow failed, then wayback - ELSE 3 -- tried all 3, none worked - END as calls_needed_sequential, - 3 as calls_parallel - FROM url_outcomes - ) - SELECT - 'Current (parallel)' as strategy, - toString(countIf(fast_success = 1 OR slow_success = 1 OR wayback_success = 1)) as urls_resolved, - '3.00' as avg_api_calls_per_url, - toString(count() * 3) as total_api_calls - FROM sequential_analysis - UNION ALL - SELECT - 'Sequential (fast->slow->wayback)' as strategy, - toString(countIf(fast_success = 1 OR slow_success = 1 OR wayback_success = 1)) as urls_resolved, - toString(round(avg(calls_needed_sequential), 2)) as avg_api_calls_per_url, - toString(sum(calls_needed_sequential)) as total_api_calls - FROM sequential_analysis - `); - console.table(sequentialSim); - - // 8. What if we removed a source entirely? - console.log("\n8. IMPACT OF REMOVING A SOURCE (last 7 days)"); - console.log("-".repeat(60)); - const removalImpact = await query<{ - scenario: string; - urls_resolved: string; - resolution_rate: string; - }>(` - WITH url_outcomes AS ( - SELECT - url, - maxIf(1, source = 'smry-fast' AND outcome = 'success') as fast_success, - maxIf(1, source = 'smry-slow' AND outcome = 'success') as slow_success, - maxIf(1, source = 'wayback' AND outcome = 'success') as wayback_success - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow', 'wayback') - AND endpoint = '/api/article' - GROUP BY url - ) - SELECT - 'All 3 sources' as scenario, - toString(countIf(fast_success = 1 OR slow_success = 1 OR wayback_success = 1)) as urls_resolved, - toString(round(countIf(fast_success = 1 OR slow_success = 1 OR wayback_success = 1) / count() * 100, 2)) as resolution_rate - FROM url_outcomes - UNION ALL - SELECT - 'Without smry-fast' as scenario, - toString(countIf(slow_success = 1 OR wayback_success = 1)) as urls_resolved, - toString(round(countIf(slow_success = 1 OR wayback_success = 1) / count() * 100, 2)) as resolution_rate - FROM url_outcomes - UNION ALL - SELECT - 'Without smry-slow' as scenario, - toString(countIf(fast_success = 1 OR wayback_success = 1)) as urls_resolved, - toString(round(countIf(fast_success = 1 OR wayback_success = 1) / count() * 100, 2)) as resolution_rate - FROM url_outcomes - UNION ALL - SELECT - 'Without wayback' as scenario, - toString(countIf(fast_success = 1 OR slow_success = 1)) as urls_resolved, - toString(round(countIf(fast_success = 1 OR slow_success = 1) / count() * 100, 2)) as resolution_rate - FROM url_outcomes - UNION ALL - SELECT - 'Only smry-fast' as scenario, - toString(countIf(fast_success = 1)) as urls_resolved, - toString(round(countIf(fast_success = 1) / count() * 100, 2)) as resolution_rate - FROM url_outcomes - UNION ALL - SELECT - 'Only smry-slow' as scenario, - toString(countIf(slow_success = 1)) as urls_resolved, - toString(round(countIf(slow_success = 1) / count() * 100, 2)) as resolution_rate - FROM url_outcomes - `); - console.table(removalImpact); - - // 9. Unique value: URLs where ONLY a specific source works - console.log("\n9. UNIQUE VALUE: URLs where ONLY this source works (last 7 days)"); - console.log("-".repeat(60)); - const uniqueValue = await query<{ - source: string; - exclusively_resolves: string; - percentage_of_resolutions: string; - }>(` - WITH url_outcomes AS ( - SELECT - url, - maxIf(1, source = 'smry-fast' AND outcome = 'success') as fast_success, - maxIf(1, source = 'smry-slow' AND outcome = 'success') as slow_success, - maxIf(1, source = 'wayback' AND outcome = 'success') as wayback_success - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow', 'wayback') - AND endpoint = '/api/article' - GROUP BY url - ), - totals AS ( - SELECT countIf(fast_success = 1 OR slow_success = 1 OR wayback_success = 1) as total_resolved - FROM url_outcomes - ) - SELECT - 'smry-fast' as source, - toString(countIf(fast_success = 1 AND slow_success = 0 AND wayback_success = 0)) as exclusively_resolves, - toString(round(countIf(fast_success = 1 AND slow_success = 0 AND wayback_success = 0) / (SELECT total_resolved FROM totals) * 100, 2)) as percentage_of_resolutions - FROM url_outcomes - UNION ALL - SELECT - 'smry-slow' as source, - toString(countIf(fast_success = 0 AND slow_success = 1 AND wayback_success = 0)) as exclusively_resolves, - toString(round(countIf(fast_success = 0 AND slow_success = 1 AND wayback_success = 0) / (SELECT total_resolved FROM totals) * 100, 2)) as percentage_of_resolutions - FROM url_outcomes - UNION ALL - SELECT - 'wayback' as source, - toString(countIf(fast_success = 0 AND slow_success = 0 AND wayback_success = 1)) as exclusively_resolves, - toString(round(countIf(fast_success = 0 AND slow_success = 0 AND wayback_success = 1) / (SELECT total_resolved FROM totals) * 100, 2)) as percentage_of_resolutions - FROM url_outcomes - `); - console.table(uniqueValue); - - await client.close(); - console.log("\n=== Analysis Complete ==="); -} - -main().catch(console.error); diff --git a/scripts/source-analysis-queries.sql b/scripts/source-analysis-queries.sql deleted file mode 100644 index 69935022..00000000 --- a/scripts/source-analysis-queries.sql +++ /dev/null @@ -1,282 +0,0 @@ --- ============================================================================= --- SOURCE EFFECTIVENESS ANALYSIS QUERIES --- Run these in ClickHouse console to understand source behavior --- ============================================================================= - --- ----------------------------------------------------------------------------- --- 1. OVERALL SOURCE SUCCESS RATES (baseline) --- ----------------------------------------------------------------------------- -SELECT - source, - count() as total, - countIf(outcome = 'success') as successes, - round(countIf(outcome = 'success') / count() * 100, 2) as success_rate -FROM request_events -WHERE timestamp > now() - INTERVAL 7 DAY - AND source != '' - AND endpoint = '/api/article' -GROUP BY source -ORDER BY total DESC; - --- ----------------------------------------------------------------------------- --- 2. HOW OFTEN DOES ONLY ONE SOURCE WORK? (and which one?) --- This answers: "For articles where multiple sources were tried, --- how often was only one successful?" --- ----------------------------------------------------------------------------- -WITH url_outcomes AS ( - SELECT - url, - uniq(source) as sources_tried, - uniqIf(source, outcome = 'success') as sources_succeeded, - groupArrayIf(source, outcome = 'success') as successful_sources - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow', 'wayback') - AND endpoint = '/api/article' - GROUP BY url - HAVING sources_tried >= 2 -- At least 2 sources were tried -) -SELECT - CASE sources_succeeded - WHEN 0 THEN 'all_failed' - WHEN 1 THEN 'only_one_worked' - WHEN 2 THEN 'two_worked' - ELSE 'all_worked' - END as outcome_type, - count() as url_count, - round(count() / (SELECT count() FROM url_outcomes) * 100, 2) as percentage -FROM url_outcomes -GROUP BY outcome_type -ORDER BY url_count DESC; - --- ----------------------------------------------------------------------------- --- 3. WHEN ONLY ONE SOURCE WORKS, WHICH ONE IS IT? --- Shows which source is the "hero" when others fail --- ----------------------------------------------------------------------------- -WITH single_success_urls AS ( - SELECT - url, - groupArrayIf(source, outcome = 'success')[1] as successful_source - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow', 'wayback') - AND endpoint = '/api/article' - GROUP BY url - HAVING uniq(source) >= 2 AND uniqIf(source, outcome = 'success') = 1 -) -SELECT - successful_source, - count() as count, - round(count() / (SELECT count() FROM single_success_urls) * 100, 2) as percentage -FROM single_success_urls -GROUP BY successful_source -ORDER BY count DESC; - --- ----------------------------------------------------------------------------- --- 4. FALLBACK EFFECTIVENESS: When smry-fast fails, what saves the day? --- Answers: "If we called smry-fast first and it failed, how often would --- smry-slow or wayback have worked?" --- ----------------------------------------------------------------------------- -WITH url_outcomes AS ( - SELECT - url, - maxIf(1, source = 'smry-fast' AND outcome = 'success') as fast_success, - maxIf(1, source = 'smry-slow' AND outcome = 'success') as slow_success, - maxIf(1, source = 'wayback' AND outcome = 'success') as wayback_success, - maxIf(1, source = 'smry-fast') as fast_tried - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow', 'wayback') - AND endpoint = '/api/article' - GROUP BY url - HAVING fast_tried = 1 AND fast_success = 0 -- smry-fast was tried and failed -) -SELECT - CASE - WHEN slow_success = 1 AND wayback_success = 1 THEN 'both smry-slow AND wayback worked' - WHEN slow_success = 1 THEN 'only smry-slow worked' - WHEN wayback_success = 1 THEN 'only wayback worked' - ELSE 'nothing worked (hard paywall or broken)' - END as scenario, - count() as count, - round(count() / (SELECT count() FROM url_outcomes) * 100, 2) as percentage -FROM url_outcomes -GROUP BY scenario -ORDER BY count DESC; - --- ----------------------------------------------------------------------------- --- 5. SEQUENTIAL vs PARALLEL: How many API calls would we save? --- Compares current parallel strategy (always 3 calls) vs sequential --- ----------------------------------------------------------------------------- -WITH url_outcomes AS ( - SELECT - url, - maxIf(1, source = 'smry-fast' AND outcome = 'success') as fast_success, - maxIf(1, source = 'smry-slow' AND outcome = 'success') as slow_success, - maxIf(1, source = 'wayback' AND outcome = 'success') as wayback_success - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow', 'wayback') - AND endpoint = '/api/article' - GROUP BY url -), -sequential_analysis AS ( - SELECT - url, - fast_success, - slow_success, - wayback_success, - -- Sequential: stop as soon as one works - CASE - WHEN fast_success = 1 THEN 1 -- just fast - WHEN slow_success = 1 THEN 2 -- fast failed, then slow - WHEN wayback_success = 1 THEN 3 -- fast+slow failed, then wayback - ELSE 3 -- tried all 3, none worked - END as calls_needed_sequential - FROM url_outcomes -) -SELECT - 'Current (parallel)' as strategy, - countIf(fast_success = 1 OR slow_success = 1 OR wayback_success = 1) as urls_resolved, - 3.00 as avg_api_calls_per_url, - count() * 3 as total_api_calls -FROM sequential_analysis -UNION ALL -SELECT - 'Sequential (fast→slow→wayback)' as strategy, - countIf(fast_success = 1 OR slow_success = 1 OR wayback_success = 1) as urls_resolved, - round(avg(calls_needed_sequential), 2) as avg_api_calls_per_url, - sum(calls_needed_sequential) as total_api_calls -FROM sequential_analysis; - --- ----------------------------------------------------------------------------- --- 6. IMPACT OF REMOVING A SOURCE --- What's the resolution rate if we removed each source? --- ----------------------------------------------------------------------------- -WITH url_outcomes AS ( - SELECT - url, - maxIf(1, source = 'smry-fast' AND outcome = 'success') as fast_success, - maxIf(1, source = 'smry-slow' AND outcome = 'success') as slow_success, - maxIf(1, source = 'wayback' AND outcome = 'success') as wayback_success - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow', 'wayback') - AND endpoint = '/api/article' - GROUP BY url -) -SELECT 'All 3 sources' as scenario, - countIf(fast_success = 1 OR slow_success = 1 OR wayback_success = 1) as urls_resolved, - count() as total_urls, - round(countIf(fast_success = 1 OR slow_success = 1 OR wayback_success = 1) / count() * 100, 2) as resolution_rate -FROM url_outcomes -UNION ALL -SELECT 'Without smry-fast' as scenario, - countIf(slow_success = 1 OR wayback_success = 1) as urls_resolved, - count() as total_urls, - round(countIf(slow_success = 1 OR wayback_success = 1) / count() * 100, 2) as resolution_rate -FROM url_outcomes -UNION ALL -SELECT 'Without smry-slow (Diffbot)' as scenario, - countIf(fast_success = 1 OR wayback_success = 1) as urls_resolved, - count() as total_urls, - round(countIf(fast_success = 1 OR wayback_success = 1) / count() * 100, 2) as resolution_rate -FROM url_outcomes -UNION ALL -SELECT 'Without wayback' as scenario, - countIf(fast_success = 1 OR slow_success = 1) as urls_resolved, - count() as total_urls, - round(countIf(fast_success = 1 OR slow_success = 1) / count() * 100, 2) as resolution_rate -FROM url_outcomes; - --- ----------------------------------------------------------------------------- --- 7. UNIQUE VALUE: URLs where ONLY this source works (exclusive value) --- These are the URLs you'd LOSE if you removed that source --- ----------------------------------------------------------------------------- -WITH url_outcomes AS ( - SELECT - url, - maxIf(1, source = 'smry-fast' AND outcome = 'success') as fast_success, - maxIf(1, source = 'smry-slow' AND outcome = 'success') as slow_success, - maxIf(1, source = 'wayback' AND outcome = 'success') as wayback_success - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow', 'wayback') - AND endpoint = '/api/article' - GROUP BY url -) -SELECT 'smry-fast' as source, - countIf(fast_success = 1 AND slow_success = 0 AND wayback_success = 0) as exclusively_resolves, - round(countIf(fast_success = 1 AND slow_success = 0 AND wayback_success = 0) / - countIf(fast_success = 1 OR slow_success = 1 OR wayback_success = 1) * 100, 2) as pct_of_resolutions -FROM url_outcomes -UNION ALL -SELECT 'smry-slow (Diffbot)' as source, - countIf(fast_success = 0 AND slow_success = 1 AND wayback_success = 0) as exclusively_resolves, - round(countIf(fast_success = 0 AND slow_success = 1 AND wayback_success = 0) / - countIf(fast_success = 1 OR slow_success = 1 OR wayback_success = 1) * 100, 2) as pct_of_resolutions -FROM url_outcomes -UNION ALL -SELECT 'wayback' as source, - countIf(fast_success = 0 AND slow_success = 0 AND wayback_success = 1) as exclusively_resolves, - round(countIf(fast_success = 0 AND slow_success = 0 AND wayback_success = 1) / - countIf(fast_success = 1 OR slow_success = 1 OR wayback_success = 1) * 100, 2) as pct_of_resolutions -FROM url_outcomes; - --- ----------------------------------------------------------------------------- --- 8. LATENCY BY SOURCE (for sequential strategy timing estimation) --- ----------------------------------------------------------------------------- -SELECT - source, - round(avg(fetch_ms)) as avg_ms, - round(quantile(0.5)(fetch_ms)) as p50_ms, - round(quantile(0.95)(fetch_ms)) as p95_ms, - round(quantile(0.99)(fetch_ms)) as p99_ms -FROM request_events -WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow', 'wayback') - AND endpoint = '/api/article' - AND outcome = 'success' - AND fetch_ms > 0 -GROUP BY source -ORDER BY avg_ms; - --- ----------------------------------------------------------------------------- --- 9. HOSTNAME-SPECIFIC SOURCE EFFECTIVENESS --- Which sources work best for which sites? --- ----------------------------------------------------------------------------- -SELECT - hostname, - source, - count() as requests, - round(countIf(outcome = 'success') / count() * 100, 2) as success_rate -FROM request_events -WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow', 'wayback') - AND endpoint = '/api/article' - AND hostname != '' -GROUP BY hostname, source -HAVING requests >= 5 -- Only sites with enough data -ORDER BY hostname, success_rate DESC; - --- ----------------------------------------------------------------------------- --- 10. COST ANALYSIS: Diffbot API calls (smry-slow costs money) --- How many Diffbot calls could we save with sequential strategy? --- ----------------------------------------------------------------------------- -WITH url_outcomes AS ( - SELECT - url, - maxIf(1, source = 'smry-fast' AND outcome = 'success') as fast_success, - maxIf(1, source = 'smry-slow') as slow_tried - FROM request_events - WHERE timestamp > now() - INTERVAL 7 DAY - AND source IN ('smry-fast', 'smry-slow') - AND endpoint = '/api/article' - GROUP BY url -) -SELECT - countIf(slow_tried = 1) as current_diffbot_calls, - countIf(slow_tried = 1 AND fast_success = 0) as needed_diffbot_calls_sequential, - countIf(slow_tried = 1 AND fast_success = 1) as wasted_diffbot_calls, - round(countIf(slow_tried = 1 AND fast_success = 1) / countIf(slow_tried = 1) * 100, 2) as pct_wasted -FROM url_outcomes; diff --git a/server/env.ts b/server/env.ts index ed687a6f..38d29563 100644 --- a/server/env.ts +++ b/server/env.ts @@ -19,11 +19,11 @@ export const env = createEnv({ UPSTASH_REDIS_REST_URL: z.string().url(), UPSTASH_REDIS_REST_TOKEN: z.string().min(1), - // Analytics - CLICKHOUSE_URL: z.string().url(), - CLICKHOUSE_USER: z.string().min(1), - CLICKHOUSE_PASSWORD: z.string().min(1), - CLICKHOUSE_DATABASE: z.string().min(1), + // Analytics (PostHog) - optional, gracefully degrades when not set + POSTHOG_API_KEY: z.string().optional(), + POSTHOG_HOST: z.string().url().optional(), + POSTHOG_PROJECT_ID: z.string().optional(), + POSTHOG_PERSONAL_API_KEY: z.string().optional(), // Alerting ALERT_EMAIL: z.string().email(), diff --git a/server/index.test.ts b/server/index.test.ts index cea60a30..94695b78 100644 --- a/server/index.test.ts +++ b/server/index.test.ts @@ -66,7 +66,7 @@ describe("Elysia API Server", () => { new Request("http://localhost/api/article?url=https://httpbin.org/html&source=smry-fast") ); // May return 200 or 500 depending on external service - just verify route is hit - expect([200, 500]).toContain(response.status); + expect([200, 401, 500]).toContain(response.status); }); it("should accept valid smry-slow source", async () => { @@ -74,7 +74,7 @@ describe("Elysia API Server", () => { new Request("http://localhost/api/article?url=https://example.com&source=smry-slow") ); // Just verify route accepts the source - expect([200, 500]).toContain(response.status); + expect([200, 401, 500]).toContain(response.status); }); it("should accept valid wayback source", async () => { @@ -83,7 +83,7 @@ describe("Elysia API Server", () => { new Request("http://localhost/api/article?url=https://example.com&source=wayback") ); // Just verify route accepts the source (may timeout with 500) - expect([200, 500]).toContain(response.status); + expect([200, 401, 500]).toContain(response.status); }, { timeout: 15000 }); it("should block hard paywall sites", async () => { @@ -103,8 +103,8 @@ describe("Elysia API Server", () => { new Request("http://localhost/api/admin") ); - // May return 200 or 500 depending on ClickHouse availability - expect([200, 500]).toContain(response.status); + // May return 200, 401 (no token), or 500 depending on PostHog availability + expect([200, 401, 500]).toContain(response.status); if (response.status === 200) { const body = await response.json(); @@ -120,7 +120,7 @@ describe("Elysia API Server", () => { const response = await app.handle( new Request("http://localhost/api/admin?range=1h") ); - expect([200, 500]).toContain(response.status); + expect([200, 401, 500]).toContain(response.status); if (response.status === 200) { const body = await response.json(); @@ -132,7 +132,7 @@ describe("Elysia API Server", () => { const response = await app.handle( new Request("http://localhost/api/admin?range=7d") ); - expect([200, 500]).toContain(response.status); + expect([200, 401, 500]).toContain(response.status); if (response.status === 200) { const body = await response.json(); @@ -144,7 +144,7 @@ describe("Elysia API Server", () => { const response = await app.handle( new Request("http://localhost/api/admin?hostname=example.com&source=smry-fast&outcome=success") ); - expect([200, 500]).toContain(response.status); + expect([200, 401, 500]).toContain(response.status); if (response.status === 200) { const body = await response.json(); @@ -159,7 +159,7 @@ describe("Elysia API Server", () => { const response = await app.handle( new Request("http://localhost/api/admin?urlSearch=test") ); - expect([200, 500]).toContain(response.status); + expect([200, 401, 500]).toContain(response.status); if (response.status === 200) { const body = await response.json(); diff --git a/server/routes/admin.ts b/server/routes/admin.ts index 19485751..dde895b2 100644 --- a/server/routes/admin.ts +++ b/server/routes/admin.ts @@ -6,7 +6,7 @@ import { Elysia, t } from "elysia"; import { timingSafeEqual } from "crypto"; -import { queryClickhouse, getBufferStats } from "../../lib/clickhouse"; +import { queryPostHog, getBufferStats } from "../../lib/posthog"; import { env } from "../env"; /** @@ -420,7 +420,7 @@ export const adminRoutes = new Elysia({ prefix: "/api" }).get( const outcomeFilter = query.outcome || ""; const urlSearch = query.urlSearch || ""; - // Build WHERE clause for filtered queries + // Build WHERE clause for filtered queries (HogQL – properties.* prefix) const buildWhereClause = (options: { timeInterval?: string; includeFilters?: boolean; @@ -428,29 +428,30 @@ export const adminRoutes = new Elysia({ prefix: "/api" }).get( const { timeInterval = `${hours} HOUR`, includeFilters = true } = options; const conditions: string[] = []; + // Event type filter for PostHog events table + conditions.push(`event = 'request_event'`); + // Always include time filter conditions.push(`timestamp > now() - INTERVAL ${timeInterval}`); // Always filter out empty hostnames - conditions.push(`hostname != ''`); + conditions.push(`properties.hostname != ''`); if (includeFilters) { - // Escape backslashes first, then single quotes (order matters for SQL injection prevention) - const escapeForClickhouse = (str: string) => str.replace(/\\/g, "\\\\").replace(/'/g, "''"); - // For LIKE patterns, also escape % and _ wildcards (after backslash escaping) - const escapeForClickhouseLike = (str: string) => - escapeForClickhouse(str).replace(/%/g, "\\%").replace(/_/g, "\\_"); + const escapeStr = (str: string) => str.replace(/\\/g, "\\\\").replace(/'/g, "''"); + const escapeForLike = (str: string) => + escapeStr(str).replace(/%/g, "\\%").replace(/_/g, "\\_"); if (hostnameFilter) { - conditions.push(`hostname = '${escapeForClickhouse(hostnameFilter)}'`); + conditions.push(`properties.hostname = '${escapeStr(hostnameFilter)}'`); } if (sourceFilter) { - conditions.push(`source = '${escapeForClickhouse(sourceFilter)}'`); + conditions.push(`properties.source = '${escapeStr(sourceFilter)}'`); } if (outcomeFilter) { - conditions.push(`outcome = '${escapeForClickhouse(outcomeFilter)}'`); + conditions.push(`properties.outcome = '${escapeStr(outcomeFilter)}'`); } if (urlSearch) { - conditions.push(`url LIKE '%${escapeForClickhouseLike(urlSearch)}%'`); + conditions.push(`properties.url LIKE '%${escapeForLike(urlSearch)}%'`); } } @@ -506,206 +507,216 @@ export const adminRoutes = new Elysia({ prefix: "/api" }).get( adFunnelTimeSeries, ] = await Promise.all([ // 1. Which sites consistently error (top 200 by volume) - queryClickhouse(` + queryPostHog(` SELECT - hostname, + properties.hostname as hostname, count() AS total_requests, - round(countIf(outcome = 'success') / count() * 100, 2) AS success_rate, - countIf(outcome = 'error') AS error_count, - round(avg(duration_ms)) AS avg_duration_ms - FROM request_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND hostname != '' + round(countIf(properties.outcome = 'success') / count() * 100, 2) AS success_rate, + countIf(properties.outcome = 'error') AS error_count, + round(avg(toFloat64(properties.duration_ms))) AS avg_duration_ms + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.hostname != '' GROUP BY hostname ORDER BY total_requests DESC LIMIT 200 `), // 2. Which sources work for which sites (show all with at least 1 request) - queryClickhouse(` + queryPostHog(` SELECT - hostname, - source, - round(countIf(outcome = 'success') / count() * 100, 2) AS success_rate, + properties.hostname as hostname, + properties.source as source, + round(countIf(properties.outcome = 'success') / count() * 100, 2) AS success_rate, count() AS request_count - FROM request_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND hostname != '' - AND source != '' + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.hostname != '' + AND properties.source != '' GROUP BY hostname, source ORDER BY hostname, request_count DESC `), // 3. Hourly traffic pattern - queryClickhouse(` + queryPostHog(` SELECT formatDateTime(toStartOfHour(timestamp), '%Y-%m-%d %H:00') AS hour, count() AS request_count, - countIf(outcome = 'success') AS success_count, - countIf(outcome = 'error') AS error_count - FROM request_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND hostname != '' + countIf(properties.outcome = 'success') AS success_count, + countIf(properties.outcome = 'error') AS error_count + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.hostname != '' GROUP BY hour ORDER BY hour `), // 4. Error breakdown by hostname and type with error messages and upstream context - queryClickhouse(` + queryPostHog(` SELECT - hostname, - error_type, - any(error_message) AS error_message, + properties.hostname as hostname, + properties.error_type as error_type, + any(properties.error_message) AS error_message, '' AS error_severity, count() AS error_count, formatDateTime(max(timestamp), '%Y-%m-%d %H:%i:%S') AS latest_timestamp, - any(upstream_hostname) AS upstream_hostname, - any(upstream_status_code) AS upstream_status_code - FROM request_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND outcome = 'error' - AND error_type != '' + any(properties.upstream_hostname) AS upstream_hostname, + any(properties.upstream_status_code) AS upstream_status_code + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.outcome = 'error' + AND properties.error_type != '' GROUP BY hostname, error_type ORDER BY error_count DESC LIMIT 100 `), // 4b. Upstream service breakdown - which external services are causing errors - queryClickhouse(` + queryPostHog(` SELECT - upstream_hostname, - upstream_status_code, + properties.upstream_hostname as upstream_hostname, + properties.upstream_status_code as upstream_status_code, count() AS error_count, - uniq(hostname) AS affected_hostnames, - any(error_type) AS sample_error_type - FROM request_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND outcome = 'error' - AND upstream_hostname != '' + uniq(properties.hostname) AS affected_hostnames, + any(properties.error_type) AS sample_error_type + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.outcome = 'error' + AND properties.upstream_hostname != '' GROUP BY upstream_hostname, upstream_status_code ORDER BY error_count DESC LIMIT 50 `), // 5. Overall health metrics - queryClickhouse(` + queryPostHog(` SELECT count() AS total_requests_24h, - round(countIf(outcome = 'success') / count() * 100, 2) AS success_rate_24h, - round(countIf(cache_hit = 1) / count() * 100, 2) AS cache_hit_rate_24h, - round(avg(duration_ms)) AS avg_duration_ms_24h, - round(quantile(0.95)(duration_ms)) AS p95_duration_ms_24h, - round(avg(heap_used_mb)) AS avg_heap_mb, - uniq(hostname) AS unique_hostnames_24h - FROM request_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND hostname != '' + round(countIf(properties.outcome = 'success') / count() * 100, 2) AS success_rate_24h, + round(countIf(toFloat64(properties.cache_hit) = 1) / count() * 100, 2) AS cache_hit_rate_24h, + round(avg(toFloat64(properties.duration_ms))) AS avg_duration_ms_24h, + round(quantile(0.95)(toFloat64(properties.duration_ms))) AS p95_duration_ms_24h, + round(avg(toFloat64(properties.heap_used_mb))) AS avg_heap_mb, + uniq(properties.hostname) AS unique_hostnames_24h + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.hostname != '' `), // 6. Real-time popular pages (last 5 minutes) - queryClickhouse(` + queryPostHog(` SELECT - url, - hostname, + properties.url as url, + properties.hostname as hostname, count() AS count - FROM request_events - WHERE timestamp > now() - INTERVAL 5 MINUTE - AND url != '' + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL 5 MINUTE + AND properties.url != '' GROUP BY url, hostname ORDER BY count DESC LIMIT 20 `), // 7. Request explorer - individual requests for debugging (applies filters) - queryClickhouse(` + queryPostHog(` SELECT - request_id, + properties.request_id as request_id, formatDateTime(timestamp, '%Y-%m-%d %H:%i:%S') AS event_time, - url, - hostname, - source, - outcome, - status_code, - error_type, - error_message, - duration_ms, - fetch_ms, - cache_lookup_ms, - cache_save_ms, - cache_hit, - cache_status, - article_length, - article_title - FROM request_events + properties.url as url, + properties.hostname as hostname, + properties.source as source, + properties.outcome as outcome, + properties.status_code as status_code, + properties.error_type as error_type, + properties.error_message as error_message, + properties.duration_ms as duration_ms, + properties.fetch_ms as fetch_ms, + properties.cache_lookup_ms as cache_lookup_ms, + properties.cache_save_ms as cache_save_ms, + properties.cache_hit as cache_hit, + properties.cache_status as cache_status, + properties.article_length as article_length, + properties.article_title as article_title + FROM events WHERE ${buildWhereClause()} ORDER BY timestamp DESC LIMIT 200 `), // 8. Live requests (last 60 seconds for live feed - also applies filters) - queryClickhouse(` + queryPostHog(` SELECT - request_id, + properties.request_id as request_id, formatDateTime(timestamp, '%H:%i:%S') AS event_time, - url, - hostname, - source, - outcome, - duration_ms, - error_type, - cache_hit - FROM request_events + properties.url as url, + properties.hostname as hostname, + properties.source as source, + properties.outcome as outcome, + properties.duration_ms as duration_ms, + properties.error_type as error_type, + properties.cache_hit as cache_hit + FROM events WHERE ${buildWhereClause({ timeInterval: "60 SECOND" })} ORDER BY timestamp DESC LIMIT 50 `), // 9. Endpoint statistics (article, summary) - queryClickhouse(` + queryPostHog(` SELECT - endpoint, + properties.endpoint as endpoint, count() AS total_requests, - countIf(outcome = 'success') AS success_count, - countIf(outcome = 'error') AS error_count, - round(countIf(outcome = 'success') / count() * 100, 2) AS success_rate, - round(avg(duration_ms)) AS avg_duration_ms, - sum(input_tokens) AS total_input_tokens, - sum(output_tokens) AS total_output_tokens - FROM request_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND endpoint != '' + countIf(properties.outcome = 'success') AS success_count, + countIf(properties.outcome = 'error') AS error_count, + round(countIf(properties.outcome = 'success') / count() * 100, 2) AS success_rate, + round(avg(toFloat64(properties.duration_ms))) AS avg_duration_ms, + sum(toFloat64(properties.input_tokens)) AS total_input_tokens, + sum(toFloat64(properties.output_tokens)) AS total_output_tokens + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.endpoint != '' GROUP BY endpoint ORDER BY total_requests DESC `), // 10. Hourly traffic by endpoint (for trends) - queryClickhouse(` + queryPostHog(` SELECT formatDateTime(toStartOfHour(timestamp), '%Y-%m-%d %H:00') AS hour, - endpoint, + properties.endpoint as endpoint, count() AS request_count, - countIf(outcome = 'success') AS success_count, - countIf(outcome = 'error') AS error_count - FROM request_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND endpoint != '' + countIf(properties.outcome = 'success') AS success_count, + countIf(properties.outcome = 'error') AS error_count + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.endpoint != '' GROUP BY hour, endpoint ORDER BY hour, endpoint `), // 11. Universally broken hostnames - sites where ALL sources fail - queryClickhouse(` + queryPostHog(` SELECT - hostname, + properties.hostname as hostname, count() AS total_requests, - uniq(source) AS sources_tried, - arrayStringConcat(groupArray(DISTINCT source), ', ') AS sources_list, - round(countIf(outcome = 'success') / count() * 100, 2) AS overall_success_rate, - any(url) AS sample_url - FROM request_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND hostname != '' - AND source != '' + uniq(properties.source) AS sources_tried, + arrayStringConcat(groupArray(DISTINCT properties.source), ', ') AS sources_list, + round(countIf(properties.outcome = 'success') / count() * 100, 2) AS overall_success_rate, + any(properties.url) AS sample_url + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.hostname != '' + AND properties.source != '' GROUP BY hostname HAVING sources_tried >= 2 @@ -716,172 +727,182 @@ export const adminRoutes = new Elysia({ prefix: "/api" }).get( `), // 12. Source error rates over time - for observability/regression detection - queryClickhouse(` + queryPostHog(` SELECT formatDateTime(toStartOfFifteenMinutes(timestamp), '%Y-%m-%d %H:%i') AS time_bucket, - source, + properties.source as source, count() AS total_requests, - countIf(outcome = 'error') AS error_count, - round(countIf(outcome = 'error') / count() * 100, 2) AS error_rate - FROM request_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND hostname != '' - AND source != '' + countIf(properties.outcome = 'error') AS error_count, + round(countIf(properties.outcome = 'error') / count() * 100, 2) AS error_rate + FROM events + WHERE event = 'request_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.hostname != '' + AND properties.source != '' GROUP BY time_bucket, source ORDER BY time_bucket, source `), // ============================================================================= - // Ad Analytics Queries (from ad_events table) + // Ad Analytics Queries (from events WHERE event = 'ad_event') // ============================================================================= // 13. Ad health metrics - overall fill rate and performance (minute-based + device filter) - queryClickhouse(` + queryPostHog(` SELECT count() AS total_requests, - countIf(status = 'filled') AS filled_count, - countIf(status = 'no_fill') AS no_fill_count, - countIf(status = 'premium_user') AS premium_count, - countIf(status = 'error' OR status = 'gravity_error') AS error_count, - countIf(status = 'timeout') AS timeout_count, - round(countIf(status = 'filled') / countIf(status != 'premium_user') * 100, 2) AS fill_rate, - round(avg(duration_ms)) AS avg_duration_ms, - uniq(session_id) AS unique_sessions, - uniqIf(brand_name, brand_name != '') AS unique_brands - FROM ad_events - WHERE timestamp > now() - INTERVAL ${minutes} MINUTE - AND event_type = 'request' - ${adDeviceFilter ? `AND device_type = '${adDeviceFilter}'` : ''} + countIf(properties.status = 'filled') AS filled_count, + countIf(properties.status = 'no_fill') AS no_fill_count, + countIf(properties.status = 'premium_user') AS premium_count, + countIf(properties.status = 'error' OR properties.status = 'gravity_error') AS error_count, + countIf(properties.status = 'timeout') AS timeout_count, + round(countIf(properties.status = 'filled') / countIf(properties.status != 'premium_user') * 100, 2) AS fill_rate, + round(avg(toFloat64(properties.duration_ms))) AS avg_duration_ms, + uniq(properties.session_id) AS unique_sessions, + uniqIf(properties.brand_name, properties.brand_name != '') AS unique_brands + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${minutes} MINUTE + AND properties.event_type = 'request' + ${adDeviceFilter ? `AND properties.device_type = '${adDeviceFilter}'` : ''} `).catch(() => [] as AdHealthMetrics[]), // 14. Ad status breakdown - only count request events - queryClickhouse(` + queryPostHog(` SELECT - status, + properties.status as status, count() AS count, - round(count() / (SELECT count() FROM ad_events WHERE timestamp > now() - INTERVAL ${hours} HOUR AND event_type = 'request') * 100, 2) AS percentage, - round(avg(duration_ms)) AS avg_duration_ms - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND event_type = 'request' + round(count() / (SELECT count() FROM events WHERE event = 'ad_event' AND timestamp > now() - INTERVAL ${hours} HOUR AND properties.event_type = 'request') * 100, 2) AS percentage, + round(avg(toFloat64(properties.duration_ms))) AS avg_duration_ms + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.event_type = 'request' GROUP BY status ORDER BY count DESC `).catch(() => [] as AdStatusBreakdown[]), // 15. Ad fill rate by hostname - only count request events - queryClickhouse(` + queryPostHog(` SELECT - hostname, + properties.hostname as hostname, count() AS total_requests, - countIf(status = 'filled') AS filled_count, - round(countIf(status = 'filled') / countIf(status != 'premium_user') * 100, 2) AS fill_rate, - anyIf(brand_name, brand_name != '') AS top_brand - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND hostname != '' - AND event_type = 'request' + countIf(properties.status = 'filled') AS filled_count, + round(countIf(properties.status = 'filled') / countIf(properties.status != 'premium_user') * 100, 2) AS fill_rate, + anyIf(properties.brand_name, properties.brand_name != '') AS top_brand + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.hostname != '' + AND properties.event_type = 'request' GROUP BY hostname - HAVING countIf(status != 'premium_user') > 0 + HAVING countIf(properties.status != 'premium_user') > 0 ORDER BY total_requests DESC LIMIT 100 `).catch(() => [] as AdHostnameStats[]), // 16. Ad fill rate by device/browser/OS - only count request events - queryClickhouse(` + queryPostHog(` SELECT - device_type, - os, - browser, + properties.device_type as device_type, + properties.os as os, + properties.browser as browser, count() AS total_requests, - countIf(status = 'filled') AS filled_count, - round(countIf(status = 'filled') / countIf(status != 'premium_user') * 100, 2) AS fill_rate - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND device_type != '' - AND event_type = 'request' + countIf(properties.status = 'filled') AS filled_count, + round(countIf(properties.status = 'filled') / countIf(properties.status != 'premium_user') * 100, 2) AS fill_rate + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.device_type != '' + AND properties.event_type = 'request' GROUP BY device_type, os, browser - HAVING countIf(status != 'premium_user') > 0 + HAVING countIf(properties.status != 'premium_user') > 0 ORDER BY total_requests DESC LIMIT 50 `).catch(() => [] as AdDeviceStats[]), // 17. Top brands by impressions - queryClickhouse(` + queryPostHog(` SELECT - brand_name, + properties.brand_name as brand_name, count() AS impressions, - uniq(hostname) AS unique_hostnames, - uniq(session_id) AS unique_sessions, - round(avg(article_content_length)) AS avg_article_length - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND status = 'filled' - AND brand_name != '' + uniq(properties.hostname) AS unique_hostnames, + uniq(properties.session_id) AS unique_sessions, + round(avg(toFloat64(properties.article_content_length))) AS avg_article_length + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.status = 'filled' + AND properties.brand_name != '' GROUP BY brand_name ORDER BY impressions DESC LIMIT 50 `).catch(() => [] as AdBrandStats[]), // 18. Hourly ad traffic - only count request events - queryClickhouse(` + queryPostHog(` SELECT formatDateTime(toStartOfHour(timestamp), '%Y-%m-%d %H:00') AS hour, count() AS total_requests, - countIf(status = 'filled') AS filled_count, - countIf(status = 'no_fill') AS no_fill_count, - round(countIf(status = 'filled') / countIf(status != 'premium_user') * 100, 2) AS fill_rate - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND event_type = 'request' + countIf(properties.status = 'filled') AS filled_count, + countIf(properties.status = 'no_fill') AS no_fill_count, + round(countIf(properties.status = 'filled') / countIf(properties.status != 'premium_user') * 100, 2) AS fill_rate + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.event_type = 'request' GROUP BY hour ORDER BY hour `).catch(() => [] as AdHourlyTraffic[]), // 19. Ad error breakdown - queryClickhouse(` + queryPostHog(` SELECT - status, - gravity_status_code, - any(error_message) AS error_message, + properties.status as status, + properties.gravity_status_code as gravity_status_code, + any(properties.error_message) AS error_message, count() AS count, formatDateTime(max(timestamp), '%Y-%m-%d %H:%i:%S') AS latest_timestamp - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND status IN ('error', 'gravity_error', 'timeout') + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.status IN ('error', 'gravity_error', 'timeout') GROUP BY status, gravity_status_code ORDER BY count DESC LIMIT 50 `).catch(() => [] as AdErrorBreakdown[]), // 20. Recent ad events (for live debugging) - queryClickhouse(` + queryPostHog(` SELECT - event_id, + properties.event_id as event_id, formatDateTime(timestamp, '%Y-%m-%d %H:%i:%S') AS event_time, - hostname, - article_title, - status, - brand_name, - duration_ms, - device_type - FROM ad_events - WHERE timestamp > now() - INTERVAL 1 HOUR + properties.hostname as hostname, + properties.article_title as article_title, + properties.status as status, + properties.brand_name as brand_name, + properties.duration_ms as duration_ms, + properties.device_type as device_type + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL 1 HOUR ORDER BY timestamp DESC LIMIT 100 `).catch(() => [] as AdRecentEvent[]), // 21. CTR by Brand - click-through rate for each advertiser (minute-based + device filter) - queryClickhouse(` + queryPostHog(` SELECT - brand_name, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - round(countIf(event_type = 'click') / countIf(event_type = 'impression') * 100, 2) AS ctr - FROM ad_events - WHERE timestamp > now() - INTERVAL ${minutes} MINUTE - AND event_type IN ('impression', 'click') - AND brand_name != '' - ${adDeviceFilter ? `AND device_type = '${adDeviceFilter}'` : ''} + properties.brand_name as brand_name, + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks, + round(countIf(properties.event_type = 'click') / countIf(properties.event_type = 'impression') * 100, 2) AS ctr + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${minutes} MINUTE + AND properties.event_type IN ('impression', 'click') + AND properties.brand_name != '' + ${adDeviceFilter ? `AND properties.device_type = '${adDeviceFilter}'` : ''} GROUP BY brand_name HAVING impressions > 0 ORDER BY impressions DESC @@ -889,8 +910,7 @@ export const adminRoutes = new Elysia({ prefix: "/api" }).get( `).catch(() => [] as AdCTRByBrand[]), // 22. Funnel by time bucket - adapts granularity based on time range - // <1h: 5-minute buckets, <6h: 15-minute buckets, <24h: hourly, else: daily - queryClickhouse(` + queryPostHog(` SELECT ${minutes <= 60 ? `formatDateTime(toStartOfFiveMinutes(timestamp), '%Y-%m-%d %H:%i') AS hour` @@ -900,27 +920,29 @@ export const adminRoutes = new Elysia({ prefix: "/api" }).get( ? `formatDateTime(toStartOfHour(timestamp), '%Y-%m-%d %H:00') AS hour` : `formatDateTime(toStartOfDay(timestamp), '%Y-%m-%d') AS hour` }, - countIf(event_type = 'request' AND status = 'filled') AS requests, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - countIf(event_type = 'dismiss') AS dismissals - FROM ad_events - WHERE timestamp > now() - INTERVAL ${minutes} MINUTE - ${adDeviceFilter ? `AND device_type = '${adDeviceFilter}'` : ''} + countIf(properties.event_type = 'request' AND properties.status = 'filled') AS requests, + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks, + countIf(properties.event_type = 'dismiss') AS dismissals + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${minutes} MINUTE + ${adDeviceFilter ? `AND properties.device_type = '${adDeviceFilter}'` : ''} GROUP BY hour ORDER BY hour `).catch(() => [] as AdHourlyFunnel[]), - // 23. Dismiss Rate by Device - see which devices dismiss ads most - queryClickhouse(` + // 23. Dismiss Rate by Device + queryPostHog(` SELECT - device_type, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'dismiss') AS dismissals, - round(countIf(event_type = 'dismiss') / countIf(event_type = 'impression') * 100, 2) AS dismiss_rate - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND device_type != '' + properties.device_type as device_type, + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'dismiss') AS dismissals, + round(countIf(properties.event_type = 'dismiss') / countIf(properties.event_type = 'impression') * 100, 2) AS dismiss_rate + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.device_type != '' GROUP BY device_type HAVING impressions > 0 ORDER BY impressions DESC @@ -930,24 +952,25 @@ export const adminRoutes = new Elysia({ prefix: "/api" }).get( // Enhanced Granular Ad Analytics // ============================================================================= - // 24. Performance by Hour of Day - identify best performing hours - queryClickhouse(` + // 24. Performance by Hour of Day + queryPostHog(` SELECT toHour(timestamp) AS hour_of_day, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - round(countIf(event_type = 'click') / countIf(event_type = 'impression') * 100, 2) AS ctr, - round(countIf(event_type = 'request' AND status = 'filled') / - countIf(event_type = 'request' AND status != 'premium_user') * 100, 2) AS fill_rate - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks, + round(countIf(properties.event_type = 'click') / countIf(properties.event_type = 'impression') * 100, 2) AS ctr, + round(countIf(properties.event_type = 'request' AND properties.status = 'filled') / + countIf(properties.event_type = 'request' AND properties.status != 'premium_user') * 100, 2) AS fill_rate + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR GROUP BY hour_of_day - HAVING countIf(event_type = 'impression') > 0 + HAVING countIf(properties.event_type = 'impression') > 0 ORDER BY hour_of_day `).catch(() => [] as AdPerformanceByHour[]), // 25. Performance by Day of Week - queryClickhouse(` + queryPostHog(` SELECT toDayOfWeek(timestamp) AS day_of_week, CASE toDayOfWeek(timestamp) @@ -959,126 +982,133 @@ export const adminRoutes = new Elysia({ prefix: "/api" }).get( WHEN 6 THEN 'Saturday' WHEN 7 THEN 'Sunday' END AS day_name, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - round(countIf(event_type = 'click') / countIf(event_type = 'impression') * 100, 2) AS ctr - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks, + round(countIf(properties.event_type = 'click') / countIf(properties.event_type = 'impression') * 100, 2) AS ctr + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR GROUP BY day_of_week, day_name - HAVING countIf(event_type = 'impression') > 0 + HAVING countIf(properties.event_type = 'impression') > 0 ORDER BY day_of_week `).catch(() => [] as AdPerformanceByDay[]), // 26. Enhanced Brand Performance with engagement metrics - queryClickhouse(` + queryPostHog(` SELECT - brand_name, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - countIf(event_type = 'dismiss') AS dismissals, - round(countIf(event_type = 'click') / countIf(event_type = 'impression') * 100, 2) AS ctr, - round(countIf(event_type = 'dismiss') / countIf(event_type = 'impression') * 100, 2) AS dismiss_rate, - round(avgIf(duration_ms, event_type = 'click' AND duration_ms > 0)) AS avg_time_to_click_ms, - uniq(session_id) AS unique_sessions - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND brand_name != '' + properties.brand_name as brand_name, + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks, + countIf(properties.event_type = 'dismiss') AS dismissals, + round(countIf(properties.event_type = 'click') / countIf(properties.event_type = 'impression') * 100, 2) AS ctr, + round(countIf(properties.event_type = 'dismiss') / countIf(properties.event_type = 'impression') * 100, 2) AS dismiss_rate, + round(avgIf(toFloat64(properties.duration_ms), properties.event_type = 'click' AND toFloat64(properties.duration_ms) > 0)) AS avg_time_to_click_ms, + uniq(properties.session_id) AS unique_sessions + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.brand_name != '' GROUP BY brand_name - HAVING countIf(event_type = 'impression') > 0 + HAVING countIf(properties.event_type = 'impression') > 0 ORDER BY impressions DESC LIMIT 25 `).catch(() => [] as AdBrandPerformance[]), - // 27. Detailed Device Breakdown (uses minute-based time + device filter) - queryClickhouse(` + // 27. Detailed Device Breakdown + queryPostHog(` SELECT - device_type, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - countIf(event_type = 'dismiss') AS dismissals, - round(countIf(event_type = 'click') / countIf(event_type = 'impression') * 100, 2) AS ctr, - round(countIf(event_type = 'dismiss') / countIf(event_type = 'impression') * 100, 2) AS dismiss_rate, - round(countIf(event_type = 'request' AND status = 'filled') / - countIf(event_type = 'request' AND status != 'premium_user') * 100, 2) AS fill_rate - FROM ad_events - WHERE timestamp > now() - INTERVAL ${minutes} MINUTE - AND device_type != '' - ${adDeviceFilter ? `AND device_type = '${adDeviceFilter}'` : ''} + properties.device_type as device_type, + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks, + countIf(properties.event_type = 'dismiss') AS dismissals, + round(countIf(properties.event_type = 'click') / countIf(properties.event_type = 'impression') * 100, 2) AS ctr, + round(countIf(properties.event_type = 'dismiss') / countIf(properties.event_type = 'impression') * 100, 2) AS dismiss_rate, + round(countIf(properties.event_type = 'request' AND properties.status = 'filled') / + countIf(properties.event_type = 'request' AND properties.status != 'premium_user') * 100, 2) AS fill_rate + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${minutes} MINUTE + AND properties.device_type != '' + ${adDeviceFilter ? `AND properties.device_type = '${adDeviceFilter}'` : ''} GROUP BY device_type - HAVING countIf(event_type = 'impression') > 0 + HAVING countIf(properties.event_type = 'impression') > 0 ORDER BY impressions DESC `).catch(() => [] as AdDeviceBreakdown[]), // 28. Browser Performance - queryClickhouse(` + queryPostHog(` SELECT - browser, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - round(countIf(event_type = 'click') / countIf(event_type = 'impression') * 100, 2) AS ctr - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND browser != '' + properties.browser as browser, + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks, + round(countIf(properties.event_type = 'click') / countIf(properties.event_type = 'impression') * 100, 2) AS ctr + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.browser != '' GROUP BY browser - HAVING countIf(event_type = 'impression') > 0 + HAVING countIf(properties.event_type = 'impression') > 0 ORDER BY impressions DESC LIMIT 10 `).catch(() => [] as AdBrowserStats[]), // 29. OS Performance - queryClickhouse(` + queryPostHog(` SELECT - os, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - round(countIf(event_type = 'click') / countIf(event_type = 'impression') * 100, 2) AS ctr - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND os != '' + properties.os as os, + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks, + round(countIf(properties.event_type = 'click') / countIf(properties.event_type = 'impression') * 100, 2) AS ctr + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.os != '' GROUP BY os - HAVING countIf(event_type = 'impression') > 0 + HAVING countIf(properties.event_type = 'impression') > 0 ORDER BY impressions DESC LIMIT 10 `).catch(() => [] as AdOSStats[]), // 30. Hostname Performance with full funnel - queryClickhouse(` + queryPostHog(` SELECT - hostname, - countIf(event_type = 'request') AS requests, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - round(if(countIf(event_type = 'impression') > 0, countIf(event_type = 'click') / countIf(event_type = 'impression') * 100, 0), 2) AS ctr, - round(if(countIf(event_type = 'request' AND status != 'premium_user') > 0, countIf(event_type = 'request' AND status = 'filled') / - countIf(event_type = 'request' AND status != 'premium_user') * 100, 0), 2) AS fill_rate, - anyIf(brand_name, brand_name != '' AND event_type = 'impression') AS top_brand - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND hostname != '' + properties.hostname as hostname, + countIf(properties.event_type = 'request') AS requests, + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks, + round(if(countIf(properties.event_type = 'impression') > 0, countIf(properties.event_type = 'click') / countIf(properties.event_type = 'impression') * 100, 0), 2) AS ctr, + round(if(countIf(properties.event_type = 'request' AND properties.status != 'premium_user') > 0, countIf(properties.event_type = 'request' AND properties.status = 'filled') / + countIf(properties.event_type = 'request' AND properties.status != 'premium_user') * 100, 0), 2) AS fill_rate, + anyIf(properties.brand_name, properties.brand_name != '' AND properties.event_type = 'impression') AS top_brand + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.hostname != '' GROUP BY hostname - HAVING countIf(event_type = 'request') > 0 + HAVING countIf(properties.event_type = 'request') > 0 ORDER BY requests DESC LIMIT 50 `).catch(() => [] as AdHostnamePerformance[]), - // 31. Content Length Correlation - do longer articles perform better? - queryClickhouse(` + // 31. Content Length Correlation + queryPostHog(` SELECT CASE - WHEN article_content_length < 500 THEN '< 500 chars' - WHEN article_content_length < 1500 THEN '500-1.5k chars' - WHEN article_content_length < 3000 THEN '1.5k-3k chars' - WHEN article_content_length < 5000 THEN '3k-5k chars' + WHEN toFloat64(properties.article_content_length) < 500 THEN '< 500 chars' + WHEN toFloat64(properties.article_content_length) < 1500 THEN '500-1.5k chars' + WHEN toFloat64(properties.article_content_length) < 3000 THEN '1.5k-3k chars' + WHEN toFloat64(properties.article_content_length) < 5000 THEN '3k-5k chars' ELSE '5k+ chars' END AS article_length_bucket, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - round(countIf(event_type = 'click') / countIf(event_type = 'impression') * 100, 2) AS ctr - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND article_content_length > 0 + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks, + round(countIf(properties.event_type = 'click') / countIf(properties.event_type = 'impression') * 100, 2) AS ctr + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND toFloat64(properties.article_content_length) > 0 GROUP BY article_length_bucket - HAVING countIf(event_type = 'impression') > 0 + HAVING countIf(properties.event_type = 'impression') > 0 ORDER BY CASE article_length_bucket WHEN '< 500 chars' THEN 1 @@ -1089,8 +1119,8 @@ export const adminRoutes = new Elysia({ prefix: "/api" }).get( END `).catch(() => [] as AdContentCorrelation[]), - // 32. Session Depth Analysis - do users who see more ads click more? - queryClickhouse(` + // 32. Session Depth Analysis + queryPostHog(` SELECT session_ad_count, count() AS session_count, @@ -1099,13 +1129,14 @@ export const adminRoutes = new Elysia({ prefix: "/api" }).get( round(if(sum(impressions) > 0, sum(clicks) / sum(impressions) * 100, 0), 2) AS avg_ctr FROM ( SELECT - session_id, - countIf(event_type = 'impression') AS session_ad_count, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - AND session_id != '' + properties.session_id as session_id, + countIf(properties.event_type = 'impression') AS session_ad_count, + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + AND properties.session_id != '' GROUP BY session_id HAVING session_ad_count > 0 ) @@ -1115,99 +1146,101 @@ export const adminRoutes = new Elysia({ prefix: "/api" }).get( `).catch(() => [] as AdSessionDepth[]), // 33. Conversion Funnel Summary - queryClickhouse(` + queryPostHog(` SELECT stage, count, round(if(first_value(count) OVER (ORDER BY stage_order) > 0, count / first_value(count) OVER (ORDER BY stage_order) * 100, 0), 2) AS rate_from_previous FROM ( - SELECT 'Requests' AS stage, 1 AS stage_order, countIf(event_type = 'request') AS count - FROM ad_events WHERE timestamp > now() - INTERVAL ${hours} HOUR + SELECT 'Requests' AS stage, 1 AS stage_order, countIf(properties.event_type = 'request') AS count + FROM events WHERE event = 'ad_event' AND timestamp > now() - INTERVAL ${hours} HOUR UNION ALL - SELECT 'Filled' AS stage, 2 AS stage_order, countIf(event_type = 'request' AND status = 'filled') AS count - FROM ad_events WHERE timestamp > now() - INTERVAL ${hours} HOUR + SELECT 'Filled' AS stage, 2 AS stage_order, countIf(properties.event_type = 'request' AND properties.status = 'filled') AS count + FROM events WHERE event = 'ad_event' AND timestamp > now() - INTERVAL ${hours} HOUR UNION ALL - SELECT 'Impressions' AS stage, 3 AS stage_order, countIf(event_type = 'impression') AS count - FROM ad_events WHERE timestamp > now() - INTERVAL ${hours} HOUR + SELECT 'Impressions' AS stage, 3 AS stage_order, countIf(properties.event_type = 'impression') AS count + FROM events WHERE event = 'ad_event' AND timestamp > now() - INTERVAL ${hours} HOUR UNION ALL - SELECT 'Clicks' AS stage, 4 AS stage_order, countIf(event_type = 'click') AS count - FROM ad_events WHERE timestamp > now() - INTERVAL ${hours} HOUR + SELECT 'Clicks' AS stage, 4 AS stage_order, countIf(properties.event_type = 'click') AS count + FROM events WHERE event = 'ad_event' AND timestamp > now() - INTERVAL ${hours} HOUR ) ORDER BY stage_order `).catch(() => [] as AdConversionFunnel[]), - // 34. Bot Detection - identify filled requests without device info (likely bots/curl) - queryClickhouse(` + // 34. Bot Detection + queryPostHog(` SELECT CASE - WHEN device_type = '' OR browser = '' THEN 'No Device Info (Likely Bot)' + WHEN properties.device_type = '' OR properties.browser = '' THEN 'No Device Info (Likely Bot)' ELSE 'Has Device Info (Real User)' END as category, - countIf(event_type = 'request' AND status = 'filled') AS filled_count, - countIf(event_type = 'impression') AS impression_count, - round(if(countIf(event_type = 'request' AND status = 'filled') > 0, - countIf(event_type = 'impression') / countIf(event_type = 'request' AND status = 'filled') * 100, 0), 1) AS impression_rate, - uniq(session_id) AS unique_sessions - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR + countIf(properties.event_type = 'request' AND properties.status = 'filled') AS filled_count, + countIf(properties.event_type = 'impression') AS impression_count, + round(if(countIf(properties.event_type = 'request' AND properties.status = 'filled') > 0, + countIf(properties.event_type = 'impression') / countIf(properties.event_type = 'request' AND properties.status = 'filled') * 100, 0), 1) AS impression_rate, + uniq(properties.session_id) AS unique_sessions + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR GROUP BY category ORDER BY filled_count DESC `).catch(() => [] as AdBotDetection[]), // 35. CTR by Hour of Day with Device Breakdown - queryClickhouse(` + queryPostHog(` SELECT toHour(timestamp) AS hour_of_day, - if(device_type = '', 'unknown', device_type) AS device_type, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - round(if(countIf(event_type = 'impression') > 0, - countIf(event_type = 'click') / countIf(event_type = 'impression') * 100, 0), 2) AS ctr, - round(if(countIf(event_type = 'request' AND status != 'premium_user') > 0, - countIf(event_type = 'request' AND status = 'filled') / - countIf(event_type = 'request' AND status != 'premium_user') * 100, 0), 2) AS fill_rate - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR - ${adDeviceFilter ? `AND device_type = '${adDeviceFilter}'` : ''} + if(properties.device_type = '', 'unknown', properties.device_type) AS device_type, + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks, + round(if(countIf(properties.event_type = 'impression') > 0, + countIf(properties.event_type = 'click') / countIf(properties.event_type = 'impression') * 100, 0), 2) AS ctr, + round(if(countIf(properties.event_type = 'request' AND properties.status != 'premium_user') > 0, + countIf(properties.event_type = 'request' AND properties.status = 'filled') / + countIf(properties.event_type = 'request' AND properties.status != 'premium_user') * 100, 0), 2) AS fill_rate + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR + ${adDeviceFilter ? `AND properties.device_type = '${adDeviceFilter}'` : ''} GROUP BY hour_of_day, device_type - HAVING countIf(event_type = 'impression') > 0 + HAVING countIf(properties.event_type = 'impression') > 0 ORDER BY hour_of_day, device_type `).catch(() => [] as AdCTRByHourDevice[]), - // 36. Filled vs Impression Gap Analysis - identify where impressions are lost - queryClickhouse(` + // 36. Filled vs Impression Gap Analysis + queryPostHog(` SELECT - if(device_type = '', 'unknown', device_type) AS device_type, - if(browser = '', 'unknown', browser) AS browser, - countIf(event_type = 'request' AND status = 'filled') AS filled_count, - countIf(event_type = 'impression') AS impression_count, - countIf(event_type = 'request' AND status = 'filled') - countIf(event_type = 'impression') AS gap_count, - round(if(countIf(event_type = 'request' AND status = 'filled') > 0, - countIf(event_type = 'impression') / countIf(event_type = 'request' AND status = 'filled') * 100, 0), 1) AS impression_rate - FROM ad_events - WHERE timestamp > now() - INTERVAL ${hours} HOUR + if(properties.device_type = '', 'unknown', properties.device_type) AS device_type, + if(properties.browser = '', 'unknown', properties.browser) AS browser, + countIf(properties.event_type = 'request' AND properties.status = 'filled') AS filled_count, + countIf(properties.event_type = 'impression') AS impression_count, + countIf(properties.event_type = 'request' AND properties.status = 'filled') - countIf(properties.event_type = 'impression') AS gap_count, + round(if(countIf(properties.event_type = 'request' AND properties.status = 'filled') > 0, + countIf(properties.event_type = 'impression') / countIf(properties.event_type = 'request' AND properties.status = 'filled') * 100, 0), 1) AS impression_rate + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${hours} HOUR GROUP BY device_type, browser - HAVING countIf(event_type = 'request' AND status = 'filled') > 0 + HAVING countIf(properties.event_type = 'request' AND properties.status = 'filled') > 0 ORDER BY gap_count DESC LIMIT 20 `).catch(() => [] as AdFilledImpressionGap[]), - // 37. Ad Funnel Time Series - minute-level granularity for real-time monitoring - // Tracks the full funnel: requests -> filled -> impressions -> clicks/dismissals - // Also tracks Gravity forwarding success for revenue assurance - queryClickhouse(` + // 37. Ad Funnel Time Series + queryPostHog(` SELECT formatDateTime(toStartOfMinute(timestamp), '%Y-%m-%d %H:%i') AS time_bucket, - countIf(event_type = 'request') AS requests, - countIf(event_type = 'request' AND status = 'filled') AS filled, - countIf(event_type = 'impression') AS impressions, - countIf(event_type = 'click') AS clicks, - countIf(event_type = 'dismiss') AS dismissals, - countIf(event_type = 'impression' AND gravity_forwarded = 1) AS gravity_forwarded, - countIf(event_type = 'impression' AND gravity_forwarded = 0) AS gravity_failed - FROM ad_events - WHERE timestamp > now() - INTERVAL ${minutes} MINUTE - ${adDeviceFilter ? `AND device_type = '${adDeviceFilter}'` : ''} + countIf(properties.event_type = 'request') AS requests, + countIf(properties.event_type = 'request' AND properties.status = 'filled') AS filled, + countIf(properties.event_type = 'impression') AS impressions, + countIf(properties.event_type = 'click') AS clicks, + countIf(properties.event_type = 'dismiss') AS dismissals, + countIf(properties.event_type = 'impression' AND toFloat64(properties.gravity_forwarded) = 1) AS gravity_forwarded, + countIf(properties.event_type = 'impression' AND toFloat64(properties.gravity_forwarded) = 0) AS gravity_failed + FROM events + WHERE event = 'ad_event' + AND timestamp > now() - INTERVAL ${minutes} MINUTE + ${adDeviceFilter ? `AND properties.device_type = '${adDeviceFilter}'` : ''} GROUP BY time_bucket ORDER BY time_bucket `).catch(() => [] as AdFunnelTimeSeries[]), diff --git a/server/routes/gravity.ts b/server/routes/gravity.ts index 767a705c..56db2539 100644 --- a/server/routes/gravity.ts +++ b/server/routes/gravity.ts @@ -3,7 +3,7 @@ * * /api/context - Fetches contextual ads from Gravity AI for free users. * /api/px - Unified tracking for impressions, clicks, dismissals. - * For impressions, wraps Gravity forwarding + ClickHouse logging atomically. + * For impressions, wraps Gravity forwarding + PostHog logging atomically. * * Endpoint names are neutral to avoid content blockers (no "ad" or "track" in names). */ @@ -13,7 +13,7 @@ import { getAuthInfo } from "../middleware/auth"; import { env } from "../env"; import { extractClientIp } from "../../lib/request-context"; import { createLogger } from "../../lib/logger"; -import { trackAdEvent, type AdEventStatus } from "../../lib/clickhouse"; +import { trackAdEvent, type AdEventStatus } from "../../lib/posthog"; const logger = createLogger("api:gravity"); @@ -133,7 +133,7 @@ export const gravityRoutes = new Elysia({ prefix: "/api" }) * Unified tracking endpoint for impressions, clicks, and dismissals. * * CRITICAL: For impressions, this endpoint WRAPS the Gravity impression pixel call. - * This ensures ClickHouse accurately reflects whether Gravity received the impression. + * This ensures PostHog accurately reflects whether Gravity received the impression. * Without this, we'd log impressions locally without knowing if we got paid. * * Named "/px" to avoid ad blocker detection (no "ad" or "track" in the name). @@ -151,7 +151,7 @@ export const gravityRoutes = new Elysia({ prefix: "/api" }) gravityResult = await forwardImpressionToGravity(impUrl); } - // Now track to ClickHouse WITH the Gravity result + // Now track to PostHog WITH the Gravity result try { trackAdEvent({ event_type: type,