Umami/umami
1
0
Fork 0
mirror of https://github.com/umami-software/umami.git synced 2026-05-30 06:47:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

add clickhouse+s3 workflow for heatmaps

Browse Source
This commit is contained in:
Francis Cao
2026-05-19 22:41:59 -07:00
parent 5bd92ce950
commit 07e40db5f4
6 changed files with 865 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/clickhouse/migrations/12_add_heatmap.sql
+22
View File
@@ -26,3 +26,25 @@ ENGINE = MergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (website_id, url_path, event_type, created_at)
SETTINGS index_granularity = 8192;
-- Create heatmap_snapshot
CREATE TABLE umami.heatmap_snapshot
(
snapshot_id UUID,
website_id UUID,
url_path String,
viewport_w UInt32,
viewport_h UInt32,
page_w UInt32,
page_h UInt32,
status UInt8,
mime_type LowCardinality(String),
object_key String,
image_size Nullable(UInt32),
error Nullable(String),
created_at DateTime('UTC')
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (website_id, url_path, viewport_w, viewport_h, created_at)
SETTINGS index_granularity = 8192;
db/clickhouse/schema.sql
+22
View File
@@ -428,3 +428,25 @@ ENGINE = MergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (website_id, url_path, event_type, created_at)
SETTINGS index_granularity = 8192;
-- Create heatmap_snapshot
CREATE TABLE umami.heatmap_snapshot
(
snapshot_id UUID,
website_id UUID,
url_path String,
viewport_w UInt32,
viewport_h UInt32,
page_w UInt32,
page_h UInt32,
status UInt8,
mime_type LowCardinality(String),
object_key String,
image_size Nullable(UInt32),
error Nullable(String),
created_at DateTime('UTC')
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (website_id, url_path, viewport_w, viewport_h, created_at)
SETTINGS index_granularity = 8192;
package.json
+1
View File
@@ -53,6 +53,7 @@
".next/cache"
],
"dependencies": {
"@aws-sdk/client-s3": "^3.924.0",
"@clickhouse/client": "^1.18.5",
"@date-fns/utc": "^2.1.1",
"@dicebear/collection": "^9.4.2",
pnpm-lock.yaml
Generated
+517
View File
@@ -8,6 +8,9 @@ importers:
.:
dependencies:
'@aws-sdk/client-s3':
specifier: ^3.924.0
version: 3.1050.0
'@clickhouse/client':
specifier: ^1.18.5
version: 1.18.5
@@ -352,6 +355,125 @@ packages:
'@asamuzakjp/nwsapi@2.3.9':
resolution: {integrity: sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==}
'@aws-crypto/crc32@5.2.0':
resolution: {integrity: sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==}
engines: {node: '>=16.0.0'}
'@aws-crypto/crc32c@5.2.0':
resolution: {integrity: sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==}
'@aws-crypto/sha1-browser@5.2.0':
resolution: {integrity: sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==}
'@aws-crypto/sha256-browser@5.2.0':
resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==}
'@aws-crypto/sha256-js@5.2.0':
resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==}
engines: {node: '>=16.0.0'}
'@aws-crypto/supports-web-crypto@5.2.0':
resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==}
'@aws-crypto/util@5.2.0':
resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==}
'@aws-sdk/client-s3@3.1050.0':
resolution: {integrity: sha512-9kgtv+bXZQrOIJT2INPPBCezrJu1FlgGrzEat/ut4A4V53IT00LynsBZgp12eFKbjJuNCeTo7iPSKjPsX8ub+A==}
engines: {node: '>=20.0.0'}
'@aws-sdk/core@3.974.12':
resolution: {integrity: sha512-qrqgioqYFjwR6LatVNS1L2Vk++EwRIxqSQXPKNv5Ofux2D8UNgqMQ1znnMyEImXquVPTtbf71fc128pvmU6y9A==}
engines: {node: '>=20.0.0'}
'@aws-sdk/crc64-nvme@3.972.8':
resolution: {integrity: sha512-fVfUCL/Xh2zINYMPZvj+iBn6XWouQf0DAnjaWCI9MkmqXzL2Iy5FoQB8O7syFe6gN6AH1ecDDU58T51Ou0kFkA==}
engines: {node: '>=20.0.0'}
'@aws-sdk/credential-provider-env@3.972.38':
resolution: {integrity: sha512-m3WjZEgPtioMhPmwqUt+DhlTJ2i9ufR6DhfkyXojb9puEvfR+ur2U5shavu5/Cc9WHHsDCvALi6UFHgcqjhQ5w==}
engines: {node: '>=20.0.0'}
'@aws-sdk/credential-provider-http@3.972.40':
resolution: {integrity: sha512-D78L/m2Dr6cJnnSvWoAudPhQmCwmJ7j6APXsPYmFpPaKfQTfCSu0rdm8j14Np+VmXF9z8Aj8HE3xFpsrwtfgeg==}
engines: {node: '>=20.0.0'}
'@aws-sdk/credential-provider-ini@3.972.42':
resolution: {integrity: sha512-Mu5ESvFXeinafVM8jTIvRqcvK2Ehj4kz3auT39yUcHwu1Vfxo6xRlmUafdKLW4tusjAJukQwK09sCSMgOm7OKg==}
engines: {node: '>=20.0.0'}
'@aws-sdk/credential-provider-login@3.972.42':
resolution: {integrity: sha512-O6WkZga3kf0yqyJYd1dbeJqVhEgJx/x1UaLgtbR+XuL/YP+K5y6QTxQKL7ka9z3jnQASESKGAPnRyt4D5hQrxA==}
engines: {node: '>=20.0.0'}
'@aws-sdk/credential-provider-node@3.972.43':
resolution: {integrity: sha512-D/DJmbrWRP5BXEO3FH+ar4el+2n6OlGofiud7dQun2jES+AQEJjczenp1jBb4MBN7CpGpS8nsWGQLtuzc9tQbA==}
engines: {node: '>=20.0.0'}
'@aws-sdk/credential-provider-process@3.972.38':
resolution: {integrity: sha512-EnbYVajGgbkb24s0K1eo4VNAPV5mHIET7LSvirTaFCwkfrfaOJxtSE+wY/tJdKDS21cEYkZs2ruCaAm+W4iblg==}
engines: {node: '>=20.0.0'}
'@aws-sdk/credential-provider-sso@3.972.42':
resolution: {integrity: sha512-RVV/9NbFwI8ZHEH5dn39lGyFmSbSVj1+orZdr6QsOe1mW9DCglmlen0cFaNZmCcqkqc7erNRHNBduxbeZuHAnw==}
engines: {node: '>=20.0.0'}
'@aws-sdk/credential-provider-web-identity@3.972.42':
resolution: {integrity: sha512-/67fXX0ddllD4u2Nujc5PvT4byHgpMUfz6+RxIKi/0nFIckeorm7JvXgzBuDyVKw0s58EbofmETDWUf9vTEuHQ==}
engines: {node: '>=20.0.0'}
'@aws-sdk/middleware-bucket-endpoint@3.972.14':
resolution: {integrity: sha512-Aaj0d+xbo1jJquBWJP0/9V/XZRYukO3LWIRp3dOLHmoFrYKb4YZ0aLefgVHfGcNOVBS2ZTq7L/n5JcrE7DaC+Q==}
engines: {node: '>=20.0.0'}
'@aws-sdk/middleware-expect-continue@3.972.12':
resolution: {integrity: sha512-dA5pKTom/Ls9mgeyeaRBNQrRIVOLVjv4AmKOB0/e4yaiXEUy0gSz2d3liP8JHtYoCAEWySU1jWnyzwLOREN+4g==}
engines: {node: '>=20.0.0'}
'@aws-sdk/middleware-flexible-checksums@3.974.20':
resolution: {integrity: sha512-NdnMVQCR1YjIcqFAiNLdBiOwr2DyQDB2IiXQrBhzolKOv32ae4d4Ll7IzLMi04eMHiq/o/Y/GjFuVjF9HuG0QA==}
engines: {node: '>=20.0.0'}
'@aws-sdk/middleware-location-constraint@3.972.10':
resolution: {integrity: sha512-rI3NZvJcEvjoD0+0PI0iUAwlPw2IlSlhyvgBK/3WkKJQE/YiKFedd9dMN2lVacdNxPNhxL/jzQaKQdrGtQagjQ==}
engines: {node: '>=20.0.0'}
'@aws-sdk/middleware-sdk-s3@3.972.41':
resolution: {integrity: sha512-M4T2I2WPuH5WQpU8Tsp+u2bcO29zGRkU14ATzuqb9I4xh8tzsLqtp4hzaJM5aO2dhMZnHDzyQwSFVgc3XbnoGg==}
engines: {node: '>=20.0.0'}
'@aws-sdk/middleware-ssec@3.972.10':
resolution: {integrity: sha512-Gli9A0u8EVVb+5bFDGS/QbSVg28w/wpEidg1ggVcSj65BDTdGR6punsOcVjqdiu1i42WHWo51MCvARPIIz9juw==}
engines: {node: '>=20.0.0'}
'@aws-sdk/nested-clients@3.997.10':
resolution: {integrity: sha512-FtQ/Bt327peZJuyo4WZSOLVUTw9ujRxntepiC7L65FxA2P82Xlq0g14T22BuqBUeMjDoxa9nvwiMHjLIfP3eUg==}
engines: {node: '>=20.0.0'}
'@aws-sdk/signature-v4-multi-region@3.996.27':
resolution: {integrity: sha512-0Phbz4t6HI3D3skxvG2uI+VWU034/nSIw1T8d+FPzzQG9EQTrw94o9mOKO2Gv3n3Oc8P7JD7RAUxkoneLWv5Eg==}
engines: {node: '>=20.0.0'}
'@aws-sdk/token-providers@3.1049.0':
resolution: {integrity: sha512-r7+d0lQMTHKypkmaF5jRTBYLYHCUHzt3gaVoN9SidLhQeWhCmHk3AKrboDTpPF5b7Pt7vKu3+oeMjznM2Eu1ow==}
engines: {node: '>=20.0.0'}
'@aws-sdk/types@3.973.8':
resolution: {integrity: sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==}
engines: {node: '>=20.0.0'}
'@aws-sdk/util-locate-window@3.965.5':
resolution: {integrity: sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==}
engines: {node: '>=20.0.0'}
'@aws-sdk/xml-builder@3.972.24':
resolution: {integrity: sha512-V8z5YcDPfsvzrBlj0xR1vhRtocblhYbqdreCJB/voGd4Sr5zjNAeWxexbnqVtskTJe0vFb5KMqbSL++ePl+zRw==}
engines: {node: '>=20.0.0'}
'@aws/lambda-invoke-store@0.2.4':
resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==}
engines: {node: '>=18.0.0'}
'@babel/code-frame@7.27.1':
resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==}
engines: {node: '>=6.9.0'}
@@ -1753,6 +1875,9 @@ packages:
cpu: [x64]
os: [win32]
'@nodable/entities@2.1.0':
resolution: {integrity: sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==}
'@nodelib/fs.scandir@2.1.5':
resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
engines: {node: '>= 8'}
@@ -2430,6 +2555,42 @@ packages:
resolution: {integrity: sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==}
engines: {node: '>=18'}
'@smithy/core@3.24.3':
resolution: {integrity: sha512-Ep/7tPamGY8mgESE3LyLKtxJyy6U52WWAqr/3wial47Sj4u3PiIF73AOGI27UyLy9duTkhZbgzodOfLV4TduZg==}
engines: {node: '>=18.0.0'}
'@smithy/credential-provider-imds@4.3.3':
resolution: {integrity: sha512-I2Bti0DKFo2IJyN28ijCsx51BAumEYR4/1yZ1FXyBygy9MqbnMqCev4JPth/MbpRfBSRAX35hITSnAdJRo1u5w==}
engines: {node: '>=18.0.0'}
'@smithy/fetch-http-handler@5.4.3':
resolution: {integrity: sha512-F+DRf8IJazRJgYog2A/yJK7eYVc0rqTlRzO+5ZxjJd4WkZoKz0IJRncf7G6t1pdVT3kryJcwuTFhN1c5m6N47A==}
engines: {node: '>=18.0.0'}
'@smithy/is-array-buffer@2.2.0':
resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==}
engines: {node: '>=14.0.0'}
'@smithy/node-http-handler@4.7.3':
resolution: {integrity: sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==}
engines: {node: '>=18.0.0'}
'@smithy/signature-v4@5.4.3':
resolution: {integrity: sha512-53+75QuPl6DL+ct6vVEB51FDO5oulXr20TPV46VvJZg76lIlXNWfxi8j+G2V/t0I2qxCBOa3vX/8bmjrpFVo9g==}
engines: {node: '>=18.0.0'}
'@smithy/types@4.14.2':
resolution: {integrity: sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==}
engines: {node: '>=18.0.0'}
'@smithy/util-buffer-from@2.2.0':
resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==}
engines: {node: '>=14.0.0'}
'@smithy/util-utf8@2.3.0':
resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==}
engines: {node: '>=14.0.0'}
'@standard-schema/spec@1.1.0':
resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==}
@@ -2906,6 +3067,9 @@ packages:
boolbase@1.0.0:
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
bowser@2.14.1:
resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==}
brace-expansion@1.1.12:
resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==}
@@ -3551,6 +3715,13 @@ packages:
fast-wrap-ansi@0.2.0:
resolution: {integrity: sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w==}
fast-xml-builder@1.2.0:
resolution: {integrity: sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==}
fast-xml-parser@5.7.3:
resolution: {integrity: sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==}
hasBin: true
fastq@1.19.1:
resolution: {integrity: sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==}
@@ -4512,6 +4683,10 @@ packages:
path-browserify@1.0.1:
resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==}
path-expression-matcher@1.5.0:
resolution: {integrity: sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==}
engines: {node: '>=14.0.0'}
path-is-absolute@1.0.1:
resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==}
engines: {node: '>=0.10.0'}
@@ -5615,6 +5790,9 @@ packages:
resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==}
engines: {node: '>=8'}
strnum@2.3.0:
resolution: {integrity: sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==}
style-inject@0.3.0:
resolution: {integrity: sha512-IezA2qp+vcdlhJaVm5SOdPPTUu0FCEqfNSli2vRuSIBbu5Nq5UvygTk/VzeCqfLz2Atj3dVII5QBKGZRZ0edzw==}
@@ -6065,6 +6243,10 @@ packages:
resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==}
engines: {node: '>=18'}
xml-naming@0.1.0:
resolution: {integrity: sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==}
engines: {node: '>=16.0.0'}
xmlchars@2.2.0:
resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==}
@@ -6152,6 +6334,271 @@ snapshots:
'@asamuzakjp/nwsapi@2.3.9': {}
'@aws-crypto/crc32@5.2.0':
dependencies:
'@aws-crypto/util': 5.2.0
'@aws-sdk/types': 3.973.8
tslib: 2.8.1
'@aws-crypto/crc32c@5.2.0':
dependencies:
'@aws-crypto/util': 5.2.0
'@aws-sdk/types': 3.973.8
tslib: 2.8.1
'@aws-crypto/sha1-browser@5.2.0':
dependencies:
'@aws-crypto/supports-web-crypto': 5.2.0
'@aws-crypto/util': 5.2.0
'@aws-sdk/types': 3.973.8
'@aws-sdk/util-locate-window': 3.965.5
'@smithy/util-utf8': 2.3.0
tslib: 2.8.1
'@aws-crypto/sha256-browser@5.2.0':
dependencies:
'@aws-crypto/sha256-js': 5.2.0
'@aws-crypto/supports-web-crypto': 5.2.0
'@aws-crypto/util': 5.2.0
'@aws-sdk/types': 3.973.8
'@aws-sdk/util-locate-window': 3.965.5
'@smithy/util-utf8': 2.3.0
tslib: 2.8.1
'@aws-crypto/sha256-js@5.2.0':
dependencies:
'@aws-crypto/util': 5.2.0
'@aws-sdk/types': 3.973.8
tslib: 2.8.1
'@aws-crypto/supports-web-crypto@5.2.0':
dependencies:
tslib: 2.8.1
'@aws-crypto/util@5.2.0':
dependencies:
'@aws-sdk/types': 3.973.8
'@smithy/util-utf8': 2.3.0
tslib: 2.8.1
'@aws-sdk/client-s3@3.1050.0':
dependencies:
'@aws-crypto/sha1-browser': 5.2.0
'@aws-crypto/sha256-browser': 5.2.0
'@aws-crypto/sha256-js': 5.2.0
'@aws-sdk/core': 3.974.12
'@aws-sdk/credential-provider-node': 3.972.43
'@aws-sdk/middleware-bucket-endpoint': 3.972.14
'@aws-sdk/middleware-expect-continue': 3.972.12
'@aws-sdk/middleware-flexible-checksums': 3.974.20
'@aws-sdk/middleware-location-constraint': 3.972.10
'@aws-sdk/middleware-sdk-s3': 3.972.41
'@aws-sdk/middleware-ssec': 3.972.10
'@aws-sdk/signature-v4-multi-region': 3.996.27
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/fetch-http-handler': 5.4.3
'@smithy/node-http-handler': 4.7.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/core@3.974.12':
dependencies:
'@aws-sdk/types': 3.973.8
'@aws-sdk/xml-builder': 3.972.24
'@aws/lambda-invoke-store': 0.2.4
'@smithy/core': 3.24.3
'@smithy/signature-v4': 5.4.3
'@smithy/types': 4.14.2
bowser: 2.14.1
tslib: 2.8.1
'@aws-sdk/crc64-nvme@3.972.8':
dependencies:
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/credential-provider-env@3.972.38':
dependencies:
'@aws-sdk/core': 3.974.12
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/credential-provider-http@3.972.40':
dependencies:
'@aws-sdk/core': 3.974.12
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/fetch-http-handler': 5.4.3
'@smithy/node-http-handler': 4.7.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/credential-provider-ini@3.972.42':
dependencies:
'@aws-sdk/core': 3.974.12
'@aws-sdk/credential-provider-env': 3.972.38
'@aws-sdk/credential-provider-http': 3.972.40
'@aws-sdk/credential-provider-login': 3.972.42
'@aws-sdk/credential-provider-process': 3.972.38
'@aws-sdk/credential-provider-sso': 3.972.42
'@aws-sdk/credential-provider-web-identity': 3.972.42
'@aws-sdk/nested-clients': 3.997.10
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/credential-provider-imds': 4.3.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/credential-provider-login@3.972.42':
dependencies:
'@aws-sdk/core': 3.974.12
'@aws-sdk/nested-clients': 3.997.10
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/credential-provider-node@3.972.43':
dependencies:
'@aws-sdk/credential-provider-env': 3.972.38
'@aws-sdk/credential-provider-http': 3.972.40
'@aws-sdk/credential-provider-ini': 3.972.42
'@aws-sdk/credential-provider-process': 3.972.38
'@aws-sdk/credential-provider-sso': 3.972.42
'@aws-sdk/credential-provider-web-identity': 3.972.42
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/credential-provider-imds': 4.3.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/credential-provider-process@3.972.38':
dependencies:
'@aws-sdk/core': 3.974.12
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/credential-provider-sso@3.972.42':
dependencies:
'@aws-sdk/core': 3.974.12
'@aws-sdk/nested-clients': 3.997.10
'@aws-sdk/token-providers': 3.1049.0
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/credential-provider-web-identity@3.972.42':
dependencies:
'@aws-sdk/core': 3.974.12
'@aws-sdk/nested-clients': 3.997.10
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/middleware-bucket-endpoint@3.972.14':
dependencies:
'@aws-sdk/core': 3.974.12
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/middleware-expect-continue@3.972.12':
dependencies:
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/middleware-flexible-checksums@3.974.20':
dependencies:
'@aws-crypto/crc32': 5.2.0
'@aws-crypto/crc32c': 5.2.0
'@aws-crypto/util': 5.2.0
'@aws-sdk/core': 3.974.12
'@aws-sdk/crc64-nvme': 3.972.8
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/middleware-location-constraint@3.972.10':
dependencies:
'@aws-sdk/types': 3.973.8
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/middleware-sdk-s3@3.972.41':
dependencies:
'@aws-sdk/core': 3.974.12
'@aws-sdk/signature-v4-multi-region': 3.996.27
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/signature-v4': 5.4.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/middleware-ssec@3.972.10':
dependencies:
'@aws-sdk/types': 3.973.8
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/nested-clients@3.997.10':
dependencies:
'@aws-crypto/sha256-browser': 5.2.0
'@aws-crypto/sha256-js': 5.2.0
'@aws-sdk/core': 3.974.12
'@aws-sdk/signature-v4-multi-region': 3.996.27
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/fetch-http-handler': 5.4.3
'@smithy/node-http-handler': 4.7.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/signature-v4-multi-region@3.996.27':
dependencies:
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/signature-v4': 5.4.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/token-providers@3.1049.0':
dependencies:
'@aws-sdk/core': 3.974.12
'@aws-sdk/nested-clients': 3.997.10
'@aws-sdk/types': 3.973.8
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/types@3.973.8':
dependencies:
'@smithy/types': 4.14.2
tslib: 2.8.1
'@aws-sdk/util-locate-window@3.965.5':
dependencies:
tslib: 2.8.1
'@aws-sdk/xml-builder@3.972.24':
dependencies:
'@nodable/entities': 2.1.0
'@smithy/types': 4.14.2
fast-xml-parser: 5.7.3
tslib: 2.8.1
'@aws/lambda-invoke-store@0.2.4': {}
'@babel/code-frame@7.27.1':
dependencies:
'@babel/helper-validator-identifier': 7.27.1
@@ -7381,6 +7828,8 @@ snapshots:
'@next/swc-win32-x64-msvc@16.2.6':
optional: true
'@nodable/entities@2.1.0': {}
'@nodelib/fs.scandir@2.1.5':
dependencies:
'@nodelib/fs.stat': 2.0.5
@@ -7941,6 +8390,54 @@ snapshots:
'@sindresorhus/merge-streams@2.3.0': {}
'@smithy/core@3.24.3':
dependencies:
'@aws-crypto/crc32': 5.2.0
'@smithy/types': 4.14.2
tslib: 2.8.1
'@smithy/credential-provider-imds@4.3.3':
dependencies:
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@smithy/fetch-http-handler@5.4.3':
dependencies:
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@smithy/is-array-buffer@2.2.0':
dependencies:
tslib: 2.8.1
'@smithy/node-http-handler@4.7.3':
dependencies:
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@smithy/signature-v4@5.4.3':
dependencies:
'@smithy/core': 3.24.3
'@smithy/types': 4.14.2
tslib: 2.8.1
'@smithy/types@4.14.2':
dependencies:
tslib: 2.8.1
'@smithy/util-buffer-from@2.2.0':
dependencies:
'@smithy/is-array-buffer': 2.2.0
tslib: 2.8.1
'@smithy/util-utf8@2.3.0':
dependencies:
'@smithy/util-buffer-from': 2.2.0
tslib: 2.8.1
'@standard-schema/spec@1.1.0': {}
'@svgr/babel-plugin-add-jsx-attribute@8.0.0(@babel/core@7.28.3)':
@@ -8412,6 +8909,8 @@ snapshots:
boolbase@1.0.0: {}
bowser@2.14.1: {}
brace-expansion@1.1.12:
dependencies:
balanced-match: 1.0.2
@@ -9169,6 +9668,18 @@ snapshots:
dependencies:
fast-string-width: 3.0.2
fast-xml-builder@1.2.0:
dependencies:
path-expression-matcher: 1.5.0
xml-naming: 0.1.0
fast-xml-parser@5.7.3:
dependencies:
'@nodable/entities': 2.1.0
fast-xml-builder: 1.2.0
path-expression-matcher: 1.5.0
strnum: 2.3.0
fastq@1.19.1:
dependencies:
reusify: 1.1.0
@@ -10109,6 +10620,8 @@ snapshots:
path-browserify@1.0.1: {}
path-expression-matcher@1.5.0: {}
path-is-absolute@1.0.1: {}
path-key@2.0.1: {}
@@ -11342,6 +11855,8 @@ snapshots:
dependencies:
min-indent: 1.0.1
strnum@2.3.0: {}
style-inject@0.3.0: {}
styled-jsx@5.1.6(@babel/core@7.29.0)(react@19.2.6):
@@ -11786,6 +12301,8 @@ snapshots:
xml-name-validator@5.0.0: {}
xml-naming@0.1.0: {}
xmlchars@2.2.0: {}
xtend@4.0.2: {}
src/lib/heatmap-s3.ts
+56
View File
@@ -0,0 +1,56 @@
import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
let client: S3Client | null = null;
function getBucket() {
const bucket = process.env.S3_HEATMAP_BUCKET;
if (!bucket) {
throw new Error('S3_HEATMAP_BUCKET is not set.');
}
return bucket;
}
function getClient() {
if (!client) {
client = new S3Client({
region: process.env.AWS_REGION,
});
}
return client;
}
export async function putHeatmapSnapshot(
objectKey: string,
imageData: Buffer,
mimeType: string,
) {
await getClient().send(
new PutObjectCommand({
Bucket: getBucket(),
Key: objectKey,
Body: imageData,
ContentType: mimeType,
}),
);
}
export async function getHeatmapSnapshot(objectKey: string) {
const response = await getClient().send(
new GetObjectCommand({
Bucket: getBucket(),
Key: objectKey,
}),
);
if (!response.Body) {
return null;
}
return {
mimeType: response.ContentType || 'application/octet-stream',
imageData: Buffer.from(await response.Body.transformToByteArray()),
};
}
src/queries/sql/heatmap/ensureHeatmapSnapshot.ts
+247 -2
View File
@@ -1,5 +1,7 @@
import clickhouse from '@/lib/clickhouse';
import prisma from '@/lib/prisma';
import { uuid } from '@/lib/crypto';
import { getHeatmapSnapshot, putHeatmapSnapshot } from '@/lib/heatmap-s3';
import { getWebsite } from '@/queries/prisma';
const SNAPSHOT_STATUS = {
@@ -10,7 +12,6 @@ const SNAPSHOT_STATUS = {
const SNAPSHOT_RETRY_DELAY_MS = 15 * 60 * 1000;
const SNAPSHOT_PENDING_WINDOW_MS = 30 * 1000;
export type HeatmapSnapshotStatus = (typeof SNAPSHOT_STATUS)[keyof typeof SNAPSHOT_STATUS];
export interface HeatmapSnapshotImage {
@@ -35,6 +36,7 @@ interface SnapshotRecord {
pageH: number;
status: HeatmapSnapshotStatus;
mimeType: string | null;
objectKey: string | null;
imageSize: number | null;
error: string | null;
hasImage: boolean;
@@ -57,6 +59,12 @@ interface CaptureResult {
pageH: number;
}
const CLICKHOUSE_SNAPSHOT_STATUS = {
pending: 0,
ready: 1,
failed: 2,
} as const;
async function measurePage(page: any) {
return page.evaluate(() => {
const doc = document.documentElement;
@@ -143,6 +151,19 @@ async function findSnapshot(
urlPath: string,
viewportW: number,
viewportH: number,
): Promise<SnapshotRecord | null> {
if (clickhouse.enabled) {
return findClickhouseSnapshot(websiteId, urlPath, viewportW, viewportH);
}
return findRelationalSnapshot(websiteId, urlPath, viewportW, viewportH);
}
async function findRelationalSnapshot(
websiteId: string,
urlPath: string,
viewportW: number,
viewportH: number,
): Promise<SnapshotRecord | null> {
const rows = await prisma.rawQuery(
`
@@ -156,6 +177,7 @@ async function findSnapshot(
page_h as "pageH",
status,
mime_type as "mimeType",
null as "objectKey",
image_size as "imageSize",
error,
image_data is not null as "hasImage",
@@ -174,6 +196,78 @@ async function findSnapshot(
return rows?.[0] ?? null;
}
async function findClickhouseSnapshot(
websiteId: string,
urlPath: string,
viewportW: number,
viewportH: number,
): Promise<SnapshotRecord | null> {
const rows = await clickhouse.rawQuery<
{
id: string;
websiteId: string;
urlPath: string;
viewportW: number;
viewportH: number;
pageW: number;
pageH: number;
status: number;
mimeType: string | null;
objectKey: string;
imageSize: number | null;
error: string | null;
createdAt: string;
}[]
>(
`
select
snapshot_id as id,
website_id as websiteId,
url_path as urlPath,
viewport_w as viewportW,
viewport_h as viewportH,
page_w as pageW,
page_h as pageH,
status,
mime_type as mimeType,
object_key as objectKey,
image_size as imageSize,
error,
created_at as createdAt
from heatmap_snapshot
where website_id = {websiteId:UUID}
and url_path = {urlPath:String}
and viewport_w = {viewportW:UInt32}
and viewport_h = {viewportH:UInt32}
order by created_at desc
limit 1
`,
{ websiteId, urlPath, viewportW, viewportH },
'findHeatmapSnapshot',
);
const row = rows?.[0];
if (!row) {
return null;
}
const status = Object.entries(CLICKHOUSE_SNAPSHOT_STATUS).find(([, value]) => value === row.status)?.[0];
if (!status) {
return null;
}
return {
...row,
status: SNAPSHOT_STATUS[status as keyof typeof SNAPSHOT_STATUS],
mimeType: row.mimeType || null,
objectKey: row.objectKey || null,
hasImage: row.status === CLICKHOUSE_SNAPSHOT_STATUS.ready && Boolean(row.objectKey),
updatedAt: row.createdAt,
};
}
function getSnapshotImageUrl(websiteId: string, snapshotId: string) {
return `/api/websites/${websiteId}/heatmaps/snapshots/${snapshotId}`;
}
@@ -231,6 +325,65 @@ export function shouldSkipSnapshot(urlPath: string) {
}
async function upsertSnapshotRecord({
id,
websiteId,
urlPath,
viewportW,
viewportH,
pageW,
pageH,
status,
mimeType,
imageData,
objectKey,
error,
}: {
id: string;
websiteId: string;
urlPath: string;
viewportW: number;
viewportH: number;
pageW: number;
pageH: number;
status: HeatmapSnapshotStatus;
mimeType: string | null;
imageData: Buffer | null;
objectKey?: string | null;
error: string | null;
}) {
if (clickhouse.enabled) {
return insertClickhouseSnapshotRecord({
id,
websiteId,
urlPath,
viewportW,
viewportH,
pageW,
pageH,
status,
mimeType,
objectKey: objectKey ?? null,
imageSize: imageData?.byteLength ?? null,
error,
});
}
return upsertRelationalSnapshotRecord({
id,
websiteId,
urlPath,
viewportW,
viewportH,
pageW,
pageH,
status,
mimeType,
imageData,
error,
});
}
async function upsertRelationalSnapshotRecord({
id,
websiteId,
urlPath,
@@ -317,6 +470,56 @@ async function upsertSnapshotRecord({
);
}
async function insertClickhouseSnapshotRecord({
id,
websiteId,
urlPath,
viewportW,
viewportH,
pageW,
pageH,
status,
mimeType,
objectKey,
imageSize,
error,
}: {
id: string;
websiteId: string;
urlPath: string;
viewportW: number;
viewportH: number;
pageW: number;
pageH: number;
status: HeatmapSnapshotStatus;
mimeType: string | null;
objectKey: string | null;
imageSize: number | null;
error: string | null;
}) {
return clickhouse.insert('heatmap_snapshot', [
{
snapshot_id: id,
website_id: websiteId,
url_path: urlPath,
viewport_w: viewportW,
viewport_h: viewportH,
page_w: pageW,
page_h: pageH,
status: CLICKHOUSE_SNAPSHOT_STATUS[status],
mime_type: mimeType || '',
object_key: objectKey || '',
image_size: imageSize,
error,
created_at: clickhouse.getUTCString(),
},
]);
}
function getSnapshotObjectKey(websiteId: string, snapshotId: string, viewportW: number, viewportH: number) {
return `heatmaps/${websiteId}/${viewportW}x${viewportH}/${snapshotId}.png`;
}
async function captureSnapshot(
url: string,
viewportW: number,
@@ -456,6 +659,14 @@ export async function ensureHeatmapSnapshot({
try {
const capture = await captureSnapshot(captureUrl, viewportW, viewportH, pageW);
const objectKey =
clickhouse.enabled
? getSnapshotObjectKey(websiteId, snapshotId, viewportW, viewportH)
: null;
if (objectKey) {
await putHeatmapSnapshot(objectKey, capture.imageData, capture.mimeType);
}
await upsertSnapshotRecord({
id: snapshotId,
@@ -467,7 +678,8 @@ export async function ensureHeatmapSnapshot({
pageH: capture.pageH,
status: SNAPSHOT_STATUS.ready,
mimeType: capture.mimeType,
imageData: capture.imageData,
imageData: clickhouse.enabled ? null : capture.imageData,
objectKey,
error: null,
});
} catch (error) {
@@ -495,6 +707,39 @@ export async function getHeatmapSnapshotImage(
websiteId: string,
snapshotId: string,
): Promise<{ mimeType: string; imageData: Buffer } | null> {
if (clickhouse.enabled) {
const rows = await clickhouse.rawQuery<
{ mimeType: string; objectKey: string }[]
>(
`
select
mime_type as mimeType,
object_key as objectKey
from heatmap_snapshot
where snapshot_id = {snapshotId:UUID}
and website_id = {websiteId:UUID}
and status = {status:UInt8}
and object_key != ''
order by created_at desc
limit 1
`,
{
websiteId,
snapshotId,
status: CLICKHOUSE_SNAPSHOT_STATUS.ready,
},
'getHeatmapSnapshotImage',
);
const row = rows?.[0];
if (!row?.objectKey) {
return null;
}
return getHeatmapSnapshot(row.objectKey);
}
const rows = await prisma.rawQuery(
`
select