Umami/umami
1
0
Fork 0
mirror of https://github.com/umami-software/umami.git synced 2026-05-30 06:47:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

performance updates checkpoint

Browse Source
This commit is contained in:
Francis Cao
2026-03-05 10:12:30 -08:00
parent 8563fe1d30
commit f6a6d2cc31
25 changed files with 488 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/clickhouse/migrations/09_add_performance.sql
+100 -67
View File
@@ -1,72 +1,105 @@
-- Create Performance
CREATE TABLE umami.website_performance
(
website_id UUID,
session_id UUID,
visit_id UUID,
url_path String,
lcp Nullable(Decimal(10, 1)),
inp Nullable(Decimal(10, 1)),
cls Nullable(Decimal(10, 4)),
fcp Nullable(Decimal(10, 1)),
ttfb Nullable(Decimal(10, 1)),
created_at DateTime('UTC')
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (website_id, toStartOfHour(created_at), session_id)
SETTINGS index_granularity = 8192;
-- Add performance columns to website_event
ALTER TABLE umami.website_event ADD COLUMN lcp Nullable(Decimal(10, 1)) AFTER twclid;
ALTER TABLE umami.website_event ADD COLUMN inp Nullable(Decimal(10, 1)) AFTER lcp;
ALTER TABLE umami.website_event ADD COLUMN cls Nullable(Decimal(10, 4)) AFTER inp;
ALTER TABLE umami.website_event ADD COLUMN fcp Nullable(Decimal(10, 1)) AFTER cls;
ALTER TABLE umami.website_event ADD COLUMN ttfb Nullable(Decimal(10, 1)) AFTER fcp;
-- Performance hourly aggregation
CREATE TABLE umami.website_performance_hourly
(
website_id UUID,
url_path String,
lcp_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
lcp_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
lcp_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
inp_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
inp_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
inp_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
cls_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 4))),
cls_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 4))),
cls_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 4))),
fcp_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
fcp_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
fcp_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
ttfb_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
ttfb_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
ttfb_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
sample_count SimpleAggregateFunction(sum, UInt64),
created_at DateTime('UTC')
)
ENGINE = AggregatingMergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (website_id, toStartOfHour(created_at), url_path)
SETTINGS index_granularity = 8192;
-- Update materialized view to exclude performance events from view counts
DROP TABLE umami.website_event_stats_hourly_mv;
CREATE MATERIALIZED VIEW umami.website_performance_hourly_mv
TO umami.website_performance_hourly
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
TO umami.website_event_stats_hourly
AS
SELECT
website_id,
url_path,
quantileState(0.5)(lcp) as lcp_p50,
quantileState(0.75)(lcp) as lcp_p75,
quantileState(0.95)(lcp) as lcp_p95,
quantileState(0.5)(inp) as inp_p50,
quantileState(0.75)(inp) as inp_p75,
quantileState(0.95)(inp) as inp_p95,
quantileState(0.5)(cls) as cls_p50,
quantileState(0.75)(cls) as cls_p75,
quantileState(0.95)(cls) as cls_p95,
quantileState(0.5)(fcp) as fcp_p50,
quantileState(0.75)(fcp) as fcp_p75,
quantileState(0.95)(fcp) as fcp_p95,
quantileState(0.5)(ttfb) as ttfb_p50,
quantileState(0.75)(ttfb) as ttfb_p75,
quantileState(0.95)(ttfb) as ttfb_p95,
count() as sample_count,
toStartOfHour(created_at) as created_at
FROM umami.website_performance
GROUP BY website_id, url_path, created_at;
session_id,
visit_id,
hostnames as hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
entry_url,
exit_url,
url_paths as url_path,
url_query,
utm_source,
utm_medium,
utm_campaign,
utm_content,
utm_term,
referrer_domain,
page_title,
gclid,
fbclid,
msclkid,
ttclid,
li_fat_id,
twclid,
event_type,
event_name,
views,
min_time,
max_time,
tag,
distinct_id,
timestamp as created_at
FROM (SELECT
website_id,
session_id,
visit_id,
arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
browser,
os,
device,
screen,
language,
country,
region,
city,
argMinState(url_path, created_at) entry_url,
argMaxState(url_path, created_at) exit_url,
arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
arrayFilter(x -> x != '', groupArray(url_query)) url_query,
arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
arrayFilter(x -> x != '', groupArray(twclid)) twclid,
event_type,
if(event_type = 2, groupArray(event_name), []) event_name,
sumIf(1, event_type NOT IN (2, 5)) views,
min(created_at) min_time,
max(created_at) max_time,
arrayFilter(x -> x != '', groupArray(tag)) tag,
distinct_id,
toStartOfHour(created_at) timestamp
FROM umami.website_event
GROUP BY website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
event_type,
distinct_id,
timestamp);
db/clickhouse/schema.sql
+7 -74
View File
@@ -34,6 +34,12 @@ CREATE TABLE umami.website_event
ttclid String,
li_fat_id String,
twclid String,
--performance
lcp Nullable(Decimal(10, 1)),
inp Nullable(Decimal(10, 1)),
cls Nullable(Decimal(10, 4)),
fcp Nullable(Decimal(10, 1)),
ttfb Nullable(Decimal(10, 1)),
--events
event_type UInt32,
event_name String,
@@ -209,7 +215,7 @@ FROM (SELECT
arrayFilter(x -> x != '', groupArray(twclid)) twclid,
event_type,
if(event_type = 2, groupArray(event_name), []) event_name,
sumIf(1, event_type != 2) views,
sumIf(1, event_type NOT IN (2, 5)) views,
min(created_at) min_time,
max(created_at) max_time,
arrayFilter(x -> x != '', groupArray(tag)) tag,
@@ -282,79 +288,6 @@ JOIN (SELECT event_id, string_value as currency
ON c.event_id = ed.event_id
WHERE positionCaseInsensitive(data_key, 'revenue') > 0;
-- Create Performance
CREATE TABLE umami.website_performance
(
website_id UUID,
session_id UUID,
visit_id UUID,
url_path String,
lcp Nullable(Decimal(10, 1)),
inp Nullable(Decimal(10, 1)),
cls Nullable(Decimal(10, 4)),
fcp Nullable(Decimal(10, 1)),
ttfb Nullable(Decimal(10, 1)),
created_at DateTime('UTC')
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (website_id, toStartOfHour(created_at), session_id)
SETTINGS index_granularity = 8192;
-- Performance hourly aggregation
CREATE TABLE umami.website_performance_hourly
(
website_id UUID,
url_path String,
lcp_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
lcp_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
lcp_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
inp_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
inp_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
inp_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
cls_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 4))),
cls_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 4))),
cls_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 4))),
fcp_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
fcp_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
fcp_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
ttfb_p50 AggregateFunction(quantile(0.5), Nullable(Decimal(10, 1))),
ttfb_p75 AggregateFunction(quantile(0.75), Nullable(Decimal(10, 1))),
ttfb_p95 AggregateFunction(quantile(0.95), Nullable(Decimal(10, 1))),
sample_count SimpleAggregateFunction(sum, UInt64),
created_at DateTime('UTC')
)
ENGINE = AggregatingMergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (website_id, toStartOfHour(created_at), url_path)
SETTINGS index_granularity = 8192;
CREATE MATERIALIZED VIEW umami.website_performance_hourly_mv
TO umami.website_performance_hourly
AS
SELECT
website_id,
url_path,
quantileState(0.5)(lcp) as lcp_p50,
quantileState(0.75)(lcp) as lcp_p75,
quantileState(0.95)(lcp) as lcp_p95,
quantileState(0.5)(inp) as inp_p50,
quantileState(0.75)(inp) as inp_p75,
quantileState(0.95)(inp) as inp_p95,
quantileState(0.5)(cls) as cls_p50,
quantileState(0.75)(cls) as cls_p75,
quantileState(0.95)(cls) as cls_p95,
quantileState(0.5)(fcp) as fcp_p50,
quantileState(0.75)(fcp) as fcp_p75,
quantileState(0.95)(fcp) as fcp_p95,
quantileState(0.5)(ttfb) as ttfb_p50,
quantileState(0.75)(ttfb) as ttfb_p75,
quantileState(0.95)(ttfb) as ttfb_p95,
count() as sample_count,
toStartOfHour(created_at) as created_at
FROM umami.website_performance
GROUP BY website_id, url_path, created_at;
-- Create session_replay
CREATE TABLE umami.session_replay
(
src/app/(main)/websites/[websiteId]/(reports)/performance/Performance.tsx
+114 -17
View File
@@ -1,13 +1,26 @@
'use client';
import { Column, Grid, Row, Text } from '@umami/react-zen';
import {
Column,
Grid,
Heading,
ListItem,
Row,
Select,
Tab,
TabList,
TabPanel,
Tabs,
Text,
} from '@umami/react-zen';
import { colord } from 'colord';
import { useCallback, useMemo, useState } from 'react';
import { BarChart } from '@/components/charts/BarChart';
import { GridRow } from '@/components/common/GridRow';
import { LoadingPanel } from '@/components/common/LoadingPanel';
import { Panel } from '@/components/common/Panel';
import { TypeIcon } from '@/components/common/TypeIcon';
import { useLocale, useMessages, useResultQuery } from '@/components/hooks';
import { ListTable } from '@/components/metrics/ListTable';
import { MetricsBar } from '@/components/metrics/MetricsBar';
import { PerformanceCard } from '@/components/metrics/PerformanceCard';
import { renderDateLabels } from '@/lib/charts';
import { CHART_COLORS, WEB_VITALS_THRESHOLDS } from '@/lib/constants';
@@ -39,8 +52,15 @@ function formatMetricValue(metric: string, value: number): string {
return `${Math.round(value)} ms`;
}
const PERCENTILES = [
{ id: 'p50', label: 'p50 — Median' },
{ id: 'p75', label: 'p75 — 75th Percentile' },
{ id: 'p95', label: 'p95 — 95th Percentile' },
] as const;
export function Performance({ websiteId, startDate, endDate, unit }: PerformanceProps) {
const [selectedMetric, setSelectedMetric] = useState<string>('lcp');
const [selectedPercentile, setSelectedPercentile] = useState<'p50' | 'p75' | 'p95'>('p75');
const { t, labels } = useMessages();
const { locale, dateLocale } = useLocale();
@@ -121,6 +141,19 @@ export function Performance({ websiteId, startDate, endDate, unit }: Performance
return (
<Column gap>
<Grid columns="280px" gap>
<Select
label="Percentile"
value={selectedPercentile}
onChange={(value: string) => setSelectedPercentile(value as 'p50' | 'p75' | 'p95')}
>
{PERCENTILES.map(({ id, label }) => (
<ListItem key={id} id={id}>
{label}
</ListItem>
))}
</Select>
</Grid>
<LoadingPanel data={data} isLoading={isLoading} error={error}>
{data && (
<Column gap>
@@ -129,7 +162,7 @@ export function Performance({ websiteId, startDate, endDate, unit }: Performance
<PerformanceCard
key={metric}
metric={metric}
value={Number(data.summary?.[metric]?.p75 || 0)}
value={Number(data.summary?.[metric]?.[selectedPercentile] || 0)}
label={t(labels[metric]) || metric.toUpperCase()}
formatValue={(n: number) => formatMetricValue(metric, n)}
onClick={() => setSelectedMetric(metric)}
@@ -162,20 +195,84 @@ export function Performance({ websiteId, startDate, endDate, unit }: Performance
/>
</Column>
</Panel>
<Panel>
<ListTable
title={t(labels.pages)}
metric={t(labels[selectedMetric]) || selectedMetric.toUpperCase()}
data={data.pages?.map(
({ urlPath, p75, count }: { urlPath: string; p75: number; count: number }) => ({
label: urlPath,
count: Number(p75),
percent: 0,
}),
)}
renderLabel={({ label }: { label: string }) => <Text>{label}</Text>}
/>
</Panel>
<GridRow layout="two">
<Panel>
<Tabs>
<Heading size="2xl">{t(labels.pages)}</Heading>
<TabList>
<Tab id="path">{t(labels.path)}</Tab>
<Tab id="title">{t(labels.pageTitle)}</Tab>
</TabList>
<TabPanel id="path">
<ListTable
metric={t(labels[selectedMetric]) || selectedMetric.toUpperCase()}
showPercentage={false}
data={data.pages?.map(({ name, p50, p75, p95 }: any) => ({
label: name,
count: Number({ p50, p75, p95 }[selectedPercentile]),
percent: 0,
}))}
renderLabel={({ label }: { label: string }) => <Text>{label}</Text>}
/>
</TabPanel>
<TabPanel id="title">
<ListTable
metric={t(labels[selectedMetric]) || selectedMetric.toUpperCase()}
showPercentage={false}
data={data.pageTitles?.map(({ name, p50, p75, p95 }: any) => ({
label: name,
count: Number({ p50, p75, p95 }[selectedPercentile]),
percent: 0,
}))}
renderLabel={({ label }: { label: string }) => <Text>{label}</Text>}
/>
</TabPanel>
</Tabs>
</Panel>
<Panel>
<Tabs>
<Heading size="2xl">{t(labels.environment)}</Heading>
<TabList>
<Tab id="device">{t(labels.device)}</Tab>
<Tab id="browser">{t(labels.browser)}</Tab>
</TabList>
<TabPanel id="device">
<ListTable
metric={t(labels[selectedMetric]) || selectedMetric.toUpperCase()}
showPercentage={false}
data={data.devices?.map(({ name, p50, p75, p95 }: any) => ({
label: name,
count: Number({ p50, p75, p95 }[selectedPercentile]),
percent: 0,
}))}
renderLabel={({ label }: { label: string }) => (
<Row gap="2" alignItems="center">
<TypeIcon type="device" value={label} />
<Text>{label}</Text>
</Row>
)}
/>
</TabPanel>
<TabPanel id="browser">
<ListTable
metric={t(labels[selectedMetric]) || selectedMetric.toUpperCase()}
showPercentage={false}
data={data.browsers?.map(({ name, p50, p75, p95 }: any) => ({
label: name,
count: Number({ p50, p75, p95 }[selectedPercentile]),
percent: 0,
}))}
renderLabel={({ label }: { label: string }) => (
<Row gap="2" alignItems="center">
<TypeIcon type="browser" value={label} />
<Text>{label}</Text>
</Row>
)}
/>
</TabPanel>
</Tabs>
</Panel>
</GridRow>
</Column>
)}
</LoadingPanel>
src/app/api/reports/performance/route.ts
+15 -2
View File
@@ -3,6 +3,7 @@ import { json, unauthorized } from '@/lib/response';
import { reportResultSchema } from '@/lib/schema';
import { canViewWebsite } from '@/permissions';
import { getPerformance, type PerformanceParameters } from '@/queries/sql/reports/getPerformance';
import { getPerformanceMetrics } from '@/queries/sql/reports/getPerformanceMetrics';
export async function POST(request: Request) {
const { auth, body, error } = await parseRequest(request, reportResultSchema);
@@ -20,7 +21,19 @@ export async function POST(request: Request) {
const parameters = await setWebsiteDate(websiteId, body.parameters);
const filters = await getQueryFilters(body.filters, websiteId);
const data = await getPerformance(websiteId, parameters as PerformanceParameters, filters);
const [{ chart, summary }, pages, pageTitles, devices, browsers] = await Promise.all([
getPerformance(websiteId, parameters as PerformanceParameters, filters),
getPerformanceMetrics(websiteId, parameters as PerformanceParameters, filters, 'url_path', 500),
getPerformanceMetrics(
websiteId,
parameters as PerformanceParameters,
filters,
'page_title',
500,
),
getPerformanceMetrics(websiteId, parameters as PerformanceParameters, filters, 'device'),
getPerformanceMetrics(websiteId, parameters as PerformanceParameters, filters, 'browser', 500),
]);
return json(data);
return json({ chart, summary, pages, pageTitles, devices, browsers });
}
src/app/api/send/route.ts
+12 -2
View File
@@ -12,7 +12,7 @@ import { parseRequest } from '@/lib/request';
import { badRequest, forbidden, json, serverError } from '@/lib/response';
import { anyObjectParam, urlOrPathParam } from '@/lib/schema';
import { safeDecodeURI, safeDecodeURIComponent } from '@/lib/url';
import { createSession, saveEvent, savePerformance, saveSessionData } from '@/queries/sql';
import { createSession, saveEvent, saveSessionData } from '@/queries/sql';
interface Cache {
websiteId: string;
@@ -284,11 +284,21 @@ export async function POST(request: Request) {
const currentUrl = new URL(url, base);
const urlPath = currentUrl.pathname === '/undefined' ? '' : currentUrl.pathname;
await savePerformance({
await saveEvent({
websiteId: sourceId,
sessionId,
visitId,
urlPath,
pageTitle: safeDecodeURIComponent(title),
eventType: EVENT_TYPE.performance,
browser,
os,
device,
screen,
language,
country,
region,
city,
lcp,
inp,
cls,
src/components/metrics/ListTable.tsx
+6 -1
View File
@@ -68,7 +68,12 @@ export function ListTable({
return (
<Column gap>
<Grid alignItems="center" justifyContent="space-between" paddingLeft="2" columns="1fr 100px">
<Grid
alignItems="center"
justifyContent="space-between"
paddingLeft="2"
columns={`1fr ${showPercentage ? '100px' : '150px'}`}
>
<Text weight="bold">{title}</Text>
<Text weight="bold" align="center">
{metric}
src/components/metrics/PerformanceCard.module.css
+8 -2
View File
@@ -1,9 +1,15 @@
.card {
transition: box-shadow 0.2s;
transition: box-shadow 0.2s ease;
}
.card:hover {
box-shadow: 0 0 0 1px color-mix(in srgb, var(--primary) 15%, transparent);
}
.card.selected {
box-shadow: 0 0 0 2px var(--base-color-primary);
box-shadow:
0 0 0 1px color-mix(in srgb, var(--primary) 40%, transparent),
0 0 6px color-mix(in srgb, var(--primary) 15%, transparent);
}
.good .rating {
src/components/metrics/PerformanceCard.tsx
+1 -4
View File
@@ -1,6 +1,4 @@
import { useSpring } from '@react-spring/web';
import { Column, Text } from '@umami/react-zen';
import { AnimatedDiv } from '@/components/common/AnimatedDiv';
import { useMessages } from '@/components/hooks';
import { WEB_VITALS_THRESHOLDS } from '@/lib/constants';
import { formatNumber } from '@/lib/format';
@@ -33,7 +31,6 @@ export const PerformanceCard = ({
}: PerformanceCardProps) => {
const { t, labels } = useMessages();
const rating = getRating(metric, value);
const props = useSpring({ x: Number(value) || 0, from: { x: 0 } });
return (
<Column
@@ -52,7 +49,7 @@ export const PerformanceCard = ({
{label}
</Text>
<Text size="4xl" weight="bold" wrap="nowrap">
<AnimatedDiv title={value?.toString()}>{props?.x?.to(x => formatValue(x))}</AnimatedDiv>
{formatValue(value)}
</Text>
<Text size="sm" className={styles.rating}>
{t(labels[rating === 'needs-improvement' ? 'needsImprovement' : rating])}
src/lib/constants.ts
+1
View File
@@ -114,6 +114,7 @@ export const EVENT_TYPE = {
customEvent: 2,
linkEvent: 3,
pixelEvent: 4,
performance: 5,
} as const;
export const ENTITY_TYPE = {
src/queries/sql/events/saveEvent.ts
+33 -6
View File
@@ -53,6 +53,13 @@ export interface SaveEventArgs {
ttclid?: string;
lifatid?: string;
twclid?: string;
// Performance
lcp?: number;
inp?: number;
cls?: number;
fcp?: number;
ttfb?: number;
}
export async function saveEvent(args: SaveEventArgs) {
@@ -89,6 +96,11 @@ async function relationalQuery({
ttclid,
lifatid,
twclid,
lcp,
inp,
cls,
fcp,
ttfb,
}: SaveEventArgs) {
const websiteEventId = uuid();
@@ -119,6 +131,11 @@ async function relationalQuery({
eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
tag,
hostname,
lcp,
inp,
cls,
fcp,
ttfb,
createdAt,
},
});
@@ -186,6 +203,11 @@ async function clickhouseQuery({
ttclid,
lifatid,
twclid,
lcp,
inp,
cls,
fcp,
ttfb,
}: SaveEventArgs) {
const { insert, getUTCString } = clickhouse;
const { sendMessage } = kafka;
@@ -221,12 +243,17 @@ async function clickhouseQuery({
tag: tag,
distinct_id: distinctId,
created_at: getUTCString(createdAt),
browser,
os,
device,
screen,
language,
hostname,
browser: browser,
os: os,
device: device,
screen: screen,
language: language,
hostname: hostname,
lcp: lcp,
inp: inp,
cls: cls,
fcp: fcp,
ttfb: ttfb,
};
if (kafka.enabled) {
src/queries/sql/getChannelExpandedMetrics.ts
+3 -3
View File
@@ -68,7 +68,7 @@ async function relationalQuery(
${excludeBounceQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${dateQuery}
${filterQuery}
group by prefix,
@@ -163,7 +163,7 @@ async function clickhouseQuery(
VIDEO_DOMAINS,
)}]) != 0 or position(lower(utm_medium), 'video') > 0 then concat(prefix, 'Video')
when referrer_domain != hostname and referrer_domain != '' then 'referral'
else '' end AS name,
else '' end AS "name",
session_id,
visit_id,
count(*) c,
@@ -174,7 +174,7 @@ async function clickhouseQuery(
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
and name != ''
${filterQuery}
group by prefix, name, session_id, visit_id
src/queries/sql/getChannelMetrics.ts
+2 -2
View File
@@ -45,7 +45,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
${excludeBounceQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${dateQuery}
${filterQuery}),
@@ -123,7 +123,7 @@ async function clickhouseQuery(
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and event_type != 2
and event_type NOT IN (2, 5)
${dateQuery}
${filterQuery}
group by 1, 2
src/queries/sql/getWebsiteStats.ts
+3 -3
View File
@@ -58,7 +58,7 @@ async function relationalQuery(
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by 1, 2
@@ -103,7 +103,7 @@ async function clickhouseQuery(
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by session_id, visit_id
) as t;
@@ -127,7 +127,7 @@ async function clickhouseQuery(
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by session_id, visit_id
) as t;
src/queries/sql/getWeeklyTraffic.ts
+3 -3
View File
@@ -33,7 +33,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by time
order by 1
@@ -63,7 +63,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by time
order by time
@@ -78,7 +78,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by time
order by time
src/queries/sql/index.ts
-1
View File
@@ -24,7 +24,6 @@ export * from './pageviews/getPageviewExpandedMetrics';
export * from './pageviews/getPageviewMetrics';
export * from './pageviews/getPageviewStats';
export * from './performance/getPerformanceStats';
export * from './performance/savePerformance';
export * from './replays/getReplayChunks';
export * from './replays/getSessionReplays';
export * from './replays/saveRecording';
src/queries/sql/pageviews/getPageviewExpandedMetrics.ts
+4 -4
View File
@@ -68,7 +68,7 @@ async function relationalQuery(
from website_event
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
group by visit_id
) x
on x.visit_id = website_event.visit_id
@@ -100,7 +100,7 @@ async function relationalQuery(
${entryExitQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${excludeDomain}
${filterQuery}
group by ${column}, website_event.session_id, website_event.visit_id
@@ -149,7 +149,7 @@ async function clickhouseQuery(
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
group by visit_id) x
ON x.visit_id = website_event.visit_id`;
}
@@ -177,7 +177,7 @@ async function clickhouseQuery(
${entryExitQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
and name != ''
${excludeDomain}
${filterQuery}
src/queries/sql/pageviews/getPageviewMetrics.ts
+5 -5
View File
@@ -63,7 +63,7 @@ async function relationalQuery(
from website_event
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
order by visit_id, created_at ${order}
) x
on x.visit_id = website_event.visit_id
@@ -81,7 +81,7 @@ async function relationalQuery(
${entryExitQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${excludeDomain}
${filterQuery}
group by 1
@@ -127,7 +127,7 @@ async function clickhouseQuery(
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
group by visit_id) x
ON x.visit_id = website_event.visit_id`;
}
@@ -141,7 +141,7 @@ async function clickhouseQuery(
${entryExitQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${excludeDomain}
${filterQuery}
group by x
@@ -180,7 +180,7 @@ async function clickhouseQuery(
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${excludeDomain}
${filterQuery}
${groupByQuery}) as g
src/queries/sql/pageviews/getPageviewStats.ts
+3 -3
View File
@@ -33,7 +33,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by 1
order by 1
@@ -70,7 +70,7 @@ async function clickhouseQuery(
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by t
) as g
@@ -90,7 +90,7 @@ async function clickhouseQuery(
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by t
) as g
src/queries/sql/performance/getPerformanceStats.ts
+22 -12
View File
@@ -23,8 +23,11 @@ async function relationalQuery(
websiteId: string,
filters: QueryFilters,
): Promise<PerformanceStatsResult> {
const { rawQuery } = prisma;
const { startDate, endDate } = filters;
const { rawQuery, parseFilters } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
const result = await rawQuery(
`
@@ -35,11 +38,15 @@ async function relationalQuery(
percentile_cont(0.75) within group (order by fcp) as fcp,
percentile_cont(0.75) within group (order by ttfb) as ttfb,
count(*) as count
from performance
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
from website_event
${cohortQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.event_type = 5
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
`,
{ websiteId, startDate, endDate },
queryParams,
);
return result?.[0] || { lcp: 0, inp: 0, cls: 0, fcp: 0, ttfb: 0, count: 0 };
@@ -49,8 +56,8 @@ async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<PerformanceStatsResult> {
const { rawQuery } = clickhouse;
const { startDate, endDate } = filters;
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId });
const result = await rawQuery<PerformanceStatsResult>(
`
@@ -61,11 +68,14 @@ async function clickhouseQuery(
quantile(0.75)(fcp) as fcp,
quantile(0.75)(ttfb) as ttfb,
count() as count
from website_performance
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
from website_event
${cohortQuery}
where website_event.website_id = {websiteId:UUID}
and website_event.event_type = 5
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
`,
{ websiteId, startDate, endDate },
queryParams,
);
return result?.[0] || { lcp: 0, inp: 0, cls: 0, fcp: 0, ttfb: 0, count: 0 };
src/queries/sql/performance/savePerformance.ts
-70
View File
@@ -1,70 +0,0 @@
import clickhouse from '@/lib/clickhouse';
import { uuid } from '@/lib/crypto';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import kafka from '@/lib/kafka';
import prisma from '@/lib/prisma';
export interface SavePerformanceArgs {
websiteId: string;
sessionId: string;
visitId: string;
urlPath: string;
lcp?: number;
inp?: number;
cls?: number;
fcp?: number;
ttfb?: number;
createdAt: Date;
}
export async function savePerformance(args: SavePerformanceArgs) {
return runQuery({
[PRISMA]: () => relationalQuery(args),
[CLICKHOUSE]: () => clickhouseQuery(args),
});
}
async function relationalQuery(data: SavePerformanceArgs) {
const { websiteId, sessionId, visitId, urlPath, lcp, inp, cls, fcp, ttfb, createdAt } = data;
await prisma.client.performance.create({
data: {
id: uuid(),
websiteId,
sessionId,
visitId,
urlPath: urlPath?.substring(0, 500),
lcp,
inp,
cls,
fcp,
ttfb,
createdAt,
},
});
}
async function clickhouseQuery(data: SavePerformanceArgs) {
const { websiteId, sessionId, visitId, urlPath, lcp, inp, cls, fcp, ttfb, createdAt } = data;
const { insert, getUTCString } = clickhouse;
const { sendMessage } = kafka;
const message = {
website_id: websiteId,
session_id: sessionId,
visit_id: visitId,
url_path: urlPath?.substring(0, 500),
lcp: lcp ?? null,
inp: inp ?? null,
cls: cls ?? null,
fcp: fcp ?? null,
ttfb: ttfb ?? null,
created_at: getUTCString(createdAt),
};
if (kafka.enabled) {
await sendMessage('performance', message);
} else {
await insert('website_performance', [message]);
}
}
src/queries/sql/reports/getPerformance.ts
+40 -54
View File
@@ -13,7 +13,6 @@ export interface PerformanceParameters {
export interface PerformanceResult {
chart: { t: string; p50: number; p75: number; p95: number }[];
pages: { urlPath: string; p75: number; count: number }[];
summary: {
lcp: { p50: number; p75: number; p95: number };
inp: { p50: number; p75: number; p95: number };
@@ -39,7 +38,11 @@ async function relationalQuery(
filters: QueryFilters,
): Promise<PerformanceResult> {
const { startDate, endDate, unit = 'day', timezone = 'utc', metric = 'lcp' } = parameters;
const { getDateSQL, rawQuery } = prisma;
const { getDateSQL, rawQuery, parseFilters } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
const chart = await rawQuery(
`
@@ -48,29 +51,17 @@ async function relationalQuery(
percentile_cont(0.5) within group (order by ${metric}) as p50,
percentile_cont(0.75) within group (order by ${metric}) as p75,
percentile_cont(0.95) within group (order by ${metric}) as p95
from performance
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
from website_event
${cohortQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.event_type = 5
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by t
order by t
`,
{ websiteId, startDate, endDate },
);
const pages = await rawQuery(
`
select
url_path as "urlPath",
percentile_cont(0.75) within group (order by ${metric}) as p75,
count(*) as count
from performance
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
group by url_path
order by p75 desc
limit 100
`,
{ websiteId, startDate, endDate },
{ ...queryParams, startDate, endDate },
);
const summaryResult = await rawQuery(
@@ -92,11 +83,15 @@ async function relationalQuery(
percentile_cont(0.75) within group (order by ttfb) as ttfb_p75,
percentile_cont(0.95) within group (order by ttfb) as ttfb_p95,
count(*) as count
from performance
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
from website_event
${cohortQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.event_type = 5
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
`,
{ websiteId, startDate, endDate },
{ ...queryParams, startDate, endDate },
).then(result => result?.[0]);
const summary = {
@@ -128,7 +123,7 @@ async function relationalQuery(
count: Number(summaryResult?.count || 0),
};
return { chart, pages, summary };
return { chart, summary };
}
async function clickhouseQuery(
@@ -137,38 +132,26 @@ async function clickhouseQuery(
filters: QueryFilters,
): Promise<PerformanceResult> {
const { startDate, endDate, unit = 'day', timezone = 'utc', metric = 'lcp' } = parameters;
const { getDateSQL, rawQuery } = clickhouse;
const { getDateSQL, rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId });
const chart = await rawQuery<{ t: string; p50: number; p75: number; p95: number }[]>(
`
select
${getDateSQL('created_at', unit, timezone)} t,
${getDateSQL('created_at', 'minute', timezone)} t,
quantile(0.5)(${metric}) as p50,
quantile(0.75)(${metric}) as p75,
quantile(0.95)(${metric}) as p95
from website_performance
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
from website_event
${cohortQuery}
where website_event.website_id = {websiteId:UUID}
and website_event.event_type = 5
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
group by t
order by t
`,
{ websiteId, startDate, endDate },
);
const pages = await rawQuery<{ urlPath: string; p75: number; count: number }[]>(
`
select
url_path as "urlPath",
quantile(0.75)(${metric}) as p75,
count() as count
from website_performance
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by url_path
order by p75 desc
limit 100
`,
{ websiteId, startDate, endDate },
{ ...queryParams, startDate, endDate },
);
const summaryResult = await rawQuery<any>(
@@ -190,11 +173,14 @@ async function clickhouseQuery(
quantile(0.75)(ttfb) as ttfb_p75,
quantile(0.95)(ttfb) as ttfb_p95,
count() as count
from website_performance
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
from website_event
${cohortQuery}
where website_event.website_id = {websiteId:UUID}
and website_event.event_type = 5
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
`,
{ websiteId, startDate, endDate },
{ ...queryParams, startDate, endDate },
).then(result => result?.[0]);
const summary = {
@@ -226,5 +212,5 @@ async function clickhouseQuery(
count: Number(summaryResult?.count || 0),
};
return { chart, pages, summary };
return { chart, summary };
}
src/queries/sql/reports/getPerformanceMetrics.ts
+98
View File
@@ -0,0 +1,98 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
import type { PerformanceParameters } from './getPerformance';
export interface PerformanceMetricsData {
name: string;
p50: number;
p75: number;
p95: number;
count: number;
}
export async function getPerformanceMetrics(
...args: [
websiteId: string,
parameters: PerformanceParameters,
filters: QueryFilters,
column: string,
limit?: number,
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
parameters: PerformanceParameters,
filters: QueryFilters,
column: string,
limit?: number,
): Promise<PerformanceMetricsData[]> {
const { startDate, endDate, metric = 'lcp' } = parameters;
const { rawQuery, parseFilters } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
return rawQuery(
`
select
${column} as "name",
percentile_cont(0.5) within group (order by ${metric}) as p50,
percentile_cont(0.75) within group (order by ${metric}) as p75,
percentile_cont(0.95) within group (order by ${metric}) as p95,
count(*) as count
from website_event
${cohortQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.event_type = 5
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by ${column}
order by p75 desc
${limit ? `limit ${limit}` : ''}
`,
{ ...queryParams, startDate, endDate },
);
}
async function clickhouseQuery(
websiteId: string,
parameters: PerformanceParameters,
filters: QueryFilters,
column: string,
limit?: number,
): Promise<PerformanceMetricsData[]> {
const { startDate, endDate, metric = 'lcp' } = parameters;
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId });
return rawQuery<PerformanceMetricsData[]>(
`
select
${column} as "name",
quantile(0.5)(${metric}) as p50,
quantile(0.75)(${metric}) as p75,
quantile(0.95)(${metric}) as p95,
count() as count
from website_event
${cohortQuery}
where website_event.website_id = {websiteId:UUID}
and website_event.event_type = 5
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
group by ${column}
order by p75 desc
${limit ? `limit ${limit}` : ''}
`,
{ ...queryParams, startDate, endDate },
);
}
src/queries/sql/sessions/getSessionExpandedMetrics.ts
+2 -2
View File
@@ -79,7 +79,7 @@ async function relationalQuery(
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by name, website_event.session_id, website_event.visit_id
${includeCountry ? ', country' : ''}
@@ -138,7 +138,7 @@ async function clickhouseQuery(
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
and name != ''
${filterQuery}
group by name, session_id, visit_id
src/queries/sql/sessions/getSessionMetrics.ts
+3 -3
View File
@@ -57,7 +57,7 @@ async function relationalQuery(
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by 1
${includeCountry ? ', 3' : ''}
@@ -101,7 +101,7 @@ async function clickhouseQuery(
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by x
${includeCountry ? ', country' : ''}
@@ -120,7 +120,7 @@ async function clickhouseQuery(
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by x
${includeCountry ? ', country' : ''}
src/queries/sql/sessions/getSessionStats.ts
+3 -3
View File
@@ -33,7 +33,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by 1
order by 1
@@ -70,7 +70,7 @@ async function clickhouseQuery(
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by t
) as g
@@ -90,7 +90,7 @@ async function clickhouseQuery(
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by t
) as g