diff --git a/apps/server/src/checker/checker.ts b/apps/server/src/checker/checker.ts index 7d60736b7..fb992a1bd 100644 --- a/apps/server/src/checker/checker.ts +++ b/apps/server/src/checker/checker.ts @@ -10,6 +10,7 @@ export const publishPingRetryPolicy = async ({ payload, latency, statusCode, + message, }: PublishPingType) => { try { console.log( @@ -17,14 +18,25 @@ export const publishPingRetryPolicy = async ({ payload, )} with latency ${latency} and status code ${statusCode}`, ); - await publishPing({ payload, statusCode, latency }); + await publishPing({ + payload, + statusCode, + latency, + message, + }); } catch { try { console.log( "2️⃣ try publish ping to tb - attempt 2 ", JSON.stringify(payload), ); - await publishPing({ payload, statusCode, latency }); + + await publishPing({ + payload, + statusCode, + latency, + message, + }); } catch (e) { throw e; } @@ -40,6 +52,7 @@ const run = async (data: Payload, retry: number) => { let startTime = 0; let endTime = 0; let res = null; + let message = undefined; // We are doing these for wrong urls try { startTime = Date.now(); @@ -47,6 +60,7 @@ const run = async (data: Payload, retry: number) => { endTime = Date.now(); } catch (e) { endTime = Date.now(); + message = `${e}`; console.log( `🚨 error on pingEndpoint for ${JSON.stringify(data)} error: `, e, @@ -59,6 +73,7 @@ const run = async (data: Payload, retry: number) => { payload: data, latency, statusCode: res.status, + message: undefined, }); if (data?.status === "error") { await updateMonitorStatus({ @@ -80,8 +95,10 @@ const run = async (data: Payload, retry: number) => { await publishPingRetryPolicy({ payload: data, latency, - statusCode: res?.status || 0, + statusCode: res?.status, + message: message, }); + if (data?.status === "active") { await updateMonitorStatus({ monitorId: data.monitorId, diff --git a/apps/server/src/checker/ping.ts b/apps/server/src/checker/ping.ts index 6bfe5e318..84559ee22 100644 --- a/apps/server/src/checker/ping.ts +++ b/apps/server/src/checker/ping.ts @@ -1,5 +1,3 @@ -import { nanoid } from "nanoid"; - import { publishPingResponse } from "@openstatus/tinybird"; import { env } from "../env"; @@ -40,14 +38,16 @@ export async function pingEndpoint(data: Payload) { export type PublishPingType = { payload: Payload; - statusCode: number; latency: number; + statusCode?: number | undefined; + message?: string | undefined; }; export async function publishPing({ payload, - statusCode, latency, + message, + statusCode, }: PublishPingType) { const { monitorId, cronTimestamp, url, workspaceId } = payload; @@ -56,7 +56,6 @@ export async function publishPing({ process.env.NODE_ENV === "test" ) { const res = await publishPingResponse({ - id: nanoid(), // TBD: we don't need it timestamp: Date.now(), statusCode, latency, @@ -65,6 +64,7 @@ export async function publishPing({ monitorId, cronTimestamp, workspaceId, + message: message, }); if (res.successful_rows === 0) { throw new Error(`error 0 rows on publish ping for ${payload.monitorId}`); diff --git a/apps/server/src/checker/utils.ts b/apps/server/src/checker/utils.ts index 6d7e4b804..c4454c12c 100644 --- a/apps/server/src/checker/utils.ts +++ b/apps/server/src/checker/utils.ts @@ -18,7 +18,8 @@ type SendNotification = ({ monitor: Monitor; notification: Notification; region: keyof typeof flyRegionsDict; - statusCode: number; + statusCode?: number; + message?: string; }) => Promise; export const providerToFunction = { diff --git a/apps/web/src/app/_components/input-search.tsx b/apps/web/src/app/_components/input-search.tsx index 3ae0512c4..cc95269cb 100644 --- a/apps/web/src/app/_components/input-search.tsx +++ b/apps/web/src/app/_components/input-search.tsx @@ -59,7 +59,7 @@ export function InputSearch({ }, // defaultState { limit: [10, 25, 50], status: [], region: [] } as { - status: number[]; + status: (number | null)[]; limit: number[]; region: string[]; }, diff --git a/apps/web/src/app/api/checker/regions/_checker.ts b/apps/web/src/app/api/checker/regions/_checker.ts index 4b08a8847..6069f1077 100644 --- a/apps/web/src/app/api/checker/regions/_checker.ts +++ b/apps/web/src/app/api/checker/regions/_checker.ts @@ -33,7 +33,6 @@ const monitor = async ( const { monitorId, cronTimestamp, url, workspaceId } = payload; await publishPingResponse({ - id: nanoid(), // TBD: we don't need it timestamp: Date.now(), statusCode, monitorId, diff --git a/apps/web/src/app/monitor/[id]/page.tsx b/apps/web/src/app/monitor/[id]/page.tsx index 9946f1535..8a23e6c34 100644 --- a/apps/web/src/app/monitor/[id]/page.tsx +++ b/apps/web/src/app/monitor/[id]/page.tsx @@ -1,6 +1,6 @@ import * as z from "zod"; -import { availableRegions } from "@openstatus/tinybird"; +import { availableRegions } from "@openstatus/utils"; import { columns } from "@/components/data-table/columns"; import { DataTable } from "@/components/data-table/data-table"; diff --git a/apps/web/src/app/play/@modal/(..)monitor/[id]/page.tsx b/apps/web/src/app/play/@modal/(..)monitor/[id]/page.tsx index 6866c2fbd..04a7b2f43 100644 --- a/apps/web/src/app/play/@modal/(..)monitor/[id]/page.tsx +++ b/apps/web/src/app/play/@modal/(..)monitor/[id]/page.tsx @@ -1,6 +1,6 @@ import * as z from "zod"; -import { availableRegions } from "@openstatus/tinybird"; +import { availableRegions } from "@openstatus/utils"; import { columns } from "@/components/data-table/columns"; import { DataTable } from "@/components/data-table/data-table"; diff --git a/apps/web/src/components/marketing/cards.tsx b/apps/web/src/components/marketing/cards.tsx index dc8277051..891147177 100644 --- a/apps/web/src/components/marketing/cards.tsx +++ b/apps/web/src/components/marketing/cards.tsx @@ -20,12 +20,12 @@ export function Cards(props: Feature) { return (

  • -

    + {feature.catchline.replace(".", "")} {" "} -

    + {feature.description} diff --git a/apps/web/src/components/status-page/status-check.tsx b/apps/web/src/components/status-page/status-check.tsx index 1fc6ee3ae..e558138e5 100644 --- a/apps/web/src/components/status-page/status-check.tsx +++ b/apps/web/src/components/status-page/status-check.tsx @@ -52,6 +52,7 @@ export async function StatusCheck({ function calcStatus() { const { count, ok } = monitorsData.flat(1).reduce( (prev, curr) => { + if (!curr.statusCode) return prev; // TODO: handle this better const isOk = curr.statusCode <= 299 && curr.statusCode >= 200; return { count: prev.count + 1, ok: prev.ok + (isOk ? 1 : 0) }; }, diff --git a/apps/web/src/content/changelog/sms-notification.mdx b/apps/web/src/content/changelog/sms-notification.mdx index 368814166..3effa6109 100644 --- a/apps/web/src/content/changelog/sms-notification.mdx +++ b/apps/web/src/content/changelog/sms-notification.mdx @@ -1,7 +1,7 @@ --- title: Get alerted on your cell phone description: - Receive alerts on your cell phone via SMS when your monitor is done. + Receive alerts on your cell phone via SMS when your monitor is down. publishedAt: 2023-11-13 image: /assets/changelog/sms-notification.png --- diff --git a/packages/notifications/discord/src/index.ts b/packages/notifications/discord/src/index.ts index 982d4ed13..f7a7d871c 100644 --- a/packages/notifications/discord/src/index.ts +++ b/packages/notifications/discord/src/index.ts @@ -20,11 +20,13 @@ export const sendDiscordMessage = async ({ notification, region, statusCode, + message, }: { monitor: Monitor; notification: Notification; - statusCode: number; + statusCode?: number; region: string; + message?: string; }) => { const notificationData = JSON.parse(notification.data); const { discord: webhookUrl } = notificationData; // webhook url @@ -34,7 +36,9 @@ export const sendDiscordMessage = async ({ await postToWebhook( `Your monitor ${name} is down 🚨 - Your monitor with url ${monitor.url} is down in ${region} with status code ${statusCode}.`, + Your monitor with url ${monitor.url} is down in ${region} with ${ + statusCode ? `status code ${statusCode}` : `error message ${message}` + }.`, webhookUrl, ); } catch (err) { diff --git a/packages/notifications/email/src/index.ts b/packages/notifications/email/src/index.ts index 0a3898163..261bf0673 100644 --- a/packages/notifications/email/src/index.ts +++ b/packages/notifications/email/src/index.ts @@ -8,11 +8,13 @@ export const send = async ({ notification, region, statusCode, + message, }: { monitor: Monitor; notification: Notification; - statusCode: number; + statusCode?: number; region: string; + message?: string; }) => { // FIXME: const config = EmailConfigurationSchema.parse(JSON.parse(notification.data)); @@ -29,7 +31,13 @@ export const send = async ({ from: "Notifications ", subject: `Your monitor ${monitor.name} is down 🚨`, - html: `

    Hi,

    Your monitor ${monitor.name} is down in ${region}.

    URL : ${monitor.url}

    Status Code: ${statusCode}

    OpenStatus 🏓

    `, + html: `

    Hi,

    Your monitor ${ + monitor.name + } is down in ${region}.

    URL : ${monitor.url}

    ${ + statusCode + ? `

    Status Code: ${statusCode}

    ` + : `

    Error message: ${message}

    ` + }

    OpenStatus 🏓

    `, }), }); diff --git a/packages/notifications/slack/src/index.ts b/packages/notifications/slack/src/index.ts index 3e6dd8bbd..bbd7984fc 100644 --- a/packages/notifications/slack/src/index.ts +++ b/packages/notifications/slack/src/index.ts @@ -17,11 +17,13 @@ export const sendSlackMessage = async ({ notification, region, statusCode, + message, }: { monitor: Monitor; notification: Notification; - statusCode: number; + statusCode?: number; region: string; + message?: string; }) => { const notificationData = JSON.parse(notification.data); const { slack: webhookUrl } = notificationData; // webhook url @@ -35,7 +37,13 @@ export const sendSlackMessage = async ({ type: "section", text: { type: "mrkdwn", - text: `Your monitor <${monitor.url}/|${name}> is down in ${region} with status code ${statusCode} 🚨 \n\n Powered by .`, + text: `Your monitor <${ + monitor.url + }/|${name}> is down in ${region} with ${ + statusCode + ? `status code ${statusCode}` + : `error message ${message}` + } 🚨 \n\n Powered by .`, }, accessory: { type: "button", diff --git a/packages/notifications/twillio-sms/src/index.ts b/packages/notifications/twillio-sms/src/index.ts index 173fed415..f06e6f355 100644 --- a/packages/notifications/twillio-sms/src/index.ts +++ b/packages/notifications/twillio-sms/src/index.ts @@ -8,11 +8,13 @@ export const sendTextMessage = async ({ notification, region, statusCode, + message, }: { monitor: Monitor; notification: Notification; - statusCode: number; + statusCode?: number; region: string; + message?: string; }) => { const notificationData = SmsConfigurationSchema.parse( JSON.parse(notification.data), @@ -24,7 +26,9 @@ export const sendTextMessage = async ({ body.set("From", "+14807252613"); body.set( "Body", - `Your monitor ${name} / ${monitor.url} is down in ${region} with status code ${statusCode}`, + `Your monitor ${name} / ${monitor.url} is down in ${region} with ${ + statusCode ? `status code ${statusCode}` : `error: ${message}` + }`, ); try { diff --git a/packages/tinybird/README.md b/packages/tinybird/README.md new file mode 100644 index 000000000..00a23b079 --- /dev/null +++ b/packages/tinybird/README.md @@ -0,0 +1,35 @@ +### A guide on how to migrate your tinybird datasource + +> What to do when you want to add/remove/update a column in your `datasource`. + +The `_migration` folder includes: + +- `ping_response__v4.datasource` which represents the `VERSION 4` of our + datasource +- `ping_response.datasource` which has the upgraded schema and a new `VERSION 5` +- `tb_backfill_populate.pipe` will fill the datasource with all the data until a + given timestamp +- `tb_materialized_until_change_ingest.pipe` will fill the data from a given + timestamp + +``` +tb push _migration/ping_response.datasource +tb push _migration/tb_materialized_until_change_ingest.pipe +# after the given ts, it is time to run the backfill populate +tb push _migration/tb_backfill_populate.pipe --populate --wait +# after populate ends, it is time to remove the pipe +tb pipe rm tb_backfill_populate --yes +``` + +Check if all the rows have been migrated: + +``` +tb pipe _migration/tb_datasource_union.pipe +# after checking the result of the pipe +tb pipe rm tb_datasource_union.pipe --yes +``` + +--- + +Link to the [issue](https://github.com/openstatusHQ/openstatus/issues/278) from +Gonzalo as reference. diff --git a/packages/tinybird/_migration/ping_response.datasource b/packages/tinybird/_migration/ping_response.datasource new file mode 100644 index 000000000..52cbe6ab2 --- /dev/null +++ b/packages/tinybird/_migration/ping_response.datasource @@ -0,0 +1,16 @@ +VERSION 5 + +SCHEMA > + `latency` Int64 `json:$.latency`, + `monitorId` String `json:$.monitorId`, + `region` LowCardinality(String) `json:$.region`, + `statusCode` Nullable(Int16) `json:$.statusCode`, + `timestamp` Int64 `json:$.timestamp`, + `url` String `json:$.url`, + `workspaceId` String `json:$.workspaceId`, + `cronTimestamp` Int64 `json:$.cronTimestamp`, + `message` Nullable(String) `json:$.message` + +ENGINE "MergeTree" +ENGINE_SORTING_KEY "monitorId, cronTimestamp" +ENGINE_PARTITION_KEY "toYYYYMM(fromUnixTimestamp64Milli(cronTimestamp))" \ No newline at end of file diff --git a/packages/tinybird/_migration/ping_response__v4.datasource b/packages/tinybird/_migration/ping_response__v4.datasource new file mode 100644 index 000000000..dc55dea56 --- /dev/null +++ b/packages/tinybird/_migration/ping_response__v4.datasource @@ -0,0 +1,16 @@ +VERSION 4 + +SCHEMA > + `id` String `json:$.id`, + `latency` Int16 `json:$.latency`, + `monitorId` String `json:$.monitorId`, + `region` LowCardinality(String) `json:$.region`, + `statusCode` Int16 `json:$.statusCode`, + `timestamp` Int64 `json:$.timestamp`, + `url` String `json:$.url`, + `workspaceId` String `json:$.workspaceId`, + `cronTimestamp` Int64 `json:$.cronTimestamp` + +ENGINE "MergeTree" +ENGINE_SORTING_KEY "monitorId, cronTimestamp" +ENGINE_PARTITION_KEY "toYYYYMM(fromUnixTimestamp64Milli(cronTimestamp))" \ No newline at end of file diff --git a/packages/tinybird/_migration/tb_backfill_populate.pipe b/packages/tinybird/_migration/tb_backfill_populate.pipe new file mode 100644 index 000000000..ef71f5a2a --- /dev/null +++ b/packages/tinybird/_migration/tb_backfill_populate.pipe @@ -0,0 +1,17 @@ +NODE mat_node +SQL > + + SELECT + latency, + monitorId, + toLowCardinality(region) region, + statusCode, + timestamp, + url, + workspaceId, + cronTimestamp, + FROM ping_response__v4 + WHERE fromUnixTimestamp64Milli(cronTimestamp) <= '2023-11-13 18:16:00.000' + +TYPE materialized +DATASOURCE ping_response__v5 \ No newline at end of file diff --git a/packages/tinybird/_migration/tb_datasource_union.pipe b/packages/tinybird/_migration/tb_datasource_union.pipe new file mode 100644 index 000000000..390b5daf7 --- /dev/null +++ b/packages/tinybird/_migration/tb_datasource_union.pipe @@ -0,0 +1,6 @@ +NODE union_all +SQL > + + SELECT count() FROM ping_response__v5 + UNION ALL + SELECT count() FROM ping_response__v4 diff --git a/packages/tinybird/_migration/tb_materialized_until_change_ingest.pipe b/packages/tinybird/_migration/tb_materialized_until_change_ingest.pipe new file mode 100644 index 000000000..16142297a --- /dev/null +++ b/packages/tinybird/_migration/tb_materialized_until_change_ingest.pipe @@ -0,0 +1,17 @@ +NODE mat_node +SQL > + + SELECT + latency, + monitorId, + toLowCardinality(region) region, + statusCode, + timestamp, + url, + workspaceId, + cronTimestamp, + FROM ping_response__v4 + WHERE fromUnixTimestamp64Milli(cronTimestamp) > '2023-11-13 18:16:00.000' + +TYPE materialized +DATASOURCE ping_response__v5 \ No newline at end of file diff --git a/packages/tinybird/package.json b/packages/tinybird/package.json index 9e96d7735..da5385634 100644 --- a/packages/tinybird/package.json +++ b/packages/tinybird/package.json @@ -9,6 +9,7 @@ }, "devDependencies": { "@openstatus/tsconfig": "workspace:*", + "@openstatus/utils": "workspace:*", "@types/node": "20.8.0", "typescript": "5.2.2" } diff --git a/packages/tinybird/pipes/home_stats.pipe b/packages/tinybird/pipes/home_stats.pipe index 5955f688a..4a5573d85 100644 --- a/packages/tinybird/pipes/home_stats.pipe +++ b/packages/tinybird/pipes/home_stats.pipe @@ -5,7 +5,7 @@ SQL > % SELECT COUNT(*) as count - FROM ping_response__v4 + FROM ping_response__v5 {% if defined(period) %} {% if String(period) == "1h" %} WHERE cronTimestamp > toUnixTimestamp(now() - INTERVAL 1 HOUR) * 1000 diff --git a/packages/tinybird/pipes/response_list.pipe b/packages/tinybird/pipes/response_list.pipe index 593c2c35f..d2fc2b126 100644 --- a/packages/tinybird/pipes/response_list.pipe +++ b/packages/tinybird/pipes/response_list.pipe @@ -1,11 +1,11 @@ -VERSION 1 +VERSION 2 NODE response_list_0 SQL > % - SELECT id, latency, monitorId, region, statusCode, timestamp, url, workspaceId, cronTimestamp - FROM ping_response + SELECT latency, monitorId, region, statusCode, timestamp, url, workspaceId, cronTimestamp + FROM ping_response__v5 WHERE monitorId = {{ String(monitorId, 'openstatusPing') }} {% if defined(region) %} AND region = {{ String(region) }} diff --git a/packages/tinybird/pipes/status_timezone.pipe b/packages/tinybird/pipes/status_timezone.pipe index 820f06b85..bc86bd481 100644 --- a/packages/tinybird/pipes/status_timezone.pipe +++ b/packages/tinybird/pipes/status_timezone.pipe @@ -16,7 +16,7 @@ SQL > avg(latency) AS avgLatency, count() AS count, count(multiIf((statusCode >= 200) AND (statusCode <= 299), 1, NULL)) AS ok - FROM ping_response__v4 + FROM ping_response__v5 WHERE (day IS NOT NULL) AND (day != 0) diff --git a/packages/tinybird/src/client.ts b/packages/tinybird/src/client.ts index 2b5109f51..7dc0adb52 100644 --- a/packages/tinybird/src/client.ts +++ b/packages/tinybird/src/client.ts @@ -14,13 +14,13 @@ import { const tb = new Tinybird({ token: process.env.TINY_BIRD_API_KEY! }); export const publishPingResponse = tb.buildIngestEndpoint({ - datasource: "ping_response__v4", + datasource: "ping_response__v5", event: tbIngestPingResponse, }); export function getResponseList(tb: Tinybird) { return tb.buildPipe({ - pipe: "response_list__v1", + pipe: "response_list__v2", parameters: tbParameterResponseList, data: tbBuildResponseList, opts: { diff --git a/packages/tinybird/src/validation.ts b/packages/tinybird/src/validation.ts index e7d1766ee..2c5e98264 100644 --- a/packages/tinybird/src/validation.ts +++ b/packages/tinybird/src/validation.ts @@ -1,62 +1,39 @@ import * as z from "zod"; -export const vercelRegions = [ - "arn1", - "bom1", - "cdg1", - "cle1", - "cpt1", - "dub1", - "fra1", - "gru1", - "hkg1", - "hnd1", - "iad1", - "icn1", - "kix1", - "lhr1", - "pdx1", - "sfo1", - "sin1", - "syd1", -] as const; - -export const flyRegions = ["ams", "iad", "hkg", "jnb", "syd", "gru"] as const; - -export const availableRegions = [...vercelRegions, ...flyRegions] as const; +import { availableRegions } from "@openstatus/utils"; /** - * Values for the datasource ping_response__v4 + * Values for the datasource ping_response */ export const tbIngestPingResponse = z.object({ - id: z.string(), workspaceId: z.string(), monitorId: z.string(), timestamp: z.number().int(), - statusCode: z.number().int(), + statusCode: z.number().int().nullable().optional(), latency: z.number(), // in ms cronTimestamp: z.number().int().optional().nullable().default(Date.now()), url: z.string().url(), region: z.string().min(3).max(4), // REMINDER: won't work on fy + message: z.string().nullable().optional(), }); /** - * Values from the pipe response_list__v1 + * Values from the pipe response_list */ export const tbBuildResponseList = z.object({ - id: z.string(), workspaceId: z.string(), monitorId: z.string(), timestamp: z.number().int(), - statusCode: z.number().int(), + statusCode: z.number().int().nullable().default(null), latency: z.number().int(), // in ms cronTimestamp: z.number().int().nullable().default(Date.now()), url: z.string().url(), region: z.enum(availableRegions), + message: z.string().nullable().optional(), }); /** - * Params for pipe response_list__v1 + * Params for pipe response_list */ export const tbParameterResponseList = z.object({ monitorId: z.string().default(""), // REMINDER: remove default once alpha @@ -68,7 +45,7 @@ export const tbParameterResponseList = z.object({ }); /** - * Params for pipe status_timezone__v0 + * Params for pipe status_timezone */ export const tbParameterMonitorList = z.object({ monitorId: z.string(), @@ -77,7 +54,7 @@ export const tbParameterMonitorList = z.object({ }); /** - * Values from the pipe status_timezone__v0 + * Values from the pipe status_timezone */ export const tbBuildMonitorList = z.object({ count: z.number().int(), @@ -87,7 +64,7 @@ export const tbBuildMonitorList = z.object({ }); /** - * Params for pipe home_stats__v0 + * Params for pipe home_stats */ export const tbParameterHomeStats = z.object({ cronTimestamp: z.number().int().optional(), @@ -95,7 +72,7 @@ export const tbParameterHomeStats = z.object({ }); /** - * Values from the pipe home_stats__v0 + * Values from the pipe home_stats */ export const tbBuildHomeStats = z.object({ count: z.number().int(), diff --git a/packages/utils/index.ts b/packages/utils/index.ts index 71ef19586..5c12d4694 100644 --- a/packages/utils/index.ts +++ b/packages/utils/index.ts @@ -160,4 +160,6 @@ export const vercelRegions = [ export const flyRegions = ["ams", "iad", "hkg", "jnb", "syd", "gru"] as const; +export const availableRegions = [...vercelRegions, ...flyRegions] as const; + export const regionsDict = { ...vercelRegionsDict, ...flyRegionsDict } as const; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 049a1c313..7db3890ec 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -754,6 +754,9 @@ importers: '@openstatus/tsconfig': specifier: workspace:* version: link:../tsconfig + '@openstatus/utils': + specifier: workspace:* + version: link:../utils '@types/node': specifier: 20.8.0 version: 20.8.0