diff --git a/apps/dashboard/src/components/chat/tool-renderers/index.tsx b/apps/dashboard/src/components/chat/tool-renderers/index.tsx index 65ff7a0a..2100153b 100644 --- a/apps/dashboard/src/components/chat/tool-renderers/index.tsx +++ b/apps/dashboard/src/components/chat/tool-renderers/index.tsx @@ -181,6 +181,14 @@ export const toolRenderers: ToolRendererRegistry = { ), summary: (o) => `resolved · ID ${o.id}`, }, + set_incident_status: { + renderDraft: (input) => [ + { field: "incidentId", after: input.id }, + { field: "status", after: input.status }, + ...(input.note ? [{ field: "note", after: input.note }] : []), + ], + summary: (o) => `${o.status} · ID ${o.id}`, + }, add_incident_note: { summary: (o) => `note added to incident ${o.incidentId}`, }, diff --git a/apps/server/src/routes/mcp/handler.test.ts b/apps/server/src/routes/mcp/handler.test.ts index 6a180317..199196d8 100644 --- a/apps/server/src/routes/mcp/handler.test.ts +++ b/apps/server/src/routes/mcp/handler.test.ts @@ -146,6 +146,7 @@ describe("MCP transport", () => { "resolve_status_report", "search_content", "search_docs", + "set_incident_status", "update_incident", "update_status_report", ]); diff --git a/apps/server/src/routes/mcp/tools/incident.ts b/apps/server/src/routes/mcp/tools/incident.ts index ae11e998..c69239ac 100644 --- a/apps/server/src/routes/mcp/tools/incident.ts +++ b/apps/server/src/routes/mcp/tools/incident.ts @@ -9,6 +9,7 @@ import { getIncidentTool, listIncidentsTool, resolveIncidentTool, + setIncidentStatusTool, updateIncidentTool, } from "@openstatus/services/agent-tools"; @@ -24,6 +25,7 @@ export function registerIncidentTools( declareIncidentTool, updateIncidentTool, resolveIncidentTool, + setIncidentStatusTool, addIncidentNoteTool, ]); } diff --git a/apps/server/src/routes/slack/commands.test.ts b/apps/server/src/routes/slack/commands.test.ts index f417389d..f77d1152 100644 --- a/apps/server/src/routes/slack/commands.test.ts +++ b/apps/server/src/routes/slack/commands.test.ts @@ -1,5 +1,15 @@ import crypto from "node:crypto"; +import { and, db, eq, or, sql } from "@openstatus/db"; +import { + auditLog, + incident, + incidentEvent, + selectWorkspaceSchema, + type Workspace, + workspace as workspaceTable, +} from "@openstatus/db/src/schema"; +import { declareIncident } from "@openstatus/services/incident"; import { beforeEach, describe, expect, test } from "@openstatus/test-utils"; import { Hono } from "hono"; @@ -23,12 +33,13 @@ function post( app: ReturnType, text: string, user: string, + channelId = "C1", ) { const body = new URLSearchParams({ text, team_id: "T_KNOWN", user_id: user, - channel_id: "C1", + channel_id: channelId, }).toString(); const timestamp = Math.floor(Date.now() / 1000); const sig = crypto @@ -92,3 +103,115 @@ describe("handleSlackCommand (members only)", () => { expect(json.blocks).toBeUndefined(); }); }); + +describe("/openstatus incident", () => { + const app = createTestApp(); + const redisStore = (globalThis as Record) + .__testRedisStore as Map; + let workspace: Workspace; + + beforeEach(async () => { + slackTestState.calls = []; + slackTestState.usersInfoImpl = () => + Promise.resolve({ + ok: true, + user: { profile: { email: "ping@openstatus.dev" } }, + }); + workspace = selectWorkspaceSchema.parse( + await db.query.workspace.findFirst({ where: eq(workspaceTable.id, 1) }), + ); + slackTestState.resolveWorkspace = (teamId: string) => + teamId === "T_KNOWN" + ? Promise.resolve({ + workspace: { + ...workspace, + limits: { ...workspace.limits, "slack-agent": true }, + }, + botToken: "xoxb-test", + botUserId: "UBOT", + }) + : Promise.resolve(null); + }); + + test("declare posts an approval card instead of declaring", async () => { + redisStore.clear(); + const res = await post( + app, + "incident declare Checkout down --sev critical", + `U_${crypto.randomUUID()}`, + ); + const json = (await res.json()) as { text: string }; + expect(json.text).toContain("approval card"); + const card = slackTestState.calls.find( + (c) => c.method === "update" && Array.isArray(c.args.blocks), + ); + expect(card).toBeDefined(); + const pending = [...redisStore.entries()].find(([key]) => + key.startsWith("slack:action:"), + ); + expect(pending?.[1]).toContain("declare_incident"); + expect(pending?.[1]).toContain("critical"); + }); + + test("note in a bound channel lands on the timeline", async () => { + const channelId = `C_INC_${crypto.randomUUID()}`; + const created = await declareIncident({ + ctx: { workspace, actor: { type: "system", job: "test" } }, + input: { title: "Bound", severity: "minor" }, + }); + await db + .update(incident) + .set({ slackTeamId: "T_KNOWN", slackChannelId: channelId }) + .where(eq(incident.id, created.id)); + try { + const res = await post( + app, + "incident note rolled back the deploy", + `U_${crypto.randomUUID()}`, + channelId, + ); + const json = (await res.json()) as { text: string }; + expect(json.text).toContain("Added to the timeline"); + const events = await db + .select() + .from(incidentEvent) + .where(eq(incidentEvent.incidentId, created.id)) + .all(); + expect(events.some((e) => e.message === "rolled back the deploy")).toBe( + true, + ); + } finally { + await db + .delete(auditLog) + .where( + and( + eq(auditLog.workspaceId, workspace.id), + or( + and( + eq(auditLog.entityType, "incident"), + eq(auditLog.entityId, String(created.id)), + ), + and( + eq(auditLog.action, "incident_event.create"), + sql`json_extract(${auditLog.metadata}, '$.incidentId') = ${created.id}`, + ), + ), + ), + ); + await db + .delete(incidentEvent) + .where(eq(incidentEvent.incidentId, created.id)); + await db.delete(incident).where(eq(incident.id, created.id)); + } + }); + + test("note outside an incident channel explains where to run it", async () => { + const res = await post( + app, + "incident note hello", + `U_${crypto.randomUUID()}`, + ); + const json = (await res.json()) as { text: string }; + expect(json.text).toContain("incident's channel"); + }); +}); diff --git a/apps/server/src/routes/slack/commands.ts b/apps/server/src/routes/slack/commands.ts index d5762db5..ea67c2fd 100644 --- a/apps/server/src/routes/slack/commands.ts +++ b/apps/server/src/routes/slack/commands.ts @@ -16,6 +16,7 @@ import { LINK_ACCOUNT_TEXT, } from "./blocks"; import type { SlackConfig, SlackEnv } from "./config"; +import { runIncidentCommand } from "./incident-commands"; import { linkAccountUrl, planRequiredMessage, @@ -43,6 +44,7 @@ const HELP = [ "• `/openstatus subscribe ` — subscribe this channel to a status page", "• `/openstatus unsubscribe ` — unsubscribe", "• `/openstatus subscriptions` — show this channel's subscriptions", + "• `/openstatus incident help` — declare and run incidents", ].join("\n"); type CommandReply = { text: string; blocks?: Block[] }; @@ -92,7 +94,12 @@ export function handleSlackCommand(c: Context) { // `help` — and anything unrecognised, which falls through to it — needs no // I/O, so it is answered in the ack itself. - if (sub !== "subscribe" && sub !== "unsubscribe" && sub !== "subscriptions") { + if ( + sub !== "subscribe" && + sub !== "unsubscribe" && + sub !== "subscriptions" && + sub !== "incident" + ) { return ephemeral(c, { text: HELP }); } @@ -166,6 +173,19 @@ async function runMemberCommand( }); return { text: LINK_ACCOUNT_TEXT, blocks: buildLinkAccountBlocks(url) }; } + if (subcommand(command) === "incident") { + const words = command.text.trim().split(/\s+/).filter(Boolean).slice(1); + return { + text: await runIncidentCommand({ + words, + teamId: command.team_id, + channelId: command.channel_id, + resolved, + actor, + config, + }), + }; + } return { text: await runCommand(command) }; } diff --git a/apps/server/src/routes/slack/confirmation-card.ts b/apps/server/src/routes/slack/confirmation-card.ts new file mode 100644 index 00000000..dc707237 --- /dev/null +++ b/apps/server/src/routes/slack/confirmation-card.ts @@ -0,0 +1,73 @@ +import type { ServiceContext } from "@openstatus/services"; +import type { WebClient } from "@slack/web-api"; + +import { + buildConfirmationBlocks, + getConfirmationText, + type RefResolvers, +} from "./blocks"; +import { store } from "./confirmation-store"; +import { + getComponentNames, + getPageDashboardLink, + getStatusReportLink, +} from "./page-urls"; +import { deriveDraftSchema, getRegistryTool } from "./registry-runner"; + +export function makeRefResolvers(workspaceId: number): RefResolvers { + return { + page: (pageId) => getPageDashboardLink(workspaceId, pageId), + statusReport: (statusReportId) => + getStatusReportLink(workspaceId, statusReportId), + componentNames: (ids) => getComponentNames(workspaceId, ids), + }; +} + +/** + * Posts an approval card for a registry write outside an agent turn (slash + * commands, follow-ups). Nothing runs until the initiator clicks Approve. + */ +export async function postConfirmationCard(args: { + slack: WebClient; + ctx: ServiceContext; + teamId: string; + channel: string; + threadTs?: string; + slackUserId: string; + toolName: string; + input: object; +}): Promise { + const { slack, ctx, teamId, channel, slackUserId, toolName } = args; + const tool = getRegistryTool(toolName); + if (!tool) throw new Error(`slack: unknown tool "${toolName}"`); + + const input = deriveDraftSchema(tool).parse(args.input); + const displayInput = tool.approval?.prepareDraftInput + ? await tool.approval.prepareDraftInput({ ctx, input }) + : input; + const text = getConfirmationText({ tool, input: displayInput }); + + const posted = await slack.chat.postMessage({ + channel, + text, + ...(args.threadTs ? { thread_ts: args.threadTs } : {}), + }); + if (!posted.ts) throw new Error("chat.postMessage returned no ts"); + + const actionId = await store({ + workspaceId: ctx.workspace.id, + teamId, + channelId: channel, + threadTs: args.threadTs ?? posted.ts, + messageTs: posted.ts, + userId: slackUserId, + payload: { toolName, input }, + }); + const blocks = await buildConfirmationBlocks({ + actionId, + tool, + input: displayInput, + resolvers: makeRefResolvers(ctx.workspace.id), + }); + await slack.chat.update({ channel, ts: posted.ts, text, blocks }); +} diff --git a/apps/server/src/routes/slack/handler.test.ts b/apps/server/src/routes/slack/handler.test.ts index 17c4bcee..b9bbbac4 100644 --- a/apps/server/src/routes/slack/handler.test.ts +++ b/apps/server/src/routes/slack/handler.test.ts @@ -1,7 +1,7 @@ import crypto from "node:crypto"; import { db, eq } from "@openstatus/db"; -import { integration } from "@openstatus/db/src/schema"; +import { incident, integration } from "@openstatus/db/src/schema"; import { createTestWorkspace } from "@openstatus/db/src/test/factories"; import { beforeEach, describe, expect, test } from "@openstatus/test-utils"; import { Hono } from "hono"; @@ -1965,3 +1965,61 @@ describe("reconnect banner", () => { expect(view.blocks[0].text?.text).toContain("Reconnect openstatus"); }); }); + +describe("incident channel context", () => { + const app = createTestApp(); + + beforeEach(resetSlackTestState); + + test("a mention in a bound channel tells the agent which incident it is", async () => { + const channelId = `C_CTX_${crypto.randomUUID()}`; + const [row] = await db + .insert(incident) + .values({ + workspaceId: 1, + title: "Context incident", + severity: "major", + declaredAt: new Date(), + startedAt: new Date(), + slackTeamId: "T_KNOWN", + slackChannelId: channelId, + }) + .returning(); + let contextNote: string | undefined; + slackTestState.runAgentOverride = (options) => { + contextNote = (options as { contextNote?: string } | undefined) + ?.contextNote; + return Promise.resolve({ + text: "ok", + toolResults: [], + finishReason: "stop", + stepCount: 1, + hitStepLimit: false, + aborted: false, + }); + }; + try { + await signAndPost(app, { + type: "event_callback", + team_id: "T_KNOWN", + event_id: `evt_ctx_${Date.now()}`, + event: { + type: "app_mention", + text: "<@UBOT> note that we rolled back", + user: "U1", + channel: channelId, + channel_type: "channel", + ts: `${Date.now()}.71`, + }, + }); + const deadline = Date.now() + 2000; + while (contextNote === undefined && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 20)); + } + expect(contextNote).toContain(`id ${row.id}`); + expect(contextNote).toContain("Context incident"); + } finally { + await db.delete(incident).where(eq(incident.id, row.id)); + } + }); +}); diff --git a/apps/server/src/routes/slack/handler.ts b/apps/server/src/routes/slack/handler.ts index f2cafd77..3cff0b6f 100644 --- a/apps/server/src/routes/slack/handler.ts +++ b/apps/server/src/routes/slack/handler.ts @@ -1,5 +1,6 @@ import { getLogger } from "@logtape/logtape"; import { isFeatureEnabled } from "@openstatus/services"; +import { getIncidentBySlackChannel } from "@openstatus/services/incident"; import { missingSlackScopes, uninstallSlackTeam, @@ -31,6 +32,7 @@ import { rememberContext, } from "./channel-context"; import type { SlackConfig, SlackEnv } from "./config"; +import { makeRefResolvers } from "./confirmation-card"; import { draftKey, findByThread, replace, store } from "./confirmation-store"; import type { PendingPayload } from "./confirmation-store"; import { publishHomeView, publishLinkAccountView } from "./home"; @@ -51,6 +53,7 @@ import { planRequiredMessage, releaseLinkCardWindow, requireSlackMember, + type SlackActor, slackAgentAllowed, } from "./require-slack-member"; import { abortTurn, broadcastStop, endTurn, startTurn } from "./running-turns"; @@ -60,16 +63,7 @@ import { markThreadTitled, renameThread, } from "./thread-title"; -import { resolveWorkspace } from "./workspace-resolver"; - -function makeRefResolvers(workspaceId: number): RefResolvers { - return { - page: (pageId) => getPageDashboardLink(workspaceId, pageId), - statusReport: (statusReportId) => - getStatusReportLink(workspaceId, statusReportId), - componentNames: (ids) => getComponentNames(workspaceId, ids), - }; -} +import { resolveWorkspace, type SlackWorkspace } from "./workspace-resolver"; const logger = getLogger("api-server"); @@ -268,6 +262,25 @@ export async function handleSlackEvent(c: Context) { return c.json({ ok: true }); } +/** Tells the agent which incident a channel belongs to, when it has one. */ +async function boundIncidentNote(args: { + workspace: SlackWorkspace["workspace"]; + actor: SlackActor; + teamId: string; + channelId: string | undefined; +}): Promise { + if (!args.channelId) return undefined; + const bound = await getIncidentBySlackChannel({ + ctx: { workspace: args.workspace, actor: args.actor }, + input: { teamId: args.teamId, channelId: args.channelId }, + }).catch(() => undefined); + if (!bound) return undefined; + const report = bound.statusReport + ? ` Its status report is "${bound.statusReport.title}" (id ${bound.statusReport.id}, ${bound.statusReport.status}).` + : " It has no status report yet."; + return `Incident channel: <#${args.channelId}> belongs to managed incident "${bound.title}" (id ${bound.id}, ${bound.severity}, ${bound.status}${bound.closedAt ? ", closed" : ""}).${report} Notes and status changes discussed here are about this incident: use id ${bound.id} without asking.`; +} + /** * Tells an unlinked Slack user how to link their account. Passive surfaces * (mentions, DMs) send it at most once per window, so a chatty user isn't @@ -680,6 +693,12 @@ async function processEvent(body: SlackEvent, config: SlackConfig) { const context = contextChannel ? channelContextTooling({ slack, channelId: contextChannel }) : undefined; + const incidentNote = await boundIncidentNote({ + workspace: resolved.workspace, + actor, + teamId, + channelId: isAgentThread ? contextChannel : event.channel, + }); logger.info("slack agent invoked", { teamId, @@ -699,7 +718,9 @@ async function processEvent(body: SlackEvent, config: SlackConfig) { events: reply.progress, signal: turn.signal, tools: context?.tools, - contextNote: context?.contextNote, + contextNote: + [context?.contextNote, incidentNote].filter(Boolean).join("\n\n") || + undefined, }, ); diff --git a/apps/server/src/routes/slack/incident-analytics.ts b/apps/server/src/routes/slack/incident-analytics.ts new file mode 100644 index 00000000..6f5bd75a --- /dev/null +++ b/apps/server/src/routes/slack/incident-analytics.ts @@ -0,0 +1,26 @@ +import { Events, setupAnalytics } from "@openstatus/analytics"; +import { type ServiceContext, tryGetActorUserId } from "@openstatus/services"; + +const EVENTS = { + declare: Events.DeclareManagedIncident, + status: Events.ChangeManagedIncidentStatus, + note: Events.AddManagedIncidentNote, +} as const; + +/** Fire-and-forget: analytics never fails a Slack action. */ +export function trackSlackIncident( + ctx: ServiceContext, + kind: keyof typeof EVENTS, + props: Record = {}, +): void { + const userId = tryGetActorUserId(ctx.actor); + setupAnalytics({ + userId: userId ? `usr_${userId}` : `slack_${ctx.workspace.id}`, + workspaceId: String(ctx.workspace.id), + plan: ctx.workspace.plan, + }) + .then((analytics) => + analytics.track({ ...EVENTS[kind], source: "slack", ...props }), + ) + .catch(() => undefined); +} diff --git a/apps/server/src/routes/slack/incident-commands.test.ts b/apps/server/src/routes/slack/incident-commands.test.ts new file mode 100644 index 00000000..30faf870 --- /dev/null +++ b/apps/server/src/routes/slack/incident-commands.test.ts @@ -0,0 +1,41 @@ +import { expect } from "@std/expect"; +import { describe, test } from "@std/testing/bdd"; + +import { parseDeclare, parseTarget } from "./incident-commands"; + +describe("parseDeclare", () => { + test("title with a severity flag anywhere", () => { + expect(parseDeclare(["API", "down", "--sev", "critical"])).toEqual({ + title: "API down", + severity: "critical", + }); + expect(parseDeclare(["--severity", "MINOR", "Slow", "search"])).toEqual({ + title: "Slow search", + severity: "minor", + }); + }); + + test("defaults to major and ignores an unknown severity", () => { + expect(parseDeclare(["Checkout", "--sev", "huge"])).toEqual({ + title: "Checkout", + severity: "major", + }); + }); +}); + +describe("parseTarget", () => { + test("leading #id or number names the incident", () => { + expect(parseTarget(["#12", "rolled", "back"])).toEqual({ + id: 12, + rest: "rolled back", + }); + expect(parseTarget(["7"])).toEqual({ id: 7, rest: "" }); + }); + + test("otherwise everything is the note", () => { + expect(parseTarget(["rolled", "back"])).toEqual({ + id: null, + rest: "rolled back", + }); + }); +}); diff --git a/apps/server/src/routes/slack/incident-commands.ts b/apps/server/src/routes/slack/incident-commands.ts new file mode 100644 index 00000000..1e34dcb2 --- /dev/null +++ b/apps/server/src/routes/slack/incident-commands.ts @@ -0,0 +1,210 @@ +import { getLogger } from "@logtape/logtape"; +import { + type IncidentStatus, + isFeatureEnabled, + type ServiceContext, + ServiceError, +} from "@openstatus/services"; +import { + addIncidentNote, + displayName, + getIncident, + getIncidentBySlackChannel, + listIncidents, +} from "@openstatus/services/incident"; +import { WebClient } from "@slack/web-api"; + +import type { SlackConfig } from "./config"; +import { postConfirmationCard } from "./confirmation-card"; +import { trackSlackIncident } from "./incident-analytics"; +import { getIncidentDashboardUrl } from "./page-urls"; +import type { SlackActor } from "./require-slack-member"; +import type { SlackWorkspace } from "./workspace-resolver"; + +const logger = getLogger(["api-server", "slack", "incident-commands"]); + +export const INCIDENT_HELP = [ + "*Incidents*", + "• `/openstatus incident declare [--sev critical|major|minor]` — declare an incident (approval card)", + "• `/openstatus incident note <text>` — add to the timeline (in an incident channel)", + "• `/openstatus incident mitigate|resolve|cancel|reopen [#id] [note]` — change its status (approval card)", + "• `/openstatus incident status [#id]` — where it stands", + "• `/openstatus incident list` — open incidents", +].join("\n"); + +const SEVERITIES = ["critical", "major", "minor"] as const; +type Severity = (typeof SEVERITIES)[number]; + +const STATUS_BY_VERB: Record<string, IncidentStatus> = { + mitigate: "mitigated", + resolve: "resolved", + cancel: "canceled", + reopen: "open", +}; + +/** `declare API down --sev critical` → title and severity, flag anywhere. */ +export function parseDeclare(words: string[]): { + title: string; + severity: Severity; +} { + let severity: Severity = "major"; + const title: string[] = []; + for (let i = 0; i < words.length; i++) { + const word = words[i]; + if (word === "--sev" || word === "--severity") { + const value = SEVERITIES.find((s) => s === words[i + 1]?.toLowerCase()); + if (value) severity = value; + i++; + continue; + } + title.push(word); + } + return { title: title.join(" ").trim(), severity }; +} + +/** A leading `#12` or `12` names the incident; the rest is the note. */ +export function parseTarget(words: string[]): { + id: number | null; + rest: string; +} { + const match = words[0]?.match(/^#?(\d+)$/); + if (!match) return { id: null, rest: words.join(" ").trim() }; + return { id: Number(match[1]), rest: words.slice(1).join(" ").trim() }; +} + +async function postCard(args: { + slack: WebClient; + ctx: ServiceContext; + teamId: string; + channelId: string; + slackUserId: string; + toolName: string; + input: object; +}): Promise<string> { + try { + await postConfirmationCard({ + ...args, + channel: args.channelId, + }); + return "Review the approval card in this channel."; + } catch (error) { + if ( + !(error instanceof Error) || + !error.message.includes("not_in_channel") + ) { + throw error; + } + try { + await args.slack.conversations.join({ channel: args.channelId }); + await postConfirmationCard({ ...args, channel: args.channelId }); + return "Review the approval card in this channel."; + } catch { + return "Invite @openstatus to this channel first (`/invite @openstatus`), then try again."; + } + } +} + +export async function runIncidentCommand(args: { + words: string[]; + teamId: string; + channelId: string; + resolved: SlackWorkspace; + actor: SlackActor; + config: SlackConfig; +}): Promise<string> { + const { words, teamId, channelId, resolved, actor } = args; + const ctx: ServiceContext = { workspace: resolved.workspace, actor }; + if (!isFeatureEnabled(resolved.workspace, "incident-management")) { + return "Incident management isn't available for this workspace yet."; + } + const slack = new WebClient(resolved.botToken); + const [verb = "help", ...rest] = words; + const bound = await getIncidentBySlackChannel({ + ctx, + input: { teamId, channelId }, + }); + + try { + switch (verb.toLowerCase()) { + case "declare": { + const { title, severity } = parseDeclare(rest); + if (!title) { + return "Usage: `/openstatus incident declare <title> [--sev critical|major|minor]`"; + } + return postCard({ + slack, + ctx, + teamId, + channelId, + slackUserId: actor.slackUserId, + toolName: "declare_incident", + input: { title, severity }, + }); + } + case "note": { + const message = rest.join(" ").trim(); + if (!bound) { + return "Run `note` in an incident's channel, or mention @openstatus and say which incident."; + } + if (!message) return "Usage: `/openstatus incident note <text>`"; + await addIncidentNote({ ctx, input: { id: bound.id, message } }); + trackSlackIncident(ctx, "note"); + return `Added to the timeline of *${bound.title}*.`; + } + case "mitigate": + case "resolve": + case "cancel": + case "reopen": { + const { id, rest: note } = parseTarget(rest); + const incidentId = id ?? bound?.id; + if (!incidentId) { + return `Run this in an incident's channel, or name it: \`/openstatus incident ${verb} #12\`.`; + } + return postCard({ + slack, + ctx, + teamId, + channelId, + slackUserId: actor.slackUserId, + toolName: "set_incident_status", + input: { + id: incidentId, + status: STATUS_BY_VERB[verb.toLowerCase()], + ...(note ? { note } : {}), + }, + }); + } + case "status": { + const { id } = parseTarget(rest); + const target = + id !== null ? await getIncident({ ctx, input: { id } }) : bound; + if (!target) { + return "No incident here. Name one: `/openstatus incident status #12`."; + } + const commander = target.commander + ? displayName(target.commander) + : "none"; + return `*${target.title}* · ${target.severity} · ${target.status}${target.closedAt ? " (closed)" : ""}\nCommander: ${commander}${target.statusReport ? `\nStatus report: ${target.statusReport.title} (${target.statusReport.status})` : ""}\n<${getIncidentDashboardUrl(target.id)}|Open in openstatus>`; + } + case "list": { + const open = await listIncidents({ + ctx, + input: { status: ["open", "mitigated"], limit: 20 }, + }); + if (open.length === 0) return "No open incidents."; + return open + .map( + (i) => + `• #${i.id} <${getIncidentDashboardUrl(i.id)}|${i.title}> · ${i.severity} · ${i.status}`, + ) + .join("\n"); + } + default: + return INCIDENT_HELP; + } + } catch (error) { + if (error instanceof ServiceError) return `:x: ${error.message}`; + logger.error("slack incident command failed", { error, verb }); + return ":x: Something went wrong. Please try again."; + } +} diff --git a/apps/server/src/routes/slack/incident-slack.ts b/apps/server/src/routes/slack/incident-slack.ts index f355b725..74e206e2 100644 --- a/apps/server/src/routes/slack/incident-slack.ts +++ b/apps/server/src/routes/slack/incident-slack.ts @@ -1,9 +1,12 @@ import { getLogger } from "@logtape/logtape"; +import { sendIncidentCommander } from "@openstatus/emails"; import { ServiceError, type ServiceContext } from "@openstatus/services"; import { announceIncidentChange, bindIncidentSlackChannel, + displayName, escapeMrkdwn, + getIncident, openIncidentSlackChannel, type SlackClientFactory, } from "@openstatus/services/incident"; @@ -11,6 +14,8 @@ import { WebClient } from "@slack/web-api"; import { z } from "zod"; import type { SlackConfig } from "./config"; +import { postConfirmationCard } from "./confirmation-card"; +import { trackSlackIncident } from "./incident-analytics"; import { requireSlackMember, slackAgentAllowed } from "./require-slack-member"; import type { SlackWorkspace } from "./workspace-resolver"; @@ -28,6 +33,17 @@ function who(ctx: ServiceContext): string { return ctx.actor.type === "slack" ? `<@${ctx.actor.slackUserId}>` : "Someone"; } +function quote(note: string | undefined): string { + return note ? `\n>${escapeMrkdwn(note).replaceAll("\n", "\n>")}` : ""; +} + +const INCIDENT_TOOLS = new Set([ + "declare_incident", + "update_incident", + "resolve_incident", + "set_incident_status", +]); + /** Slack side effects of an approved incident tool call. Best effort. */ export async function afterIncidentTool(args: { ctx: ServiceContext; @@ -35,13 +51,24 @@ export async function afterIncidentTool(args: { input: object; output: object; config: SlackConfig; + slack: WebClient; + teamId: string; + channelId: string; + threadTs: string; }): Promise<void> { const { ctx, toolName, config } = args; + if (!INCIDENT_TOOLS.has(toolName)) return; const output = incidentOutput.safeParse(args.output); if (!output.success) return; const incidentId = output.data.id; + const status = output.data.status; + const note = incidentInput.safeParse(args.input).data?.note; try { if (toolName === "declare_incident") { + trackSlackIncident(ctx, "declare"); + await notifyCommander(ctx, incidentId, config).catch((error) => + logger.warn("incident commander email failed", { error, incidentId }), + ); const result = await openIncidentSlackChannel({ ctx, incidentId, @@ -51,25 +78,102 @@ export async function afterIncidentTool(args: { logger.info("slack incident channel", { incidentId, ...result }); return; } - if (toolName === "resolve_incident" || toolName === "update_incident") { - const note = incidentInput.safeParse(args.input).data?.note; - const text = - toolName === "resolve_incident" - ? `${who(ctx)} marked the incident *resolved*.${note ? `\n>${escapeMrkdwn(note).replaceAll("\n", "\n>")}` : ""}` - : `${who(ctx)} updated the incident.`; + if (toolName === "update_incident") { await announceIncidentChange({ ctx, incidentId, - text, + text: `${who(ctx)} updated the incident.`, clientFor: slackClientFor, dashboardUrl: config.dashboardUrl, }); + return; + } + + trackSlackIncident(ctx, "status", { status }); + const closed = status === "resolved" || status === "canceled"; + const row = closed + ? await getIncident({ ctx, input: { id: incidentId } }) + : undefined; + const report = + row?.statusReport && row.statusReport.status !== "resolved" + ? row.statusReport + : undefined; + // An archived channel can't take the resolve card, so keep it open. + const cardInIncidentChannel = + !!report && row?.slackChannelId === args.channelId; + await announceIncidentChange({ + ctx, + incidentId, + text: `${who(ctx)} marked the incident *${status}*.${quote(note)}`, + clientFor: slackClientFor, + dashboardUrl: config.dashboardUrl, + archive: status === "canceled" && !cardInIncidentChannel, + }); + if (report) { + await offerStatusReportResolve({ + ...args, + statusReportId: report.id, + note, + status, + }); } } catch (error) { logger.warn("slack incident follow-up failed", { error, incidentId }); } } +/** A linked status report still open gets a resolve card; never automatic. */ +async function offerStatusReportResolve(args: { + ctx: ServiceContext; + slack: WebClient; + teamId: string; + channelId: string; + threadTs: string; + statusReportId: number; + note: string | undefined; + status: string; +}): Promise<void> { + if (args.ctx.actor.type !== "slack") return; + await postConfirmationCard({ + slack: args.slack, + ctx: args.ctx, + teamId: args.teamId, + channel: args.channelId, + threadTs: args.threadTs, + slackUserId: args.ctx.actor.slackUserId, + toolName: "resolve_status_report", + input: { + statusReportId: args.statusReportId, + message: + args.note ?? + (args.status === "resolved" + ? "This incident has been resolved." + : "This was a false alarm. Everything is operating normally."), + }, + }); +} + +async function notifyCommander( + ctx: ServiceContext, + incidentId: number, + config: SlackConfig, +): Promise<void> { + const row = await getIncident({ ctx, input: { id: incidentId } }); + const commander = row?.commander; + const actorUserId = ctx.actor.type === "slack" ? ctx.actor.userId : null; + if (!row || !commander?.email || commander.id === actorUserId) return; + const declarer = row.declaredByUser ? displayName(row.declaredByUser) : null; + await sendIncidentCommander({ + to: commander.email, + incidentTitle: row.title, + severity: row.severity, + workspaceName: ctx.workspace.name ?? ctx.workspace.slug, + assignedBy: declarer ?? "A teammate", + url: `${config.dashboardUrl}/incidents/${row.id}`, + idempotencyKey: `incident-commander:${row.id}:${commander.id}:${row.updatedAt.getTime()}`, + }).catch(() => undefined); +} + /** "Link this channel": the fallback when binding failed during declare. */ export async function bindChannelFromButton(args: { resolved: SlackWorkspace; diff --git a/apps/server/src/routes/slack/interactions.ts b/apps/server/src/routes/slack/interactions.ts index b1ee7253..b5f29e20 100644 --- a/apps/server/src/routes/slack/interactions.ts +++ b/apps/server/src/routes/slack/interactions.ts @@ -292,7 +292,17 @@ async function runAndPresent(args: { runInBackground( "incident-follow-up", () => - afterIncidentTool({ ctx, toolName: tool.name, input, output, config }), + afterIncidentTool({ + ctx, + toolName: tool.name, + input, + output, + config, + slack, + teamId: pending.teamId ?? actor.teamId, + channelId, + threadTs: pending.threadTs, + }), { toolName: tool.name }, ); } diff --git a/apps/server/src/routes/slack/presenters/incident.ts b/apps/server/src/routes/slack/presenters/incident.ts index 7842bb35..6981fe2a 100644 --- a/apps/server/src/routes/slack/presenters/incident.ts +++ b/apps/server/src/routes/slack/presenters/incident.ts @@ -25,3 +25,9 @@ export const resolveIncidentPresenter: Presenter = ({ input, output }) => { const o = output as AgentToolOutput<"resolve_incident">; return `:white_check_mark: Incident *${o.title}* resolved.${i.note ? `\n> ${i.note}` : ""}\n${link(o.id)}`; }; + +export const setIncidentStatusPresenter: Presenter = ({ input, output }) => { + const i = input as AgentToolInput<"set_incident_status">; + const o = output as AgentToolOutput<"set_incident_status">; + return `:white_check_mark: Incident *${o.title}* is now ${o.status}.${i.note ? `\n> ${i.note}` : ""}\n${link(o.id)}`; +}; diff --git a/apps/server/src/routes/slack/presenters/index.ts b/apps/server/src/routes/slack/presenters/index.ts index c53ee796..a48d1f74 100644 --- a/apps/server/src/routes/slack/presenters/index.ts +++ b/apps/server/src/routes/slack/presenters/index.ts @@ -5,6 +5,7 @@ import { defaultPresenter } from "./default"; import { declareIncidentPresenter, resolveIncidentPresenter, + setIncidentStatusPresenter, updateIncidentPresenter, } from "./incident"; import { createMaintenancePresenter } from "./maintenance"; @@ -25,6 +26,7 @@ export const presenters: Record<string, Presenter> = { declare_incident: declareIncidentPresenter, update_incident: updateIncidentPresenter, resolve_incident: resolveIncidentPresenter, + set_incident_status: setIncidentStatusPresenter, }; export async function renderToolResult(args: { diff --git a/apps/server/src/routes/slack/system-prompt.ts b/apps/server/src/routes/slack/system-prompt.ts index 8fc4db52..aa0c0c7a 100644 --- a/apps/server/src/routes/slack/system-prompt.ts +++ b/apps/server/src/routes/slack/system-prompt.ts @@ -3,11 +3,13 @@ const INCIDENT_SECTION = ` Managed incidents (internal, never published): -- Three things are called "incident". A managed incident (list_incidents, get_incident, declare_incident, update_incident, resolve_incident, add_incident_note) is the team's internal record with a severity, a commander and a timeline. A status report is the PUBLIC communication. Monitor downtime is detected automatically. -- declare_incident, update_incident and resolve_incident render approval cards like the other write tools: nothing changes until the user clicks Approve, so never report them as done. +- Three things are called "incident". A managed incident (list_incidents, get_incident, declare_incident, update_incident, resolve_incident, set_incident_status, add_incident_note) is the team's internal record with a severity, a commander and a timeline. A status report is the PUBLIC communication. Monitor downtime is detected automatically. +- declare_incident, update_incident, resolve_incident and set_incident_status render approval cards like the other write tools: nothing changes until the user clicks Approve, so never report them as done. - "declare an incident", "open an incident" -> declare_incident (publishes nothing). Then offer a status report, passing incidentId to create_status_report to link them. - "note that…", "add to the timeline" -> add_incident_note (runs right away, no card). - "the incident is resolved" with a managed incident in play -> resolve_incident; if its linked status report is still open, draft resolve_status_report too. +- "mitigated", "the bleeding stopped" -> set_incident_status mitigated; "false alarm", "declared by mistake" -> set_incident_status canceled (closes it for good). +- In an incident channel (see the note below when there is one), notes and updates belong to that incident: use its id without asking. - Before referencing a managed incident, call list_incidents. Severity: critical = major outage or data loss, major = significant degradation, minor = limited impact.`; export function buildSystemPrompt( diff --git a/packages/services/src/agent-tools/incident.ts b/packages/services/src/agent-tools/incident.ts index c5769cf2..ec271830 100644 --- a/packages/services/src/agent-tools/incident.ts +++ b/packages/services/src/agent-tools/incident.ts @@ -359,6 +359,48 @@ export const resolveIncidentTool: AgentTool< }, }; +const SetIncidentStatusInput = z.object({ + id: z.number().int().describe("Incident id, from list_incidents."), + status: z + .enum(incidentStatus) + .describe( + "mitigated: impact stopped, not fixed yet. resolved: fixed. canceled: false alarm or declared by mistake. open: reopen.", + ), + note: z.string().max(10_000).optional().describe("Why, in a sentence."), +}); + +export const setIncidentStatusTool: AgentTool< + z.infer<typeof SetIncidentStatusInput>, + z.infer<typeof IncidentWriteOutput> +> = { + name: "set_incident_status", + description: + "Move a managed incident to mitigated, resolved, canceled (false alarm) or back to open. Canceling closes it for good. Does not touch its status report.", + scope: "write", + destructive: true, + feature: FEATURE, + inputSchema: SetIncidentStatusInput, + outputSchema: IncidentWriteOutput, + approval: { + summarize: (input) => ({ + title: `Mark incident #${input.id} ${input.status}`, + lines: optionalLines([["Note", input.note]]), + }), + verb: "updated", + }, + async run({ ctx, input }) { + const row = await setIncidentStatus({ + ctx, + input: { + id: input.id, + status: input.status, + note: input.note?.trim() || undefined, + }, + }); + return writeOutput(row); + }, +}; + const AddIncidentNoteInput = z.object({ id: z.number().int().describe("Incident id, from list_incidents."), message: note.describe("The note, markdown."), diff --git a/packages/services/src/agent-tools/index.ts b/packages/services/src/agent-tools/index.ts index ec49ddd9..e2fef14a 100644 --- a/packages/services/src/agent-tools/index.ts +++ b/packages/services/src/agent-tools/index.ts @@ -11,6 +11,7 @@ import { getIncidentTool, listIncidentsTool, resolveIncidentTool, + setIncidentStatusTool, updateIncidentTool, } from "./incident"; import { createMaintenanceTool, listMaintenancesTool } from "./maintenance"; @@ -44,6 +45,7 @@ export { getIncidentTool, listIncidentsTool, resolveIncidentTool, + setIncidentStatusTool, updateIncidentTool, } from "./incident"; export { createMaintenanceTool, listMaintenancesTool } from "./maintenance"; @@ -105,6 +107,7 @@ export const agentTools = { declare_incident: declareIncidentTool, update_incident: updateIncidentTool, resolve_incident: resolveIncidentTool, + set_incident_status: setIncidentStatusTool, add_incident_note: addIncidentNoteTool, list_maintenances: listMaintenancesTool, create_maintenance: createMaintenanceTool, diff --git a/packages/services/src/agent-tools/prompt.ts b/packages/services/src/agent-tools/prompt.ts index b4d016b5..9b2b0a51 100644 --- a/packages/services/src/agent-tools/prompt.ts +++ b/packages/services/src/agent-tools/prompt.ts @@ -45,11 +45,12 @@ Exception: after get_doc_page or get_content_page, DO synthesize an answer from ? ` Managed incidents (internal): -- Three different things are called "incident". A managed incident (list_incidents, get_incident, declare_incident, update_incident, resolve_incident, add_incident_note) is the team's INTERNAL record: severity, commander, timeline. A status report is PUBLIC communication on a status page. Monitor downtime (activeIncidentCount on monitors) is detected automatically. +- Three different things are called "incident". A managed incident (list_incidents, get_incident, declare_incident, update_incident, resolve_incident, set_incident_status, add_incident_note) is the team's INTERNAL record: severity, commander, timeline. A status report is PUBLIC communication on a status page. Monitor downtime (activeIncidentCount on monitors) is detected automatically. - "declare an incident", "open an incident", "we have a SEV" → declare_incident. It publishes nothing. Offer to create a status report afterwards and pass incidentId to create_status_report to link them. - Before referencing a managed incident: call list_incidents. Never guess its id. - "note that…", "add to the timeline", "log that…" → add_incident_note (internal, runs without confirmation). - "the incident is fixed/resolved" with a managed incident in play → resolve_incident; if its linked status report is still open, ask whether to resolve that too (resolve_status_report). +- "mitigated", "the bleeding stopped" → set_incident_status mitigated; "false alarm", "declared by mistake" → set_incident_status canceled (this closes it). - severity: critical = major outage or data loss, major = significant degradation, minor = limited impact. Ask when unclear.` : ""; diff --git a/packages/services/src/incident/index.ts b/packages/services/src/incident/index.ts index 1d93c101..c307352e 100644 --- a/packages/services/src/incident/index.ts +++ b/packages/services/src/incident/index.ts @@ -10,7 +10,12 @@ export { } from "./link-status-report"; export { listIncidentEvents } from "./list-events"; export { clearIncidentCommander } from "./members"; -export { getIncident, getIncidentForStatusReport, listIncidents } from "./list"; +export { + getIncident, + getIncidentBySlackChannel, + getIncidentForStatusReport, + listIncidents, +} from "./list"; export { AddIncidentNoteInput, BindIncidentSlackChannelInput, diff --git a/packages/services/src/incident/list.ts b/packages/services/src/incident/list.ts index 0f58f721..48ec4b65 100644 --- a/packages/services/src/incident/list.ts +++ b/packages/services/src/incident/list.ts @@ -2,7 +2,8 @@ import { and, desc, eq, inArray, sql } from "@openstatus/db"; import { incident } from "@openstatus/db/src/schema"; import { type ServiceContext, getReadDb } from "../context"; -import { requireIncidentFeature } from "./internal"; +import { isFeatureEnabled } from "../features"; +import { INCIDENT_FEATURE, requireIncidentFeature } from "./internal"; import { IncidentIdInput, ListIncidentsInput } from "./schemas"; const userColumns = { @@ -81,3 +82,23 @@ export async function getIncidentForStatusReport(args: { ) .get(); } + +/** The incident bound to a Slack channel, or `undefined`. Never throws on the gate. */ +export async function getIncidentBySlackChannel(args: { + ctx: ServiceContext; + input: { teamId: string; channelId: string }; +}) { + const { ctx } = args; + if (!isFeatureEnabled(ctx.workspace, INCIDENT_FEATURE)) return undefined; + return getReadDb(ctx).query.incident.findFirst({ + where: and( + eq(incident.workspaceId, ctx.workspace.id), + eq(incident.slackTeamId, args.input.teamId), + eq(incident.slackChannelId, args.input.channelId), + ), + with: { + commander: { columns: userColumns }, + statusReport: { columns: { id: true, title: true, status: true } }, + }, + }); +}