From 65de945ded8b5fdb4388d0455d5d5ebd89ee2987 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:12:24 -0800 Subject: [PATCH 01/34] Release v1.0.0-rc.1 (#1325) --- CHANGELOG.md | 233 +++++++++++------- app/src/components/approvals/rules.tsx | 71 +++++- app/src/components/approvals/waiting.tsx | 46 +++- app/tests/approval-card-internals.test.tsx | 139 +++++++++++ app/tests/approval-rule-line.test.tsx | 122 +++++++++ app/tests/approvals-rules.test.tsx | 56 ++++- app/tests/bot-approval-rules.test.tsx | 18 +- app/tests/skins-openmuse.test.tsx | 7 +- charts/openbot/Chart.yaml | 2 +- desktop/src-tauri/Cargo.lock | 2 +- desktop/src-tauri/Cargo.toml | 2 +- extension/scripts/store-package.test.ts | 4 +- package.json | 2 +- release-policy.json | 10 +- server/src/activity/activity.ts | 11 +- server/src/approvals/wiring.ts | 3 +- server/src/delivery/opentag-agent.ts | 9 +- server/src/delivery/opentag.ts | 8 +- ...pproval-activity-words.integration.test.ts | 108 ++++++++ server/tests/approvals.test.ts | 2 +- server/tests/delivery-routes.test.ts | 74 +++++- shared/tool-name.ts | 136 ++++++++++ 22 files changed, 914 insertions(+), 151 deletions(-) create mode 100644 app/tests/approval-card-internals.test.tsx create mode 100644 app/tests/approval-rule-line.test.tsx create mode 100644 server/tests/approval-activity-words.integration.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index be69c4643d..3336a64d60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,118 +8,161 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased -- OpenBot Desktop's updates download from downloads.copilotkit.ai too. The update feed there names - that address for the Windows installer and the Mac update archives, instead of GitHub; every - signature is unchanged, and the feed is only published once each of those files answers there. +**Before upgrading.** There are no migrations: the database stays at +`0083_learning_skill_approvals`. Four things change for an existing deployment: +- OpenBot Desktop updates its back end to CopilotKit Intelligence 0.16.0-rc.10 at the next Start, + saying "Updating the back end to 0.16.0-rc.10. Its data stays." With it, a first back end setup + downloads all of its images through downloads.copilotkit.ai first, where its own images came + straight from ghcr.io and Docker Hub, and a setup step that fails reports its own error, which + OpenBot's log keeps (below). +- The Helm chart's bundled back end (the `intelligence` subchart) is 0.16.0-rc.10, from rc.9. +- The first time anybody opens OpenBot in a browser or in OpenBot Desktop after the upgrade, every + built-in component description nobody has edited takes this release's wording, so Bots read the + new Notice, Checklist, Record, Headline figures, Data table, Bar chart, Donut chart and Progress + against target descriptions (below). Until then, Bots keep the old wording. A component whose + description an admin saved, or that an admin published or unpublished, on Admin → UI components + keeps what they chose. Each change is one row in Admin → Audit, "Update a component's + description", started by this deployment. In the personal edition every built-in description + follows, and Activity leaves these rows out. +- On OpenBot Desktop, a Bot on LangChain that follows the installer has "Powered by LangChain + (LangGraph)" in its subtitle changed to "Powered by LangChain" at the next Start, keeping the rest + of the subtitle, and a Bot picked in setup before 0.3.3 and named "LangChain (LangGraph)" is + renamed "LangChain". A Bot whose connection you set yourself keeps its subtitle. + ### OpenBot Desktop -- **An update the mirror cannot send comes from GitHub.** When downloading the new version from - downloads.copilotkit.ai fails, OpenBot reads GitHub's update feed and downloads the same signed - version from there once, instead of failing until the mirror is fixed. The log says which one it - came from. - **A back end setup that stops says why.** OpenBot's log now records how every back end setup, start, update and stop ended: how long it took, its exit code, what its report says went wrong and the end of its error output. A setup that stopped used to leave nothing in the log. The card says whether the back end stopped with an error partway, quit without saying why, or could not be run, where all three said "The back end could not start." -- The first Bot to need you after OpenBot Desktop starts raises its notification. It used to raise - nothing while the app was still checking that notifications were on, so the first question after - each launch, asked while OpenBot was minimised or behind another window, went unannounced. -- A Bot with no folders, as every Bot has after OpenBot restarts, now says so in plain words: it - has no folders it can read, access ends when you stop OpenBot, and Choose folder in Settings → - Computer gives it one. It said "No host folders are granted to this Bot.", and the Bot told you - an administrator could restore it. - -- **CopilotKit Intelligence 0.16.0-rc.10.** OpenBot Desktop's back end and the Helm chart move to - Intelligence 0.16.0-rc.10 (CLI `copilotkit@647a03b`). A first back end setup now downloads its - images through downloads.copilotkit.ai too, because the CLI's own pulls read the cluster's - mirrors, and a setup step that fails now reports its own error instead of only that the command - did not finish. -- **Setup and Home say what is true.** "OpenBot is running" no longer flashes Start's own steps, and - points to the system tray on Windows instead of the menu bar; after Accept the account step says - it is setting up your apps; a relaunch says "Starting the engine machine." while it does; the - personal edition's Home no longer says nobody has shared a Bot; and Teach a Bot says "No - microphone found. Narration is off." on a computer with none, keeping your choice for when there - is one. - -### Automatic Learning and Teach a Bot - -- **Teach a Bot reads as one page**: its title and how in one sentence, the browser, and your - recordings under it, all as wide as the browser. The browser is drawn as a window (tabs on its - frame, a pill address bar with a lock, a start page for an empty tab), sized to its page so no - blank band sits under it, and the page is laid out while the browser opens. **Record** is one - button at the toolbar's end that asks for the task's name and narration in a popover, with an - (i) saying what narration is for; the narration bar shows only while recording, or to say there - is no microphone. Your recordings links to Settings → Learning, where what a recording teaches - shows up. +- **The first notification after OpenBot Desktop starts is raised**, for a Bot that needs you or + replies while OpenBot is minimised or behind another window. The first one after each launch used + to raise nothing, because the app was still checking that notifications were on. OpenBot's log + now says whether each notification was shown, never what it said. +- **Setup says what is happening.** The screen shown while OpenBot runs says you can show or stop + it from its icon in the system tray on Windows, where it said the menu bar everywhere, and no + longer flashes Start's own steps before OpenBot opens. After you accept the terms, the account + step says "Setting up your apps" while they connect, where it flashed "Sign in to CopilotKit". + Opening OpenBot after a restart says "Starting the engine machine." while it starts OpenBot's + Podman machine, and Start's steps say "OpenBot's services" where they said "Containers". +- **App updates download from downloads.copilotkit.ai.** The update feed there names + downloads.copilotkit.ai for the Windows installer and the Mac update archives, where they came + from GitHub, and every signature is unchanged. If that download fails, OpenBot Desktop reads + GitHub's update feed and downloads the same signed version from there, where the update failed. + The log says where it came from. +- **A Bot with no folders says so in plain words**, as every Bot does after OpenBot restarts: "This + Bot has no folders it can read right now. Folder access ends when you stop OpenBot. To give it a + folder, use Choose folder in Settings → Computer." It said "No host folders are granted to this + Bot." + +### Teach a Bot + +- **Teach a Bot is one page.** Its title and one line on how sit above the browser, and Your + recordings sits under it at the same width and scrolls on its own, so a long list no longer makes + the browser smaller. The page was headed "Browser". Where the browser opens as a window on your + desktop, the page is as it was. +- **The browser looks like a browser**: tabs along the top, a rounded address bar with a lock on + https pages, and a search field in an empty tab, sized so the page fills it. While it starts it + says "Opening your browser…", where the page showed no browser. +- **Record is one button** at the end of the toolbar. It opens **Record a task**, with the task's + name, the Narration switch (its (i) says what narration is for) and **Start recording**, where the + name field, the switch and Record sat in a bar above the browser. The narration line shows under + the browser only while recording, or to say "No microphone found. Narration is off." on a + computer with none, which keeps your narration choice for when there is one. +- Your recordings links to Settings → Learning ("See what your Bots learned"). ### Conversations -- **The sidebar's + opens a new conversation with To: empty** and the Bot menu open, where it - picked a Bot for you, as do Shift+N and a skin's new conversation link. A Bot's own Message link - still starts with that Bot, and a message sent before a Bot is picked stays in the box with - "Choose a Bot first." -- The Bot's browser beside a conversation is a 360px glimpse with the address it is on and Take - control on the picture, and its full-size view fits its window to the page. -- Home says which Bot is working, with a way to it, and a conversation says "Working" in the - sidebar while its Bot works. A Bot's browsing reads "Browsed example.com", where it read "Browsed - 1 page". -- Bots answer ordinary questions, summaries and short lists directly in Markdown by default. A - notice card is kept for a distinct alert that needs you to act, or for when you ask for one, and - other components for when a chart, table or card makes the answer easier to use. A Bot asked for - its top emails used to wrap the list in a notice. -- A component description nobody has edited now follows OpenBot's own when you upgrade, so the new - Notice, Checklist, Record, Headline figures, Data table, Bar chart, Donut chart and Progress - against target descriptions reach an existing deployment. The text comes from OpenBot itself, - never from a browser. A description an admin has saved, or a component an admin has published or - unpublished, keeps what they chose. Each update is one row in Admin → Audit, started by OpenBot. -- A notice card's supporting point that is itself a list is drawn as that list, without a stray dot - and an extra indent beside it. -- A long link in a card's text (a notice, a checklist, a quotation, an approval, a choice's - options) wraps, where it ran under the card's edge and lost its end. -- A choice card's summary is drawn as Markdown, as an approval's is, where it showed the raw - markers, such as `**two**`. -- A Bot can link a bar, a progress row, a headline figure or a donut chart's legend row to where - the figure came from, such as the record or the search, when it has that address. Clicking it - opens the page in a new tab, as a source under a chart does. Only web addresses become links, and - a figure without one draws as before. -- A chart whose figures are still arriving shimmers with "Drawing the chart…" at its own height, - where it said "There is no data to chart." until its first figure came in. A chart that was - stopped, or that finished with no figures, still says there is no data. -- A bar chart whose figures don't fit their bars, such as ten bars on a phone, writes them - compactly ("15K", "3.7K"), where they ran into one another. The exact figure stays in the - tooltip and is what a screen reader reads. +- **Bots answer in Markdown by default.** Ordinary questions, explanations, summaries and short + lists come back as a reply. A notice card is kept for an alert that needs you to act, or for when + you ask for one, and a chart, table or card is drawn when it makes the answer easier to use. Bots + used to be told to answer with a component whenever an answer had structure, so ordinary answers + came back as cards. +- **A figure can link to where it came from.** A bar, a progress row, a headline figure or a donut + chart's legend row links to the record or search its figure came from when the Bot has that + address, and opens it in a new tab (in OpenBot Desktop, in your own browser). Only web addresses + become links. +- A chart whose figures are still arriving shows "Drawing the chart…" at the chart's own height, + where it said "There is no data to chart." A chart that was stopped, or has no figures, still says + so. +- **+ and Shift+N open a new conversation with To: empty** and the Bot menu open, in the personal + edition and wherever there is only one Bot, where since 0.3.3 they started with your Bot picked. A + Bot's own Message link still starts with that Bot, and Send with nobody in To: still keeps the + message and says "Choose a Bot first." +- **The Bot's screen shows where it is.** In a conversation and in the panel beside it, the address + the Bot's browser is on sits above the picture. While the Bot has control, a strip over the + picture names it with **Take control**, where "The Bot has control." and Take control sat in a bar + under the picture. The panel beside a conversation is narrower, and the full-size view is sized to + the page. +- **A Bot that is working says so.** Its conversation says "Working" in the sidebar where its time + was, and Home shows that the Bot is working, with the conversation's name and **Open**. +- A Bot's browsing names the site when it stayed on one, "Browsed example.com" or "Browsed 3 pages + on example.com", where it said "Browsed 1 page". +- **Approvals for a folder or a Bot's own computer use the words of their step.** The approval + card, Needs you and a task's waiting line say "Save a file todo.txt in share", "Read a file" and + "Run a command", where they said "Write file on todo.txt in share". A file on the Bot's own + computer is named by its path, and its save step reads "Saved a file". +- An opened "Looked for a tool" step lists the tools the Bot found by name ("Send email on Gmail"), + where it showed their ids and the descriptions the search returned to the Bot. +- A browser action waiting for your approval is named the way its card names it. A Bot's Activity + row read "Approve computer_click" with "write" beside it, and now reads "Click the link “Python” + on en.wikipedia.org". The Needs you card names the link too and lists what the action would send + by label, without the page handles a click carried as raw JSON. An approval in Slack says it the + same way ("Research Bot wants to click the link “Python” on en.wikipedia.org"), and a saved rule + is titled "Click" or "Send email on Gmail" instead of its tool id. The line under it names the + folder and the Bot by name and the kind of action in words ("changes in share for Shop"), where + it read "write on host-grant-15407-… for shopper-bot". ### Personal edition -- In Activity, a Bot's call to an app you haven't connected reads "Not connected" and "Gmail isn't - connected. Connect it in Connections.", rows already recorded included. It read "The server - didn't answer" and printed raw `[connect-app]` text, which Admin → Audit no longer shows either. -- Home no longer shows an empty Explore Bots box. +- Home no longer shows an empty Explore Bots box saying "Nobody has shared a Bot with you yet." +- In Activity, a Bot's call to an app you haven't connected reads "Not connected" with "Gmail isn't + connected. Connect it in Connections.", or "No longer connected" when it needs connecting again, + rows already recorded included. It read "The server didn't answer" with raw `[connect-app]` text + under it, which Admin → Audit no longer shows either. +- Activity names a skill by its title ("Write a skill", where it said "Skill creator"), on older + rows too. An app a Bot got when you pressed Connect in a conversation reads "Let a Bot use an + app", "Allowed automatically", where it read "Approve a request for an app", as if somebody had + approved it. +- An app's page names the setting that makes a Bot ask first by its name, "Always ask before making + changes", where it said "Ask before acting". ### Smaller changes -- LangChain is listed as **LangChain**, with LangChain's own symbol, where it said "LangChain - (LangGraph)". This covers New Bot, a Bot's "Powered by" subtitle, its setup row and OpenBot - Desktop's plan note. A subtitle saved under the old name still follows a change of harness. New - Bot's custom agents are one line each, and the four named frameworks show without scrolling. -- **New Bot** shows custom agents on its first screen. Below the simple form, **Connect a custom - agent** lists every framework with its logo, Claude Agent SDK, LangChain, Google ADK and AWS - Strands first, and on a desktop its heading and first row show without scrolling. Pressing one - goes straight to its address, and Back returns to the list. Its labels are small and uppercase, - and Isolated and apps are plain rows. - The sidebar floats as frosted glass over the colours of OpenBot's orb, the orb sits beside - OpenBot's name, and the app is set in Atkinson Hyperlegible Next. A skin keeps its own faces. -- Settings → General lists your answers as rows that open a dialog. Settings → Notifications shows - a switch only for channels that are set up, and says how to set them up, or that your admin does. - In OpenBot Desktop it says those channels need OpenBot running on a server. -- **Activity, approvals and a Bot's page say things the way a person would.** Activity names a skill - by its title ("Write a skill", not "Skill creator"), says "Set up an app's sign-in" for "Register - an OAuth client", and says an app the personal edition gave a Bot on Connect was "Allowed - automatically" where it read as a person's approval. A file save's approval card reads "Save a - file todo.txt in share", as its step and its Activity row do, and an opened "Looked for a tool" - names the tools it found in words. A Bot's Your skills lists each skill by its title, and an app's - page names the "Always ask before making changes" setting by its name. + OpenBot's name, and the app is set in Atkinson Hyperlegible Next. The glass turns off with the + system's reduce transparency or increase contrast setting, and a skin keeps its own look. +- Settings → General, headed "Preferences" before, shows your name and standing instructions as + rows under About you, each opening a dialog to change it, where both were edited on the page. Dark + theme, Skin and Message list emphasis sit under Appearance. +- Settings → Notifications shows a switch only for channels that are set up, where the rest showed + as "Not set up". A choice saved for one is kept and comes back once it is set up. With none set up + it says "Updates stay in OpenBot", with **How to set them up** for admins and anyone running + OpenBot themselves, and that your admin sets them up for everyone else. In OpenBot Desktop it says + they need OpenBot running on a server. +- The skill OpenBot ships as Make a coworker is called **Make a Bot**, and Write a skill's + description says it saves a new skill rather than a `/` command. Their `/` names are unchanged. +- Activity and Admin → Audit say "Set up an app's sign-in" and "Check an app's sign-in", where they + said "Register an OAuth client" and "Read an OAuth client". +- Your skills on a Bot's page lists each skill by its title ("Tide report"), where it showed + "/tide-report". +- A notice card's supporting point that is itself a list is drawn as that list, without a stray + dot and an extra indent beside it. +- A long link or path in a card's text wraps where it was cut off at the card's edge: in a notice, + a checklist, a record, a quotation, a data table, an approval, and a choice card's summary and + options. A choice card's summary is drawn as Markdown, as an approval's is, where it showed the + raw text, such as `**two**`. +- When a bar chart's figures don't fit above their bars, every bar shows a shorter figure, such as + "15K" or "3.7K", where they ran into each other. The exact figure shows when you point at it and + is read out by screen readers. +- **New Bot** lists the frameworks a custom agent can run on under **Connect a custom agent** on its + first screen, below the form, one line each with its logo, Claude Agent SDK, LangChain, Google ADK + and AWS Strands first. Choosing one goes straight to its address, and Back returns to the list. + They were behind a Connect a custom agent button. +- LangChain is called **LangChain**, with LangChain's own mark, where it said "LangChain + (LangGraph)": in New Bot, a Bot's "Powered by" subtitle and its Runs on row, and OpenBot Desktop's + note under a ChatGPT plan. ## 0.3.4 diff --git a/app/src/components/approvals/rules.tsx b/app/src/components/approvals/rules.tsx index a3e844dd10..a76c35a4c3 100644 --- a/app/src/components/approvals/rules.tsx +++ b/app/src/components/approvals/rules.tsx @@ -1,10 +1,11 @@ +import { ruleTitle } from "@openbot/shared/tool-name"; import { IconChevronRight, IconLock, IconPlus, IconShieldCheck, } from "@tabler/icons-react"; -import type { UseMutationResult } from "@tanstack/react-query"; +import { type UseMutationResult, useQuery } from "@tanstack/react-query"; import { useId, useState } from "react"; import { Button } from "@/components/ui/button"; import { @@ -33,6 +34,7 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; +import { GONE_BOT, useBotNames } from "@/lib/agents/bot-names"; import { type ApprovalRuleInput, type ApprovalRuleRow, @@ -40,6 +42,10 @@ import { type HostCommandPolicy, type RuleBehaviour, } from "@/lib/approvals"; +import { + type HostFolderGrant, + hostAccessQueryOptions, +} from "@/lib/computers/host-access"; export const HOST_LABELS: Record = { ask: "Ask every time", @@ -97,6 +103,38 @@ const TEAM_BEHAVIOUR_LABELS = Object.fromEntries( ]), ) as Record; +/** What kind of action a rule covers, in words. Any other kind is left unsaid. */ +const EFFECT_WORDS: Record = { + read: "reading", + write: "changes", + destructive: "deleting", + delegate: "handing off work", +}; + +/** "for Expenses"; nothing until the names arrive, so a Bot is never named by its id. */ +function forBot(name: string): string | null { + if (!name) return null; + return name === GONE_BOT ? "for a Bot no longer here" : `for ${name}`; +} + +/** A rule saved from a folder's card is on the id OpenBot gave that folder (`host-access/broker.ts`). */ +const FOLDER_GRANT = "host-grant-"; + +/** + * A folder rule's place by the folder's name, as Settings → Computer shows it: "in share". A grant + * ends when OpenBot stops, so a rule on one that is gone says so rather than naming its id. + */ +function folderWords( + grantId: string, + grants: readonly HostFolderGrant[] | undefined, +): string { + if (!grants) return "in a folder"; + const grant = grants.find((candidate) => candidate.id === grantId); + return grant && !grant.revoked + ? `in ${grant.displayName}` + : "in a folder that's no longer shared"; +} + /** * One saved rule, as a row: what it matches, and what a Bot does about it. A locked rule — a team * rule seen by a member, or a personal rule the team has switched off — states its behaviour and @@ -118,12 +156,29 @@ export function RuleRow({ onChange: (behaviour: RuleBehaviour) => void; }) { const labels = team ? TEAM_BEHAVIOUR_LABELS : BEHAVIOUR_LABELS; - // The tool is the title; the rest of what the rule matches is the line beneath, never clamped, - // since two rules for one tool differ only there. + // The action in its approval card's words is the title ("Click", "Any action on Gmail"); the rest + // of what the rule matches is the line beneath, never clamped, since two rules for one action + // differ only there. The title already names an app's rule's place. + const { title, place } = ruleTitle(rule); + const botName = useBotNames(); + const folderRule = rule.scope.startsWith(FOLDER_GRANT); + const folders = useQuery({ + ...hostAccessQueryOptions(), + refetchInterval: false, + enabled: folderRule, + }); const match = [ - rule.effect !== "*" ? rule.effect : null, - rule.scope !== "*" ? `on ${rule.scope}` : null, - rule.botId !== "*" ? `for ${rule.botId}` : null, + EFFECT_WORDS[rule.effect] ?? null, + rule.scope === "*" || place + ? null + : folderRule + ? folderWords(rule.scope, folders.data?.grants) + : `on ${rule.scope}`, + rule.botId === "*" + ? null + : rule.botId.includes("*") + ? `for Bots matching ${rule.botId}` + : forBot(botName(rule.botId)), ] .filter(Boolean) .join(" "); @@ -133,7 +188,7 @@ export function RuleRow({ {locked ? : } - {rule.toolRef} + {title} {match ? ( {match} @@ -150,7 +205,7 @@ export function RuleRow({ <> + {details.map((detail) => ( +
+
{detail.label}
+
+ {detail.value} +
+
+ ))} + + ); +} + /** * An action waiting on a decision, decided on the row's own footer line, in from the edge by the * icon and its gap so the buttons line up with the text, as a task's and an item's do beside it. @@ -160,9 +194,7 @@ function RequestRow({ {request.action.policy.reason}

) : null} - 
-          {JSON.stringify(request.action.args, null, 2)}
-
+ {request.action.policy?.behaviour === "hand_off" ? ( @@ -520,11 +552,7 @@ function RequestBoardRow({ request }: { request: WaitingRequest }) { ) : null, ]} - more={ - 
-          {JSON.stringify(request.action.args, null, 2)}
-
- } + more={} open title={action} waiting diff --git a/app/tests/approval-card-internals.test.tsx b/app/tests/approval-card-internals.test.tsx new file mode 100644 index 0000000000..eff3429e21 --- /dev/null +++ b/app/tests/approval-card-internals.test.tsx @@ -0,0 +1,139 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, render } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { WaitingForYou } from "@/components/approvals/waiting"; +import { BotNeedsYou } from "@/components/needs-you/board"; +import { settleReactWork } from "./settle-react-work"; + +/* + * FOUND IN THE 1.0 RC1 MAC PASS: a pending click's "Needs you" card read "Click / en.wikipedia.org" + * over a raw block, `{ "ref": "f3e43", "snapshotId": 4 }`. Those are the handles the model cites for + * an element on a page snapshot, and nobody deciding can read them. The card names the link from the + * label the server resolved, and shows only what the action would send. + */ + +beforeAll(() => GlobalRegistrator.register()); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +const originalFetch = global.fetch; +afterEach(() => { + global.fetch = originalFetch; +}); + +/** A pending computer action as `GET /api/approvals` returns it (`publicApproval`). */ +const request = ( + id: string, + toolRef: string, + args: Record, + element: { ref: string; role: string; name: string }, +) => ({ + id, + status: "pending", + createdAt: "2026-11-08T10:00:00Z", + action: { + botId: "researcher", + toolRef, + effect: "write", + scope: "en.wikipedia.org", + threadId: `thread-${id}`, + toolCallId: `call-${id}`, + args, + target: { + pageUrl: "https://en.wikipedia.org/wiki/Main_Page", + element: { ...element, type: "" }, + snapshotId: 4, + }, + }, +}); + +function serving() { + global.fetch = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url === "/api/approvals") + return Response.json({ + enabled: true, + rules: [], + teamRules: [], + questions: [], + requests: [ + request( + "click", + "computer_click", + { ref: "f3e43", snapshotId: 4 }, + { + ref: "f3e43", + role: "link", + name: "Python (programming language)", + }, + ), + request( + "type", + "computer_type", + { ref: "f7", snapshotId: 4, text: "monty python" }, + { ref: "f7", role: "searchbox", name: "Search Wikipedia" }, + ), + ], + }); + if (url === "/api/proactive/suggestions") + return Response.json({ suggestions: [] }); + if (url === "/api/tasks") return Response.json({ tasks: [] }); + if (url.startsWith("/api/channels")) + return Response.json({ channels: [], nextCursor: null }); + return Response.json({ + agents: [{ id: "researcher", name: "Researcher" }], + }); + }) as unknown as typeof fetch; +} + +function draw(node: ReactNode) { + return render( + + {node} + , + ); +} + +/** Nothing the model cites, and no raw JSON, anywhere a person reads. */ +function expectNoInternals(text: string) { + expect(text).not.toContain("snapshotId"); + expect(text).not.toContain("f3e43"); + expect(text).not.toContain('"ref"'); + expect(text).not.toContain("computer_"); + expect(text).not.toContain("{"); +} + +test("on the Needs you board, a click names the link and shows no raw JSON", async () => { + serving(); + const view = draw(); + expect( + await view.findByText("Click the link “Python (programming language)”"), + ).toBeTruthy(); + expect(view.getByText("Fill in “Search Wikipedia”")).toBeTruthy(); + // What a typing step would send is still shown whole, by its label. + expect(view.getByText("Text")).toBeTruthy(); + expect(view.getByText("monty python")).toBeTruthy(); + expect(view.getAllByText("en.wikipedia.org").length).toBeGreaterThan(0); + expectNoInternals(view.container.textContent ?? ""); +}); + +test("on the Bot's page, a click says what the Bot wants to do and shows no raw JSON", async () => { + serving(); + const view = draw(); + expect( + await view.findByText( + "Researcher wants to click the link “Python (programming language)”", + ), + ).toBeTruthy(); + expect(view.getByText("monty python")).toBeTruthy(); + expectNoInternals(view.container.textContent ?? ""); +}); diff --git a/app/tests/approval-rule-line.test.tsx b/app/tests/approval-rule-line.test.tsx new file mode 100644 index 0000000000..b5471e85fa --- /dev/null +++ b/app/tests/approval-rule-line.test.tsx @@ -0,0 +1,122 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, render } from "@testing-library/react"; +import { RuleRow } from "@/components/approvals/rules"; +import type { ApprovalRuleRow } from "@/lib/approvals"; +import { settleReactWork } from "./settle-react-work"; + +/* + * FOUND IN THE 1.0 RC1 MAC PASS: under a saved rule's title, the line that says what else it matches + * read "write on host-grant-15407-1794112361547002000 for shopper-bot": the raw kind of action, the + * id OpenBot gave a folder, and the Bot's id. It names the folder as Settings → Computer does, the + * Bot by its name, and the kind of action in words. + */ + +beforeAll(() => GlobalRegistrator.register()); +afterEach(cleanup); +afterAll(async () => { + await settleReactWork(); + GlobalRegistrator.unregister(); +}); + +const originalFetch = global.fetch; +afterEach(() => { + global.fetch = originalFetch; +}); + +const GRANT = "host-grant-15407-1794112361547002000"; + +function serving() { + global.fetch = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url === "/api/host-access") + return Response.json({ + connected: true, + grants: [ + { + id: GRANT, + botId: "shopper-bot", + actorId: "me", + displayName: "share", + }, + ], + pending: [], + }); + if (url.startsWith("/api/agents")) + return Response.json({ agents: [{ id: "shopper-bot", name: "Shop" }] }); + return new Response(null, { status: 404 }); + }) as unknown as typeof fetch; +} + +const rule = (overrides: Partial): ApprovalRuleRow => + ({ + id: "rule", + botId: "*", + toolRef: "host/write_file", + effect: "*", + scope: "*", + behaviour: "allow", + ...overrides, + }) as ApprovalRuleRow; + +function draw(row: ApprovalRuleRow) { + return render( + + {}} + onRevoke={() => {}} + rule={row} + /> + , + ); +} + +test("a folder rule names the folder, the Bot and the kind of action in words", async () => { + serving(); + const view = draw( + rule({ id: "folder", botId: "shopper-bot", effect: "write", scope: GRANT }), + ); + expect(await view.findByText("changes in share for Shop")).toBeTruthy(); + const text = view.container.textContent ?? ""; + expect(text).not.toContain("host-grant"); + expect(text).not.toContain("shopper-bot"); + expect(text).not.toMatch(/\bwrite\b/); +}); + +test("a rule on a folder no longer shared, for a Bot no longer here, says so", async () => { + serving(); + const view = draw( + rule({ + id: "gone", + botId: "old-bot", + effect: "read", + scope: "host-grant-1-2", + }), + ); + expect( + await view.findByText( + "reading in a folder that's no longer shared for a Bot no longer here", + ), + ).toBeTruthy(); + expect(view.container.textContent ?? "").not.toContain("host-grant"); +}); + +test("a kind of action with no words of its own is left unsaid", async () => { + serving(); + const view = draw( + rule({ + id: "custom", + toolRef: "computer_click", + effect: "teleport", + scope: "shop.example", + }), + ); + expect(await view.findByText("on shop.example")).toBeTruthy(); + expect(view.container.textContent ?? "").not.toContain("teleport"); +}); diff --git a/app/tests/approvals-rules.test.tsx b/app/tests/approvals-rules.test.tsx index ee16804b4b..0e99d5e88d 100644 --- a/app/tests/approvals-rules.test.tsx +++ b/app/tests/approvals-rules.test.tsx @@ -163,7 +163,7 @@ test("team rules are locked for members, enforced auto-review is shown as requir const view = draw(); await view.findByText("Ask before taking action (team rule)"); const mine = view.getByRole("combobox", { - name: "Behaviour for mcp/gmail/*", + name: "Behaviour for Any action on Gmail", }); expect(mine.textContent).toContain("Take action without asking"); expect(view.getByText("Locked")).toBeTruthy(); @@ -179,7 +179,7 @@ test("team rules are locked for members, enforced auto-review is shown as requir test("a member changes a saved rule in place", async () => { const view = draw(); const mine = await view.findByRole("combobox", { - name: "Behaviour for mcp/gmail/*", + name: "Behaviour for Any action on Gmail", }); const user = userEvent.setup({ document }); await user.click(mine); @@ -194,7 +194,7 @@ test("a member changes a saved rule in place", async () => { }), ); expect( - view.queryByRole("combobox", { name: "Behaviour for mcp/drive/share" }), + view.queryByRole("combobox", { name: "Behaviour for Share on Drive" }), ).toBeNull(); }); @@ -282,11 +282,11 @@ test("two rules for one tool are told apart by the rest of what they match, neve }; const view = draw(); const match = await view.findByText( - "write on checkout.a-very-long-shop-name.example", + "changes on checkout.a-very-long-shop-name.example", ); expect(match.className).toContain("line-clamp-none"); - // The tool stays the row's title. - expect(view.getAllByText("computer_click")).toHaveLength(2); + // The action, in its approval card's words, stays the row's title. + expect(view.getAllByText("Click")).toHaveLength(2); }); test("with personal rules switched off by the team, they are shown as not applying and none can be added", async () => { @@ -350,7 +350,7 @@ test("rules for every Bot point to where rules for a single Bot are", async () = expect(link.getAttribute("href")).toBe("/bots"); // A rule naming one Bot lives on that Bot's page, not here. expect( - view.queryByRole("combobox", { name: "Behaviour for computer_click" }), + view.queryByRole("combobox", { name: "Behaviour for Click" }), ).toBeNull(); }); @@ -371,7 +371,7 @@ test("a rule naming no Bot this person can open is listed so it can be removed", const view = draw(); expect(await view.findByText("Rules for other Bots")).toBeTruthy(); expect( - view.getByRole("combobox", { name: "Behaviour for computer_click" }), + view.getByRole("combobox", { name: "Behaviour for Click" }), ).toBeTruthy(); }); @@ -519,3 +519,43 @@ test.each(["team", "personal"] as const)( ); }, ); + +/* + * FOUND IN THE 1.0 RC1 MAC PASS: a rule saved from an approval card's "Always allow here" was + * titled by its tool id ("computer_click", "composio-gmail/GMAIL_SEND_EMAIL" over "on + * composio-gmail"). It is titled in the words the card used for the same action. + */ +test("a saved rule is titled in its approval card's words, never by its tool id", async () => { + inboxOverride = { + rules: [ + { + id: "click-here", + botId: "*", + toolRef: "computer_click", + effect: "*", + scope: "en.wikipedia.org", + behaviour: "allow", + }, + { + id: "send-mail", + botId: "*", + toolRef: "composio-gmail/GMAIL_SEND_EMAIL", + effect: "*", + scope: "composio-gmail", + behaviour: "allow", + }, + ], + teamRules: [], + }; + const view = draw(); + expect( + await view.findByRole("combobox", { name: "Behaviour for Click" }), + ).toBeTruthy(); + expect(view.getByText("Click")).toBeTruthy(); + expect(view.getByText("on en.wikipedia.org")).toBeTruthy(); + expect(view.getByText("Send email on Gmail")).toBeTruthy(); + const text = view.container.textContent ?? ""; + expect(text).not.toContain("computer_click"); + expect(text).not.toContain("composio"); + expect(text).not.toContain("GMAIL_SEND_EMAIL"); +}); diff --git a/app/tests/bot-approval-rules.test.tsx b/app/tests/bot-approval-rules.test.tsx index 8c5d36d51b..4e8f9b4cae 100644 --- a/app/tests/bot-approval-rules.test.tsx +++ b/app/tests/bot-approval-rules.test.tsx @@ -94,17 +94,19 @@ test("only this Bot's own rules can be changed, and the team rules that reach it serving(); const view = draw(); expect( - await view.findByRole("combobox", { name: "Behaviour for mcp/gmail/*" }), + await view.findByRole("combobox", { + name: "Behaviour for Any action on Gmail", + }), ).toBeTruthy(); expect( - view.queryByRole("combobox", { name: "Behaviour for mcp/drive/*" }), + view.queryByRole("combobox", { name: "Behaviour for Any action on Drive" }), ).toBeNull(); expect( - view.queryByRole("combobox", { name: "Behaviour for mcp/slack/*" }), + view.queryByRole("combobox", { name: "Behaviour for Any action on Slack" }), ).toBeNull(); - expect(view.getByText("mcp/drive/share")).toBeTruthy(); - expect(view.getByText(/^computer_click/)).toBeTruthy(); - expect(view.queryByText(/^host\/\*/)).toBeNull(); + expect(view.getByText("Share on Drive")).toBeTruthy(); + expect(view.getByText("Click")).toBeTruthy(); + expect(view.queryByText("Any action in your folders")).toBeNull(); }); test("a rule added here is for this Bot, with no Bot to type", async () => { @@ -146,6 +148,8 @@ test("a personal rule whose Bot is a pattern matching this Bot is shown and can }) as unknown as typeof fetch; const view = draw(); expect( - await view.findByRole("combobox", { name: "Behaviour for mcp/linear/*" }), + await view.findByRole("combobox", { + name: "Behaviour for Any action on Linear", + }), ).toBeTruthy(); }); diff --git a/app/tests/skins-openmuse.test.tsx b/app/tests/skins-openmuse.test.tsx index 905bb45822..5fe9a0fb7a 100644 --- a/app/tests/skins-openmuse.test.tsx +++ b/app/tests/skins-openmuse.test.tsx @@ -872,7 +872,9 @@ describe("each tab is OpenBot's own sections for the centre Bot", () => { // OpenBot's Needs you section, under the Bot's name, with its approval's details always shown. expect(await page().findByText("Open url")).toBeTruthy(); expect(page().getAllByText("Muse").length).toBeGreaterThan(0); - expect(page().getByText(/"url": "https:\/\/example.com"/)).toBeTruthy(); + // Labelled, as the card in OpenBot lists what it would send: the URL is the target and the value. + expect(page().getByText("URL")).toBeTruthy(); + expect(page().getAllByText("https://example.com").length).toBe(2); // OpenBot's activity sections for the Bot. expect(await page().findByText("Morning briefing")).toBeTruthy(); expect(page().getByText("Draft the launch post")).toBeTruthy(); @@ -935,7 +937,8 @@ describe("each tab is OpenBot's own sections for the centre Bot", () => { const sheet = within(await page().findByRole("dialog")); expect(await sheet.findByText("Muse wants to open url")).toBeTruthy(); // What it will send is always on screen, as in OpenBot. - expect(sheet.getByText(/"url": "https:\/\/example.com"/)).toBeTruthy(); + expect(sheet.getByText("URL")).toBeTruthy(); + expect(sheet.getAllByText("https://example.com").length).toBe(2); expect(sheet.queryByText(/send email/)).toBeNull(); fireEvent.click(sheet.getByRole("button", { name: "Allow once" })); await waitFor(() => diff --git a/charts/openbot/Chart.yaml b/charts/openbot/Chart.yaml index e213ec1df4..9de41f949e 100644 --- a/charts/openbot/Chart.yaml +++ b/charts/openbot/Chart.yaml @@ -6,7 +6,7 @@ type: application version: 0.2.0 # The OpenBot release this chart's default image tag points at. Kept equal to package.json's version # (the last version published) by the release workflow, and CI refuses a tree where they have drifted. -appVersion: "0.3.4" +appVersion: "1.0.0-rc.1" home: https://github.com/CopilotKit/OpenBot sources: - https://github.com/CopilotKit/OpenBot diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index a328222521..5dd9fae457 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -2746,7 +2746,7 @@ dependencies = [ [[package]] name = "openbot-desktop" -version = "0.3.4" +version = "1.0.0-rc.1" dependencies = [ "base64 0.22.1", "block2", diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index b8f580a7c8..65097cd8fe 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "openbot-desktop" -version = "0.3.4" +version = "1.0.0-rc.1" description = "OpenBot Desktop" edition = "2021" rust-version = "1.90" diff --git a/extension/scripts/store-package.test.ts b/extension/scripts/store-package.test.ts index 5d5f1a9d32..3f6178351b 100644 --- a/extension/scripts/store-package.test.ts +++ b/extension/scripts/store-package.test.ts @@ -18,6 +18,7 @@ import { import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { unzipSync } from "fflate"; +import { toChromeManifestVersion } from "../src/manifest"; import { CHROME_PACKAGE, EDGE_PACKAGE, @@ -72,7 +73,8 @@ function builtRecorder( const manifest = { manifest_version: 3, name: "OpenBot Recorder", - version: ROOT_VERSION, + // As the build writes it: a prerelease such as 1.0.0-rc.1 is version 1.0.0 with that version_name. + ...toChromeManifestVersion(ROOT_VERSION), description: "Record a task in your browser so an OpenBot Bot can learn to do it.", icons: { "16": "icons/16.png", "128": "icons/128.png" }, diff --git a/package.json b/package.json index 30ac141992..b8e37624d7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openbot", - "version": "0.3.4", + "version": "1.0.0-rc.1", "license": "MIT", "private": true, "packageManager": "bun@1.4.3", diff --git a/release-policy.json b/release-policy.json index f2935d9c93..981f8389a5 100644 --- a/release-policy.json +++ b/release-policy.json @@ -1,19 +1,23 @@ { "$schema": "./shared/release-policy.schema.json", - "version": "v0.3.4", + "version": "v1.0.0-rc.1", "compatibility": { "launcherApi": 1, "configSchema": 1, "extensionApi": 1, "database": { - "from": ["0082_routine_goal", "0077_channel_opening"], + "from": [ + "0083_learning_skill_approvals", + "0082_routine_goal", + "0077_channel_opening" + ], "to": "0083_learning_skill_approvals", "rollbackCompatible": false }, "intelligenceCapabilities": [] }, "upgrade": { - "fromVersions": ["v0.3.3", "v0.3.2", "v0.3.1", "v0.3.0"], + "fromVersions": ["v0.3.4", "v0.3.3", "v0.3.2", "v0.3.1", "v0.3.0"], "requiresDowntime": true, "recovery": "restore-backup" } diff --git a/server/src/activity/activity.ts b/server/src/activity/activity.ts index fb84ade1e6..f3f97690bb 100644 --- a/server/src/activity/activity.ts +++ b/server/src/activity/activity.ts @@ -9,6 +9,7 @@ * EVERY QUERY IS NARROWED TO THE PERSON FIRST. A public Bot works for many people; this screen shows * the work done in the caller's name and nothing of anybody else's. */ +import { approvalPhrase } from "@openbot/shared/tool-name"; import { and, desc, @@ -447,12 +448,15 @@ export function createActivityStore(database: Database) { const approvalChannels = await channelsForThreads( approvals.map((approval) => approval.action.threadId), ); - for (const approval of approvals) + for (const approval of approvals) { + // In the words its card uses: "Click the link “Python” on en.wikipedia.org", never the + // tool's id and its effect ("Approve computer_click", "write"). + const phrase = approvalPhrase(approval.action); activity.inProgress.push({ id: `approval:${approval.id}`, kind: "approval", - title: `Approve ${approval.action.toolRef}`, - detail: approval.action.effect, + title: phrase.charAt(0).toUpperCase() + phrase.slice(1), + detail: approval.action.policy?.reason ?? null, status: "waiting for you", at: iso(approval.createdAt), channelId: approvalChannels.get(approval.action.threadId) ?? null, @@ -460,6 +464,7 @@ export function createActivityStore(database: Database) { stoppable: false, cancellable: false, }); + } // Questions this Bot asked this person and is waiting on. const questions = await database diff --git a/server/src/approvals/wiring.ts b/server/src/approvals/wiring.ts index c50389a978..df5c841c4a 100644 --- a/server/src/approvals/wiring.ts +++ b/server/src/approvals/wiring.ts @@ -1,4 +1,5 @@ import { isDeepStrictEqual } from "node:util"; +import { approvalPhrase } from "@openbot/shared/tool-name"; import type { HeadlessTool } from "@openbot/toolkit/headless-tools"; import type { IntelligenceLike } from "@openbot/toolkit/run-turn"; import { type RunAssertion, readRunAssertion } from "../agents/callback-token"; @@ -145,7 +146,7 @@ export function createServerApprovalService(deps: { id: request.id, kind: "approval", requestId: request.id, - text: `${botName} asks for approval to use ${action.toolRef}. Open the Bot's page in OpenBot to review this action.`, + text: `${botName} wants to ${approvalPhrase(action)}. Open the Bot's page in OpenBot to review it.`, }, ); } diff --git a/server/src/delivery/opentag-agent.ts b/server/src/delivery/opentag-agent.ts index 0a90c9860c..a095c296d5 100644 --- a/server/src/delivery/opentag-agent.ts +++ b/server/src/delivery/opentag-agent.ts @@ -7,6 +7,7 @@ * runs. */ import { randomUUID } from "node:crypto"; +import { approvalPhrase } from "@openbot/shared/tool-name"; import type { DrawnComponent } from "@openbot/toolkit/run-turn"; import { z } from "zod"; import { @@ -59,6 +60,9 @@ export type PairedApprovals = { botId: string; toolRef: string; effect: string; + /** Where it acts, and what it is aimed at: the card names the action by both. */ + scope: string; + target?: unknown; threadId: string; args: unknown; /** Why the gate asked; a check-in task's approval offers no "Always allow". */ @@ -530,7 +534,8 @@ async function approvalCard( const bot = await deps.store.botIdentity(record.action.botId); return confirmWriteInterrupt({ approvalId, - action: `${bot?.name ?? "Your Bot"}: ${record.action.toolRef}`, + // In the words the approval card in OpenBot uses, never the tool's id ("Your Bot: computer_click"). + action: `${bot?.name ?? "Your Bot"} wants to ${approvalPhrase(record.action)}`, approver: `${binding.transport}:${binding.identity}`, effect: record.action.effect, // The approvals preview already redacts secrets and private content. @@ -594,7 +599,7 @@ async function decideFromCard( return { text: decision === "deny" - ? `Denied. ${name} will not run ${record.action.toolRef}.` + ? `Denied. ${name} won't ${approvalPhrase(record.action)}.` : `${decision === "allow_always" ? "Always allowed" : "Allowed once"}. ${name} is continuing and will send its result here.`, }; } diff --git a/server/src/delivery/opentag.ts b/server/src/delivery/opentag.ts index 2f117cbfef..2833bc3fd5 100644 --- a/server/src/delivery/opentag.ts +++ b/server/src/delivery/opentag.ts @@ -18,6 +18,7 @@ * `/openbot/deliver`, because a managed Channel can only post inside a live delivery. */ import { timingSafeEqual } from "node:crypto"; +import { approvalDetails } from "@openbot/shared/tool-name"; import { z } from "zod"; import { cutAtCodeUnits } from "../channels/text"; import { DELIVERY_REQUEST_TIMEOUT_MS } from "../timeouts"; @@ -385,11 +386,12 @@ export function approvalFields(preview: unknown) { return preview === undefined ? [] : [{ label: "Arguments", value: clip(JSON.stringify(preview)) }]; - return Object.entries(preview) + // Labelled as the card in OpenBot labels them, without the page handles a click carries. + return approvalDetails(preview) .slice(0, 10) - .map(([label, value]) => ({ + .map(({ label, value }) => ({ label: clip(label, 60), - value: clip(typeof value === "string" ? value : JSON.stringify(value)), + value: clip(value), })); } function clip(value: string | undefined, max = 300) { diff --git a/server/tests/approval-activity-words.integration.test.ts b/server/tests/approval-activity-words.integration.test.ts new file mode 100644 index 0000000000..712aae9b47 --- /dev/null +++ b/server/tests/approval-activity-words.integration.test.ts @@ -0,0 +1,108 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { eq, inArray } from "drizzle-orm"; +import { createActivityStore } from "../src/activity/activity"; +import { createDatabase } from "../src/db/client"; +import { approvalRequests } from "../src/db/schema/approvals"; +import { users } from "../src/db/schema/core"; +import { TEST_POOL, testDatabaseUrl } from "./support/database"; + +/** + * FOUND IN THE 1.0 RC1 MAC PASS: a Bot's Activity row for a pending click read "Approve + * computer_click · Approval · needs you · 10 hours ago · write", the tool's id and its effect. It is + * said in the words its approval card uses, from the element the server resolved, on a real database + * through the store the Bot's page reads. + */ + +const database = createDatabase(testDatabaseUrl(), TEST_POOL); +const activity = createActivityStore(database); +const suite = randomUUID().slice(0, 8); +const owner = `approval-words-${suite}`; +const bot = `approval-words-bot-${suite}`; + +/** A pending computer action, as the gateway's approval gate saves it. */ +function pending( + id: string, + toolRef: string, + args: Record, + element?: { ref: string; role: string; name: string }, +) { + return { + id: `${suite}-${id}`, + ownerUserId: owner, + runId: "run", + toolCallId: `call-${id}`, + actionDigest: `digest-${id}`, + action: { + actorId: owner, + botId: bot, + toolRef, + effect: "write" as const, + scope: "en.wikipedia.org", + args, + target: { + pageUrl: "https://en.wikipedia.org/wiki/Main_Page", + ...(element + ? { element: { ...element, type: "" }, snapshotId: 4 } + : {}), + }, + runId: "run", + threadId: `thread-${suite}`, + toolCallId: `call-${id}`, + actionDigest: `digest-${id}`, + }, + }; +} + +beforeAll(async () => { + await database + .insert(users) + .values({ id: owner, email: `${owner}@example.test` }); + await database.insert(approvalRequests).values([ + pending( + "click", + "computer_click", + { ref: "f3e43", snapshotId: 4 }, + { ref: "f3e43", role: "link", name: "Python (programming language)" }, + ), + pending("click-unnamed", "computer_click", { ref: "f9", snapshotId: 4 }), + pending("open", "computer_navigate", { + url: "https://en.wikipedia.org/wiki/Monty_Python", + }), + pending("type", "computer_type", { + ref: "f7", + snapshotId: 4, + text: "monty python", + }), + ]); +}); + +afterAll(async () => { + await database + .delete(approvalRequests) + .where(eq(approvalRequests.ownerUserId, owner)); + await database.delete(users).where(inArray(users.id, [owner])); + await database.$client.end({ timeout: 5 }); +}); + +test("a pending computer approval's Activity row is said in words, with no tool id or effect", async () => { + const rows = (await activity.forBot(owner, bot)).inProgress.filter( + (item) => item.kind === "approval", + ); + const byId = Object.fromEntries( + rows.map((item) => [item.id.replace(`approval:${suite}-`, ""), item]), + ); + expect(byId.click).toMatchObject({ + title: "Click the link “Python (programming language)” on en.wikipedia.org", + detail: null, + status: "waiting for you", + needsYou: true, + }); + expect(byId["click-unnamed"]?.title).toBe("Click on en.wikipedia.org"); + expect(byId.open?.title).toBe("Open a page on en.wikipedia.org"); + expect(byId.type?.title).toBe("Fill in a field on en.wikipedia.org"); + for (const row of rows) { + expect(row.title).not.toMatch(/computer_|Approve /); + expect(row.detail).not.toBe("write"); + } +}); diff --git a/server/tests/approvals.test.ts b/server/tests/approvals.test.ts index 4724b2d7ab..2809210d9d 100644 --- a/server/tests/approvals.test.ts +++ b/server/tests/approvals.test.ts @@ -765,7 +765,7 @@ test("an approval preview shows what will be typed, written or sent, and still h approvalFields( approvalPreview({ channel: "C1", text: "We are shipping today" }), ), - ).toContainEqual({ label: "text", value: "We are shipping today" }); + ).toContainEqual({ label: "Text", value: "We are shipping today" }); // Secrets stay hidden by key and inside values. expect( approvalPreview({ diff --git a/server/tests/delivery-routes.test.ts b/server/tests/delivery-routes.test.ts index b92b44b162..412bca8171 100644 --- a/server/tests/delivery-routes.test.ts +++ b/server/tests/delivery-routes.test.ts @@ -38,6 +38,13 @@ function fixture( challengeTransport?: "slack" | "teams"; /** Why the gate asked, as the stored request carries it; "task" for a check-in task's. */ policySource?: string; + /** What the stored request asks to do, in place of opening example.com. */ + action?: { + toolRef: string; + scope: string; + args: Record; + target?: unknown; + }; } = {}, ) { const calls: unknown[] = []; @@ -132,10 +139,12 @@ function fixture( status: "pending", action: { botId: "bot", - toolRef: "browser/navigate", + toolRef: "computer_navigate", + scope: "example.com", effect: "write", threadId: "canonical", args: { url: "https://example.com" }, + ...options.action, ...(options.policySource ? { policy: { @@ -597,8 +606,8 @@ test("bound sender runs the canonical turn and an approval becomes an OpenTag ca __copilotkit_interrupt_value__: { action: "confirm_write", args: { - action: "Research Bot: browser/navigate", - fields: [{ label: "url", value: "https://example.com" }], + action: "Research Bot wants to open a page on example.com", + fields: [{ label: "URL", value: "https://example.com" }], approver: "slack:U1", effect: "write", allow_always: true, @@ -626,7 +635,7 @@ test("a check-in task's approval becomes a card without Always allow, and an old (e) => e.type === "CUSTOM" && e.name === "on_interrupt", ); expect(card.value.__copilotkit_interrupt_value__.args).toMatchObject({ - action: "Research Bot: browser/navigate", + action: "Research Bot wants to open a page on example.com", allow_always: false, }); const resumed = await run(f, { @@ -766,6 +775,63 @@ test("card resume decides through the approvals service as the binding owner onl await resume({ confirmed: false, by: "slack:T1:U1" }); expect(f.decisions[1]).toEqual(["owner", APPROVAL, "deny"]); }); +/* + * FOUND IN THE 1.0 RC1 MAC PASS: a click waiting in Slack read "Research Bot: computer_click" over + * "ref" and "snapshotId" fields, and denying it said "will not run computer_click". Both are said in + * the words the approval card in OpenBot uses, and the click's page handles are left out. + */ +test("a click waiting in Slack is named in words, with no tool id or page handles", async () => { + const f = fixture({ + action: { + toolRef: "computer_click", + scope: "en.wikipedia.org", + args: { ref: "f3e43", snapshotId: 4 }, + target: { + pageUrl: "https://en.wikipedia.org/wiki/Main_Page", + element: { ref: "f3e43", role: "link", name: "Python", type: "" }, + snapshotId: 4, + }, + }, + converse: { + kind: "waiting", + binding: bound, + inboxId: "in1", + replyText: "", + extras: { approvals: [APPROVAL], questions: [] }, + }, + }); + const asked = await run(f, { + context: [sender()], + messages: [ + { id: "m9", role: "user", content: "<@UBOT> read about Python" }, + ], + }); + const card = asked.events.find( + (e) => e.type === "CUSTOM" && e.name === "on_interrupt", + ); + expect(card.value.__copilotkit_interrupt_value__.args).toMatchObject({ + action: "Research Bot wants to click the link “Python” on en.wikipedia.org", + fields: [], + }); + const denied = await run(f, { + forwardedProps: { + command: { + resume: { + "0f8fad5bd9cb469fa16570867728950e": { + confirmed: false, + by: "slack:T1:U1", + }, + }, + }, + }, + }); + expect(said(denied.events)).toBe( + "Denied. Research Bot won't click the link “Python” on en.wikipedia.org.", + ); + expect(JSON.stringify([asked.events, denied.events])).not.toMatch( + /computer_click|snapshotId|f3e43/, + ); +}); test("SMS STOP and START record opt-out without a turn or a second reply", async () => { const f = fixture(); const signedRequest = (params: URLSearchParams) => { diff --git a/shared/tool-name.ts b/shared/tool-name.ts index 71d209a382..16a503c26a 100644 --- a/shared/tool-name.ts +++ b/shared/tool-name.ts @@ -197,6 +197,12 @@ function approvalParts(action: { joiner: "on", }; } + /* + * A browser action on the site it acts on, with the control it presses when the page named one: + * "click the link “Python” on en.wikipedia.org". Activity said "Approve computer_click". + */ + const browser = browserAction(action.toolRef, action.target); + if (browser) return { action: browser, place: action.scope, joiner: "on" }; /* * A computer's or a folder's own action in the words its step and its Activity row use. The card * said "write file on todo.txt in share" above a step that read "Save a file todo.txt in share". @@ -228,6 +234,136 @@ function approvalParts(action: { return { action: words, place: action.scope, joiner: "on" }; } +/** Handles the model cites and the run carries, which name nothing a person could check. */ +const INTERNAL_ARGS = new Set(["ref", "snapshotId", "toolCallId", "signal"]); + +/** + * What an approval would send, as labelled lines for its card: "Text: hello", "To: sam@example.test". + * The card showed the arguments as JSON, and a click read `{ "ref": "f3e43", "snapshotId": 4 }`, two + * handles to a page snapshot nobody can see. Those are left out, so a click has nothing to add to its + * title and shows no lines at all. Everything else stays whole: nobody allows what they were not shown. + */ +export function approvalDetails( + args: unknown, +): { label: string; value: string }[] { + if (!args || typeof args !== "object" || Array.isArray(args)) return []; + return Object.entries(args).flatMap(([key, value]) => { + if (INTERNAL_ARGS.has(key) || value === undefined || value === null) + return []; + const text = + typeof value === "string" + ? value + : typeof value === "boolean" + ? value + ? "Yes" + : "No" + : typeof value === "number" + ? String(value) + : Array.isArray(value) && + value.every((entry) => typeof entry === "string") + ? value.join(", ") + : JSON.stringify(value, null, 2); + const words = wordsOf(key) + .map((word) => ACRONYMS[word] ?? word) + .join(" "); + const label = words ? words.charAt(0).toUpperCase() + words.slice(1) : key; + return [{ label, value: text }]; + }); +} + +/** Argument words a person reads in capitals. */ +const ACRONYMS: Record = { url: "URL", id: "ID" }; + +/** + * A saved approval rule's title, in the words its approval card used for the action it covers: + * "Click", "Send email on Gmail", "Any action on Gmail". The list titled each rule by its tool id + * ("computer_click", "mcp/gmail/*"). `place` is true when the title already names where the rule + * applies, so the line under it need not say "on composio-gmail" again. A pattern a person typed + * that matches several actions some other way is kept as they wrote it. + */ +export function ruleTitle(rule: { toolRef: string; scope: string }): { + title: string; + place: boolean; +} { + const upper = (text: string) => text.charAt(0).toUpperCase() + text.slice(1); + if (rule.toolRef === "*") return { title: "Any action", place: false }; + if (rule.toolRef === "host/*") + return { title: "Any action in your folders", place: false }; + const ref = rule.toolRef.replace(/^mcp\//, ""); + const slash = ref.indexOf("/"); + if (slash > 0 && !ref.startsWith("host/")) { + const app = ref.slice(0, slash); + const tool = ref.slice(slash + 1); + if (app.includes("*")) return { title: rule.toolRef, place: false }; + const named = rule.scope === "*" || rule.scope === app; + if (tool === "*") + return { title: `Any action on ${connectorName(app)}`, place: named }; + if (tool.includes("*")) return { title: rule.toolRef, place: false }; + const { action, place } = approvalWords({ toolRef: ref, scope: app }); + return { title: `${upper(action)} on ${place}`, place: named }; + } + if (ref.includes("*")) return { title: rule.toolRef, place: false }; + return { + title: upper(approvalWords({ toolRef: ref, scope: rule.scope }).action), + place: false, + }; +} + +/** What a browser action asks to do, said as its step says it. */ +const BROWSER_ACTIONS: Record = { + computer_navigate: "open a page", + computer_click: "click", + computer_type: "fill in a field", + computer_key: "press a key", + computer_scroll: "scroll the page", +}; + +/** A page control's role in the words a person uses for it. */ +const CONTROL_WORDS: Record = { + link: "link", + button: "button", + checkbox: "checkbox", + radio: "option", + option: "option", + tab: "tab", + menuitem: "menu item", + switch: "switch", + textbox: "field", + searchbox: "search field", + combobox: "list", +}; + +/** + * A browser action in words, naming the control it presses from the label the server resolved + * (`target.element`), never the ref and snapshot the model cited. Undefined for any other action. + */ +function browserAction(toolRef: string, target: unknown): string | undefined { + const base = BROWSER_ACTIONS[toolRef]; + if (!base) return undefined; + const element = + target && typeof target === "object" + ? (target as { element?: { role?: unknown; name?: unknown } }).element + : undefined; + const name = + typeof element?.name === "string" + ? element.name.replace(/\s+/g, " ").trim() + : ""; + const role = + typeof element?.role === "string" ? CONTROL_WORDS[element.role] : undefined; + if (toolRef === "computer_click") { + if (name) return `click ${role ? `the ${role} ` : ""}“${clipName(name)}”`; + return role ? `click ${/^[aeiou]/.test(role) ? "an" : "a"} ${role}` : base; + } + if (toolRef === "computer_type" && name) return `fill in “${clipName(name)}”`; + return base; +} + +/** A control's label, short enough for a title. */ +function clipName(name: string): string { + const chars = Array.from(name); + return chars.length > 60 ? `${chars.slice(0, 59).join("")}…` : name; +} + /** The path a file action on the Bot's own computer names (`target.filePath`), or undefined. */ function computerFile(target: unknown): string | undefined { if (!target || typeof target !== "object") return undefined; From 0154b2ecda27fd2bbae0e6c1ccf0cdcb97adeb00 Mon Sep 17 00:00:00 2001 From: David McKay Date: Sun, 8 Nov 2026 18:41:36 -0800 Subject: [PATCH 02/34] Open "OpenBot is running" on its heading, with Updates under the buttons (#1324) --- CHANGELOG.md | 3 ++ desktop/src/App.test.tsx | 69 ++++++++++++++++++++++++++++++++++++++++ desktop/src/App.tsx | 4 ++- desktop/src/styles.css | 6 ++++ 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3336a64d60..17a4028ddf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,6 +130,9 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ### Smaller changes +- "OpenBot is running" opens on its heading again. Its Updates disclosure, and the line above it, + sit under Show OpenBot and Stop OpenBot, as on the stopped screen, instead of at the top of the + card with nothing above the line. - The sidebar floats as frosted glass over the colours of OpenBot's orb, the orb sits beside OpenBot's name, and the app is set in Atkinson Hyperlegible Next. The glass turns off with the system's reduce transparency or increase contrast setting, and a skin keeps its own look. diff --git a/desktop/src/App.test.tsx b/desktop/src/App.test.tsx index 97509bb68d..41bb8e5724 100644 --- a/desktop/src/App.test.tsx +++ b/desktop/src/App.test.tsx @@ -1706,6 +1706,75 @@ test("the running screen shows none of Start's own steps, and names the system t } }); +/* + * Seen on Windows: with Start's rows gone from the running screen, the card opened on a divider + * and "Updates", with nothing above the line, and "OpenBot is running" under them. The heading + * comes first; Updates sits under the buttons, as on the stopped screen. + */ +test("the running screen opens on its heading, with Updates under the buttons, with and without a failure", async () => { + setupConnectedLocalStack( + "/tmp/local-eval-running-order", + localStatus("ready", "valid"), + ); + let showFails = false; + const previous = invokeHandler; + invokeHandler = async (command, args) => { + if (command === "show_openbot" && showFails) + throw "OpenBot is not answering on port 3010 yet, so there is nothing to show."; + return previous(command, args); + }; + const view = await renderApp(); + await waitFor(() => expect(localSequence()).toContain("start_stack")); + await emitProgress({ step: "env", ok: true, detail: "Saved." }); + await emitProgress({ + step: "services", + ok: true, + running: true, + detail: "Starting OpenBot's services.", + }); + + /** The card's own children in order, by what each one is. */ + const order = (sheet: Element) => + [...sheet.children].map((child) => + child.tagName === "H1" + ? "heading" + : child.tagName === "DETAILS" + ? `details:${child.querySelector("summary")?.textContent}` + : child.getAttribute("aria-label") === "Setup progress" + ? "rows" + : child.classList.contains("row") + ? "buttons" + : child.tagName.toLowerCase(), + ); + + const heading = await view.findByRole("heading", { + name: "OpenBot is running", + }); + const sheet = heading.closest(".sheet") as HTMLElement; + expect(order(sheet)).toEqual(["heading", "p", "buttons", "details:Updates"]); + expect(sheet.firstElementChild === heading).toBe(true); + expect(view.queryByRole("list", { name: "Setup progress" })).toBeNull(); + + showFails = true; + await userEvent.click(view.getByRole("button", { name: "Show OpenBot" })); + const stopped = await view.findByRole("heading", { name: "OpenBot stopped" }); + const failedSheet = stopped.closest(".sheet") as HTMLElement; + // The rows still show where it stopped, between the heading and the buttons. + const rows = view.getByRole("list", { name: "Setup progress" }); + expect(within(rows).getAllByRole("listitem").length).toBeGreaterThan(0); + expect(order(failedSheet)).toEqual([ + "heading", + "rows", + expect.any(String), + "buttons", + "details:Updates", + ]); + expect(failedSheet.firstElementChild === stopped).toBe(true); + expect(view.getByRole("alert").textContent).toContain( + "OpenBot is not answering on port 3010 yet", + ); +}); + test("relaunch says when Docker Desktop did not start, and Retry opens it again", async () => { setupConnectedLocalStack( "/tmp/local-eval-docker-never", diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx index b7f9a0dbd0..b0bb307679 100644 --- a/desktop/src/App.tsx +++ b/desktop/src/App.tsx @@ -1040,7 +1040,6 @@ function Setup() { return (
- {returningToInstallation && appUpdates} {/* A failure outranks `running`. The supervisor gives up on a process and sends the window back here, and a heading that still says everything is running while the box underneath names the process that stopped is a screen arguing with itself. */} @@ -1095,6 +1094,9 @@ function Setup() { Stop OpenBot
+ {/* Under the buttons, as on the stopped screen. A disclosure draws a rule above itself, and + first in the card it opened the card on a line with nothing above it. */} + {returningToInstallation && appUpdates}
); diff --git a/desktop/src/styles.css b/desktop/src/styles.css index 1df1c5f2fa..22aaf2708b 100644 --- a/desktop/src/styles.css +++ b/desktop/src/styles.css @@ -689,6 +689,12 @@ details { margin-top: 4px; } +/* Under a screen's buttons, its rule keeps the row's own gap rather than touching them. Zero + specificity, so a disclosure that sets its own margin (`.blocker-manual`) keeps it. */ +:where(.row) + details { + margin-top: 24px; +} + details > summary { cursor: pointer; font-size: var(--text-sm); From 1b8c8f377599b7e9ea8c3657974df3a452af5b41 Mon Sep 17 00:00:00 2001 From: David McKay Date: Sun, 8 Nov 2026 18:41:56 -0800 Subject: [PATCH 03/34] Reach past the newest 50 events on Activity and Admin Audit (#1326) --- CHANGELOG.md | 2 + app/src/components/audit/audit-table.tsx | 46 +++++- app/src/components/manage/audit-page.tsx | 34 ++++- app/src/lib/audit/queries.ts | 40 ++++- app/tests/activity-older-events.test.tsx | 143 ++++++++++++++++++ app/tests/activity-page-name.test.tsx | 7 +- app/tests/activity-person-words.test.tsx | 9 +- app/tests/audit-approval-outcomes.test.tsx | 4 +- app/tests/audit-connect-app-rows.test.tsx | 4 +- app/tests/audit-dry-run-label.test.tsx | 4 +- app/tests/audit-event-words.test.tsx | 6 +- app/tests/audit-export-truncated.test.tsx | 4 +- app/tests/audit-folder-actions.test.tsx | 4 +- .../audit-grant-and-routine-rows.test.tsx | 4 +- app/tests/audit-grant-rows.test.tsx | 7 +- app/tests/audit-openbot-setup-rows.test.tsx | 4 +- app/tests/audit-plumbing-rows.test.tsx | 4 +- app/tests/audit-table-fits.test.tsx | 4 +- app/tests/audit-table.test.tsx | 4 +- app/tests/bot-activity-actions.test.tsx | 6 +- app/tests/bot-id-never-shown.test.tsx | 4 +- app/tests/personal-says-openbot.test.tsx | 7 +- app/tests/seed-audit-page.ts | 18 +++ 23 files changed, 308 insertions(+), 61 deletions(-) create mode 100644 app/tests/activity-older-events.test.tsx create mode 100644 app/tests/seed-audit-page.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 17a4028ddf..7e4c78be74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,6 +130,8 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ### Smaller changes +- Settings > Activity and Admin > Audit reach past the newest 50 events: "Show older events" under + the last row loads the next page, and the count says "Newest 50 events" until nothing is older. - "OpenBot is running" opens on its heading again. Its Updates disclosure, and the line above it, sit under Show OpenBot and Stop OpenBot, as on the stopped screen, instead of at the top of the card with nothing above the line. diff --git a/app/src/components/audit/audit-table.tsx b/app/src/components/audit/audit-table.tsx index 988dcb972f..ba06f5d14c 100644 --- a/app/src/components/audit/audit-table.tsx +++ b/app/src/components/audit/audit-table.tsx @@ -97,6 +97,8 @@ const FACETS = [ export function AuditTable({ deploymentName = "this deployment", events, + footer, + moreToLoad = false, nameFor, query, storedNames = false, @@ -105,13 +107,18 @@ export function AuditTable({ /** What OpenBot itself is called in a row it started: "this deployment" in Admin. */ deploymentName?: string; events: AuditEvent[]; + /** Drawn under the last row, inside the scroll: where older events are asked for. */ + footer?: React.ReactNode; + /** Whether the server has older events than `events`, so the count says these are the newest. */ + moreToLoad?: boolean; nameFor: NameFor; query: string; /** Whether a row's What shows the trail's stored name on hover: Admin's, not a person's. */ storedNames?: boolean; /** - * Drawn above the table, given how many rows it shows ("3 events", "1 of 3 events") and the filter - * buttons to place beside the search. It stays put; only the rows under it scroll. + * Drawn above the table, given how many rows it shows ("3 events", "1 of 3 events", "Newest 50 + * events" while older ones are still to load) and the filter buttons to place beside the search. + * It stays put; only the rows under it scroll. */ toolbar: (parts: { count: string; @@ -144,7 +151,7 @@ export function AuditTable({ return ( <> {toolbar({ - count: countLabel(rows.length, events.length), + count: countLabel(rows.length, events.length, moreToLoad), filters: FACETS.map((facet) => { const column = table.getColumn(facet.id); const selected = (column?.getFilterValue() as string[]) ?? []; @@ -217,6 +224,16 @@ export function AuditTable({ ); })} + {footer && rows.length > 0 ? ( + + + {footer} + + + ) : null} {events.length > 0 && rows.length === 0 ? ( @@ -224,6 +241,7 @@ export function AuditTable({ Nothing loaded matches this search and these filters. + {footer} ) : null} @@ -315,10 +333,24 @@ function Facet({ ); } -/** How many rows the table is showing, for the toolbar. */ -function countLabel(shown: number, total: number): string { - const noun = total === 1 ? "event" : "events"; - return shown === total ? `${total} ${noun}` : `${shown} of ${total} ${noun}`; +/** + * How many rows the table is showing, for the toolbar. While the server has older events than those + * loaded, the count names them the newest, so it never reads as the size of the whole trail. + */ +function countLabel( + shown: number, + loaded: number, + moreToLoad: boolean, +): string { + const noun = loaded === 1 ? "event" : "events"; + if (moreToLoad) { + return shown === loaded + ? `Newest ${loaded} ${noun}` + : `${shown} of the newest ${loaded} ${noun}`; + } + return shown === loaded + ? `${loaded} ${noun}` + : `${shown} of ${loaded} ${noun}`; } function auditColumns( diff --git a/app/src/components/manage/audit-page.tsx b/app/src/components/manage/audit-page.tsx index 5acc77a62a..3ebd3cafb4 100644 --- a/app/src/components/manage/audit-page.tsx +++ b/app/src/components/manage/audit-page.tsx @@ -9,7 +9,7 @@ import { IconSearch, IconX, } from "@tabler/icons-react"; -import { useQuery } from "@tanstack/react-query"; +import { useInfiniteQuery, useQuery } from "@tanstack/react-query"; import { type ReactNode, useState } from "react"; import type { AuditEvent } from "@/components/audit/audit-cells"; import { AuditTable } from "@/components/audit/audit-table"; @@ -18,10 +18,10 @@ import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { useBotNames } from "@/lib/agents/bot-names"; import { + CALLS_RECORDED_AS_CHANGES, COMPUTER_ACTION_EVENT_TYPES, DID_NOT_HAPPEN_EVENT_TYPES, eventTypeFilter, - CALLS_RECORDED_AS_CHANGES, OPENBOT_WORKING_EVENT_TYPES, oneRowPerAction, REFUSED_EVENT_TYPES, @@ -146,14 +146,15 @@ export function AuditPage({ ? withoutOpenBotWorking(withBot(view, bot)) : withBot(view, bot); const [query, setQuery] = useState(""); - const events = useQuery(auditEventsQueryOptions(search)); + // A page at a time, newest first; "Show older events" under the last row asks for the next. + const events = useInfiniteQuery(auditEventsQueryOptions(search)); // Activity names a skill by the title its page shows; the trail keeps only its slug. const skills = useQuery({ ...pluginsPageQueryOptions(), enabled: edition === "personal", }).data?.skills; const rows = withSkillTitles( - oneRowPerAction((events.data?.events ?? []) as AuditEvent[]), + oneRowPerAction(events.data?.pages.flatMap((page) => page.events) ?? []), skills, ); const nameFor = useBotNames(); @@ -211,7 +212,8 @@ export function AuditPage({
- {events.isPending ? null : events.isError ? ( + {events.isPending ? null : events.isError && + !events.isFetchNextPageError ? (

Couldn't load these events. Reload the page.

@@ -219,6 +221,28 @@ export function AuditPage({ + {events.isFetchNextPageError ? ( +

+ Couldn't load older events. Try again. +

+ ) : null} + +
+ ) : null + } + moreToLoad={events.hasNextPage} nameFor={nameFor} query={query} storedNames={edition !== "personal"} diff --git a/app/src/lib/audit/queries.ts b/app/src/lib/audit/queries.ts index 60ed55bc86..8658a8ce0e 100644 --- a/app/src/lib/audit/queries.ts +++ b/app/src/lib/audit/queries.ts @@ -1,17 +1,41 @@ -import { queryOptions } from "@tanstack/react-query"; +import { infiniteQueryOptions } from "@tanstack/react-query"; +import type { AuditEvent } from "@/components/audit/audit-cells"; import { client } from "@/lib/client"; -export const auditKeys = { all: ["audit-events"] as const }; +export const auditKeys = { + all: ["audit-events"] as const, + list: (search = "") => ["audit-events", "list", { search }] as const, +}; +/** One page of the trail as `GET /api/admin/audit-events` answers it, newest first. */ +export type AuditEventsPage = { + events: AuditEvent[]; + /** Where the next, older page starts. Absent on the last page. */ + nextCursor?: string; +}; + +/** + * The trail for one view (`search`, e.g. `?eventType=...`), a page at a time. + * + * Paged because the trail only grows. This used to ask once and never follow `nextCursor`, so the + * page showed the newest 50 events, called them the whole trail, and had no way to reach the rest. + */ export function auditEventsQueryOptions(search = "") { - return queryOptions({ - queryKey: [...auditKeys.all, search] as const, - queryFn: async () => { - const response = await client(`/api/admin/audit-events${search}`, { - fallback: "Could not load audit events", - }); + return infiniteQueryOptions({ + queryKey: auditKeys.list(search), + initialPageParam: "", + queryFn: async ({ pageParam }): Promise => { + // `search` is empty or starts with `?`; an older page carries the cursor after it. + const cursor = pageParam + ? `${search ? "&" : "?"}cursor=${encodeURIComponent(pageParam)}` + : ""; + const response = await client( + `/api/admin/audit-events${search}${cursor}`, + { fallback: "Could not load audit events" }, + ); return response.json(); }, + getNextPageParam: (page: AuditEventsPage) => page.nextCursor || undefined, }); } diff --git a/app/tests/activity-older-events.test.tsx b/app/tests/activity-older-events.test.tsx new file mode 100644 index 0000000000..6e8d20c061 --- /dev/null +++ b/app/tests/activity-older-events.test.tsx @@ -0,0 +1,143 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { + createMemoryHistory, + createRootRoute, + createRouter, + RouterProvider, +} from "@tanstack/react-router"; +import { cleanup, fireEvent, render, waitFor } from "@testing-library/react"; +import type { ComponentType } from "react"; +import { agentKeys } from "@/lib/agents/queries"; +import { deploymentKeys } from "@/lib/deployment/queries"; +import { pluginKeys } from "@/lib/plugins/queries"; +import { Route as ActivityRoute } from "@/routes/_authed/settings/_personal/activity"; +import { settleReactWork } from "./settle-react-work"; + +/* + * FOUND ON A USED INSTALL (personal edition, 1.0 RC1 candidate, 298 events in the trail): Settings > + * Activity showed the newest 50 events as "49 events", as if that were everything, with no way to + * see an older one. The server answered `nextCursor` and the page never asked for it. + */ + +beforeAll(() => GlobalRegistrator.register()); +afterEach(() => cleanup()); + +const originalFetch = globalThis.fetch; +let asked: URL[] = []; + +afterAll(async () => { + globalThis.fetch = originalFetch; + await settleReactWork(); + await GlobalRegistrator.unregister(); +}); + +/** A row as `GET /api/admin/audit-events` answers it: the stored columns, `createdAt` as ISO. */ +function storedRow(n: number) { + return { + id: `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`, + actorUserId: "dev", + initiatorKind: "person", + initiatorId: null, + eventType: "computer.action_allowed", + targetType: "computer", + targetId: "personal-assistant", + payload: { + bot: "personal-assistant", + action: "computer_click", + element: { name: `Button ${n}`, role: "button" }, + }, + // Newest first, a minute apart: row 1 is the newest. + createdAt: new Date( + Date.UTC(2026, 10, 8, 12, 0) - n * 60_000, + ).toISOString(), + }; +} + +// The server's own cursor encoding (`encodeCursor` in server/src/audit.ts): the last row's time and id. +const CURSOR = Buffer.from( + JSON.stringify({ createdAt: storedRow(50).createdAt, id: storedRow(50).id }), +).toString("base64url"); + +/** The server's two pages: 50 rows and where the next starts, then the last 10 with no cursor. */ +function serve(url: URL) { + if (!url.searchParams.has("cursor")) { + return { + events: Array.from({ length: 50 }, (_, i) => storedRow(i + 1)), + nextCursor: CURSOR, + }; + } + // `nextCursor` is undefined on the last page, which `context.json` leaves out of the body. + return { events: Array.from({ length: 10 }, (_, i) => storedRow(i + 51)) }; +} + +test("Activity names what it shows as the newest events, and Show older events reaches the rest", async () => { + asked = []; + globalThis.fetch = Object.assign( + async (input: RequestInfo | URL) => { + const url = new URL(String(input), "http://127.0.0.1"); + if (url.pathname === "/api/admin/audit-events") { + asked.push(url); + return Response.json(serve(url)); + } + return Response.json({ error: "not served" }, { status: 404 }); + }, + { preconnect: originalFetch.preconnect }, + ); + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false, staleTime: Number.POSITIVE_INFINITY }, + }, + }); + queryClient.setQueryData(deploymentKeys.capabilities(), { + generativeUi: true, + selfHostBanner: false, + localEvaluation: true, + edition: "personal", + }); + queryClient.setQueryData(pluginKeys.page(), { + catalogue: [], + servers: [], + skills: [], + }); + queryClient.setQueryData(agentKeys.list(false), [ + { id: "personal-assistant", name: "Personal Assistant" }, + ]); + queryClient.setQueryData(agentKeys.list(true), []); + const Page = ActivityRoute.options.component as ComponentType; + const router = createRouter({ + routeTree: createRootRoute({ component: () => }), + history: createMemoryHistory({ initialEntries: ["/"] }), + }); + const view = render( + + + , + ); + + // The first page, and a count that does not pass itself off as the whole trail. + await view.findByText("Newest 50 events"); + expect(view.container.textContent).toContain("Button 50"); + expect(view.container.textContent).not.toContain("Button 51"); + + fireEvent.click(view.getByRole("button", { name: "Show older events" })); + + // The older page, asked for with the same view and the server's cursor, under the newer rows. + await view.findByText("60 events"); + expect(view.container.textContent).toContain("Button 60"); + expect(view.container.querySelectorAll("tbody tr[data-outcome]").length).toBe( + 60, + ); + expect(asked).toHaveLength(2); + expect(asked[1]?.searchParams.get("cursor")).toBe(CURSOR); + expect(asked[1]?.searchParams.get("excludeEventType")).toBe( + asked[0]?.searchParams.get("excludeEventType") ?? "missing", + ); + // The last page has no cursor, so there is nothing older to offer. + await waitFor(() => + expect( + view.queryByRole("button", { name: "Show older events" }), + ).toBeNull(), + ); +}); diff --git a/app/tests/activity-page-name.test.tsx b/app/tests/activity-page-name.test.tsx index 21297e9c72..a5959a7deb 100644 --- a/app/tests/activity-page-name.test.tsx +++ b/app/tests/activity-page-name.test.tsx @@ -12,11 +12,11 @@ import type { ComponentType } from "react"; import { withoutOpenBotWorking } from "@/components/manage/audit-page"; import { settingsSidebarItems } from "@/components/settings/settings-sidebar"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; import { deploymentKeys } from "@/lib/deployment/queries"; import { managePaths } from "@/lib/manage-paths"; import { Route as AdminAuditRoute } from "@/routes/_authed/admin/audit"; import { Route as ActivityRoute } from "@/routes/_authed/settings/_personal/activity"; +import { seedAuditPage } from "./seed-audit-page"; import { settleReactWork } from "./settle-react-work"; /* @@ -45,8 +45,9 @@ async function renderAs(edition: "team" | "personal", Page: ComponentType) { edition, }); // Activity asks without OpenBot's own working; Admin asks for everything. - queryClient.setQueryData( - [...auditKeys.all, edition === "personal" ? withoutOpenBotWorking("") : ""], + seedAuditPage( + queryClient, + edition === "personal" ? withoutOpenBotWorking("") : "", { events: [] }, ); queryClient.setQueryData(agentKeys.list(false), []); diff --git a/app/tests/activity-person-words.test.tsx b/app/tests/activity-person-words.test.tsx index b59e7f8b72..faaf3efcf0 100644 --- a/app/tests/activity-person-words.test.tsx +++ b/app/tests/activity-person-words.test.tsx @@ -8,15 +8,15 @@ import { RouterProvider, } from "@tanstack/react-router"; import { cleanup, fireEvent, render } from "@testing-library/react"; +import { ServerToolLine } from "@/components/channels/chat-transcript"; import { AuditPage, withoutOpenBotWorking, } from "@/components/manage/audit-page"; -import { ServerToolLine } from "@/components/channels/chat-transcript"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; import { deploymentKeys } from "@/lib/deployment/queries"; import { pluginKeys } from "@/lib/plugins/queries"; +import { seedAuditPage } from "./seed-audit-page"; import { settleReactWork } from "./settle-react-work"; /* @@ -137,8 +137,9 @@ async function renderAs(edition: "team" | "personal") { localEvaluation: edition === "personal", edition, }); - queryClient.setQueryData( - [...auditKeys.all, edition === "personal" ? withoutOpenBotWorking("") : ""], + seedAuditPage( + queryClient, + edition === "personal" ? withoutOpenBotWorking("") : "", { events: ROWS }, ); queryClient.setQueryData(pluginKeys.page(), { diff --git a/app/tests/audit-approval-outcomes.test.tsx b/app/tests/audit-approval-outcomes.test.tsx index d6581ae95c..6f9ff1f75f 100644 --- a/app/tests/audit-approval-outcomes.test.tsx +++ b/app/tests/audit-approval-outcomes.test.tsx @@ -16,7 +16,7 @@ import { eventTypeFilter, REFUSED_EVENT_TYPES, } from "@/lib/audit/outcome"; -import { auditKeys } from "@/lib/audit/queries"; +import { seedAuditPage } from "./seed-audit-page"; /** * Settings › Activity after a Bot was asked to write two files with "Ask before making changes" on, @@ -140,7 +140,7 @@ async function renderActivity() { queries: { retry: false, staleTime: Number.POSITIVE_INFINITY }, }, }); - queryClient.setQueryData([...auditKeys.all, ""], { events: ROWS }); + seedAuditPage(queryClient, "", { events: ROWS }); queryClient.setQueryData(agentKeys.list(false), []); const router = createRouter({ routeTree: createRootRoute({ component: () => }), diff --git a/app/tests/audit-connect-app-rows.test.tsx b/app/tests/audit-connect-app-rows.test.tsx index 758a07b926..72a0603a27 100644 --- a/app/tests/audit-connect-app-rows.test.tsx +++ b/app/tests/audit-connect-app-rows.test.tsx @@ -11,10 +11,10 @@ import { import { cleanup, render } from "@testing-library/react"; import type { ComponentType } from "react"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; import { deploymentKeys } from "@/lib/deployment/queries"; import { Route as AdminAuditRoute } from "@/routes/_authed/admin/audit"; import { Route as ActivityRoute } from "@/routes/_authed/settings/_personal/activity"; +import { seedAuditPage } from "./seed-audit-page"; import { settleReactWork } from "./settle-react-work"; /* @@ -116,7 +116,7 @@ async function renderPage( edition, }); for (const [search, events] of Object.entries(audit)) - queryClient.setQueryData([...auditKeys.all, search], { events }); + seedAuditPage(queryClient, search, { events }); queryClient.setQueryData(agentKeys.list(false), [ { id: BOT, name: "Personal Assistant" }, ]); diff --git a/app/tests/audit-dry-run-label.test.tsx b/app/tests/audit-dry-run-label.test.tsx index b206fd330c..5447ef6509 100644 --- a/app/tests/audit-dry-run-label.test.tsx +++ b/app/tests/audit-dry-run-label.test.tsx @@ -9,8 +9,8 @@ import { } from "@tanstack/react-router"; import { cleanup, render, within } from "@testing-library/react"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; import { Route as AuditRoute } from "@/routes/_authed/admin/audit"; +import { seedAuditPage } from "./seed-audit-page"; beforeAll(() => GlobalRegistrator.register()); @@ -44,7 +44,7 @@ async function renderAuditRow(row: Record) { queries: { retry: false, staleTime: Number.POSITIVE_INFINITY }, }, }); - queryClient.setQueryData([...auditKeys.all, ""], { events: [row] }); + seedAuditPage(queryClient, "", { events: [row] }); queryClient.setQueryData(agentKeys.list(false), []); const rootRoute = createRootRoute({ component: AuditRoute.options.component, diff --git a/app/tests/audit-event-words.test.tsx b/app/tests/audit-event-words.test.tsx index 9aaa3e7597..b8aa91f03d 100644 --- a/app/tests/audit-event-words.test.tsx +++ b/app/tests/audit-event-words.test.tsx @@ -15,8 +15,8 @@ import { } from "@/components/manage/audit-page"; import { agentKeys } from "@/lib/agents/queries"; import { OPENBOT_WORKING_EVENT_TYPES } from "@/lib/audit/outcome"; -import { auditKeys } from "@/lib/audit/queries"; import { deploymentKeys } from "@/lib/deployment/queries"; +import { seedAuditPage } from "./seed-audit-page"; import { settleReactWork } from "./settle-react-work"; /* @@ -123,10 +123,10 @@ async function renderAs(edition: "team" | "personal", rows = ROWS) { }); // Activity asks without OpenBot's own working, and the server answers without it. if (edition === "personal") - queryClient.setQueryData([...auditKeys.all, withoutOpenBotWorking("")], { + seedAuditPage(queryClient, withoutOpenBotWorking(""), { events: rows.filter((row) => !LEFT_OUT.has(row.eventType)), }); - else queryClient.setQueryData([...auditKeys.all, ""], { events: rows }); + else seedAuditPage(queryClient, "", { events: rows }); queryClient.setQueryData(agentKeys.list(false), [ { id: "personal-assistant", name: "Personal Assistant" }, ]); diff --git a/app/tests/audit-export-truncated.test.tsx b/app/tests/audit-export-truncated.test.tsx index 7111a1297e..7b4f9e6d52 100644 --- a/app/tests/audit-export-truncated.test.tsx +++ b/app/tests/audit-export-truncated.test.tsx @@ -9,8 +9,8 @@ import { } from "@tanstack/react-router"; import { cleanup, fireEvent, render } from "@testing-library/react"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; import { Route as AuditRoute } from "@/routes/_authed/admin/audit"; +import { seedAuditPage } from "./seed-audit-page"; beforeAll(() => GlobalRegistrator.register()); @@ -34,7 +34,7 @@ async function exportWith(headers: Record) { queries: { retry: false, staleTime: Number.POSITIVE_INFINITY }, }, }); - queryClient.setQueryData([...auditKeys.all, ""], { events: [] }); + seedAuditPage(queryClient, "", { events: [] }); queryClient.setQueryData(agentKeys.list(false), []); const router = createRouter({ routeTree: createRootRoute({ component: AuditRoute.options.component }), diff --git a/app/tests/audit-folder-actions.test.tsx b/app/tests/audit-folder-actions.test.tsx index acdb49e9b7..32c85c31c7 100644 --- a/app/tests/audit-folder-actions.test.tsx +++ b/app/tests/audit-folder-actions.test.tsx @@ -13,8 +13,8 @@ import { withoutOpenBotWorking, } from "@/components/manage/audit-page"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; import { deploymentKeys } from "@/lib/deployment/queries"; +import { seedAuditPage } from "./seed-audit-page"; import { settleReactWork } from "./settle-react-work"; /* @@ -117,7 +117,7 @@ async function renderActivity() { localEvaluation: true, edition: "personal", }); - queryClient.setQueryData([...auditKeys.all, withoutOpenBotWorking("")], { + seedAuditPage(queryClient, withoutOpenBotWorking(""), { events: ROWS, }); queryClient.setQueryData(agentKeys.list(false), [ diff --git a/app/tests/audit-grant-and-routine-rows.test.tsx b/app/tests/audit-grant-and-routine-rows.test.tsx index 84402a0319..7d148dd7af 100644 --- a/app/tests/audit-grant-and-routine-rows.test.tsx +++ b/app/tests/audit-grant-and-routine-rows.test.tsx @@ -19,11 +19,11 @@ import { cleanup, render } from "@testing-library/react"; import type { ComponentType } from "react"; import { GALLERY as ACTIVITY } from "@/components/gallery/activity"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; import * as gallery from "@/lib/copilot/gallery-registry"; import { deploymentKeys } from "@/lib/deployment/queries"; import { Route as AdminAuditRoute } from "@/routes/_authed/admin/audit"; import { Route as ActivityRoute } from "@/routes/_authed/settings/_personal/activity"; +import { seedAuditPage } from "./seed-audit-page"; import { settleReactWork } from "./settle-react-work"; /* @@ -119,7 +119,7 @@ async function renderPage( edition, }); for (const [search, events] of Object.entries(audit)) - queryClient.setQueryData([...auditKeys.all, search], { events }); + seedAuditPage(queryClient, search, { events }); queryClient.setQueryData(agentKeys.list(false), [ { id: BOT, name: "Personal Assistant" }, ]); diff --git a/app/tests/audit-grant-rows.test.tsx b/app/tests/audit-grant-rows.test.tsx index b5c28c3852..811f5aa002 100644 --- a/app/tests/audit-grant-rows.test.tsx +++ b/app/tests/audit-grant-rows.test.tsx @@ -14,8 +14,8 @@ import { } from "@/components/manage/audit-page"; import { agentKeys } from "@/lib/agents/queries"; import { OPENBOT_WORKING_EVENT_TYPES } from "@/lib/audit/outcome"; -import { auditKeys } from "@/lib/audit/queries"; import { deploymentKeys } from "@/lib/deployment/queries"; +import { seedAuditPage } from "./seed-audit-page"; import { settleReactWork } from "./settle-react-work"; /* @@ -142,8 +142,9 @@ async function renderAs(edition: "team" | "personal") { edition, }); // Activity asks without OpenBot's own working; Admin asks for everything. - queryClient.setQueryData( - [...auditKeys.all, edition === "personal" ? withoutOpenBotWorking("") : ""], + seedAuditPage( + queryClient, + edition === "personal" ? withoutOpenBotWorking("") : "", // What the server answers each: Activity's query leaves a build's components out. { events: diff --git a/app/tests/audit-openbot-setup-rows.test.tsx b/app/tests/audit-openbot-setup-rows.test.tsx index 2ce4b514cb..a0efb45811 100644 --- a/app/tests/audit-openbot-setup-rows.test.tsx +++ b/app/tests/audit-openbot-setup-rows.test.tsx @@ -20,11 +20,11 @@ import type { ComponentType } from "react"; import { GALLERY as ACTIVITY } from "@/components/gallery/activity"; import { GALLERY as TABLE } from "@/components/gallery/table"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; import * as gallery from "@/lib/copilot/gallery-registry"; import { deploymentKeys } from "@/lib/deployment/queries"; import { Route as AdminAuditRoute } from "@/routes/_authed/admin/audit"; import { Route as ActivityRoute } from "@/routes/_authed/settings/_personal/activity"; +import { seedAuditPage } from "./seed-audit-page"; import { settleReactWork } from "./settle-react-work"; /* @@ -204,7 +204,7 @@ async function renderPage( edition, }); for (const [search, events] of Object.entries(audit)) - queryClient.setQueryData([...auditKeys.all, search], { events }); + seedAuditPage(queryClient, search, { events }); queryClient.setQueryData(agentKeys.list(false), [ { id: BOT, name: "Personal Assistant" }, ]); diff --git a/app/tests/audit-plumbing-rows.test.tsx b/app/tests/audit-plumbing-rows.test.tsx index 661c6f0fc9..d7bc17f9c0 100644 --- a/app/tests/audit-plumbing-rows.test.tsx +++ b/app/tests/audit-plumbing-rows.test.tsx @@ -11,10 +11,10 @@ import { cleanup, render, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import type { ComponentType } from "react"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; import { deploymentKeys } from "@/lib/deployment/queries"; import { Route as AdminAuditRoute } from "@/routes/_authed/admin/audit"; import { Route as ActivityRoute } from "@/routes/_authed/settings/_personal/activity"; +import { seedAuditPage } from "./seed-audit-page"; import { settleReactWork } from "./settle-react-work"; /* @@ -86,7 +86,7 @@ async function renderPage( edition, }); for (const [search, events] of Object.entries(audit)) - queryClient.setQueryData([...auditKeys.all, search], { events }); + seedAuditPage(queryClient, search, { events }); queryClient.setQueryData(agentKeys.list(false), [ { id: BOT, name: "Personal Assistant" }, ]); diff --git a/app/tests/audit-table-fits.test.tsx b/app/tests/audit-table-fits.test.tsx index acd5c3d974..a85d8ce2bd 100644 --- a/app/tests/audit-table-fits.test.tsx +++ b/app/tests/audit-table-fits.test.tsx @@ -11,7 +11,7 @@ import { cleanup, render, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { AuditPage } from "@/components/manage/audit-page"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; +import { seedAuditPage } from "./seed-audit-page"; /** * The Audit page in the Desktop's default window, about 1000px wide with Settings' own column @@ -80,7 +80,7 @@ async function renderAudit() { queries: { retry: false, staleTime: Number.POSITIVE_INFINITY }, }, }); - queryClient.setQueryData([...auditKeys.all, ""], { events: ROWS }); + seedAuditPage(queryClient, "", { events: ROWS }); queryClient.setQueryData(agentKeys.list(false), [ { id: "bot-a", name: "Claude Agent SDK" }, ]); diff --git a/app/tests/audit-table.test.tsx b/app/tests/audit-table.test.tsx index be5dfa6b35..70526b6621 100644 --- a/app/tests/audit-table.test.tsx +++ b/app/tests/audit-table.test.tsx @@ -17,7 +17,7 @@ import { } from "@/components/audit/audit-cells"; import { AuditPage } from "@/components/manage/audit-page"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; +import { seedAuditPage } from "./seed-audit-page"; /** * Admin → Audit as a log viewer: one window tall, the table across the whole pane and the only thing @@ -72,7 +72,7 @@ async function renderAudit(sections?: ReactNode) { queries: { retry: false, staleTime: Number.POSITIVE_INFINITY }, }, }); - queryClient.setQueryData([...auditKeys.all, ""], { events: ROWS }); + seedAuditPage(queryClient, "", { events: ROWS }); queryClient.setQueryData(agentKeys.list(false), []); const router = createRouter({ routeTree: createRootRoute({ diff --git a/app/tests/bot-activity-actions.test.tsx b/app/tests/bot-activity-actions.test.tsx index f1d2fe65aa..e3c94bf88d 100644 --- a/app/tests/bot-activity-actions.test.tsx +++ b/app/tests/bot-activity-actions.test.tsx @@ -16,10 +16,10 @@ import { withoutOpenBotWorking, } from "@/components/manage/audit-page"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; import { authKeys } from "@/lib/auth/queries"; import { botLifecycleKeys } from "@/lib/bot-lifecycle/queries"; import { deploymentKeys } from "@/lib/deployment/queries"; +import { seedAuditPage } from "./seed-audit-page"; import { settleReactWork } from "./settle-react-work"; /** @@ -41,7 +41,7 @@ async function renderWith( options: { edition: "personal" | "team"; role: "admin" | "user"; - audit?: Record; + audit?: Record; ready?: string; }, ) { @@ -67,7 +67,7 @@ async function renderWith( { id: BOT, name: "Claude Agent SDK" }, ]); for (const [search, events] of Object.entries(options.audit ?? {})) - queryClient.setQueryData([...auditKeys.all, search], { events }); + seedAuditPage(queryClient, search, { events }); const router = createRouter({ routeTree: createRootRoute({ component: () => ui }), history: createMemoryHistory({ initialEntries: ["/"] }), diff --git a/app/tests/bot-id-never-shown.test.tsx b/app/tests/bot-id-never-shown.test.tsx index b9cde04147..1c91cd23a2 100644 --- a/app/tests/bot-id-never-shown.test.tsx +++ b/app/tests/bot-id-never-shown.test.tsx @@ -14,7 +14,7 @@ import { ChannelAvatar } from "@/components/channels/avatar"; import { AuditPage } from "@/components/manage/audit-page"; import { useBotNames } from "@/lib/agents/bot-names"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; +import { seedAuditPage } from "./seed-audit-page"; import { settleReactWork } from "./settle-react-work"; /* @@ -57,7 +57,7 @@ function everythingOn(root: Element): string { } async function renderActivity(queryClient: QueryClient) { - queryClient.setQueryData([...auditKeys.all, ""], { + seedAuditPage(queryClient, "", { events: [ { id: "e1", diff --git a/app/tests/personal-says-openbot.test.tsx b/app/tests/personal-says-openbot.test.tsx index 82da417c68..f2ec65b40e 100644 --- a/app/tests/personal-says-openbot.test.tsx +++ b/app/tests/personal-says-openbot.test.tsx @@ -15,7 +15,6 @@ import { } from "@/components/manage/audit-page"; import { PluginPage } from "@/components/manage/plugin-page"; import { agentKeys } from "@/lib/agents/queries"; -import { auditKeys } from "@/lib/audit/queries"; import { deploymentKeys } from "@/lib/deployment/queries"; import { managePaths } from "@/lib/manage-paths"; import { @@ -28,6 +27,7 @@ import { pluginKeys, } from "@/lib/plugins/queries"; import { Route as ActivityRoute } from "@/routes/_authed/settings/_personal/activity"; +import { seedAuditPage } from "./seed-audit-page"; import { settleReactWork } from "./settle-react-work"; /* @@ -133,8 +133,9 @@ function draw(edition: "team" | "personal", screen: () => ReactNode) { client.setQueryData(agentKeys.list(false), []); client.setQueryData(agentKeys.list(true), []); // Activity asks without OpenBot's own working; Admin asks for everything. - client.setQueryData( - [...auditKeys.all, edition === "personal" ? withoutOpenBotWorking("") : ""], + seedAuditPage( + client, + edition === "personal" ? withoutOpenBotWorking("") : "", { events: [startedByOpenBot] }, ); const router = createRouter({ diff --git a/app/tests/seed-audit-page.ts b/app/tests/seed-audit-page.ts new file mode 100644 index 0000000000..080339c213 --- /dev/null +++ b/app/tests/seed-audit-page.ts @@ -0,0 +1,18 @@ +import type { QueryClient } from "@tanstack/react-query"; +import { auditKeys } from "@/lib/audit/queries"; + +/** + * One page of the trail in the cache, as `GET /api/admin/audit-events` answers it (`{ events }`, and + * `nextCursor` when older events remain), under the key the page asks for the view `search` with. + * The page reads it as the first page of its paged query. + */ +export function seedAuditPage( + queryClient: QueryClient, + search: string, + page: { events: unknown[]; nextCursor?: string }, +): void { + queryClient.setQueryData(auditKeys.list(search), { + pages: [page], + pageParams: [""], + }); +} From 09dde85aff0084c0bdad428f86d80b1ee3372b48 Mon Sep 17 00:00:00 2001 From: David McKay Date: Sun, 8 Nov 2026 18:42:26 -0800 Subject: [PATCH 04/34] Skip the narration question on a computer with no microphone (#1327) --- CHANGELOG.md | 2 ++ app/src/components/browser/recording.tsx | 12 +++++-- app/tests/browser-page.test.tsx | 46 ++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e4c78be74..12d994ee6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,8 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. the browser only while recording, or to say "No microphone found. Narration is off." on a computer with none, which keeps your narration choice for when there is one. - Your recordings links to Settings → Learning ("See what your Bots learned"). +- On a computer with no microphone, Record no longer asks "Narrate while you record?". It records + without narration and asks the first time you record with a microphone. ### Conversations diff --git a/app/src/components/browser/recording.tsx b/app/src/components/browser/recording.tsx index 91cf99b712..9884e44aa7 100644 --- a/app/src/components/browser/recording.tsx +++ b/app/src/components/browser/recording.tsx @@ -37,6 +37,7 @@ import { import { cn } from "@/lib/utils"; import { queryClient } from "@/query-client"; import type { NarrationChoice } from "./narration-choice"; +import { useMicrophone } from "./use-microphone"; /** The language narration is heard in: the person's own browser's. */ export function narrationLanguage(): string { @@ -130,7 +131,9 @@ export type RecordingStart = { * * Record asks the person whether to narrate when they have never said (`choice`, their account's preference, the same * in every browser they record in), in a dialog whose answer records and is kept, and tells the Recorder the answer - * with every recording, so a microphone opens only for a person who chose to narrate. + * with every recording, so a microphone opens only for a person who chose to narrate. On a computer with no microphone + * there is nothing to narrate with, so it asks nothing and records without narration, the choice left unmade for when + * one is plugged in. */ export function useRecordingWith({ recorder, @@ -151,6 +154,7 @@ export function useRecordingWith({ const [finishing, setFinishing] = useState(false); /** The narration question is on screen, for the task named when Record was pressed. */ const [asking, setAsking] = useState(false); + const noMicrophone = useMicrophone() === false; const start = useMutation(startOptions); const stop = useMutation(stopOptions); const phase = recorder?.phase ?? "idle"; @@ -198,11 +202,13 @@ export function useRecordingWith({ if (!name) return; // Asked only of a person who never chose: a choice the page has not heard yet is read first. const narration = await choice.read(); - if (narration === null) { + // No microphone: the page already says narration is off, so nothing is offered that could not be heard. + if (narration === null && !noMicrophone) { setAsking(true); return; } - // Unreadable: recorded without narration, the microphone closed, and the choice left as it is. + // Unreadable, or never made on a computer with no microphone: recorded without narration, the microphone + // closed, and the choice left as it is. begin(name, narration === "on"); }, /** The narration question, while it is asked: its dialog's props. */ diff --git a/app/tests/browser-page.test.tsx b/app/tests/browser-page.test.tsx index 87a5b30539..3334b1e59d 100644 --- a/app/tests/browser-page.test.tsx +++ b/app/tests/browser-page.test.tsx @@ -1746,6 +1746,52 @@ test("Record asks whether to narrate before the first recording, keeps the answe expect(view.queryByRole("dialog")).toBeNull(); }); +test("in the OpenBot Desktop window on a computer with no microphone, Record asks nothing about narration and records without it; with a microphone it still asks", async () => { + // Windows with no microphone: the page said "No microphone found. Narration is off." with its switch off, yet Start + // recording still asked "Narrate while you record?", offering Record with narration first. + narrationChoice = null; + state = screenState(); + openAnswer = answerFor("screen"); + installChrome(null); + installDesktopShell(); + const devices = installDevices(["audioinput", "audiooutput"]); + try { + const view = draw(); + const user = userEvent.setup({ document }); + await user.type(await taskField(view, user), "Submit an expense report"); + + // With a microphone, the first Record asks, exactly as before. + await pressRecord(view, user); + const question = await view.findByRole("dialog", { + name: "Narrate while you record?", + }); + expect( + within(question).getByRole("button", { name: "Record with narration" }), + ).toBeTruthy(); + await user.keyboard("{Escape}"); + await waitFor(() => expect(view.queryByRole("dialog")).toBeNull()); + expect(starts()).toEqual([]); + + // The microphone unplugged: Record records at once, without narration, and the choice stays unmade. + devices.kinds = ["audiooutput"]; + await act(async () => { + devices.dispatchEvent(new Event("devicechange")); + }); + expect( + await view.findByText("No microphone found. Narration is off."), + ).toBeTruthy(); + await pressRecord(view, user); + await waitFor(() => expect(starts()).toHaveLength(1)); + expect(starts()[0]?.body).toMatchObject({ narrate: false }); + expect( + view.queryByRole("dialog", { name: "Narrate while you record?" }), + ).toBeNull(); + expect(preferenceWrites()).toEqual([]); + } finally { + delete (navigator as { mediaDevices?: unknown }).mediaDevices; + } +}); + test("a choice still being read when Record is pressed is read, not asked again", async () => { // The person chose narration on long ago. Their preferences took a moment to arrive, and a Record pressed meanwhile // asked them again, as if they never had. From b93699c3316048e0bbeec3c9864c9b0c5b7b4b3b Mon Sep 17 00:00:00 2001 From: David McKay Date: Sun, 8 Nov 2026 18:42:38 -0800 Subject: [PATCH 05/34] Fit New Bot's four named frameworks above the footer at 800px (#1328) --- .github/workflows/ci.yml | 7 +- CHANGELOG.md | 3 + app/src/components/agents/new-bot-sheet.tsx | 32 ++-- app/tests/new-bot-sheet-fit.fixture.tsx | 40 ++++ app/tests/new-bot-sheet-fit.test.tsx | 196 ++++++++++++++++++++ 5 files changed, 260 insertions(+), 18 deletions(-) create mode 100644 app/tests/new-bot-sheet-fit.fixture.tsx create mode 100644 app/tests/new-bot-sheet-fit.test.tsx diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fe1770a9e..f96577d1e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,15 +157,16 @@ jobs: OPENBOT_COMPUTER_BROWSER: '1' COMPUTER_BROWSER_MODE: headless # The app's real-browser tests, here because this job has the browser: a running step's words - # under reduced motion, where onboarding's Back lands beside the Mac window's buttons, and the name - # step's orb drawn whole under the step dashes, which only a browser's layout resolves. The root + # under reduced motion, where onboarding's Back lands beside the Mac window's buttons, the name + # step's orb drawn whole under the step dashes, and the New Bot sheet's first four frameworks + # clearing its footer in an 800px window, which only a browser's layout resolves. The root # playwright-core is the same version as # agent-computer's playwright, so it finds the Chromium installed above. The onboarding frame # reads the generated app config, which `bun test` alone doesn't write (the test script's # pretest does). - run: | bun run generate:app-config - bun test app/tests/tool-line-reduced-motion.test.tsx app/tests/onboarding-window-buttons.test.tsx app/tests/onboarding-orb-room.test.tsx + bun test app/tests/tool-line-reduced-motion.test.tsx app/tests/onboarding-window-buttons.test.tsx app/tests/onboarding-orb-room.test.tsx app/tests/new-bot-sheet-fit.test.tsx env: OPENBOT_APP_BROWSER: '1' # Headed is the default for hosted computers, so it is tested as one: the computer starting its diff --git a/CHANGELOG.md b/CHANGELOG.md index 12d994ee6e..ebe412ae9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -167,6 +167,9 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. first screen, below the form, one line each with its logo, Claude Agent SDK, LangChain, Google ADK and AWS Strands first. Choosing one goes straight to its address, and Back returns to the list. They were behind a Connect a custom agent button. +- In an 800px-tall window, OpenBot Desktop's usual height on a Mac, New Bot's first screen shows all + four of those frameworks above Cancel and Create Bot before you scroll, where only Claude Agent + SDK did. - LangChain is called **LangChain**, with LangChain's own mark, where it said "LangChain (LangGraph)": in New Bot, a Bot's "Powered by" subtitle and its Runs on row, and OpenBot Desktop's note under a ChatGPT plan. diff --git a/app/src/components/agents/new-bot-sheet.tsx b/app/src/components/agents/new-bot-sheet.tsx index 3571ae9923..42e28cd76a 100644 --- a/app/src/components/agents/new-bot-sheet.tsx +++ b/app/src/components/agents/new-bot-sheet.tsx @@ -120,7 +120,7 @@ export function NewBotSheet({ const STEPS = ["about", "runtime", "endpoint"] as const; /** * The web list as it comes: the frameworks people look for first, and "An agent you already run" - * last, so the row that peeks above the footer on the first screen is a framework they know. + * last, so the rows that show above the footer on the first screen are frameworks they know. */ const CONNECTION_OPTIONS = WEB_HARNESSES; type Step = (typeof STEPS)[number]; @@ -438,7 +438,7 @@ function NewBotFlow({ onCreated }: { onCreated: (agentId: string) => void }) { submit(); }} > -
+
{/* Shares its layout id with the orb on the created view, so it travels there. */}
@@ -467,7 +467,7 @@ function NewBotFlow({ onCreated }: { onCreated: (agentId: string) => void }) { + Name @@ -776,7 +776,7 @@ function AboutStep({ {personal ? ( ) : apps.length > 0 ? ( -
+
Apps it can use @@ -786,7 +786,7 @@ function AboutStep({
{apps.map((app) => (
); + } else if (recording.stopped && savedToLearning(recording.stopped)) { + body = setStep(2)} />; } else if (recording.stopped) { const stopped = recording.stopped; body = ( @@ -236,6 +251,49 @@ function StepDots({ step }: { step: Step }) { ); } +/** + * The last step once a recording is saved: what it held, what Learning does with it next (Learn now, then Review or + * Try it once a skill from it is there), and the way on from here, to the skills in Settings, Learning or another + * recording. A saved recording is not a skill yet, so the step says what turns it into one. + */ +function SavedStep({ + stopped, + another, +}: { + stopped: Stopped; + another: () => void; +}) { + const learning = useLearningNext(stopped); + const { heading, sentence } = learningNextWords(learning, stopped.title); + const learned = learning.next.kind === "learned"; + return ( + + + +
+ } + > + + + {learning.loaded ? ( + <> + + + + ) : null} + + + ); +} + /** One step: what it shows in the middle of the stage, and its heading, sentence and button along the bottom. */ function Slide({ title, @@ -266,8 +324,9 @@ function Slide({
{children}
-
-
+ {/* The buttons go under the words once the two can't share a line without squeezing the sentence. */} +
+

{title}

{description} diff --git a/app/src/lib/learning/queries.ts b/app/src/lib/learning/queries.ts index b52cc3f644..fd3efa07bd 100644 --- a/app/src/lib/learning/queries.ts +++ b/app/src/lib/learning/queries.ts @@ -10,6 +10,7 @@ import type { PeopleProposals, PersonalLearningSharing, PersonalReview, + RecordingSkills, SharingRequest, } from "@openbot/shared/learning"; import { queryOptions } from "@tanstack/react-query"; @@ -25,6 +26,8 @@ export const learningKeys = { sharingRequests: ["learning", "sharing-requests"] as const, personProposal: (userId: string, proposalId: string) => ["learning", "person-proposal", { userId, proposalId }] as const, + recordingSkills: (title: string, since: number, revision: string) => + ["learning", "recording-skills", { title, since, revision }] as const, personalProposal: (container: string, proposalId: string) => ["learning", "personal-proposal", { container, proposalId }] as const, proposals: (agentId?: string) => @@ -306,3 +309,26 @@ export function sharingRequestsQueryOptions() { }), }); } + +/** + * Teach a Bot, after a recording saves: the skills Learning proposed from it, named `title` and stopped at `since` + * (milliseconds). `revision` changes whenever the person's review does (a run moves on, a proposal is approved), which + * is when there can be something new to find. + */ +export function recordingSkillsQueryOptions( + title: string, + since: number, + revision: string, +) { + return queryOptions({ + queryKey: learningKeys.recordingSkills(title, since, revision), + retry: false, + queryFn: async (): Promise => + ( + await client( + `/api/settings/learning/recording-skills?title=${encodeURIComponent(title)}&since=${since}`, + { fallback: "Couldn't read your skills just now. Try again." }, + ) + ).json(), + }); +} diff --git a/app/src/routes/_authed/_app/index.tsx b/app/src/routes/_authed/_app/index.tsx index 0ba3b3f3ed..7db793fd1e 100644 --- a/app/src/routes/_authed/_app/index.tsx +++ b/app/src/routes/_authed/_app/index.tsx @@ -4,8 +4,10 @@ import { Link, useNavigate, useRouter, + useSearch, } from "@tanstack/react-router"; import { useEffect, useRef, useState } from "react"; +import { z } from "zod"; import { AgentCard } from "@/components/agents/agent-card"; import { Composer, @@ -45,6 +47,11 @@ import { useAnyBotSkillCommands } from "@/lib/plugins/skill-commands"; const EXPLORE_SHOWN = 4; export const Route = createFileRoute("/_authed/_app/")({ + /* + * `ask`: words to start the box with, as Teach a Bot's Try it hands over a task a Bot just learned. Only put in the + * box: nothing is sent until the person sends it. + */ + validateSearch: z.object({ ask: z.string().max(2000).optional() }), // The edition decides whether there is an Explore section at all, so it is read before the page draws and the // composer never moves when it arrives. beforeLoad: async ({ context }) => { @@ -76,6 +83,9 @@ function RouteComponent() { // under it, the way a failed send always left them. Taken on mount, and again whenever one fails // while this page is open (somebody who came back here before it failed is looking at it). const [restored, setRestored] = useState(takeFailedStart); + // Read once, as the box reads its words once: the words a link handed over (`ask`). + const ask = useSearch({ strict: false }).ask; + const [asked] = useState(ask); const [restores, setRestores] = useState(0); const [error, setError] = useState(restored?.error ?? null); /* @@ -203,7 +213,11 @@ function RouteComponent() { // The box reads its words once, when it mounts; a start that fails while this page is // open is a new box with them in it. key={restores} - {...(restored ? { initialValue: restored.text } : {})} + {...(restored + ? { initialValue: restored.text } + : asked + ? { initialValue: asked, autoFocus: true } + : {})} onSubmit={async (draft) => { // The conversation opens NOW, as a placeholder showing the message, and the Bot is // picked and the channel made behind it; see `pending-start.ts`. Waiting here for diff --git a/app/tests/browser-page.test.tsx b/app/tests/browser-page.test.tsx index 3334b1e59d..d3e41f6d8d 100644 --- a/app/tests/browser-page.test.tsx +++ b/app/tests/browser-page.test.tsx @@ -17,6 +17,11 @@ import { browserTabsRequestSchema, type LauncherState, } from "@openbot/shared/openbot-browser"; +import type { + LearningRunSummary, + PersonalReview, + RecordingSkills, +} from "@openbot/shared/learning"; import type { RecordingRow } from "@openbot/shared/recorder-api"; import { QueryClientProvider } from "@tanstack/react-query"; import { @@ -96,6 +101,68 @@ let recordingRows: RecordingRow[] = []; let recordingsHeld: Promise | null = null; let recordingsAsked = 0; +/** The person's own Learning review, as Settings, Learning reads it: one space of their recordings, nothing run yet. */ +const RECORDINGS_SPACE = "openbot-person-ada"; +function freshReview(): PersonalReview { + return { + canReview: true, + autoApprove: true, + approved: 0, + sources: [ + { + containerId: RECORDINGS_SPACE, + kind: "recordings", + label: "Your recordings", + proposals: [], + latestRun: null, + }, + ], + }; +} +let review: PersonalReview = freshReview(); +/** The skills Learning proposed from the recording the page asks about, and what it asked. */ +let recordingSkills: RecordingSkills["skills"] = []; +const skillsAsked: string[] = []; +/** Learn now's refusal, in the server's sentence, while set. */ +let learnRefusal: string | null = null; + +/** A run of Learning over the person's recordings, made a minute after now. */ +function learningRun( + status: LearningRunSummary["status"], + failure: string | null = null, +): LearningRunSummary { + const at = new Date(Date.now() + 60_000).toISOString(); + return { + id: "run-after", + status, + trigger: "manual", + proposals: null, + insights: null, + failureCode: failure ? "BatchInsightAgentFailed" : null, + failure, + createdAt: at, + startedAt: at, + completedAt: status === "succeeded" || status === "failed" ? at : null, + }; +} + +/** A skill Learning proposed from "Find my meeting". */ +function meetingSkill( + status: "pending_review" | "approved", +): RecordingSkills["skills"][number] { + return { + id: "4b9f2a8e-1c3d-4e5f-8a9b-0c1d2e3f4a5b", + title: "find-my-meeting", + description: "Find a meeting on Google Calendar by name and open it.", + reason: "You did this in a recording.", + operation: "add", + status, + createdAt: new Date().toISOString(), + reviewedAt: null, + container: RECORDINGS_SPACE, + }; +} + function nativeState(): BrowserApiState { return { available: true, @@ -185,6 +252,16 @@ async function server(path: string, init?: RequestInit): Promise { }, }); } + if (path === "/api/settings/learning/review") return Response.json(review); + if (path.startsWith("/api/settings/learning/recording-skills")) { + skillsAsked.push(path); + return Response.json({ skills: recordingSkills }); + } + if (path.startsWith("/api/settings/learning/runs") && method === "POST") { + if (learnRefusal) + return Response.json({ error: learnRefusal }, { status: 409 }); + return Response.json({ run: { id: "run-new", status: "queued" } }); + } if (path === "/api/recorder/recordings") { recordingsAsked += 1; await recordingsHeld; @@ -376,6 +453,10 @@ beforeEach(() => { opening = null; preferencesHeld = null; preferencesFail = false; + review = freshReview(); + recordingSkills = []; + skillsAsked.length = 0; + learnRefusal = null; recordingRows = []; recordingsHeld = null; recordingsAsked = 0; @@ -410,6 +491,9 @@ const recording = (title: string): RecordingRow => ({ reason: null, }); +/** The router the page was last drawn in, to read where a link took it. */ +let drawnRouter: { state: { location: { pathname: string } } } | undefined; + /** Drawn as the app draws it: strict mode, the app's one query client, a router, and its toasts. */ function draw(ui: ReactNode, entry = "/") { const router = createRouter({ @@ -423,6 +507,7 @@ function draw(ui: ReactNode, entry = "/") { ), }), }); + drawnRouter = router; return render( @@ -802,6 +887,229 @@ test("on OpenBot Desktop the introduction shows until a recording is saved, Cont expect(localStorage.getItem(INTRODUCED_KEY)).toBe("1"); }); +/** OpenBot Desktop, after Stop saved "Find my meeting": the Recorder's own report, as a reload reads it. */ +function savedMeeting(): BrowserApiState { + const base = nativeState(); + return { + ...base, + recorder: { + ...(base.recorder as NonNullable), + lastStopped: { + recordingId: "rec-1", + title: "Find my meeting", + stoppedAt: Date.now() - 5_000, + actions: 6, + phrases: 6, + }, + }, + }; +} + +/** The link a button on the page draws, by its words. */ +function linkNamed(view: ReturnType, name: string) { + const link = view.getByText(name, { exact: true }).closest("a"); + if (!link) throw new Error(`${name} is not a link`); + return link; +} + +// A teammate on OpenBot Desktop 0.3.3, after Stop: "I'm not sure what to do next. It doesn't tell me to go back." +// The page said only "Saved to your Learning space: 6 actions and 6 narrated phrases." +test("on OpenBot Desktop a saved recording says what happens next, Learn now starts Learning on the person's recordings, and See your skills opens Settings, Learning", async () => { + state = savedMeeting(); + const view = draw(); + const user = userEvent.setup({ document }); + expect( + await view.findByRole("heading", { name: "Saved “Find my meeting”" }), + ).toBeTruthy(); + expect( + view.getByText( + "OpenBot learns a skill from it, and you'll find it in Settings, Learning. Once it's learned, ask any Bot to do it.", + ), + ).toBeTruthy(); + expect(await view.findByText("Learning runs on its own")).toBeTruthy(); + expect(view.getByRole("button", { name: "Record another" })).toBeTruthy(); + expect(linkNamed(view, "See your skills").getAttribute("href")).toBe( + "/settings/learning", + ); + + // The same run as Settings, Learning's Learn now, on the person's own recordings. + await user.click(view.getByRole("button", { name: "Learn now" })); + await waitFor(() => + expect(writes).toContainEqual({ + path: `/api/settings/learning/runs?container=${RECORDINGS_SPACE}`, + method: "POST", + body: {}, + }), + ); + expect(await view.findByText("Learning from your recordings")).toBeTruthy(); + expect(view.queryByRole("button", { name: "Learn now" })).toBeNull(); + + await user.click(view.getByText("See your skills", { exact: true })); + await waitFor(() => + expect(drawnRouter?.state.location.pathname).toBe("/settings/learning"), + ); +}); + +test("on OpenBot Desktop a skill learned from the saved recording says so, with Try it opening a new conversation with the task", async () => { + state = savedMeeting(); + review.sources[0] = { + ...(review.sources[0] as PersonalReview["sources"][number]), + latestRun: learningRun("succeeded"), + }; + // Waiting for the person's review first. + recordingSkills = [meetingSkill("pending_review")]; + const view = draw(); + expect( + await view.findByRole("heading", { + name: "Your new skill is ready to review", + }), + ).toBeTruthy(); + expect(view.getByText("Ready to review")).toBeTruthy(); + expect(linkNamed(view, "Review").getAttribute("href")).toBe( + "/settings/learning", + ); + // Asked about this recording, by its name and when it stopped. + expect(skillsAsked[0]).toMatch( + /^\/api\/settings\/learning\/recording-skills\?title=Find%20my%20meeting&since=\d+$/, + ); + + // Approved: the page says the Bots learned it, and Try it starts a conversation with the task in the box. + cleanup(); + queryClient.clear(); + recordingSkills = [meetingSkill("approved")]; + const learned = draw(); + expect( + await learned.findByRole("heading", { name: "Your Bots learned it" }), + ).toBeTruthy(); + expect(learned.getByText("Skill learned")).toBeTruthy(); + expect( + learned.getByText("Find a meeting on Google Calendar by name and open it."), + ).toBeTruthy(); + expect(linkNamed(learned, "Try it").getAttribute("href")).toMatch( + /^\/\?ask=Find(\+|%20)my(\+|%20)meeting$/, + ); +}); + +test("on OpenBot Desktop a saved recording says why Learning can't run, didn't finish, or didn't start", async () => { + const noModel = + "Learning can't run on the model you signed in with. To use it, sign OpenBot in with an OpenAI or Anthropic API key, or a Claude or ChatGPT plan."; + state = savedMeeting(); + review.modelProblem = noModel; + let view = draw(); + expect(await view.findByText("Learning can't run yet")).toBeTruthy(); + expect(view.getByText(noModel)).toBeTruthy(); + expect(view.queryByRole("button", { name: "Learn now" })).toBeNull(); + expect( + view.getByText( + "Your recording is kept. OpenBot learns a skill from it once Learning can run.", + ), + ).toBeTruthy(); + + cleanup(); + queryClient.clear(); + const why = + "Your model plan didn't answer Learning. Try Learn now again in a few minutes."; + review = freshReview(); + review.sources[0] = { + ...(review.sources[0] as PersonalReview["sources"][number]), + latestRun: learningRun("failed", why), + }; + view = draw(); + expect(await view.findByText("Learning didn't finish")).toBeTruthy(); + expect(view.getByText(why)).toBeTruthy(); + + // Learn now again, refused with #1238's reason: said where it was pressed. + learnRefusal = noModel; + const user = userEvent.setup({ document }); + await user.click(view.getByRole("button", { name: "Learn now" })); + expect(await view.findByText("Learning didn't start")).toBeTruthy(); + expect(view.getByText(noModel)).toBeTruthy(); +}); + +/** The same saved recording where the browser is drawn in the page (OpenBot Desktop since #1306, and hosted OpenBot). */ +function savedMeetingOnScreen(): BrowserApiState { + const base = screenState(); + return { + ...base, + recorder: { + ...(base.recorder as NonNullable), + lastStopped: savedMeeting().recorder?.lastStopped, + }, + }; +} + +// Teach a Bot became one page with Record in the browser's toolbar (#1306), and after Stop it said only "Saved “Find my +// meeting” to your Learning space: 6 actions and 6 narrated phrases." under the toolbar. +test("with the browser in the page, a saved recording says what happens next under the toolbar, Learn now starts Learning, and See your skills opens Settings, Learning", async () => { + state = savedMeetingOnScreen(); + openAnswer = answerFor("screen"); + const view = draw(); + const user = userEvent.setup({ document }); + expect(await stopSays(view)).toBe( + "Saved “Find my meeting” to your Learning space: 6 actions and 6 narrated phrases.", + ); + expect( + await view.findByText( + "OpenBot learns a skill from it, and you'll find it in Settings, Learning. Once it's learned, ask any Bot to do it.", + ), + ).toBeTruthy(); + expect(await view.findByText("Learning runs on its own")).toBeTruthy(); + expect(linkNamed(view, "See your skills").getAttribute("href")).toBe( + "/settings/learning", + ); + + await user.click(view.getByRole("button", { name: "Learn now" })); + await waitFor(() => + expect(writes).toContainEqual({ + path: `/api/settings/learning/runs?container=${RECORDINGS_SPACE}`, + method: "POST", + body: {}, + }), + ); + expect(await view.findByText("Learning from your recordings")).toBeTruthy(); + + await user.click(view.getByText("See your skills", { exact: true })); + await waitFor(() => + expect(drawnRouter?.state.location.pathname).toBe("/settings/learning"), + ); +}); + +test("with the browser in the page, a skill learned from the saved recording says so, with Try it", async () => { + state = savedMeetingOnScreen(); + openAnswer = answerFor("screen"); + recordingSkills = [meetingSkill("approved")]; + const view = draw(); + expect(await view.findByText("Skill learned")).toBeTruthy(); + expect( + view.getByText( + "Your Bots can do “Find my meeting” now. Try it, or ask any Bot to do it.", + ), + ).toBeTruthy(); + expect(linkNamed(view, "Try it").getAttribute("href")).toMatch( + /^\/\?ask=Find(\+|%20)my(\+|%20)meeting$/, + ); +}); + +test("with the browser in the page, a recording still on its way has no next step yet", async () => { + const base = savedMeetingOnScreen(); + const stopped = base.recorder?.lastStopped; + if (!stopped) throw new Error("savedMeetingOnScreen has no recording"); + state = { + ...base, + recorder: { + ...(base.recorder as NonNullable), + lastStopped: { ...stopped, sending: true }, + }, + }; + openAnswer = answerFor("screen"); + const view = draw(); + expect(await stopSays(view)).toBe( + "Saving “Find my meeting” to your Learning space: 6 actions and 6 narrated phrases.", + ); + expect(view.queryByText("See your skills")).toBeNull(); + expect(view.queryByText("Learning runs on its own")).toBeNull(); +}); + test("on OpenBot Desktop a recording discarded from the Recorder's mark in the browser is said to be discarded, never saved", async () => { state = { ...nativeState(), diff --git a/app/tests/home-fallback-routing.test.tsx b/app/tests/home-fallback-routing.test.tsx index be988e4b3e..a0443871a3 100644 --- a/app/tests/home-fallback-routing.test.tsx +++ b/app/tests/home-fallback-routing.test.tsx @@ -73,7 +73,7 @@ function queryClientWithAgents(agents: AgentProfile[]) { return queryClient; } -function renderHome(queryClient: QueryClient) { +function renderHome(queryClient: QueryClient, entry = "/") { // Home and the placeholder it opens on send, so a start that fails behind the placeholder comes // back to a freshly mounted home page, as it does in the app. const rootRoute = createRootRoute({ component: Outlet }); @@ -81,6 +81,7 @@ function renderHome(queryClient: QueryClient) { createRoute({ getParentRoute: () => rootRoute, path: "/", + validateSearch: HomeRoute.options.validateSearch, component: HomeRoute.options.component, }), createRoute({ @@ -91,7 +92,7 @@ function renderHome(queryClient: QueryClient) { ]); const router = createRouter({ routeTree, - history: createMemoryHistory({ initialEntries: ["/"] }), + history: createMemoryHistory({ initialEntries: [entry] }), }); return Object.assign( render( @@ -143,6 +144,31 @@ function installHomeRoutingFetch(options: { ); } +// Teach a Bot's Try it, once a Bot learned a task from a recording: a new conversation with the task in the box. +test("/ starts the box with the words a link hands over, and sends them only when the person does", async () => { + const own = agent({ + id: "own-public", + mine: true, + name: "Own Public", + visibility: "public", + }); + const starts: string[][] = []; + installHomeRoutingFetch({ + routeResponse: Response.json({ error: "router down" }, { status: 503 }), + starts, + }); + const view = renderHome( + queryClientWithAgents([own]), + "/?ask=Find%20my%20meeting", + ); + const box = await view.findByRole("textbox", { name: "Message" }); + await waitFor(() => expect(box.textContent).toBe("Find my meeting")); + expect(starts).toEqual([]); + const user = userEvent.setup({ document: view.container.ownerDocument }); + await user.click(await view.findByRole("button", { name: "Send message" })); + await waitFor(() => expect(starts).toEqual([["own-public"]])); +}); + test("/ fallback routing matches the server default when route selection is unavailable", async () => { const ownPublic = agent({ id: "own-public", diff --git a/server/src/index.ts b/server/src/index.ts index ae3c987886..59a645bb6d 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -3174,6 +3174,7 @@ const app = createApp( invalidate: () => openSkillIndex.invalidate(), sharingAllowed: skillSharingAllowed, manage: learningProbeIntelligence.experimental.learning, + modelUnavailable: learningModelUnavailable(process.env), recordingTitles: recorderExtension.recordingTitles, settings: learningSettings, scopesFor: async (actor) => diff --git a/server/src/learning/sharing-routes.ts b/server/src/learning/sharing-routes.ts index 6fb566f29b..4b20e716b9 100644 --- a/server/src/learning/sharing-routes.ts +++ b/server/src/learning/sharing-routes.ts @@ -6,6 +6,7 @@ import { type LearningSource, type PersonalLearningSharing, type PersonalReview, + type RecordingSkills, } from "@openbot/shared/learning"; import { Hono, type MiddlewareHandler } from "hono"; import { bodyLimit } from "hono/body-limit"; @@ -20,7 +21,9 @@ import { import { CANDIDATE_ID, type LearningManagement, + type LearningModelUnavailable, learningFailure, + noLearningModel, readProposal, readProposals, readSkillRecordings, @@ -81,8 +84,18 @@ export type PersonalLearningRouteDeps = { userId: string, ids: string[], ) => Promise>; + /** + * Why OpenBot Desktop could not give its local Intelligence's Learning agent a model, from + * `OPENBOT_LEARNING_MODEL_UNAVAILABLE`. See `learningModelUnavailable`. + */ + modelUnavailable?: LearningModelUnavailable; }; +/** The most proposals one recording's skills are looked for in, newest first. Each costs a read of its evidence. */ +const RECORDING_SKILL_PROPOSALS = 5; +/** How much earlier than the recording's stop a proposal may be dated and still count: the two clocks can differ. */ +const RECORDING_CLOCK_SLACK_MS = 60_000; + /** What a person reads when they try to open their Skills while an administrator has not allowed it. */ const SHARING_OFF = "An admin hasn't turned on sharing skills with every Bot, so your skills stay with your own Bots."; @@ -356,11 +369,79 @@ export function createPersonalLearningRoutes( ? ((await deps.settings.read()).autoApprove ?? true) : null, approved, + ...(deps.modelUnavailable + ? { modelProblem: noLearningModel(deps.modelUnavailable) } + : {}), sources, }; return context.json(answer); }); + /** + * The skills Learning proposed from one recording, for Teach a Bot once it saves one: `title` is the recording's name + * and `since` when it stopped (milliseconds). Read from the proposals made since then in the person's own recording + * spaces, newest first and a few at most, kept where the evidence cites a recording of theirs by that name. Recordings + * that aren't theirs never match: `recordingTitles` names only their own. + */ + routes.get("/recording-skills", async (context) => { + const title = context.req.query("title")?.trim() ?? ""; + const since = Number(context.req.query("since")); + if (!title || title.length > 200 || !Number.isFinite(since)) { + return context.json({ error: "Name a recording to look for." }, 400); + } + const manage = deps.manage; + const recordingTitles = deps.recordingTitles; + if (!manage || !recordingTitles) { + return context.json( + { error: "Couldn't reach the back end just now. Try again." }, + 503, + ); + } + const actor = context.var.actor; + const from = since - RECORDING_CLOCK_SLACK_MS; + const titles = (ids: string[]) => recordingTitles(actor.id, ids); + const sources = (await deps.scopesFor(actor)).filter( + (source) => source.kind === "recordings", + ); + const answer: RecordingSkills = { skills: [] }; + for (const source of sources) { + const listed = await settle(readProposals(manage, source.containerId)); + // A space not made yet holds nothing; any other failure is said, so the page doesn't read it as "none yet". + if (listed.failed) { + if (listed.missing) continue; + if (listed.off) return context.json(answer); + return context.json( + { error: "Couldn't read your skills just now. Try again." }, + 502, + ); + } + const recent = listed.value + .filter( + (proposal) => + Date.parse(proposal.createdAt) >= from && + proposal.status !== "rejected" && + proposal.operation !== "remove", + ) + .slice(0, RECORDING_SKILL_PROPOSALS); + for (const proposal of recent) { + const detail = await readProposal( + manage, + source.containerId, + proposal.id, + titles, + ).catch(() => null); + const cited = detail?.insights.some((insight) => + (insight.evidence ?? []).some( + (item) => item.kind === "recording" && item.title?.trim() === title, + ), + ); + if (cited) + answer.skills.push({ ...proposal, container: source.containerId }); + } + } + return context.json(answer); + }); + routes.get("/proposals/:candidateId", async (context) => { const candidateId = context.req.param("candidateId"); if (!CANDIDATE_ID.test(candidateId)) { @@ -391,7 +472,7 @@ export function createPersonalLearningRoutes( ), }); } catch (error) { - const failure = learningFailure(error); + const failure = learningFailure(error, deps.modelUnavailable); return context.json({ error: failure.error }, failure.status); } }); @@ -466,7 +547,7 @@ export function createPersonalLearningRoutes( if (error instanceof UnnamedSkillError) { return context.json({ error: error.message }, 422); } - const failure = learningFailure(error); + const failure = learningFailure(error, deps.modelUnavailable); return context.json({ error: failure.error }, failure.status); } await audit({ @@ -513,7 +594,7 @@ export function createPersonalLearningRoutes( try { run = await manage.startLearningRun({ containerId }); } catch (error) { - const failure = learningFailure(error); + const failure = learningFailure(error, deps.modelUnavailable); return context.json({ error: failure.error }, failure.status); } await audit({ diff --git a/server/tests/learning-recording-skills.test.ts b/server/tests/learning-recording-skills.test.ts new file mode 100644 index 0000000000..3fa9994a93 --- /dev/null +++ b/server/tests/learning-recording-skills.test.ts @@ -0,0 +1,189 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import type { PersonalReview, RecordingSkills } from "@openbot/shared/learning"; +import type { MiddlewareHandler } from "hono"; +import type { AppVariables } from "../src/auth/guards"; +import type { SkillApprovalStore } from "../src/learning/approvals"; +import { runtimeKeyClient } from "../src/learning/auto-assign"; +import { personalLearningContainerId } from "../src/learning/personal-space"; +import type { LearningSettingsStore } from "../src/learning/settings"; +import type { LearningSharingStore } from "../src/learning/sharing"; +import { + createPersonalLearningRoutes, + type PersonalLearningRouteDeps, +} from "../src/learning/sharing-routes"; +import { + type FakeIntelligence, + fakeIntelligence, + RUNTIME_KEY, +} from "./support/intelligence-learning"; + +/* + * Teach a Bot, once a recording saves: which skills Learning proposed from that recording, and why Learning can't run + * when OpenBot Desktop could give it no model. The person's routes are built directly over a fake Intelligence holding + * their recordings space; nothing here reads OpenBot's database. + */ + +const ada = "recording-skills-ada"; +const SPACE = personalLearningContainerId(ada); +const PROPOSAL = "0198a012-a20f-77c0-9e33-8992f18e74d1"; +/** The recording the fake Intelligence's evidence cites (support/intelligence-learning.ts `EVIDENCE`). */ +const CITED = "0198a012-a20f-77c0-9e33-8992f18e74b0"; +/** When the fake Intelligence made its proposal. */ +const PROPOSED_AT = Date.parse("2026-11-04T12:00:00.000Z"); +const served: FakeIntelligence[] = []; + +afterEach(() => { + for (const intelligence of served.splice(0)) intelligence.stop(); +}); + +function routes( + options: { + /** The person's own recordings, by id, as `recordingTitles` names them: anybody else's are left out. */ + theirs?: Record; + noModel?: boolean; + modelUnavailable?: PersonalLearningRouteDeps["modelUnavailable"]; + } = {}, +) { + const intelligence = fakeIntelligence("current", { + containers: [SPACE], + candidates: [PROPOSAL], + ...(options.noModel ? { noModel: true } : {}), + }); + served.push(intelligence); + const client = runtimeKeyClient( + intelligence.url, + RUNTIME_KEY, + "ws://127.0.0.1:9", + ); + const requireUser: MiddlewareHandler<{ Variables: AppVariables }> = async ( + context, + next, + ) => { + context.set("actor", { + id: ada, + email: `${ada}@example.test`, + role: "user", + }); + await next(); + }; + const theirs = options.theirs ?? { [CITED]: "Find my meeting" }; + const deps: PersonalLearningRouteDeps = { + store: {} as LearningSharingStore, + approvals: {} as SkillApprovalStore, + skillsOf: async () => ({ skills: [] }), + invalidate: () => {}, + sharingAllowed: async () => true, + manage: client.experimental.learning, + recordingTitles: async (userId, ids) => + new Map( + userId === ada + ? ids.flatMap((id) => { + const title = theirs[id]; + return title === undefined ? [] : [[id, title] as const]; + }) + : [], + ), + ...(options.modelUnavailable + ? { modelUnavailable: options.modelUnavailable } + : {}), + settings: { + read: async () => ({ + enabled: true, + defaultTarget: null, + agents: {}, + personalReview: "people", + }), + } as unknown as LearningSettingsStore, + scopesFor: async () => [ + { containerId: SPACE, kind: "recordings", label: "Your recordings" }, + ], + }; + return { app: createPersonalLearningRoutes(deps, requireUser), intelligence }; +} + +const skillsFrom = async ( + app: ReturnType["app"], + title: string, + since: number, +) => { + const response = await app.request( + `/recording-skills?title=${encodeURIComponent(title)}&since=${since}`, + ); + expect(response.status).toBe(200); + return ((await response.json()) as RecordingSkills).skills; +}; + +describe("the skills learned from one recording", () => { + test("a proposal made since the recording stopped, citing the person's recording by its name, is that recording's skill", async () => { + const { app } = routes(); + const skills = await skillsFrom(app, "Find my meeting", PROPOSED_AT - 1); + expect(skills).toMatchObject([ + { + id: PROPOSAL, + status: "pending_review", + operation: "add", + container: SPACE, + }, + ]); + }); + + test("a proposal from another recording, one made before this one stopped, or a recording that isn't theirs is not its skill", async () => { + expect( + await skillsFrom(routes().app, "Submit an expense report", 0), + ).toEqual([]); + // Made more than the clocks' slack before this recording stopped: it was learned from something earlier. + expect( + await skillsFrom(routes().app, "Find my meeting", PROPOSED_AT + 120_000), + ).toEqual([]); + // The evidence cites somebody else's recording, so it is never named for this person. + expect( + await skillsFrom(routes({ theirs: {} }).app, "Find my meeting", 0), + ).toEqual([]); + }); + + test("an approved proposal is said to be approved, and a rejected one is left out", async () => { + const { app, intelligence } = routes(); + const proposal = intelligence.candidates.get(PROPOSAL); + if (!proposal) throw new Error("no proposal"); + intelligence.candidates.set(PROPOSAL, { ...proposal, status: "approved" }); + expect(await skillsFrom(app, "Find my meeting", 0)).toMatchObject([ + { id: PROPOSAL, status: "approved" }, + ]); + intelligence.candidates.set(PROPOSAL, { ...proposal, status: "rejected" }); + expect(await skillsFrom(app, "Find my meeting", 0)).toEqual([]); + }); + + test("a request without a recording's name or when it stopped is refused", async () => { + const { app } = routes(); + expect((await app.request("/recording-skills?since=1")).status).toBe(400); + expect( + (await app.request("/recording-skills?title=Find%20my%20meeting")).status, + ).toBe(400); + }); +}); + +describe("Learning with no model on OpenBot Desktop", () => { + const unsupported = + "Learning can't run on the model you signed in with. To use it, sign OpenBot in with an OpenAI or Anthropic API key, or a Claude or ChatGPT plan."; + + test("the person's review says why Learning can't run, before anyone presses Learn now", async () => { + const { app } = routes({ modelUnavailable: "unsupported" }); + const review = (await ( + await app.request("/review") + ).json()) as PersonalReview; + expect(review.modelProblem).toBe(unsupported); + const without = (await ( + await routes().app.request("/review") + ).json()) as PersonalReview; + expect(without.modelProblem).toBeUndefined(); + }); + + test("Learn now on the person's own recordings says the same, not the back end's setup", async () => { + const { app } = routes({ noModel: true, modelUnavailable: "unsupported" }); + const response = await app.request(`/runs?container=${SPACE}`, { + method: "POST", + }); + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ error: unsupported }); + }); +}); diff --git a/shared/learning.ts b/shared/learning.ts index f71ca84af0..27d0776da3 100644 --- a/shared/learning.ts +++ b/shared/learning.ts @@ -291,6 +291,11 @@ export type PersonalReview = { * always sends it; optional so a reader written before it still type-checks. */ approved?: number; + /** + * Why Learning can't run on this machine at all, in a sentence for the person (OpenBot Desktop signed in with a model + * Learning can't use). Absent when nothing is known to be wrong. + */ + modelProblem?: string; sources: (LearningSource & { proposals: LearningProposal[] | null; latestRun: LearningRunSummary | null; @@ -299,6 +304,15 @@ export type PersonalReview = { })[]; }; +/** + * The skills Learning proposed from one of the person's recordings, newest first: each proposal made since the + * recording stopped whose evidence cites a recording of theirs by that name. Rejected proposals and removals are left + * out. `container` is the Learning space it is in, for opening or reviewing it. + */ +export type RecordingSkills = { + skills: (LearningProposal & { container: string })[]; +}; + /** What an administrator reviews for each person, where people's Learning is reviewed by admins. */ export type PeopleProposals = { people: { From a0c68dcfcdc534c3489e637b6f3ea1bec3c5531a Mon Sep 17 00:00:00 2001 From: David McKay Date: Sun, 8 Nov 2026 20:05:44 -0800 Subject: [PATCH 09/34] Keep Install's bar short of full while items unpack, and say Give work to Bots on Welcome (#1332) --- CHANGELOG.md | 6 ++++ app/src/routes/_authed/onboarding.tsx | 2 +- app/tests/onboarding-route.test.tsx | 12 +++++++ desktop/src/SetupProgress.test.tsx | 47 +++++++++++++++++++++++++++ desktop/src/SetupProgress.tsx | 40 ++++++++++++++++++----- 5 files changed, 98 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebe412ae9b..2af670c45f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,10 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. Bot has no folders it can read right now. Folder access ends when you stop OpenBot. To give it a folder, use Choose folder in Settings → Computer." It said "No host folders are granted to this Bot." +- **Install's bar is full only when it is done.** Once every byte has arrived and the last items + are still unpacking, the bar moves without filling and the line says "1.1 GB downloaded · + Unpacking the last item". It showed a full bar beside "1.1 GB of 1.1 GB · 1 of 5 items + remaining". ### Teach a Bot @@ -173,6 +177,8 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. - LangChain is called **LangChain**, with LangChain's own mark, where it said "LangChain (LangGraph)": in New Bot, a Bot's "Powered by" subtitle and its Runs on row, and OpenBot Desktop's note under a ChatGPT plan. +- Onboarding's Welcome says "Give work to Bots in plain words.", where it said "Hand work to + Bots". ## 0.3.4 diff --git a/app/src/routes/_authed/onboarding.tsx b/app/src/routes/_authed/onboarding.tsx index 2ee7fec549..c07a5295cb 100644 --- a/app/src/routes/_authed/onboarding.tsx +++ b/app/src/routes/_authed/onboarding.tsx @@ -138,7 +138,7 @@ function welcomeStep(computer: boolean): OnboardingStep { } > -

Hand work to Bots in plain words. They tell you when it's done.

+

Give work to Bots in plain words. They tell you when it's done.

), demo: () => , diff --git a/app/tests/onboarding-route.test.tsx b/app/tests/onboarding-route.test.tsx index adf7677201..7c98a49a01 100644 --- a/app/tests/onboarding-route.test.tsx +++ b/app/tests/onboarding-route.test.tsx @@ -315,6 +315,18 @@ test("arriving without a step writes the first one into the address, in place", expect(view.history.length).toBe(1); }); +// "Hand work to Bots" was read with "Hand" as a noun. "Give" can only be the verb. +test("Welcome says to give work to Bots in plain words", async () => { + const view = open("/onboarding?step=welcome"); + await heading(view, "Welcome to OpenBot"); + expect( + view.getByText( + "Give work to Bots in plain words. They tell you when it's done.", + ), + ).toBeTruthy(); + expect(view.queryByText(/Hand work to Bots/)).toBeNull(); +}); + test("Continue adds a step to the address and to history, and saves it", async () => { const view = open("/onboarding?step=welcome"); await heading(view, "Welcome to OpenBot"); diff --git a/desktop/src/SetupProgress.test.tsx b/desktop/src/SetupProgress.test.tsx index 0240c9958a..93fab1833b 100644 --- a/desktop/src/SetupProgress.test.tsx +++ b/desktop/src/SetupProgress.test.tsx @@ -80,6 +80,53 @@ test("install progress is the bytes downloaded out of the whole download, and di expect(row.queryByRole("progressbar")).toBeNull(); }); +// Reported on 1.0.0-rc.1: a full bar read "1.1 GB of 1.1 GB · 1 of 5 items remaining" while the +// last image was still being unpacked, so a person took the step for done. +test("with every byte in and an item still unpacking, the bar is not full and says it is unpacking", () => { + const size = 1_100_000_000; + const view = render( + , + ); + const row = within( + view.getByRole("listitem", { name: "Installing OpenBot" }), + ); + const said = "1.1 GB downloaded · Unpacking the last item"; + let bar = row.getByRole("progressbar", { name: "Download progress" }); + // No value: the indeterminate bar, not a full one. + expect(bar.hasAttribute("value")).toBe(false); + expect(bar.getAttribute("aria-valuetext")).toBe(said); + expect(row.getByText(said)).toBeTruthy(); + expect(row.queryByText(/remaining/)).toBeNull(); + + view.rerender( + , + ); + bar = row.getByRole("progressbar", { name: "Download progress" }); + expect(bar.hasAttribute("value")).toBe(false); + expect( + row.getByText("1.1 GB downloaded · Unpacking the last 3 items"), + ).toBeTruthy(); + + // Bytes still arriving: the bar shows its fraction, as before. + view.rerender( + , + ); + bar = row.getByRole("progressbar", { name: "Download progress" }); + expect(bar.getAttribute("value")).toBe("1000000000"); + expect( + row.getByText("1.0 GB of 1.1 GB · 1 of 5 items remaining"), + ).toBeTruthy(); +}); + test("a download whose size is not known yet says so, and its bar does not guess", () => { const view = render( 0; + const known = + downloads.size != null && downloads.size > 0 && !unpacking(downloads); return (
{known ? ( @@ -281,13 +286,32 @@ function DownloadProgress({ downloads }: { downloads: Downloads }) { ); } -/** "1.2 GB of 3.4 GB · 1 of 5 items remaining", or what is known of it. */ -export function downloadSummary({ - bytes, - size, - done, - total, -}: Downloads): string { +/** + * Every byte is in, and some items are still being unpacked, imported or tagged. That brings no + * more bytes, so the bytes alone would say the step is done. + */ +function unpacking({ bytes, size, done, total }: Downloads): boolean { + return ( + size != null && + size > 0 && + bytes >= size && + total != null && + (done ?? 0) < total + ); +} + +/** + * "1.2 GB of 3.4 GB · 1 of 5 items remaining", or what is known of it. Once every byte is in: + * "3.4 GB downloaded · Unpacking the last item". + */ +export function downloadSummary(downloads: Downloads): string { + const { bytes, size, done, total } = downloads; + if (unpacking(downloads)) { + const remaining = (total ?? 0) - (done ?? 0); + return `${formatDownloadBytes(bytes)} downloaded · Unpacking ${ + remaining === 1 ? "the last item" : `the last ${remaining} items` + }`; + } const parts: string[] = []; if (size != null) { if (size > 0) From dfb0c9a7ce1b33184f6e26f92de714bafccb54c1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:28:50 -0800 Subject: [PATCH 10/34] Release v1.0.0-rc.2 (#1330) --- CHANGELOG.md | 105 +++++++++++++++++++++++++---------- charts/openbot/Chart.yaml | 2 +- desktop/src-tauri/Cargo.lock | 2 +- desktop/src-tauri/Cargo.toml | 2 +- package.json | 2 +- release-policy.json | 4 +- 6 files changed, 82 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2af670c45f..8d9d58ddd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,16 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased -**Before upgrading.** There are no migrations: the database stays at -`0083_learning_skill_approvals`. Four things change for an existing deployment: +**Before upgrading.** Two migrations run at start, and the database moves to +`0085_approval_rule_account`. `0084_brokered_accounts` creates a new, empty `brokered_accounts` +table, which fills as people open an app's page in Connections. `0085_approval_rule_account` adds +a nullable `account` column to `approval_rules`. No existing row is rewritten or backfilled, +nothing is dropped and no existing constraint changes. A person's existing connection to an app is +untouched, and calls through it run as before. Going back to 1.0.0-rc.1 or 0.3.4 after people +connect more than one account of an app is not seamless: a rule saved for one account applies to +every account of that app, and Disconnect can leave an extra account connected at Arcade, so +restore the backup taken before the upgrade instead. Four things change for an existing +deployment: - OpenBot Desktop updates its back end to CopilotKit Intelligence 0.16.0-rc.10 at the next Start, saying "Updating the back end to 0.16.0-rc.10. Its data stays." With it, a first back end setup downloads all of its images through downloads.copilotkit.ai first, where its own images came @@ -29,6 +37,34 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. of the subtitle, and a Bot picked in setup before 0.3.3 and named "LangChain (LangGraph)" is renamed "LangChain". A Bot whose connection you set yourself keeps its subtitle. +### Apps and connections + +- **Connect more than one account of an app.** Once an app is connected, its page in Connections, + and Admin → Plugins for an admin's own account, offers **Connect another account**, on Composio + and Arcade. Pressing Connect again used to be refused with "You already have an account connected + to Gmail. Disconnect it first if you want to connect a different one." With several accounts the + page lists each by its address, with **Make default** and a Disconnect that ends only that + account. An app whose team account is shared still has one account. +- **A Bot can use the account a request is about.** It is told the addresses of your accounts at + that app; with none named it uses your default, the first you connected until you pick another. + A name that isn't one of your accounts is refused, and nothing is sent. +- **An approval names the account** ("Research Bot wants to send email from … on Gmail"), and Always + allow here saves the rule for that account only, shown as "from" and the address under the rule. + Once you connect a second account of an app, your own rules that let Bots act on that app without + asking stop applying, so Bots ask again for each account. Rules that ask or hand off, and team + rules, still cover every account. +- **Apps that need a key say so before you connect** (on Composio). In Browse apps, an app you + connect with a key reads "Needs a key from 1Password", and apps you sign in to are listed first, + where the list was by name alone. Its Connections tile says the same until it is connected, and + its page says "Needs a key from 1Password. Connect shows where to get one.", where it said the app + "connects with a key you already have". +- **The key form says where the key comes from.** Above the boxes, Connect says "1Password doesn't + offer a sign-in here, so it connects with a key from your 1Password account.", what to do at the + app, and **Get a key from 1Password**, which opens the app's own page, for 33 of the most-used key + apps and any app whose listing publishes a link. Box labels read in plain words ("Access token" + for "Bearer Token", "Web address" for "Base URL"), and help text that named Composio says + OpenBot. + ### OpenBot Desktop - **A back end setup that stops says why.** OpenBot's log now records how every back end setup, @@ -36,6 +72,10 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. and the end of its error output. A setup that stopped used to leave nothing in the log. The card says whether the back end stopped with an error partway, quit without saying why, or could not be run, where all three said "The back end could not start." +- When OpenBot Desktop is signed in with a model Learning can't use, Learn now in Settings → + Learning says "Learning can't run on the model you signed in with. To use it, sign OpenBot in with + an OpenAI or Anthropic API key, or a Claude or ChatGPT plan.", where it told you to give the back + end's Learning agent a model and key. - **The first notification after OpenBot Desktop starts is raised**, for a Bot that needs you or replies while OpenBot is minimised or behind another window. The first one after each launch used to raise nothing, because the app was still checking that notifications were on. OpenBot's log @@ -46,6 +86,12 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. step says "Setting up your apps" while they connect, where it flashed "Sign in to CopilotKit". Opening OpenBot after a restart says "Starting the engine machine." while it starts OpenBot's Podman machine, and Start's steps say "OpenBot's services" where they said "Containers". +- **Install's bar is full only when it is done.** Once every byte has arrived and the last items are + still unpacking, the bar keeps moving without filling and the line says "1.1 GB downloaded · + Unpacking the last item", where it showed a full bar beside "1.1 GB of 1.1 GB · 1 of 5 items + remaining". +- The screen shown while OpenBot runs opens on "OpenBot is running", with its Updates section under + Show OpenBot and Stop OpenBot, as on the stopped screen, where it sat above the heading. - **App updates download from downloads.copilotkit.ai.** The update feed there names downloads.copilotkit.ai for the Windows installer and the Mac update archives, where they came from GitHub, and every signature is unchanged. If that download fails, OpenBot Desktop reads @@ -55,13 +101,17 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. Bot has no folders it can read right now. Folder access ends when you stop OpenBot. To give it a folder, use Choose folder in Settings → Computer." It said "No host folders are granted to this Bot." -- **Install's bar is full only when it is done.** Once every byte has arrived and the last items - are still unpacking, the bar moves without filling and the line says "1.1 GB downloaded · - Unpacking the last item". It showed a full bar beside "1.1 GB of 1.1 GB · 1 of 5 items - remaining". ### Teach a Bot +- **Teach a Bot says what happens after a recording saves**: "OpenBot learns a skill from it, and + you'll find it in Settings, Learning. Once it's learned, ask any Bot to do it." A row under the + recording follows it through Learning: **Learn now** before a run, "Learning from your + recordings" during one, "Ready to review" with **Review**, and "Skill learned" with **Try it**, + which opens Home with the task in the message box, not sent. A failed run, Learning with no + model, or a refused Learn now says why, and **See your skills** opens Settings → Learning. With + the page in the background and notifications allowed, a notification says when the skill + arrives. It used to say only that the recording was saved to your Learning space. - **Teach a Bot is one page.** Its title and one line on how sit above the browser, and Your recordings sits under it at the same width and scrolls on its own, so a long list no longer makes the browser smaller. The page was headed "Browser". Where the browser opens as a window on your @@ -73,10 +123,9 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. name, the Narration switch (its (i) says what narration is for) and **Start recording**, where the name field, the switch and Record sat in a bar above the browser. The narration line shows under the browser only while recording, or to say "No microphone found. Narration is off." on a - computer with none, which keeps your narration choice for when there is one. + computer with none. There, Record no longer asks "Narrate while you record?": it records without + narration and asks the first time you record with a microphone. - Your recordings links to Settings → Learning ("See what your Bots learned"). -- On a computer with no microphone, Record no longer asks "Narrate while you record?". It records - without narration and asks the first time you record with a microphone. ### Conversations @@ -109,16 +158,16 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. card, Needs you and a task's waiting line say "Save a file todo.txt in share", "Read a file" and "Run a command", where they said "Write file on todo.txt in share". A file on the Bot's own computer is named by its path, and its save step reads "Saved a file". +- **A Bot's Activity names an action waiting for your approval the way its card does**, "Click + the link “Python” on en.wikipedia.org" or "Send email on Gmail", where it read "Approve + computer_click" with "write" beside it. Approval cards list what an action would send as + labelled lines, where they showed raw JSON, and a click shows none. In Slack and Microsoft Teams + the card says it the same way ("Research Bot wants to click the link “Python” on + en.wikipedia.org"). A saved rule is titled "Click" or "Send email on Gmail" instead of its tool + id, and the line under it reads "changes in share for Shop", where it read "write on + host-grant-15407-… for shopper-bot". - An opened "Looked for a tool" step lists the tools the Bot found by name ("Send email on Gmail"), where it showed their ids and the descriptions the search returned to the Bot. -- A browser action waiting for your approval is named the way its card names it. A Bot's Activity - row read "Approve computer_click" with "write" beside it, and now reads "Click the link “Python” - on en.wikipedia.org". The Needs you card names the link too and lists what the action would send - by label, without the page handles a click carried as raw JSON. An approval in Slack says it the - same way ("Research Bot wants to click the link “Python” on en.wikipedia.org"), and a saved rule - is titled "Click" or "Send email on Gmail" instead of its tool id. The line under it names the - folder and the Bot by name and the kind of action in words ("changes in share for Shop"), where - it read "write on host-grant-15407-… for shopper-bot". ### Personal edition @@ -136,11 +185,6 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ### Smaller changes -- Settings > Activity and Admin > Audit reach past the newest 50 events: "Show older events" under - the last row loads the next page, and the count says "Newest 50 events" until nothing is older. -- "OpenBot is running" opens on its heading again. Its Updates disclosure, and the line above it, - sit under Show OpenBot and Stop OpenBot, as on the stopped screen, instead of at the top of the - card with nothing above the line. - The sidebar floats as frosted glass over the colours of OpenBot's orb, the orb sits beside OpenBot's name, and the app is set in Atkinson Hyperlegible Next. The glass turns off with the system's reduce transparency or increase contrast setting, and a skin keeps its own look. @@ -169,16 +213,19 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. is read out by screen readers. - **New Bot** lists the frameworks a custom agent can run on under **Connect a custom agent** on its first screen, below the form, one line each with its logo, Claude Agent SDK, LangChain, Google ADK - and AWS Strands first. Choosing one goes straight to its address, and Back returns to the list. - They were behind a Connect a custom agent button. -- In an 800px-tall window, OpenBot Desktop's usual height on a Mac, New Bot's first screen shows all - four of those frameworks above Cancel and Create Bot before you scroll, where only Claude Agent - SDK did. + and AWS Strands first, all four above Cancel and Create Bot in OpenBot Desktop's default window, + 800px tall. Choosing one goes straight to its address, and Back returns to the list. They were + behind a Connect a custom agent button. - LangChain is called **LangChain**, with LangChain's own mark, where it said "LangChain (LangGraph)": in New Bot, a Bot's "Powered by" subtitle and its Runs on row, and OpenBot Desktop's note under a ChatGPT plan. -- Onboarding's Welcome says "Give work to Bots in plain words.", where it said "Hand work to - Bots". +- Settings → Activity and Admin → Audit reach past the newest 50 events. **Show older events** under + the last row loads the next page, and the count reads "Newest 50 events", growing as more load, + until nothing older is left. They stopped at the newest 50 and counted them as if that were + everything. +- Apps the broker lists in lowercase are named with capitals in Browse apps and when added, such as + "1Password" where it said "1password". An app added before the upgrade keeps its name. +- Onboarding's Welcome says "Give work to Bots in plain words.", where it said "Hand work to Bots". ## 0.3.4 diff --git a/charts/openbot/Chart.yaml b/charts/openbot/Chart.yaml index 9de41f949e..752b571ae7 100644 --- a/charts/openbot/Chart.yaml +++ b/charts/openbot/Chart.yaml @@ -6,7 +6,7 @@ type: application version: 0.2.0 # The OpenBot release this chart's default image tag points at. Kept equal to package.json's version # (the last version published) by the release workflow, and CI refuses a tree where they have drifted. -appVersion: "1.0.0-rc.1" +appVersion: "1.0.0-rc.2" home: https://github.com/CopilotKit/OpenBot sources: - https://github.com/CopilotKit/OpenBot diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 5dd9fae457..7b24faeee3 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -2746,7 +2746,7 @@ dependencies = [ [[package]] name = "openbot-desktop" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" dependencies = [ "base64 0.22.1", "block2", diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 65097cd8fe..3e3178f7eb 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "openbot-desktop" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" description = "OpenBot Desktop" edition = "2021" rust-version = "1.90" diff --git a/package.json b/package.json index b8e37624d7..eccb1a9e80 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openbot", - "version": "1.0.0-rc.1", + "version": "1.0.0-rc.2", "license": "MIT", "private": true, "packageManager": "bun@1.4.3", diff --git a/release-policy.json b/release-policy.json index 981f8389a5..a801ff9f84 100644 --- a/release-policy.json +++ b/release-policy.json @@ -1,6 +1,6 @@ { "$schema": "./shared/release-policy.schema.json", - "version": "v1.0.0-rc.1", + "version": "v1.0.0-rc.2", "compatibility": { "launcherApi": 1, "configSchema": 1, @@ -11,7 +11,7 @@ "0082_routine_goal", "0077_channel_opening" ], - "to": "0083_learning_skill_approvals", + "to": "0085_approval_rule_account", "rollbackCompatible": false }, "intelligenceCapabilities": [] From dea620fcde895e09f6fdd8e7efc969ef80bc878a Mon Sep 17 00:00:00 2001 From: David McKay Date: Sun, 8 Nov 2026 20:51:55 -0800 Subject: [PATCH 11/34] Ask the Google Maps starter for step-by-step walking directions (#1336) --- CHANGELOG.md | 2 ++ app/src/components/starter-tour/suggestions.ts | 2 +- app/tests/starter-tour.test.tsx | 2 +- docs/testing/first-use-tour.md | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d9d58ddd9..3bf12fb2a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -226,6 +226,8 @@ deployment: - Apps the broker lists in lowercase are named with capitals in Browse apps and when added, such as "1Password" where it said "1password". An app added before the upgrade keeps its name. - Onboarding's Welcome says "Give work to Bots in plain words.", where it said "Hand work to Bots". +- The "Browse Google Maps" starter task asks the Bot to list the walking directions from King's + Cross step by step, where a Bot could reply with only the walking time and distance. ## 0.3.4 diff --git a/app/src/components/starter-tour/suggestions.ts b/app/src/components/starter-tour/suggestions.ts index b86295559f..5d70f7e6bc 100644 --- a/app/src/components/starter-tour/suggestions.ts +++ b/app/src/components/starter-tour/suggestions.ts @@ -26,7 +26,7 @@ export const STARTER_SUGGESTIONS: CopilotChatSuggestionViewProps["suggestions"] { title: "Browse Google Maps", message: - "Use your browser to open Google Maps and check the British Museum's Saturday opening hours. Then show walking directions from King's Cross station and tell me when to leave to arrive at opening time. Don't sign in or change anything.", + "Use your browser to open Google Maps and check the British Museum's Saturday opening hours. Then get walking directions from King's Cross station, list them step by step, and tell me when to leave to arrive at opening time. Don't sign in or change anything.", isLoading: false, }, { diff --git a/app/tests/starter-tour.test.tsx b/app/tests/starter-tour.test.tsx index b9e3683ca5..e3ed130071 100644 --- a/app/tests/starter-tour.test.tsx +++ b/app/tests/starter-tour.test.tsx @@ -159,7 +159,7 @@ test("the starters tour search with a chart, the Bot's browser, Gmail and an int { title: "Browse Google Maps", message: - "Use your browser to open Google Maps and check the British Museum's Saturday opening hours. Then show walking directions from King's Cross station and tell me when to leave to arrive at opening time. Don't sign in or change anything.", + "Use your browser to open Google Maps and check the British Museum's Saturday opening hours. Then get walking directions from King's Cross station, list them step by step, and tell me when to leave to arrive at opening time. Don't sign in or change anything.", }, { title: "Triage my inbox", diff --git a/docs/testing/first-use-tour.md b/docs/testing/first-use-tour.md index 7aef4ebf8c..159e370c9e 100644 --- a/docs/testing/first-use-tour.md +++ b/docs/testing/first-use-tour.md @@ -39,7 +39,7 @@ commit, backend release/image and OS for each result; the desktop shell can run | Suggestion | Result that proves useful work | Mac candidate | Windows x64 candidate | | --- | --- | --- | --- | | Chart Hacker News | A bar chart of the current Hacker News front page by points, then the same stories listed with points and links. Compare both with the live front page. Inspect the activity log for web search or fetch tools. | Not run | Not run | -| Browse Google Maps | Watch the Bot open Google Maps in its browser, read the British Museum's Saturday opening hours, open walking directions from King's Cross, and give a leave-by time that matches the hours and walking time Maps shows. Nothing is signed in to or changed. | Not run | Not run | +| Browse Google Maps | Watch the Bot open Google Maps in its browser, read the British Museum's Saturday opening hours, open walking directions from King's Cross, list the route step by step, and give a leave-by time that matches the hours and walking time Maps shows. Nothing is signed in to or changed. | Not run | Not run | | Triage my inbox | Gmail's connect card appears in the conversation before any mail is read. Connect, finish consent, return to the same thread, and check the two emails it picks against the last 24 hours of mail. An empty day must be reported accurately. Test cancellation and retry. Nothing is marked read or changed. | Not run | Not run | | Build a trip budget | Generated controls start at $100 + $200 + $80 = $380. Change lodging to $250 and observe $430 without another model request. | Not run | Not run | From 6f71a6d7536fff5b63a58909c3b83fb345ec5a24 Mon Sep 17 00:00:00 2001 From: David McKay Date: Sun, 8 Nov 2026 20:52:34 -0800 Subject: [PATCH 12/34] Book a demo: only Not now under the calendar until a call is booked (#1333) --- CHANGELOG.md | 3 + .../onboarding/onboarding-frame.tsx | 67 ++++++++++--------- .../onboarding/onboarding-wizard.tsx | 31 +++++---- app/src/routes/_authed/onboarding.tsx | 20 ++++-- app/tests/onboarding-route.test.tsx | 23 +++++-- 5 files changed, 90 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bf12fb2a1..3ff3e9b4f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -226,6 +226,9 @@ deployment: - Apps the broker lists in lowercase are named with capitals in Browse apps and when added, such as "1Password" where it said "1password". An app added before the upgrade keeps its name. - Onboarding's Welcome says "Give work to Bots in plain words.", where it said "Hand work to Bots". +- Onboarding's Book a demo step has one button under the calendar, **Not now**, which moves on + without booking. It had Continue beside Not now, which read as going on with the booking. Once + you book a time in the calendar, Not now becomes **Continue**. - The "Browse Google Maps" starter task asks the Bot to list the walking directions from King's Cross step by step, where a Bot could reply with only the walking time and distance. diff --git a/app/src/components/onboarding/onboarding-frame.tsx b/app/src/components/onboarding/onboarding-frame.tsx index 3cf5ff9c4f..edfcd6c540 100644 --- a/app/src/components/onboarding/onboarding-frame.tsx +++ b/app/src/components/onboarding/onboarding-frame.tsx @@ -41,7 +41,9 @@ import { cn } from "@/lib/utils"; * * AN OPTIONAL STEP HAS A SECOND BUTTON. "Not now" goes beside the primary (under it on a phone, the * same size and shape, outlined), and its place is kept when it goes (`reserveSecondary`), so the - * primary does not move mid-step. A footnote, where a step has one, sits just above the buttons. + * primary does not move mid-step. A step can also have no primary at all, and then "Not now" is + * the only way on: it takes the primary's place and keeps its outline on a desktop too, so it never + * reads as a link. A footnote, where a step has one, sits just above the buttons. */ export function OnboardingFrame({ titles, @@ -66,7 +68,8 @@ export function OnboardingFrame({ reached: number; onBack: () => void; onPick: (index: number) => void; - primary: { + /** The way forward. Absent on a step whose only way on is the second button. */ + primary?: { label: string; onClick: () => void; disabled?: boolean; @@ -201,39 +204,43 @@ export function OnboardingFrame({ Back ) : null} - + {primary ? ( + + ) : null} {secondary ? ( +
+ `; + +async function call>( + path: string, + body?: unknown, + signal?: AbortSignal, +): Promise<{ status: number; json: T }> { + const response = await fetch(`${base}${path}`, { + method: body === undefined ? "GET" : "POST", + headers: { + "content-type": "application/json", + "x-openbot-computer-token": TOKEN, + "x-openbot-bot-id": BOT, + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + ...(signal ? { signal } : {}), + }); + return { status: response.status, json: (await response.json()) as T }; +} + +type Snapshot = { + snapshotId: number; + elements: Array<{ ref: string; role: string; name: string; value?: string }>; +}; + +async function openForm(): Promise { + expect((await call("/navigate", { url: `${origin}/new` })).status).toBe(200); + const { status, json } = await call("/snapshot", {}); + expect(status).toBe(200); + return json; +} + +function refOf(snap: Snapshot, name: string): string { + const element = snap.elements.find((each) => each.name === name); + if (!element) throw new Error(`No ${name} on the form.`); + return element.ref; +} + +async function values(): Promise> { + const { json } = await call("/snapshot", {}); + return Object.fromEntries( + FIELDS.map((name) => [ + name, + json.elements.find((each) => each.name === name)?.value, + ]), + ); +} + +beforeAll(async () => { + if (!asked) return; + root = await mkdtemp(join(tmpdir(), "parallel-actions-")); + site = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: (request) => + new URL(request.url).pathname === "/new" + ? new Response(FORM, { headers: { "content-type": "text/html" } }) + : new Response("Not found", { status: 404 }), + }); + origin = `http://127.0.0.1:${site.port}`; + const port = await reservePort(); + base = `http://127.0.0.1:${port}`; + child = Bun.spawn([process.execPath, "src/index.ts"], { + cwd: join(import.meta.dir, ".."), + env: { + ...process.env, + COMPUTER_TOKEN: TOKEN, + PORT: String(port), + COMPUTER_BROWSER_BACKEND: "managed", + COMPUTER_BROWSER_MODE: "headless", + ACTION_TIMEOUT_MS: String(ACTION_TIMEOUT_MS), + // No API server pushes this computer a network policy, so it browses before one arrives. + EGRESS_POLICY_REQUIRED: "0", + PROFILES_DIR: join(root, "profiles"), + WORKSPACE_DIR: join(root, "workspace"), + }, + stdout: "ignore", + stderr: "inherit", + }); + const deadline = Date.now() + 15_000; + for (;;) { + if (child.exitCode !== null) + throw new Error(`The computer exited with ${child.exitCode}.`); + const ready = await fetch(`${base}/health`) + .then((response) => response.ok) + .catch(() => false); + if (ready) break; + if (Date.now() > deadline) throw new Error("The computer did not start."); + await Bun.sleep(50); + } +}); + +afterAll(async () => { + if (!asked) return; + child?.kill(); + if (child) await child.exited; + await site?.stop(true); + await rm(root, { recursive: true, force: true }); +}); + +describe.skipIf(!asked)("actions sent together, on one browser", () => { + test("a field that shows a placeholder reads back what was typed in it", async () => { + // One at a time, so only the reading is under test. Read as empty, a Bot checking its work typed the value again. + const snap = await openForm(); + for (const name of ["Website", "Employees"]) { + const typed = await call("/type", { + ref: refOf(snap, name), + snapshotId: snap.snapshotId, + text: name === "Website" ? "halvorsen.com" : "2,400", + }); + expect(typed.status).toBe(200); + } + const read = await values(); + expect([read.Website, read.Employees, read.Headquarters]).toEqual([ + "halvorsen.com", + "2,400", + undefined, + ]); + }); + + test("a form's worth of typing sent at once keeps every value in its own field", async () => { + // Long enough that typing one value takes several frames, as a real address or a note does, and short enough + // that a snapshot reads it back whole (it cuts a value at 200 characters). + const wanted = Object.fromEntries( + FIELDS.map((name, index) => [ + name, + `${name} value ${index} `.repeat(6).trim(), + ]), + ); + // Several rounds, because a race that loses a value only sometimes is still the bug. + for (let round = 0; round < 3; round += 1) { + const snap = await openForm(); + const sent = await Promise.all( + FIELDS.map((name) => + call("/type", { + ref: refOf(snap, name), + snapshotId: snap.snapshotId, + text: wanted[name], + }), + ), + ); + expect(sent.map((each) => each.status)).toEqual(FIELDS.map(() => 200)); + expect(await values()).toEqual(wanted); + } + }); + + test("a snapshot sent beside typing reads the typing that was sent before it", async () => { + const snap = await openForm(); + const [typed, read] = await Promise.all([ + call("/type", { + ref: refOf(snap, "Notes"), + snapshotId: snap.snapshotId, + text: "Met at the expo. ".repeat(10), + }), + // Sent a moment later, as the next call in the same step is. + Bun.sleep(5).then(() => call<{ elements: unknown[] }>("/snapshot", {})), + ]); + expect(typed.status).toBe(200); + expect(read.status).toBe(200); + expect((await values()).Notes).toBe("Met at the expo. ".repeat(10).trim()); + }); + + test("an action that never finishes gives way to the ones behind it", async () => { + const snap = await openForm(); + // The Save button sits under a transparent overlay, so a click on it waits out the whole action timeout. + const started = Date.now(); + const [stuck, typed] = await Promise.all([ + call<{ error?: string }>("/click", { + ref: refOf(snap, "Save"), + snapshotId: snap.snapshotId, + }), + Bun.sleep(5).then(() => + call("/type", { + ref: refOf(snap, "Website"), + snapshotId: snap.snapshotId, + text: "example.com", + }), + ), + ]); + expect(stuck.status).not.toBe(200); + expect(typed.status).toBe(200); + expect((await values()).Website).toBe("example.com"); + // One action timeout and the typing, not a queue held open indefinitely. + expect(Date.now() - started).toBeLessThan(ACTION_TIMEOUT_MS * 3); + }); + + test("an action stopped while it waits its turn never runs", async () => { + const snap = await openForm(); + const stop = new AbortController(); + const stuck = call("/click", { + ref: refOf(snap, "Save"), + snapshotId: snap.snapshotId, + }); + await Bun.sleep(50); + const waiting = call( + "/type", + { + ref: refOf(snap, "Source"), + snapshotId: snap.snapshotId, + text: "Referral", + }, + stop.signal, + ).catch(() => "stopped"); + await Bun.sleep(50); + stop.abort(); + expect(await waiting).toBe("stopped"); + expect((await stuck).status).not.toBe(200); + expect((await values()).Source).toBeUndefined(); + }); +}); diff --git a/agent-langgraph-agui/src/tool_runtime.py b/agent-langgraph-agui/src/tool_runtime.py index 40d595312f..c4bbc64377 100644 --- a/agent-langgraph-agui/src/tool_runtime.py +++ b/agent-langgraph-agui/src/tool_runtime.py @@ -491,7 +491,27 @@ async def execute_tools(state): raise ValueError( "This turn must return to the surface before tools can execute." ) - results = await asyncio.gather( - *[_call_tool(call, context) for call in state["messages"][-1].tool_calls] + calls = state["messages"][-1].tool_calls + # A Bot has one browser. Its computer calls go one after another, in the order the model made + # them: a form's fields sent together took the focus from each other, and a click on Save could + # overtake the typing before it. Everything else still runs alongside them. + computer = [call for call in calls if call["name"].startswith("computer_")] + + async def in_order(): + return [await _call_tool(call, context) for call in computer] + + ordered, *others = await asyncio.gather( + in_order(), + *[ + _call_tool(call, context) + for call in calls + if not call["name"].startswith("computer_") + ], ) + answered = iter(ordered) + elsewhere = iter(others) + results = [ + next(answered if call["name"].startswith("computer_") else elsewhere) + for call in calls + ] return {"messages": results} diff --git a/agent-langgraph-agui/tests/test_tool_protocol.py b/agent-langgraph-agui/tests/test_tool_protocol.py index 59ae4dde0d..2a1bb6f050 100644 --- a/agent-langgraph-agui/tests/test_tool_protocol.py +++ b/agent-langgraph-agui/tests/test_tool_protocol.py @@ -7,9 +7,10 @@ import asyncio import json import sys +import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from threading import Thread +from threading import Lock, Thread from uuid import uuid4 import httpx @@ -585,6 +586,48 @@ async def test_http_refusal_tells_the_model_the_deployments_reason( assert boundary["model"][-1]["messages"][-1]["content"] == expected +@pytest.mark.asyncio +async def test_computer_calls_made_together_run_one_at_a_time_in_order(boundary): + # A model that calls tools in parallel sends a form's fields in one step. A Bot has one browser, + # and side by side the fields took the focus from each other; a click could overtake the typing. + names = ["computer_type", "computer_click", "computer_key", "granted_lookup"] + computer = names[:3] + running = {"now": 0, "most": 0} + spans = {} + lock = Lock() + + def authorize(body, _token): + started = time.monotonic() + with lock: + if body["name"] in computer: + running["now"] += 1 + running["most"] = max(running["most"], running["now"]) + # Long enough that calls sent together would overlap. + time.sleep(0.15) + with lock: + if body["name"] in computer: + running["now"] -= 1 + spans[body["name"]] = (started, time.monotonic()) + return 200 + + boundary["callback_authorize"] = authorize + events = await run_protocol(run_input(names, deployment=names)) + called = [each["body"]["name"] for each in boundary["callback"]] + assert [name for name in called if name in computer] == computer + assert running["most"] == 1 + # Everything else still runs beside them: the lookup overlaps the computer's calls rather than waiting for them. + lookup_start, lookup_end = spans["granted_lookup"] + assert any( + start < lookup_end and lookup_start < end + for name, (start, end) in spans.items() + if name in computer + ) + # The model reads each answer against the call it made, in the order it made them. + assert [ + m["toolCallId"] for m in snapshot(events) if m.get("role") == "tool" + ] == ["call-" + name for name in names] + + @pytest.mark.asyncio async def test_next_request_does_not_inherit_old_tool_offer(boundary): body = run_input(["computer_navigate"]) diff --git a/agent-langgraph/src/browser-order.ts b/agent-langgraph/src/browser-order.ts new file mode 100644 index 0000000000..fd32ecda56 --- /dev/null +++ b/agent-langgraph/src/browser-order.ts @@ -0,0 +1,24 @@ +/** + * Answer one step's tool calls: the computer's one after another, in the order the model made them, and everything + * else alongside. + * + * A Bot has one browser. A model that calls tools in parallel sends a form's fields in one step, and side by side they + * took the focus from each other; a click on Save could overtake the typing before it. The computer also takes its + * actions one at a time, but only in the order they reach it, which calls sent together do not promise. + * + * The answers come back in the order of the calls, whichever finished first. + */ +export async function answerInOrder( + calls: readonly Call[], + answer: (call: Call) => Promise, +): Promise { + let browser: Promise = Promise.resolve(); + return Promise.all( + calls.map((call) => { + if (!call.name.startsWith("computer_")) return answer(call); + const turn = browser.then(() => answer(call)); + browser = turn.catch(() => undefined); + return turn; + }), + ); +} diff --git a/agent-langgraph/src/index.ts b/agent-langgraph/src/index.ts index 945cf97775..43d775d4d0 100644 --- a/agent-langgraph/src/index.ts +++ b/agent-langgraph/src/index.ts @@ -21,6 +21,7 @@ import { keyVariableFor, requiresResponsesApi, } from "../../shared/model-providers"; +import { answerInOrder } from "./browser-order"; import { toLangChainMessages } from "./history"; import { readReasoningEffort } from "./model-options"; import { streamRun } from "./stream"; @@ -342,26 +343,26 @@ function buildGraph(input: RunAgentInput) { })) .addNode("tools", async (state) => { const last = state.messages.at(-1) as AIMessage; - const results = await Promise.all( + // The computer's calls one after another, in the order the model made them (browser-order.ts). + const results = await answerInOrder( /* * Only this deployment's own tools. A component is drawn by the surface, and a decision is * answered there by a person, so neither is executed here and neither gets a result invented * here. The run ends instead, and the surface starts the next one carrying what it produced. */ - (last.tool_calls ?? []) - .filter((call) => ours.has(call.name)) - .map(async (call) => { - const text = await callTool( - run, - call.name, - (call.args ?? {}) as Record, - ); - return new ToolMessage({ - content: text, - tool_call_id: call.id ?? call.name, - name: call.name, - }); - }), + (last.tool_calls ?? []).filter((call) => ours.has(call.name)), + async (call) => { + const text = await callTool( + run, + call.name, + (call.args ?? {}) as Record, + ); + return new ToolMessage({ + content: text, + tool_call_id: call.id ?? call.name, + name: call.name, + }); + }, ); return { messages: results }; }) diff --git a/agent-langgraph/tests/browser-order.test.ts b/agent-langgraph/tests/browser-order.test.ts new file mode 100644 index 0000000000..6d736cd349 --- /dev/null +++ b/agent-langgraph/tests/browser-order.test.ts @@ -0,0 +1,39 @@ +import { expect, test } from "bun:test"; +import { answerInOrder } from "../src/browser-order"; + +/** + * One step's tool calls, as a model that calls tools in parallel sends a form: a Bot has one browser, so its computer + * calls run one after another in the order they were made, while anything else runs beside them. + */ +test("computer calls made together run one at a time, in order, and the rest alongside", async () => { + const calls = [ + { name: "computer_type", id: "name" }, + { name: "computer_type", id: "website" }, + { name: "search_web", id: "lookup" }, + { name: "computer_click", id: "save" }, + ]; + const started: string[] = []; + let running = 0; + let most = 0; + let lookupBeside = false; + const answers = await answerInOrder(calls, async (call) => { + started.push(call.id); + const computer = call.name.startsWith("computer_"); + if (computer) most = Math.max(most, ++running); + // Long enough that calls sent together would overlap; the first is the slowest, so nothing can finish ahead of it + // by chance. + await Bun.sleep(call.id === "name" ? 60 : 20); + if (computer) running -= 1; + // The lookup finishes while the first field is still being typed. + else lookupBeside = running > 0; + return `answered ${call.id}`; + }); + expect(most).toBe(1); + expect(started.filter((id) => id !== "lookup")).toEqual([ + "name", + "website", + "save", + ]); + expect(lookupBeside).toBe(true); + expect(answers).toEqual(calls.map((call) => `answered ${call.id}`)); +}); diff --git a/app/package.json b/app/package.json index 79d9fcf96b..784454e3ef 100644 --- a/app/package.json +++ b/app/package.json @@ -18,7 +18,6 @@ "@base-ui/react": "^1.6.0", "@better-auth/sso": "^1.7.1", "@copilotkit/react-core": "1.85.0", - "@fontsource-variable/atkinson-hyperlegible-next": "5.3.0", "@fontsource-variable/inter": "^5.3.0", "@openbot/shared": "workspace:*", "@shadcn/react": "^0.3.0", diff --git a/app/src/components/admin/admin-sidebar.tsx b/app/src/components/admin/admin-sidebar.tsx index 498f395ca2..526887c4b7 100644 --- a/app/src/components/admin/admin-sidebar.tsx +++ b/app/src/components/admin/admin-sidebar.tsx @@ -15,6 +15,7 @@ import { IconShieldCheck, IconSlash, IconUsers, + IconZoomQuestion, } from "@tabler/icons-react"; import { Link, type LinkOptions } from "@tanstack/react-router"; import type * as React from "react"; @@ -142,6 +143,11 @@ const GROUPS: { icon: IconListDetails, linkOptions: { to: "/admin/audit" }, }, + { + title: "What would happen", + icon: IconZoomQuestion, + linkOptions: { to: "/admin/what-if" }, + }, ], }, ]; diff --git a/app/src/components/admin/controls/capabilities-section.tsx b/app/src/components/admin/controls/capabilities-section.tsx index df6f704adf..a0a848ec43 100644 --- a/app/src/components/admin/controls/capabilities-section.tsx +++ b/app/src/components/admin/controls/capabilities-section.tsx @@ -1,9 +1,11 @@ import { IconChevronRight, + IconSparkles, IconToggleRight, IconUsersGroup, } from "@tabler/icons-react"; import { useMutation } from "@tanstack/react-query"; +import { Link } from "@tanstack/react-router"; import { useState } from "react"; import { PageRows } from "@/components/layout/page-shell"; import { Button } from "@/components/ui/button"; @@ -113,6 +115,24 @@ export function CapabilitiesSection() { + + } + size="sm" + > + + + + + Describe it in words + + Say who may use what, and your model writes the changes. + + + + + + = { allow_all: "Allow all network access", @@ -57,7 +69,12 @@ const MODE_NAMES: Record = { allowlist_only: "Team allowlist only", }; -/** One rule per line: `example.com`, `*.corp.example`, or `10.0.0.0/8 443,8443`. */ +/** + * One rule per line: `example.com`, `*.corp.example`, `intranet.example 8443` or `10.0.0.0/8 443,8443`. + * + * A domain keeps its ports, as an IP range always has. It used to drop them, so opening a policy that + * held `intranet.example 8443` and pressing Save widened it to every port without a word. + */ function rulesFromText(text: string): EgressRule[] { return text .split("\n") @@ -65,21 +82,19 @@ function rulesFromText(text: string): EgressRule[] { .filter(Boolean) .map((line) => { const [value = "", ports] = line.split(/\s+/, 2); - return /^[\d.:a-fA-F]+(\/\d+)?$/.test(value) && + const type = + /^[\d.:a-fA-F]+(\/\d+)?$/.test(value) && /[\d]/.test(value) && /[.:]/.test(value) - ? { type: "cidr" as const, value, ...(ports ? { ports } : {}) } - : { type: "domain" as const, value }; + ? ("cidr" as const) + : ("domain" as const); + return { type, value, ...(ports ? { ports } : {}) }; }); } function rulesToText(rules: EgressRule[]): string { return rules - .map((rule) => - rule.type === "domain" - ? rule.value - : `${rule.value}${rule.ports ? ` ${rule.ports}` : ""}`, - ) + .map((rule) => `${rule.value}${rule.ports ? ` ${rule.ports}` : ""}`) .join("\n"); } @@ -287,11 +302,42 @@ function NetworkDialog({ ); const [rules, setRules] = useState(rulesToText(existing?.rules ?? [])); const [locked, setLocked] = useState(existing?.locked ?? false); + const [written, setWritten] = useState(null); const save = useMutation(setNetworkPolicyMutationOptions(queryClient)); const remove = useMutation(removeNetworkPolicyMutationOptions(queryClient)); const scopeId = initial.scopeKind === "group" ? group : ""; const formId = useId(); + const entries = rulesFromText(rules); + const edited = + written !== null && + (mode !== written.mode || + JSON.stringify(entries) !== JSON.stringify(written.rules)); + /* The model's list, unedited, that didn't parse. Its problems are already on screen, under it. */ + const broken = written !== null && !written.check.ok && !edited; + /* + * The policy as the dialog holds it, replayed once it differs from the one in force. Each part of + * the scope matters: a group's policy covers its members' Bots, and a locked organization policy + * covers everybody's. + */ + const unchanged = + existing !== undefined && + existing.mode === mode && + existing.locked === locked && + JSON.stringify(existing.rules) === JSON.stringify(entries); + const candidate = + unchanged || broken || (initial.scopeKind === "group" && !scopeId) + ? null + : { scopeKind: initial.scopeKind, scopeId, mode, rules: entries, locked }; + + /* The model's list goes into the Allowlist field and its mode into Mode, where the admin reads both. */ + const takeWritten = (next: WrittenAllowlist) => { + save.reset(); + setWritten(next); + setMode(next.mode); + setRules(rulesToText(next.rules)); + }; + return ( (open ? undefined : onClose())} open> @@ -306,96 +352,113 @@ function NetworkDialog({ {/* The submit button is in the footer and reaches this form by id, so DialogBody stays a direct child of DialogContent and keeps scrolling. */}
{ event.preventDefault(); + if (broken) return; save.mutate( { scopeKind: initial.scopeKind, scopeId, mode, - rules: rulesFromText(rules), + rules: entries, locked, + ...(written + ? { + authoring: { + source: "english" as const, + text: written.text, + }, + } + : {}), }, { onSuccess: onClose }, ); }} > - {initial.scopeKind === "group" && !initial.scopeId ? ( -
- + + {initial.scopeKind === "group" && !initial.scopeId ? ( + + Group + + + ) : null} + + + Mode -
- ) : null} -
- - -
-
- -