This repository has no description
1// The real engine. Cut the recording into 1-frame-per-second screenshots (with the
2// timestamp burned into each), hand them + the app's source code to Claude under a
3// single system prompt, and let the model itself find the UX bugs, locate them in
4// the source, and write the fix (as the full corrected file). No spoon-feeding.
5
6import { execFile } from "node:child_process";
7import { existsSync, statSync } from "node:fs";
8import { mkdir, readFile, readdir } from "node:fs/promises";
9import { resolve } from "node:path";
10import { promisify } from "node:util";
11import Anthropic from "@anthropic-ai/sdk";
12import type { Config } from "./config.js";
13import type { Finding, WebhookEvent } from "./types.js";
14
15const exec = promisify(execFile);
16
17const MAC_FONTS = [
18 "/System/Library/Fonts/Supplemental/Arial.ttf",
19 "/System/Library/Fonts/Helvetica.ttc",
20 "/Library/Fonts/Arial.ttf",
21];
22
23const CODE_EXT = /\.(html|htm|js|ts|jsx|tsx|css|svelte|vue|go|py|rb|templ|gohtml|tmpl)$/i;
24const MAX_SOURCE_BYTES = 256 * 1024; // bound the source we feed so we don't blow context
25
26function mmss(sec: number): string {
27 return `${Math.floor(sec / 60)}:${String(Math.floor(sec % 60)).padStart(2, "0")}`;
28}
29
30/** Sweep the whole recording at 1 fps → downscaled PNGs with the timestamp drawn on. */
31async function sweepFrames(videoPath: string, outDir: string): Promise<{ path: string; at: string }[]> {
32 await mkdir(outDir, { recursive: true });
33 const font = MAC_FONTS.find(existsSync);
34 const draw = font
35 ? `,drawtext=fontfile='${font}':text='%{pts\\:hms}':x=12:y=12:fontsize=30:fontcolor=yellow:box=1:boxcolor=black@0.65:boxborderw=8`
36 : "";
37 const pattern = resolve(outDir, "f_%04d.png");
38 try {
39 await exec("ffmpeg", ["-y", "-i", videoPath, "-vf", `fps=1,scale='min(1280,iw)':-2${draw}`, pattern, "-loglevel", "error"]);
40 } catch {
41 await exec("ffmpeg", ["-y", "-i", videoPath, "-vf", `fps=1,scale='min(1280,iw)':-2`, pattern, "-loglevel", "error"]);
42 }
43 const files = (await readdir(outDir)).filter((f) => f.startsWith("f_") && f.endsWith(".png")).sort();
44 // fps=1 emits frames at t = 0,1,2,… so frame index i (0-based) ≈ i seconds.
45 return files.map((f, i) => ({ path: resolve(outDir, f), at: mmss(i) }));
46}
47
48/** Read the configured source paths (files or dirs), bounded by total bytes. */
49async function readSource(cfg: Config, log: (m: string) => void): Promise<{ rel: string; code: string }[]> {
50 const out: { rel: string; code: string }[] = [];
51 let budget = MAX_SOURCE_BYTES;
52 const addFile = async (rel: string) => {
53 if (budget <= 0) return;
54 const abs = resolve(cfg.repoRoot, rel);
55 try {
56 const code = await readFile(abs, "utf8");
57 if (code.length > budget) return;
58 budget -= code.length;
59 out.push({ rel, code });
60 } catch {
61 /* skip unreadable */
62 }
63 };
64 const walk = async (relDir: string) => {
65 let entries;
66 try {
67 entries = await readdir(resolve(cfg.repoRoot, relDir), { withFileTypes: true });
68 } catch {
69 return;
70 }
71 for (const e of entries) {
72 const rel = relDir ? `${relDir}/${e.name}` : e.name;
73 if (e.isDirectory()) await walk(rel);
74 else if (CODE_EXT.test(e.name)) await addFile(rel);
75 }
76 };
77 for (const p of cfg.sourcePaths) {
78 const abs = resolve(cfg.repoRoot, p);
79 if (existsSync(abs) && statSync(abs).isDirectory()) await walk(p);
80 else await addFile(p);
81 }
82 log(`source: ${out.length} file(s), ${MAX_SOURCE_BYTES - budget} bytes`);
83 return out;
84}
85
86const SYS = `You are loup, a senior engineer who reviews real user session recordings, finds where the user hit a bug or got confused, and fixes it in the code.
87
88You receive:
891. SOURCE CODE — the app's relevant source files (path + full contents).
902. SCREENSHOTS — frames sampled at 1 per second from a screen recording of a real user, each labeled with its timestamp.
91
92Find EVERY distinct issue the recording reveals — there are usually several, of different kinds:
93- visual/layout problems (misalignment, off-center labels, cramped or clipped controls),
94- broken flows (a step that leaves the user stuck, controls that vanish, no way to recover/retry),
95- missing safeguards (no validation before submit, no inline feedback, confusing errors).
96Report each as its OWN finding. Do not stop at the first. Watch the frames in order and cross-reference the SOURCE CODE for the root cause.
97
98For each finding decide:
99- kind "pr": you can fix it in the given source. Put the COMPLETE corrected file in "newContent" — the entire file, byte-for-byte identical to the source EXCEPT your fix. Do not reformat or touch unrelated lines. The fix must be a real, minimal code change that resolves the issue.
100- kind "issue": it needs human judgement and has no clear single code fix. "newContent" is null.
101
102Respond with ONLY a JSON array, no prose, no code fences. Each item:
103{
104 "kind": "pr" | "issue",
105 "title": string, // concise, imperative
106 "at": "m:ss", // timestamp of the frame that shows it
107 "file": string | null, // exact source file path (from the SOURCE CODE headers) the issue lives in
108 "evidence": string, // one line: exactly what you saw in the frame(s)
109 "body": string, // what the user hit, why it matters, and what your fix does
110 "newContent": string | null // for "pr": the full corrected file contents. null for "issue".
111}`;
112
113interface RawFinding {
114 kind: "pr" | "issue";
115 title: string;
116 at: string;
117 file: string | null;
118 evidence?: string;
119 body: string;
120 newContent: string | null;
121}
122
123/** Build model-specific thinking + effort params (Opus/Sonnet 4.6+ use adaptive). */
124function reasoningParams(model: string): Record<string, unknown> {
125 const adaptive = /opus-4-[678]|sonnet-4-6|fable-5|mythos-5/.test(model);
126 if (adaptive) {
127 const effort = process.env.LOUP_EFFORT ?? "high";
128 return { thinking: { type: "adaptive" }, output_config: { effort } };
129 }
130 const budget = Number(process.env.LOUP_THINK ?? 1024);
131 return { thinking: budget > 0 ? { type: "enabled", budget_tokens: budget } : { type: "disabled" } };
132}
133
134export async function analyzeWithVision(
135 event: WebhookEvent,
136 cfg: Config,
137 framesDir: string,
138 log: (m: string) => void = () => {},
139): Promise<Finding[]> {
140 if (!cfg.anthropicKey) throw new Error("no ANTHROPIC_API_KEY");
141
142 const frames = await sweepFrames(event.videoPath, framesDir);
143 const source = await readSource(cfg, log);
144 log(`vision: ${frames.length} frame(s) @ 1fps; model ${cfg.model}`);
145
146 const content: Anthropic.MessageParam["content"] = [{ type: "text", text: "SOURCE CODE:\n" }];
147 for (const s of source) content.push({ type: "text", text: `\n===== ${s.rel} =====\n${s.code}\n` });
148 if (event.text.trim()) {
149 content.push({
150 type: "text",
151 text: `\nOPERATOR NOTES (optional context — verify against the frames, do not just trust):\n${event.text.trim()}\n`,
152 });
153 }
154 content.push({ type: "text", text: "\nSCREENSHOTS (1 fps, timestamp tagged):" });
155 for (const fr of frames) {
156 const data = (await readFile(fr.path)).toString("base64");
157 content.push({ type: "text", text: `Frame @ ${fr.at}` });
158 content.push({ type: "image", source: { type: "base64", media_type: "image/png", data } });
159 }
160
161 const client = new Anthropic({ apiKey: cfg.anthropicKey });
162 const params = {
163 model: cfg.model,
164 max_tokens: 8192,
165 system: SYS,
166 messages: [{ role: "user", content }],
167 ...reasoningParams(cfg.model),
168 } as unknown as Anthropic.MessageCreateParamsNonStreaming;
169
170 const resp = await client.messages.create(params);
171 const text = resp.content.find((b) => b.type === "text");
172 const raw = text && "text" in text ? text.text : "[]";
173 const parsed = JSON.parse(stripFences(raw)) as RawFinding[];
174
175 const findings: Finding[] = [];
176 for (const f of parsed) {
177 log(`vision: [${f.kind}] "${f.title}" @ ${f.at}${f.evidence ? ` — ${f.evidence}` : ""}`);
178 findings.push({
179 kind: f.kind,
180 title: f.title,
181 at: f.at,
182 file: f.file ?? null,
183 evidence: f.evidence,
184 body: f.evidence ? `${f.body}\n\n*Verified from the recording at ${f.at}: ${f.evidence}*` : f.body,
185 newContent: f.kind === "pr" ? f.newContent : null,
186 });
187 }
188 return findings;
189}
190
191function stripFences(s: string): string {
192 const m = s.match(/```(?:json)?\s*([\s\S]*?)```/);
193 return (m ? m[1] : s).trim();
194}
195
196/** Warm the model connection (TLS/auth) at startup so the first real send is fast. */
197export async function prewarm(cfg: Config): Promise<void> {
198 if (!cfg.anthropicKey || cfg.visionOff) return;
199 try {
200 const client = new Anthropic({ apiKey: cfg.anthropicKey });
201 await client.messages.create({ model: cfg.model, max_tokens: 1, messages: [{ role: "user", content: "hi" }] });
202 } catch {
203 /* best-effort warmup */
204 }
205}