Compare commits
2 Commits
35ad932bbf
...
647352e49c
| Author | SHA1 | Date | |
|---|---|---|---|
 mrosati
|
647352e49c | ||
|
mrosati
|
3ecc07aeab |
@@ -7,4 +7,5 @@ sessions
|
|||||||
.mcp.json
|
.mcp.json
|
||||||
mcp-cache.json
|
mcp-cache.json
|
||||||
mcp-npx-cache.json
|
mcp-npx-cache.json
|
||||||
|
mcp-onboarding.json
|
||||||
run-history.jsonl
|
run-history.jsonl
|
||||||
|
|||||||
@@ -1,376 +0,0 @@
|
|||||||
import { spawn } from "node:child_process";
|
|
||||||
import * as fs from "node:fs";
|
|
||||||
import * as os from "node:os";
|
|
||||||
import * as path from "node:path";
|
|
||||||
import type { Message } from "@earendil-works/pi-ai";
|
|
||||||
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
||||||
import { TDM_PROMPT, DEV_PROMPT, ANA_PROMPT } from "./prompts";
|
|
||||||
import { AgentRunResult, AnaVerdict } from "./types";
|
|
||||||
|
|
||||||
|
|
||||||
const MAX_ITERATIONS = 5;
|
|
||||||
|
|
||||||
// UI Symbols:
|
|
||||||
// ┌ ┐ └ ┘ │ ─ ├ ┤
|
|
||||||
|
|
||||||
const TDM_UI = `
|
|
||||||
┌─────┐ ┌─────┐ ┌─────┐
|
|
||||||
│ **TDM** │ --> │ DEV │ --> │ ANA │
|
|
||||||
└─────┘ └─────┘ └─────┘
|
|
||||||
↗
|
|
||||||
`
|
|
||||||
|
|
||||||
const DEV_UI = `
|
|
||||||
┌─────┐ ┌─────┐ ┌─────┐
|
|
||||||
│ TDM │ --> │ **DEV** │ --> │ ANA │
|
|
||||||
└─────┘ └─────┘ └─────┘
|
|
||||||
↗
|
|
||||||
`
|
|
||||||
|
|
||||||
const ANA_UI = `
|
|
||||||
┌─────┐ ┌─────┐ ┌─────┐
|
|
||||||
│ TDM │ --> │ DEV │ --> │ **ANA** │
|
|
||||||
└─────┘ └─────┘ └─────┘
|
|
||||||
↗
|
|
||||||
`
|
|
||||||
|
|
||||||
const ANA_REWORK_UI = `
|
|
||||||
┌─────┐ ┌─────┐ ┌─────┐
|
|
||||||
│ TDM │ --> │ DEV │ --> │ **ANA** │
|
|
||||||
└─────┘ └─────┘ └─────┘
|
|
||||||
↖
|
|
||||||
`
|
|
||||||
|
|
||||||
function getPiInvocation(args: string[]): { command: string; args: string[] } {
|
|
||||||
const currentScript = process.argv[1];
|
|
||||||
const isBunVirtualScript = currentScript?.startsWith("/$bunfs/root/");
|
|
||||||
if (currentScript && !isBunVirtualScript && fs.existsSync(currentScript)) {
|
|
||||||
return { command: process.execPath, args: [currentScript, ...args] };
|
|
||||||
}
|
|
||||||
|
|
||||||
const execName = path.basename(process.execPath).toLowerCase();
|
|
||||||
const isGenericRuntime = /^(node|bun)(\.exe)?$/.test(execName);
|
|
||||||
if (!isGenericRuntime) return { command: process.execPath, args };
|
|
||||||
return { command: "pi", args };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function writePromptToTempFile(
|
|
||||||
name: string,
|
|
||||||
prompt: string,
|
|
||||||
): Promise<{ dir: string; filePath: string }> {
|
|
||||||
const tmpDir = await fs.promises.mkdtemp(
|
|
||||||
path.join(os.tmpdir(), "pi-multiagent-"),
|
|
||||||
);
|
|
||||||
const safeName = name.replace(/[^\w.-]+/g, "_");
|
|
||||||
const filePath = path.join(tmpDir, `${safeName}.md`);
|
|
||||||
await fs.promises.writeFile(filePath, prompt, {
|
|
||||||
encoding: "utf-8",
|
|
||||||
mode: 0o600,
|
|
||||||
});
|
|
||||||
return { dir: tmpDir, filePath };
|
|
||||||
}
|
|
||||||
|
|
||||||
function getFinalText(messages: Message[]): string {
|
|
||||||
for (let i = messages.length - 1; i >= 0; i--) {
|
|
||||||
const msg = messages[i];
|
|
||||||
if (msg.role !== "assistant") continue;
|
|
||||||
const text = msg.content.find((c) => c.type === "text");
|
|
||||||
if (text && text.type === "text") return text.text;
|
|
||||||
}
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
|
|
||||||
async function runAgent(
|
|
||||||
cwd: string,
|
|
||||||
systemPrompt: string,
|
|
||||||
task: string,
|
|
||||||
tools: string,
|
|
||||||
signal?: AbortSignal,
|
|
||||||
): Promise<AgentRunResult> {
|
|
||||||
const tmp = await writePromptToTempFile("system", systemPrompt);
|
|
||||||
const args = [
|
|
||||||
"--mode",
|
|
||||||
"json",
|
|
||||||
"-p",
|
|
||||||
"--no-session",
|
|
||||||
"--tools",
|
|
||||||
tools,
|
|
||||||
"--append-system-prompt",
|
|
||||||
tmp.filePath,
|
|
||||||
task,
|
|
||||||
];
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await new Promise<AgentRunResult>((resolve) => {
|
|
||||||
const invocation = getPiInvocation(args);
|
|
||||||
const proc = spawn(invocation.command, invocation.args, {
|
|
||||||
cwd,
|
|
||||||
shell: false,
|
|
||||||
stdio: ["ignore", "pipe", "pipe"],
|
|
||||||
});
|
|
||||||
|
|
||||||
const messages: Message[] = [];
|
|
||||||
let stderr = "";
|
|
||||||
let buffer = "";
|
|
||||||
|
|
||||||
const processLine = (line: string) => {
|
|
||||||
if (!line.trim()) return;
|
|
||||||
try {
|
|
||||||
const event = JSON.parse(line);
|
|
||||||
if (event.type === "message_end" && event.message)
|
|
||||||
messages.push(event.message as Message);
|
|
||||||
if (event.type === "tool_result_end" && event.message)
|
|
||||||
messages.push(event.message as Message);
|
|
||||||
} catch {
|
|
||||||
// ignore malformed lines
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
proc.stdout.on("data", (d) => {
|
|
||||||
buffer += d.toString();
|
|
||||||
const lines = buffer.split("\n");
|
|
||||||
buffer = lines.pop() || "";
|
|
||||||
for (const l of lines) processLine(l);
|
|
||||||
});
|
|
||||||
|
|
||||||
proc.stderr.on("data", (d) => {
|
|
||||||
stderr += d.toString();
|
|
||||||
});
|
|
||||||
|
|
||||||
proc.on("close", (code) => {
|
|
||||||
if (buffer.trim()) processLine(buffer);
|
|
||||||
resolve({ exitCode: code ?? 1, messages, stderr });
|
|
||||||
});
|
|
||||||
|
|
||||||
if (signal) {
|
|
||||||
const killProc = () => {
|
|
||||||
proc.kill("SIGTERM");
|
|
||||||
setTimeout(() => {
|
|
||||||
if (!proc.killed) proc.kill("SIGKILL");
|
|
||||||
}, 5000);
|
|
||||||
};
|
|
||||||
if (signal.aborted) killProc();
|
|
||||||
else signal.addEventListener("abort", killProc, { once: true });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} finally {
|
|
||||||
try {
|
|
||||||
fs.unlinkSync(tmp.filePath);
|
|
||||||
} catch { }
|
|
||||||
try {
|
|
||||||
fs.rmdirSync(tmp.dir);
|
|
||||||
} catch { }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseAnaVerdict(raw: string): AnaVerdict {
|
|
||||||
try {
|
|
||||||
const parsed = JSON.parse(raw) as AnaVerdict;
|
|
||||||
return {
|
|
||||||
approved: Boolean(parsed.approved),
|
|
||||||
feedback: String(parsed.feedback ?? "").trim(),
|
|
||||||
};
|
|
||||||
} catch {
|
|
||||||
return {
|
|
||||||
approved: false,
|
|
||||||
feedback: `ANA output was not valid JSON:\n${raw}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function startLoading(
|
|
||||||
setStatus: (text: string) => void,
|
|
||||||
baseLabel: string,
|
|
||||||
): () => void {
|
|
||||||
const frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
|
|
||||||
let i = 0;
|
|
||||||
setStatus(`${frames[0]} ${baseLabel}`);
|
|
||||||
const timer = setInterval(() => {
|
|
||||||
i = (i + 1) % frames.length;
|
|
||||||
setStatus(`${frames[i]} ${baseLabel}`);
|
|
||||||
}, 120);
|
|
||||||
|
|
||||||
return () => {
|
|
||||||
clearInterval(timer);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export default function (pi: ExtensionAPI) {
|
|
||||||
let enabled = false;
|
|
||||||
|
|
||||||
pi.registerCommand("multiagent", {
|
|
||||||
description: "Toggle multi-agent workflow (/multiagent on|off|status)",
|
|
||||||
handler: async (args, ctx) => {
|
|
||||||
const action = (args || "toggle").trim().toLowerCase();
|
|
||||||
if (action === "on") enabled = true;
|
|
||||||
else if (action === "off") enabled = false;
|
|
||||||
else if (action === "toggle") enabled = !enabled;
|
|
||||||
else if (action === "status") {
|
|
||||||
ctx.ui.notify(`Multiagent: ${enabled ? "ON" : "OFF"}`, "info");
|
|
||||||
return;
|
|
||||||
} else {
|
|
||||||
ctx.ui.notify("Usage: /multiagent [on|off|toggle|status]", "warning");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
ctx.ui.notify(
|
|
||||||
`Multiagent: ${enabled ? "ON" : "OFF"}`,
|
|
||||||
enabled ? "info" : "info",
|
|
||||||
);
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
pi.registerShortcut("ctrl+alt+m", {
|
|
||||||
description: "Toggle multi-agent workflow",
|
|
||||||
handler: async (ctx) => {
|
|
||||||
enabled = !enabled;
|
|
||||||
ctx.ui.notify(
|
|
||||||
`Multiagent: ${enabled ? "ON" : "OFF"}`,
|
|
||||||
enabled ? "info" : "info",
|
|
||||||
);
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
pi.on("input", async (event, ctx) => {
|
|
||||||
if (!enabled) return { action: "continue" as const };
|
|
||||||
if (event.source === "extension") return { action: "continue" as const };
|
|
||||||
if (event.text.trim().startsWith("/"))
|
|
||||||
return { action: "continue" as const };
|
|
||||||
|
|
||||||
ctx.ui.setStatus("multiagent", "multiagent: queued");
|
|
||||||
|
|
||||||
pi.sendMessage({
|
|
||||||
customType: "multiagent-result",
|
|
||||||
content: `### Multiagent workflow started\n\n${TDM_UI}\n\n(max ${MAX_ITERATIONS} DEV/ANA loops).`,
|
|
||||||
display: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
const userTask = event.text.trim();
|
|
||||||
|
|
||||||
const stopTdmLoader = startLoading(
|
|
||||||
(text) => ctx.ui.setStatus("multiagent", text),
|
|
||||||
"TDM analyzing request + codebase",
|
|
||||||
);
|
|
||||||
|
|
||||||
const tdm = await runAgent(
|
|
||||||
ctx.cwd,
|
|
||||||
TDM_PROMPT,
|
|
||||||
`User request:\n${userTask}`,
|
|
||||||
"read,grep,find,ls,bash",
|
|
||||||
ctx.signal,
|
|
||||||
);
|
|
||||||
|
|
||||||
stopTdmLoader();
|
|
||||||
|
|
||||||
const specs = getFinalText(tdm.messages);
|
|
||||||
|
|
||||||
if (tdm.exitCode !== 0 || !specs) {
|
|
||||||
pi.sendMessage({
|
|
||||||
customType: "multiagent-result",
|
|
||||||
content: `TDM step failed.\n\n${tdm.stderr || "No output."}`,
|
|
||||||
display: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
ctx.ui.setStatus("multiagent", "");
|
|
||||||
|
|
||||||
return { action: "handled" as const };
|
|
||||||
}
|
|
||||||
|
|
||||||
pi.sendMessage({
|
|
||||||
customType: "multiagent-result",
|
|
||||||
content: `### TDM output (specs)\n\n${specs}`,
|
|
||||||
display: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
pi.sendMessage({
|
|
||||||
customType: "multiagent-result",
|
|
||||||
content: DEV_UI,
|
|
||||||
display: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
let devContext = `TDM Specs:\n${specs}`;
|
|
||||||
let lastDevOutput = "";
|
|
||||||
let lastAnaFeedback = "";
|
|
||||||
|
|
||||||
for (let i = 1; i <= MAX_ITERATIONS; i++) {
|
|
||||||
const stopDevLoader = startLoading(
|
|
||||||
(text) => ctx.ui.setStatus("multiagent", text),
|
|
||||||
`DEV implementing iteration ${i}/${MAX_ITERATIONS}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
const dev = await runAgent(
|
|
||||||
ctx.cwd,
|
|
||||||
DEV_PROMPT,
|
|
||||||
`${devContext}\n\nProceed with the implementation the current codebase. Then summarize changes and rationale.`,
|
|
||||||
"read,grep,find,ls,bash,edit,write",
|
|
||||||
ctx.signal,
|
|
||||||
);
|
|
||||||
|
|
||||||
stopDevLoader();
|
|
||||||
|
|
||||||
lastDevOutput = getFinalText(dev.messages);
|
|
||||||
|
|
||||||
pi.sendMessage({
|
|
||||||
customType: "multiagent-result",
|
|
||||||
content: `### DEV iteration ${i} output\n\n${lastDevOutput || "(no DEV output)"}`,
|
|
||||||
display: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (dev.exitCode !== 0) {
|
|
||||||
pi.sendMessage({
|
|
||||||
customType: "multiagent-result",
|
|
||||||
content: `DEV iteration ${i} failed.\n\n${dev.stderr || "No stderr output."}`,
|
|
||||||
display: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
pi.sendMessage({
|
|
||||||
customType: "multiagent-result",
|
|
||||||
content: ANA_UI,
|
|
||||||
display: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
const stopAnaLoader = startLoading(
|
|
||||||
(text) => ctx.ui.setStatus("multiagent", text),
|
|
||||||
`ANA reviewing iteration ${i}/${MAX_ITERATIONS}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
const ana = await runAgent(
|
|
||||||
ctx.cwd,
|
|
||||||
ANA_PROMPT,
|
|
||||||
`TDM Specs:\n${specs}\n\nDEV report:\n${lastDevOutput}\n\nReview the implementation in the codebase and provide verdict JSON.`,
|
|
||||||
"read,grep,find,ls,bash",
|
|
||||||
ctx.signal,
|
|
||||||
);
|
|
||||||
|
|
||||||
stopAnaLoader();
|
|
||||||
|
|
||||||
const verdict = parseAnaVerdict(getFinalText(ana.messages));
|
|
||||||
lastAnaFeedback = verdict.feedback;
|
|
||||||
|
|
||||||
pi.sendMessage({
|
|
||||||
customType: "multiagent-result",
|
|
||||||
content: `### ANA iteration ${i} verdict\n\n- approved: **${verdict.approved ? "true" : "false"}**\n\n${verdict.feedback || "(no feedback)"}`,
|
|
||||||
display: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (verdict.approved) break;
|
|
||||||
|
|
||||||
pi.sendMessage({
|
|
||||||
customType: "multiagent-result",
|
|
||||||
content: ANA_REWORK_UI,
|
|
||||||
display: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
devContext = `TDM Specs:\n${specs}\n\nANA feedback to address:\n${verdict.feedback}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
pi.sendMessage({
|
|
||||||
customType: "multiagent-result",
|
|
||||||
content: `### Multiagent workflow completed\n\n${lastDevOutput || "(No DEV summary)"}\n\n---\n**Final ANA feedback:**\n${lastAnaFeedback || "Approved with no additional feedback."}`,
|
|
||||||
display: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
ctx.ui.setStatus("multiagent", "");
|
|
||||||
return { action: "handled" as const };
|
|
||||||
});
|
|
||||||
}
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
export const TDM_PROMPT = `You are the Technical Delivery Manager (TDM).
|
|
||||||
Your job:
|
|
||||||
1) Deeply understand user intent.
|
|
||||||
2) Clarify ambiguities by explicitly listing assumptions.
|
|
||||||
3) Produce precise, context-aware implementation specs for a developer.
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
- Focus on WHAT must be done, not implementation details.
|
|
||||||
- Be explicit, structured, and testable.
|
|
||||||
- Output markdown with sections:
|
|
||||||
- Intent
|
|
||||||
- Assumptions
|
|
||||||
- Scope
|
|
||||||
- Detailed Specs
|
|
||||||
- Acceptance Criteria`;
|
|
||||||
|
|
||||||
export const DEV_PROMPT = `You are the Developer (DEV).
|
|
||||||
Your job:
|
|
||||||
- Implement exactly the provided specs with minimal necessary changes.
|
|
||||||
- Prefer small, safe, reliable edits.
|
|
||||||
- Stay within scope.
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
- Inspect code before editing.
|
|
||||||
- Make focused changes only.
|
|
||||||
- Report exactly what was changed and why.`;
|
|
||||||
|
|
||||||
export const ANA_PROMPT = `You are the Code Analyst (ANA).
|
|
||||||
Your job:
|
|
||||||
- Critique DEV output against TDM specs.
|
|
||||||
- Find mistakes, risky choices, anti-patterns, and sub-optimal decisions.
|
|
||||||
- Suggest concrete improvements.
|
|
||||||
|
|
||||||
You MUST output valid JSON only with this schema:
|
|
||||||
{
|
|
||||||
"approved": boolean,
|
|
||||||
"feedback": string
|
|
||||||
}
|
|
||||||
|
|
||||||
Set approved=true only if implementation is solid and no significant fixes are needed.`;
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
import type { Message } from "@earendil-works/pi-ai";
|
|
||||||
|
|
||||||
export interface AgentRunResult {
|
|
||||||
exitCode: number;
|
|
||||||
messages: Message[];
|
|
||||||
stderr: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface AnaVerdict {
|
|
||||||
approved: boolean;
|
|
||||||
feedback: string;
|
|
||||||
}
|
|
||||||
@@ -1,14 +1,4 @@
|
|||||||
{
|
{
|
||||||
"mcpServers": {
|
"mcpServers": {
|
||||||
"chrome-devtools": {
|
|
||||||
"command": "npx",
|
|
||||||
"args": [
|
|
||||||
"-y",
|
|
||||||
"chrome-devtools-mcp@latest"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"open-pencil": {
|
|
||||||
"command": "openpencil-mcp"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-3
@@ -4,9 +4,7 @@
|
|||||||
"defaultModel": "gpt-5.4",
|
"defaultModel": "gpt-5.4",
|
||||||
"packages": [
|
"packages": [
|
||||||
"npm:@sherif-fanous/pi-catppuccin",
|
"npm:@sherif-fanous/pi-catppuccin",
|
||||||
"npm:pi-mcp-adapter",
|
"npm:pi-web-access"
|
||||||
"npm:pi-web-access",
|
|
||||||
"npm:pi-subagents"
|
|
||||||
],
|
],
|
||||||
"theme": "catppuccin-frappe",
|
"theme": "catppuccin-frappe",
|
||||||
"editorPaddingX": 0,
|
"editorPaddingX": 0,
|
||||||
|
|||||||
@@ -1,79 +0,0 @@
|
|||||||
---
|
|
||||||
name: brave-search
|
|
||||||
description: Web search and content extraction via Brave Search API. Use for searching documentation, facts, or any web content. Lightweight, no browser required.
|
|
||||||
---
|
|
||||||
|
|
||||||
# Brave Search
|
|
||||||
|
|
||||||
Web search and content extraction using the official Brave Search API. No browser required.
|
|
||||||
|
|
||||||
## Setup
|
|
||||||
|
|
||||||
Requires a Brave Search API account with a free subscription. A credit card is required to create the free subscription (you won't be charged).
|
|
||||||
|
|
||||||
1. Create an account at https://api-dashboard.search.brave.com/register
|
|
||||||
2. Create a "Free AI" subscription
|
|
||||||
3. Create an API key for the subscription
|
|
||||||
4. Add to your shell profile (`~/.profile` or `~/.zprofile` for zsh):
|
|
||||||
```bash
|
|
||||||
export BRAVE_API_KEY="your-api-key-here"
|
|
||||||
```
|
|
||||||
5. Install shared dependencies (run once):
|
|
||||||
```bash
|
|
||||||
cd ~/.pi/agent/extensions
|
|
||||||
npm install
|
|
||||||
```
|
|
||||||
|
|
||||||
## Search
|
|
||||||
|
|
||||||
```bash
|
|
||||||
{baseDir}/search.js "query" # Basic search (5 results)
|
|
||||||
{baseDir}/search.js "query" -n 10 # More results (max 20)
|
|
||||||
{baseDir}/search.js "query" --content # Include page content as markdown
|
|
||||||
{baseDir}/search.js "query" --freshness pw # Results from last week
|
|
||||||
{baseDir}/search.js "query" --freshness 2024-01-01to2024-06-30 # Date range
|
|
||||||
{baseDir}/search.js "query" --country DE # Results from Germany
|
|
||||||
{baseDir}/search.js "query" -n 3 --content # Combined options
|
|
||||||
```
|
|
||||||
|
|
||||||
### Options
|
|
||||||
|
|
||||||
- `-n <num>` - Number of results (default: 5, max: 20)
|
|
||||||
- `--content` - Fetch and include page content as markdown
|
|
||||||
- `--country <code>` - Two-letter country code (default: US)
|
|
||||||
- `--freshness <period>` - Filter by time:
|
|
||||||
- `pd` - Past day (24 hours)
|
|
||||||
- `pw` - Past week
|
|
||||||
- `pm` - Past month
|
|
||||||
- `py` - Past year
|
|
||||||
- `YYYY-MM-DDtoYYYY-MM-DD` - Custom date range
|
|
||||||
|
|
||||||
## Extract Page Content
|
|
||||||
|
|
||||||
```bash
|
|
||||||
{baseDir}/content.js https://example.com/article
|
|
||||||
```
|
|
||||||
|
|
||||||
Fetches a URL and extracts readable content as markdown.
|
|
||||||
|
|
||||||
## Output Format
|
|
||||||
|
|
||||||
```
|
|
||||||
--- Result 1 ---
|
|
||||||
Title: Page Title
|
|
||||||
Link: https://example.com/page
|
|
||||||
Age: 2 days ago
|
|
||||||
Snippet: Description from search results
|
|
||||||
Content: (if --content flag used)
|
|
||||||
Markdown content extracted from the page...
|
|
||||||
|
|
||||||
--- Result 2 ---
|
|
||||||
...
|
|
||||||
```
|
|
||||||
|
|
||||||
## When to Use
|
|
||||||
|
|
||||||
- Searching for documentation or API references
|
|
||||||
- Looking up facts or current information
|
|
||||||
- Fetching content from specific URLs
|
|
||||||
- Any task requiring web search without interactive browsing
|
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
import path from "node:path";
|
|
||||||
import { fileURLToPath } from "node:url";
|
|
||||||
import { createRequire } from "node:module";
|
|
||||||
|
|
||||||
const require = createRequire(import.meta.url);
|
|
||||||
const baseDir = path.dirname(fileURLToPath(import.meta.url));
|
|
||||||
const sharedNodeModules = path.resolve(baseDir, "../../extensions/node_modules");
|
|
||||||
const resolveFromShared = (pkg) => require.resolve(pkg, { paths: [sharedNodeModules] });
|
|
||||||
|
|
||||||
const { Readability } = require(resolveFromShared("@mozilla/readability"));
|
|
||||||
const { JSDOM } = require(resolveFromShared("jsdom"));
|
|
||||||
const TurndownService = require(resolveFromShared("turndown"));
|
|
||||||
const { gfm } = require(resolveFromShared("turndown-plugin-gfm"));
|
|
||||||
|
|
||||||
const url = process.argv[2];
|
|
||||||
|
|
||||||
if (!url) {
|
|
||||||
console.log("Usage: content.js <url>");
|
|
||||||
console.log("\nExtracts readable content from a webpage as markdown.");
|
|
||||||
console.log("\nExamples:");
|
|
||||||
console.log(" content.js https://example.com/article");
|
|
||||||
console.log(" content.js https://doc.rust-lang.org/book/ch04-01-what-is-ownership.html");
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
function htmlToMarkdown(html) {
|
|
||||||
const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
|
|
||||||
turndown.use(gfm);
|
|
||||||
turndown.addRule("removeEmptyLinks", {
|
|
||||||
filter: (node) => node.nodeName === "A" && !node.textContent?.trim(),
|
|
||||||
replacement: () => "",
|
|
||||||
});
|
|
||||||
return turndown
|
|
||||||
.turndown(html)
|
|
||||||
.replace(/\[\\?\[\s*\\?\]\]\([^)]*\)/g, "")
|
|
||||||
.replace(/ +/g, " ")
|
|
||||||
.replace(/\s+,/g, ",")
|
|
||||||
.replace(/\s+\./g, ".")
|
|
||||||
.replace(/\n{3,}/g, "\n\n")
|
|
||||||
.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const response = await fetch(url, {
|
|
||||||
headers: {
|
|
||||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
||||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
||||||
"Accept-Language": "en-US,en;q=0.9",
|
|
||||||
},
|
|
||||||
signal: AbortSignal.timeout(15000),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
console.error(`HTTP ${response.status}: ${response.statusText}`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
const html = await response.text();
|
|
||||||
const dom = new JSDOM(html, { url });
|
|
||||||
const reader = new Readability(dom.window.document);
|
|
||||||
const article = reader.parse();
|
|
||||||
|
|
||||||
if (article && article.content) {
|
|
||||||
if (article.title) {
|
|
||||||
console.log(`# ${article.title}\n`);
|
|
||||||
}
|
|
||||||
console.log(htmlToMarkdown(article.content));
|
|
||||||
process.exit(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback: try to extract main content
|
|
||||||
const fallbackDoc = new JSDOM(html, { url });
|
|
||||||
const body = fallbackDoc.window.document;
|
|
||||||
body.querySelectorAll("script, style, noscript, nav, header, footer, aside").forEach(el => el.remove());
|
|
||||||
|
|
||||||
const title = body.querySelector("title")?.textContent?.trim();
|
|
||||||
const main = body.querySelector("main, article, [role='main'], .content, #content") || body.body;
|
|
||||||
|
|
||||||
if (title) {
|
|
||||||
console.log(`# ${title}\n`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const text = main?.innerHTML || "";
|
|
||||||
if (text.trim().length > 100) {
|
|
||||||
console.log(htmlToMarkdown(text));
|
|
||||||
} else {
|
|
||||||
console.error("Could not extract readable content from this page.");
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
console.error(`Error: ${e.message}`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
@@ -1,208 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
import path from "node:path";
|
|
||||||
import { fileURLToPath } from "node:url";
|
|
||||||
import { createRequire } from "node:module";
|
|
||||||
|
|
||||||
const require = createRequire(import.meta.url);
|
|
||||||
const baseDir = path.dirname(fileURLToPath(import.meta.url));
|
|
||||||
const sharedNodeModules = path.resolve(baseDir, "../../extensions/node_modules");
|
|
||||||
const resolveFromShared = (pkg) => require.resolve(pkg, { paths: [sharedNodeModules] });
|
|
||||||
|
|
||||||
const { Readability } = require(resolveFromShared("@mozilla/readability"));
|
|
||||||
const { JSDOM } = require(resolveFromShared("jsdom"));
|
|
||||||
const TurndownService = require(resolveFromShared("turndown"));
|
|
||||||
const { gfm } = require(resolveFromShared("turndown-plugin-gfm"));
|
|
||||||
|
|
||||||
const args = process.argv.slice(2);
|
|
||||||
|
|
||||||
const contentIndex = args.indexOf("--content");
|
|
||||||
const fetchContent = contentIndex !== -1;
|
|
||||||
if (fetchContent) args.splice(contentIndex, 1);
|
|
||||||
|
|
||||||
let numResults = 5;
|
|
||||||
const nIndex = args.indexOf("-n");
|
|
||||||
if (nIndex !== -1 && args[nIndex + 1]) {
|
|
||||||
numResults = parseInt(args[nIndex + 1], 10);
|
|
||||||
args.splice(nIndex, 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse country option
|
|
||||||
let country = "US";
|
|
||||||
const countryIndex = args.indexOf("--country");
|
|
||||||
if (countryIndex !== -1 && args[countryIndex + 1]) {
|
|
||||||
country = args[countryIndex + 1].toUpperCase();
|
|
||||||
args.splice(countryIndex, 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse freshness option
|
|
||||||
let freshness = null;
|
|
||||||
const freshnessIndex = args.indexOf("--freshness");
|
|
||||||
if (freshnessIndex !== -1 && args[freshnessIndex + 1]) {
|
|
||||||
freshness = args[freshnessIndex + 1];
|
|
||||||
args.splice(freshnessIndex, 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
const query = args.join(" ");
|
|
||||||
|
|
||||||
if (!query) {
|
|
||||||
console.log("Usage: search.js <query> [-n <num>] [--content] [--country <code>] [--freshness <period>]");
|
|
||||||
console.log("\nOptions:");
|
|
||||||
console.log(" -n <num> Number of results (default: 5, max: 20)");
|
|
||||||
console.log(" --content Fetch readable content as markdown");
|
|
||||||
console.log(" --country <code> Country code for results (default: US)");
|
|
||||||
console.log(" --freshness <period> Filter by time: pd (day), pw (week), pm (month), py (year)");
|
|
||||||
console.log("\nEnvironment:");
|
|
||||||
console.log(" BRAVE_API_KEY Required. Your Brave Search API key.");
|
|
||||||
console.log("\nExamples:");
|
|
||||||
console.log(' search.js "javascript async await"');
|
|
||||||
console.log(' search.js "rust programming" -n 10');
|
|
||||||
console.log(' search.js "climate change" --content');
|
|
||||||
console.log(' search.js "news today" --freshness pd');
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
const apiKey = process.env.BRAVE_API_KEY;
|
|
||||||
if (!apiKey) {
|
|
||||||
console.error("Error: BRAVE_API_KEY environment variable is required.");
|
|
||||||
console.error("Get your API key at: https://api-dashboard.search.brave.com/app/keys");
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchBraveResults(query, numResults, country, freshness) {
|
|
||||||
const params = new URLSearchParams({
|
|
||||||
q: query,
|
|
||||||
count: Math.min(numResults, 20).toString(),
|
|
||||||
country: country,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (freshness) {
|
|
||||||
params.append("freshness", freshness);
|
|
||||||
}
|
|
||||||
|
|
||||||
const url = `https://api.search.brave.com/res/v1/web/search?${params.toString()}`;
|
|
||||||
|
|
||||||
const response = await fetch(url, {
|
|
||||||
headers: {
|
|
||||||
"Accept": "application/json",
|
|
||||||
"Accept-Encoding": "gzip",
|
|
||||||
"X-Subscription-Token": apiKey,
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const errorText = await response.text();
|
|
||||||
throw new Error(`HTTP ${response.status}: ${response.statusText}\n${errorText}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await response.json();
|
|
||||||
|
|
||||||
const results = [];
|
|
||||||
|
|
||||||
// Extract web results
|
|
||||||
if (data.web && data.web.results) {
|
|
||||||
for (const result of data.web.results) {
|
|
||||||
if (results.length >= numResults) break;
|
|
||||||
|
|
||||||
results.push({
|
|
||||||
title: result.title || "",
|
|
||||||
link: result.url || "",
|
|
||||||
snippet: result.description || "",
|
|
||||||
age: result.age || result.page_age || "",
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
function htmlToMarkdown(html) {
|
|
||||||
const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
|
|
||||||
turndown.use(gfm);
|
|
||||||
turndown.addRule("removeEmptyLinks", {
|
|
||||||
filter: (node) => node.nodeName === "A" && !node.textContent?.trim(),
|
|
||||||
replacement: () => "",
|
|
||||||
});
|
|
||||||
return turndown
|
|
||||||
.turndown(html)
|
|
||||||
.replace(/\[\\?\[\s*\\?\]\]\([^)]*\)/g, "")
|
|
||||||
.replace(/ +/g, " ")
|
|
||||||
.replace(/\s+,/g, ",")
|
|
||||||
.replace(/\s+\./g, ".")
|
|
||||||
.replace(/\n{3,}/g, "\n\n")
|
|
||||||
.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchPageContent(url) {
|
|
||||||
try {
|
|
||||||
const response = await fetch(url, {
|
|
||||||
headers: {
|
|
||||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
|
|
||||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
||||||
},
|
|
||||||
signal: AbortSignal.timeout(10000),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
return `(HTTP ${response.status})`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const html = await response.text();
|
|
||||||
const dom = new JSDOM(html, { url });
|
|
||||||
const reader = new Readability(dom.window.document);
|
|
||||||
const article = reader.parse();
|
|
||||||
|
|
||||||
if (article && article.content) {
|
|
||||||
return htmlToMarkdown(article.content).substring(0, 5000);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback: try to get main content
|
|
||||||
const fallbackDoc = new JSDOM(html, { url });
|
|
||||||
const body = fallbackDoc.window.document;
|
|
||||||
body.querySelectorAll("script, style, noscript, nav, header, footer, aside").forEach(el => el.remove());
|
|
||||||
const main = body.querySelector("main, article, [role='main'], .content, #content") || body.body;
|
|
||||||
const text = main?.textContent || "";
|
|
||||||
|
|
||||||
if (text.trim().length > 100) {
|
|
||||||
return text.trim().substring(0, 5000);
|
|
||||||
}
|
|
||||||
|
|
||||||
return "(Could not extract content)";
|
|
||||||
} catch (e) {
|
|
||||||
return `(Error: ${e.message})`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Main
|
|
||||||
try {
|
|
||||||
const results = await fetchBraveResults(query, numResults, country, freshness);
|
|
||||||
|
|
||||||
if (results.length === 0) {
|
|
||||||
console.error("No results found.");
|
|
||||||
process.exit(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (fetchContent) {
|
|
||||||
for (const result of results) {
|
|
||||||
result.content = await fetchPageContent(result.link);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (let i = 0; i < results.length; i++) {
|
|
||||||
const r = results[i];
|
|
||||||
console.log(`--- Result ${i + 1} ---`);
|
|
||||||
console.log(`Title: ${r.title}`);
|
|
||||||
console.log(`Link: ${r.link}`);
|
|
||||||
if (r.age) {
|
|
||||||
console.log(`Age: ${r.age}`);
|
|
||||||
}
|
|
||||||
console.log(`Snippet: ${r.snippet}`);
|
|
||||||
if (r.content) {
|
|
||||||
console.log(`Content:\n${r.content}`);
|
|
||||||
}
|
|
||||||
console.log("");
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
console.error(`Error: ${e.message}`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user