feat: add email digest ingestion for job-alert emails

README.md

@@ -179,7 +179,7 @@ npx playwright install chromium
 
 ## Scheduled job searching
 
-Beyond scraping your target companies, deckhandAI can search job boards automatically and populate the pending queue without any manual effort. Three approaches are supported — GitHub Actions, system cron, and Claude Code's built-in scheduler — depending on how you run the app.
+Beyond scraping your target companies, deckhandAI can search job boards automatically and populate the pending queue without any manual effort. Three approaches are supported — GitHub Actions, system cron, and Claude Code's built-in scheduler — depending on how you run the app. It can also parse job-alert emails you already receive (Indeed, TheLadders, Lensa, etc.) via `scripts/email-job-search.mjs`, fed by any mail source you wire up yourself.
 
 → [docs/scheduled-job-search.md](docs/scheduled-job-search.md)
 

docs/scheduled-job-search.md

@@ -184,6 +184,48 @@ You can combine both into a single trigger — Claude will run WebSearch first,
 
 ---
 
+## Email digest ingestion
+
+If you already subscribe to job-alert emails (Indeed, TheLadders, Lensa, LinkedIn, or similar),
+`scripts/email-job-search.mjs` can parse those digests directly instead of running fresh
+searches. It reuses the same preferences, dedup, and write flow as `scripts/job-search.mjs`, but
+takes emails as input rather than running its own web/Indeed search.
+
+This script is deliberately mail-provider-agnostic — it has no Gmail/IMAP/OAuth code of its own.
+You feed it a JSON array of already-exported emails, and it does the extraction, filtering,
+dedup, and write:
+
+```
+[
+  { "id": "...", "subject": "...", "from": "...", "receivedAt": "...", "body": "..." }
+]
+```
+
+```bash
+node scripts/email-job-search.mjs --input emails.json
+# or
+cat emails.json | node scripts/email-job-search.mjs
+```
+
+It requires the same env vars as `scripts/job-search.mjs` (`AI_API_KEY`, `AI_MODEL`,
+`GITHUB_TOKEN`, `GITHUB_DATA_REPO`, `GITHUB_DATA_BRANCH`, optional `AI_BASE_URL`) and prints a
+single JSON summary to stdout on success: `{ "processed", "added", "skippedDuplicate", "skippedNoMatch" }`.
+
+**How you get emails into that shape is up to you** — any of the following work:
+
+- **OpenClaw `gog` (Gmail) skill** — a scheduled OpenClaw cron job searches Gmail for
+  job-platform senders, exports matching threads as JSON, and pipes them into this script. This
+  is how the deckhandAI maintainer runs it day to day; the OpenClaw job itself is personal
+  automation (tied to one Gmail account and one data repo), not part of this repo.
+- **A plain IMAP fetch script** — any language, any provider, as long as it emits the shape above.
+- **A one-off export** — e.g. Gmail's "Download message" or Takeout, converted to the input shape
+  with a small script, for a manual/occasional pass instead of a recurring schedule.
+
+Because the extraction call runs once per email and each job-alert digest often lists several
+roles, a single run can add multiple jobs from one email.
+
+---
+
 ## Reviewing results
 
 After any scheduled run, new jobs appear in the **Pending** section of the deckhandAI UI. From there you can:

scripts/backfill-ai-summaries.mjs

@@ -30,9 +30,12 @@
 import { existsSync, readFileSync } from "fs";
 import { resolve, dirname } from "path";
 import { fileURLToPath } from "url";
+import { githubGet, githubPut } from "./lib/github-contents.mjs";
+import { callAI } from "./lib/ai-provider.mjs";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const DRY_RUN = process.argv.includes("--dry-run");
+const USER_AGENT = "deckhandAI/backfill-ai-summaries";
 
 // ---------------------------------------------------------------------------
 // Env (.env.local convenience — existing env always wins)
@@ -50,39 +53,6 @@ function loadEnvLocal() {
   }
 }
 
-// ---------------------------------------------------------------------------
-// GitHub API (same shape as scripts/job-search.mjs)
-// ---------------------------------------------------------------------------
-
-async function githubGet(repo, branch, path) {
-  const res = await fetch(
-    `https://api.github.com/repos/${repo}/contents/${path}?ref=${branch}`,
-    {
-      headers: {
-        Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
-        "User-Agent": "deckhandAI/backfill-ai-summaries",
-      },
-    }
-  );
-  if (!res.ok) throw new Error(`GitHub GET ${path}: ${res.status} ${await res.text()}`);
-  const { content, sha } = await res.json();
-  return { data: JSON.parse(Buffer.from(content, "base64").toString("utf8")), sha };
-}
-
-async function githubPut(repo, branch, path, sha, data, message) {
-  const content = Buffer.from(JSON.stringify(data, null, 2)).toString("base64");
-  const res = await fetch(`https://api.github.com/repos/${repo}/contents/${path}`, {
-    method: "PUT",
-    headers: {
-      Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
-      "Content-Type": "application/json",
-      "User-Agent": "deckhandAI/backfill-ai-summaries",
-    },
-    body: JSON.stringify({ message, content, sha, branch }),
-  });
-  if (!res.ok) throw new Error(`GitHub PUT ${path}: ${res.status} ${await res.text()}`);
-}
-
 // ---------------------------------------------------------------------------
 // AI call — endpoint resolution mirrors lib/model.ts
 // ---------------------------------------------------------------------------
@@ -102,54 +72,7 @@ ${job.notes}`;
 }
 
 async function generateSummary(job) {
-  const provider = process.env.AI_PROVIDER || "anthropic";
-  const model = process.env.AI_MODEL || "claude-sonnet-4-6";
-  const apiKey = process.env.AI_API_KEY || "";
-
-  if (provider === "anthropic") {
-    const res = await fetch("https://api.anthropic.com/v1/messages", {
-      method: "POST",
-      headers: {
-        "x-api-key": apiKey,
-        "anthropic-version": "2023-06-01",
-        "content-type": "application/json",
-      },
-      body: JSON.stringify({
-        model,
-        max_tokens: 300,
-        system: SYSTEM_PROMPT,
-        messages: [{ role: "user", content: buildUserPrompt(job) }],
-      }),
-    });
-    if (!res.ok) throw new Error(`AI provider error ${res.status}: ${await res.text()}`);
-    const json = await res.json();
-    return (json.content ?? []).filter((c) => c.type === "text").map((c) => c.text).join("");
-  }
-
-  const builtinBase = {
-    openai: "https://api.openai.com/v1",
-    gemini: "https://generativelanguage.googleapis.com/v1beta/openai",
-    grok: "https://api.x.ai/v1",
-  };
-  const baseUrl = process.env.AI_BASE_URL || builtinBase[provider] || "https://api.openai.com/v1";
-  const res = await fetch(`${baseUrl}/chat/completions`, {
-    method: "POST",
-    headers: {
-      Authorization: `Bearer ${apiKey}`,
-      "Content-Type": "application/json",
-    },
-    body: JSON.stringify({
-      model,
-      max_tokens: 300,
-      messages: [
-        { role: "system", content: SYSTEM_PROMPT },
-        { role: "user", content: buildUserPrompt(job) },
-      ],
-    }),
-  });
-  if (!res.ok) throw new Error(`AI provider error ${res.status}: ${await res.text()}`);
-  const json = await res.json();
-  return json.choices?.[0]?.message?.content ?? "";
+  return callAI({ systemPrompt: SYSTEM_PROMPT, userPrompt: buildUserPrompt(job), maxTokens: 300 });
 }
 
 // Same cleanup rules as normalizeAiSummary in lib/job-summary.ts — keep in sync.
@@ -188,7 +111,7 @@ async function main() {
   }
 
   console.log(`Reading data/jobs.json from ${repo}@${branch}…`);
-  const { data: jobs, sha } = await githubGet(repo, branch, "data/jobs.json");
+  const { data: jobs, sha } = await githubGet(repo, branch, "data/jobs.json", USER_AGENT);
 
   let generated = 0;
   let skippedExisting = 0;
@@ -234,7 +157,7 @@ async function main() {
     return;
   }
 
-  await githubPut(repo, branch, "data/jobs.json", sha, jobs, `Backfill aiSummary for ${generated} jobs`);
+  await githubPut(repo, branch, "data/jobs.json", sha, jobs, `Backfill aiSummary for ${generated} jobs`, USER_AGENT);
   console.log(`Wrote data/jobs.json to ${repo}@${branch}.`);
 }