From d56b8a86edf395337e20215763e078b21e6f9372 Mon Sep 17 00:00:00 2001 From: Brian Schwartz Date: Sat, 1 Aug 2026 00:37:10 +0000 Subject: [PATCH] feat: add email digest ingestion for job-alert emails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds scripts/email-job-search.mjs — a provider-agnostic ingestion source for job-alert digest emails (Indeed, TheLadders, Lensa, etc.). It has no mail integration of its own: it takes pre-exported email JSON as input, so any mail source can feed it (Gmail export, IMAP, or an external scheduler like OpenClaw's gog-based Gmail triage, documented as one example wiring). Extraction uses a strict, conservative filtering prompt — reject a listing whenever title tier, salary, or location eligibility is unclear rather than including it on a hopeful match, since a false positive costs real review time and a missed listing costs nothing (it stays in the source digest). Also, while building this: - Extracted shared scripts/lib/github-contents.mjs (githubGet/githubPut) and scripts/lib/ai-provider.mjs (callAI), deduplicating identical logic that was previously copy-pasted across job-search.mjs and backfill-ai-summaries.mjs. - ai-provider.mjs adds native Anthropic Messages API support (AI_PROVIDER=anthropic, the default), so an Anthropic-shaped AI_API_KEY works instead of only OpenAI-compatible chat/completions endpoints. - Added a company+role dedup fallback (scripts/lib/job-key.mjs, mirroring lib/jobs.ts's jobKey) alongside the existing url-based dedup, since some platforms (TheLadders, Lensa) send tracking-redirect URLs that mint a fresh token per send — the same posting re-appearing in a later digest wouldn't match on url alone. - Applied the same strict-filtering and dedup-fallback changes to job-search.mjs's WebSearch pass for consistency. Verified against real Gmail digest content from Indeed, LinkedIn, TheLadders, and Lensa: correctly extracts and adds clear matches, correctly rejects weak/ambiguous listings (wrong discipline, under salary floor, unconfirmed remote), and correctly catches a same-company+role duplicate under a new tracking URL. --- README.md | 2 +- docs/scheduled-job-search.md | 42 +++++ scripts/backfill-ai-summaries.mjs | 89 +---------- scripts/email-job-search.mjs | 249 ++++++++++++++++++++++++++++++ scripts/job-search.mjs | 118 ++++++-------- scripts/lib/ai-provider.mjs | 68 ++++++++ scripts/lib/github-contents.mjs | 34 ++++ scripts/lib/job-key.mjs | 9 ++ 8 files changed, 456 insertions(+), 155 deletions(-) create mode 100644 scripts/email-job-search.mjs create mode 100644 scripts/lib/ai-provider.mjs create mode 100644 scripts/lib/github-contents.mjs create mode 100644 scripts/lib/job-key.mjs diff --git a/README.md b/README.md index d3bdf39..73673af 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ npx playwright install chromium ## Scheduled job searching -Beyond scraping your target companies, deckhandAI can search job boards automatically and populate the pending queue without any manual effort. Three approaches are supported — GitHub Actions, system cron, and Claude Code's built-in scheduler — depending on how you run the app. +Beyond scraping your target companies, deckhandAI can search job boards automatically and populate the pending queue without any manual effort. Three approaches are supported — GitHub Actions, system cron, and Claude Code's built-in scheduler — depending on how you run the app. It can also parse job-alert emails you already receive (Indeed, TheLadders, Lensa, etc.) via `scripts/email-job-search.mjs`, fed by any mail source you wire up yourself. → [docs/scheduled-job-search.md](docs/scheduled-job-search.md) diff --git a/docs/scheduled-job-search.md b/docs/scheduled-job-search.md index 9782d2a..ce623f3 100644 --- a/docs/scheduled-job-search.md +++ b/docs/scheduled-job-search.md @@ -184,6 +184,48 @@ You can combine both into a single trigger — Claude will run WebSearch first, --- +## Email digest ingestion + +If you already subscribe to job-alert emails (Indeed, TheLadders, Lensa, LinkedIn, or similar), +`scripts/email-job-search.mjs` can parse those digests directly instead of running fresh +searches. It reuses the same preferences, dedup, and write flow as `scripts/job-search.mjs`, but +takes emails as input rather than running its own web/Indeed search. + +This script is deliberately mail-provider-agnostic — it has no Gmail/IMAP/OAuth code of its own. +You feed it a JSON array of already-exported emails, and it does the extraction, filtering, +dedup, and write: + +``` +[ + { "id": "...", "subject": "...", "from": "...", "receivedAt": "...", "body": "..." } +] +``` + +```bash +node scripts/email-job-search.mjs --input emails.json +# or +cat emails.json | node scripts/email-job-search.mjs +``` + +It requires the same env vars as `scripts/job-search.mjs` (`AI_API_KEY`, `AI_MODEL`, +`GITHUB_TOKEN`, `GITHUB_DATA_REPO`, `GITHUB_DATA_BRANCH`, optional `AI_BASE_URL`) and prints a +single JSON summary to stdout on success: `{ "processed", "added", "skippedDuplicate", "skippedNoMatch" }`. + +**How you get emails into that shape is up to you** — any of the following work: + +- **OpenClaw `gog` (Gmail) skill** — a scheduled OpenClaw cron job searches Gmail for + job-platform senders, exports matching threads as JSON, and pipes them into this script. This + is how the deckhandAI maintainer runs it day to day; the OpenClaw job itself is personal + automation (tied to one Gmail account and one data repo), not part of this repo. +- **A plain IMAP fetch script** — any language, any provider, as long as it emits the shape above. +- **A one-off export** — e.g. Gmail's "Download message" or Takeout, converted to the input shape + with a small script, for a manual/occasional pass instead of a recurring schedule. + +Because the extraction call runs once per email and each job-alert digest often lists several +roles, a single run can add multiple jobs from one email. + +--- + ## Reviewing results After any scheduled run, new jobs appear in the **Pending** section of the deckhandAI UI. From there you can: diff --git a/scripts/backfill-ai-summaries.mjs b/scripts/backfill-ai-summaries.mjs index 822645c..afee0f2 100644 --- a/scripts/backfill-ai-summaries.mjs +++ b/scripts/backfill-ai-summaries.mjs @@ -30,9 +30,12 @@ import { existsSync, readFileSync } from "fs"; import { resolve, dirname } from "path"; import { fileURLToPath } from "url"; +import { githubGet, githubPut } from "./lib/github-contents.mjs"; +import { callAI } from "./lib/ai-provider.mjs"; const __dirname = dirname(fileURLToPath(import.meta.url)); const DRY_RUN = process.argv.includes("--dry-run"); +const USER_AGENT = "deckhandAI/backfill-ai-summaries"; // --------------------------------------------------------------------------- // Env (.env.local convenience — existing env always wins) @@ -50,39 +53,6 @@ function loadEnvLocal() { } } -// --------------------------------------------------------------------------- -// GitHub API (same shape as scripts/job-search.mjs) -// --------------------------------------------------------------------------- - -async function githubGet(repo, branch, path) { - const res = await fetch( - `https://api.github.com/repos/${repo}/contents/${path}?ref=${branch}`, - { - headers: { - Authorization: `Bearer ${process.env.GITHUB_TOKEN}`, - "User-Agent": "deckhandAI/backfill-ai-summaries", - }, - } - ); - if (!res.ok) throw new Error(`GitHub GET ${path}: ${res.status} ${await res.text()}`); - const { content, sha } = await res.json(); - return { data: JSON.parse(Buffer.from(content, "base64").toString("utf8")), sha }; -} - -async function githubPut(repo, branch, path, sha, data, message) { - const content = Buffer.from(JSON.stringify(data, null, 2)).toString("base64"); - const res = await fetch(`https://api.github.com/repos/${repo}/contents/${path}`, { - method: "PUT", - headers: { - Authorization: `Bearer ${process.env.GITHUB_TOKEN}`, - "Content-Type": "application/json", - "User-Agent": "deckhandAI/backfill-ai-summaries", - }, - body: JSON.stringify({ message, content, sha, branch }), - }); - if (!res.ok) throw new Error(`GitHub PUT ${path}: ${res.status} ${await res.text()}`); -} - // --------------------------------------------------------------------------- // AI call — endpoint resolution mirrors lib/model.ts // --------------------------------------------------------------------------- @@ -102,54 +72,7 @@ ${job.notes}`; } async function generateSummary(job) { - const provider = process.env.AI_PROVIDER || "anthropic"; - const model = process.env.AI_MODEL || "claude-sonnet-4-6"; - const apiKey = process.env.AI_API_KEY || ""; - - if (provider === "anthropic") { - const res = await fetch("https://api.anthropic.com/v1/messages", { - method: "POST", - headers: { - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - body: JSON.stringify({ - model, - max_tokens: 300, - system: SYSTEM_PROMPT, - messages: [{ role: "user", content: buildUserPrompt(job) }], - }), - }); - if (!res.ok) throw new Error(`AI provider error ${res.status}: ${await res.text()}`); - const json = await res.json(); - return (json.content ?? []).filter((c) => c.type === "text").map((c) => c.text).join(""); - } - - const builtinBase = { - openai: "https://api.openai.com/v1", - gemini: "https://generativelanguage.googleapis.com/v1beta/openai", - grok: "https://api.x.ai/v1", - }; - const baseUrl = process.env.AI_BASE_URL || builtinBase[provider] || "https://api.openai.com/v1"; - const res = await fetch(`${baseUrl}/chat/completions`, { - method: "POST", - headers: { - Authorization: `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - model, - max_tokens: 300, - messages: [ - { role: "system", content: SYSTEM_PROMPT }, - { role: "user", content: buildUserPrompt(job) }, - ], - }), - }); - if (!res.ok) throw new Error(`AI provider error ${res.status}: ${await res.text()}`); - const json = await res.json(); - return json.choices?.[0]?.message?.content ?? ""; + return callAI({ systemPrompt: SYSTEM_PROMPT, userPrompt: buildUserPrompt(job), maxTokens: 300 }); } // Same cleanup rules as normalizeAiSummary in lib/job-summary.ts — keep in sync. @@ -188,7 +111,7 @@ async function main() { } console.log(`Reading data/jobs.json from ${repo}@${branch}…`); - const { data: jobs, sha } = await githubGet(repo, branch, "data/jobs.json"); + const { data: jobs, sha } = await githubGet(repo, branch, "data/jobs.json", USER_AGENT); let generated = 0; let skippedExisting = 0; @@ -234,7 +157,7 @@ async function main() { return; } - await githubPut(repo, branch, "data/jobs.json", sha, jobs, `Backfill aiSummary for ${generated} jobs`); + await githubPut(repo, branch, "data/jobs.json", sha, jobs, `Backfill aiSummary for ${generated} jobs`, USER_AGENT); console.log(`Wrote data/jobs.json to ${repo}@${branch}.`); } diff --git a/scripts/email-job-search.mjs b/scripts/email-job-search.mjs new file mode 100644 index 0000000..c5c1da5 --- /dev/null +++ b/scripts/email-job-search.mjs @@ -0,0 +1,249 @@ +#!/usr/bin/env node +/** + * Email digest ingestion pass for deckhandAI. + * + * Reads a batch of already-exported emails (job-alert digests from services + * like Indeed, TheLadders, or Lensa), uses your configured AI provider to + * extract any qualifying job listings against your data/config.json + * preferences, deduplicates against existing jobs, and writes new entries to + * the pending queue in your GitHub data repo. + * + * This script has no idea where the emails came from — it only knows the + * input shape below. Any tool that can export emails in that shape can feed + * it: a Gmail export, an IMAP fetch script, an mbox parser, or (as one + * example) an OpenClaw `gog`-based Gmail triage job. See + * docs/scheduled-job-search.md for a worked OpenClaw wiring example. + * + * Input (JSON array), via --input or stdin: + * [{ "id": "...", "subject": "...", "from": "...", "receivedAt": "...", "body": "..." }] + * + * Usage: + * node scripts/email-job-search.mjs --input emails.json + * cat emails.json | node scripts/email-job-search.mjs + * + * Required env vars: + * AI_API_KEY — your AI provider API key + * GITHUB_TOKEN — PAT with write access to your data repo + * GITHUB_DATA_REPO — e.g. your-org/your-private-repo + * GITHUB_DATA_BRANCH — e.g. main + * + * Optional: + * AI_PROVIDER — anthropic (default) | openai | gemini | grok | ollama | custom + * AI_MODEL — model to use (default: claude-sonnet-4-6) + * AI_BASE_URL — base URL for OpenAI-compatible endpoints; ignored for + * AI_PROVIDER=anthropic (native Messages API instead). + * Defaults per provider, falling back to + * https://api.openai.com/v1 + * + * On success, prints one JSON summary object to stdout: + * { "processed": 12, "added": [...], "skippedDuplicate": [...], "skippedNoMatch": [...] } + */ + +import { readFileSync } from "fs"; +import { resolve, dirname } from "path"; +import { fileURLToPath } from "url"; +import { githubGet, githubPut } from "./lib/github-contents.mjs"; +import { callAI } from "./lib/ai-provider.mjs"; +import { jobKey } from "./lib/job-key.mjs"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const CONFIG_PATH = resolve(__dirname, "../data/config.json"); +const USER_AGENT = "deckhandAI/email-job-search"; + +// --------------------------------------------------------------------------- +// Config + input +// --------------------------------------------------------------------------- + +function loadConfig() { + try { + return JSON.parse(readFileSync(CONFIG_PATH, "utf8")); + } catch { + console.error("Could not read data/config.json — copy data/config.sample.json to get started."); + process.exit(1); + } +} + +async function readStdin() { + const chunks = []; + for await (const chunk of process.stdin) chunks.push(chunk); + return Buffer.concat(chunks).toString("utf8"); +} + +async function loadEmails() { + const inputFlagIndex = process.argv.indexOf("--input"); + const raw = + inputFlagIndex !== -1 && process.argv[inputFlagIndex + 1] + ? readFileSync(process.argv[inputFlagIndex + 1], "utf8") + : await readStdin(); + + let emails; + try { + emails = JSON.parse(raw); + } catch { + console.error("Could not parse input as JSON — expected an array of email objects."); + process.exit(1); + } + if (!Array.isArray(emails)) { + console.error("Input must be a JSON array of email objects."); + process.exit(1); + } + return emails; +} + +// --------------------------------------------------------------------------- +// AI extraction + filtering (one call per email — digests often list several roles) +// --------------------------------------------------------------------------- + +async function extractQualifyingJobs(email, preferences) { + const prompt = `You are helping filter a job-alert email for a candidate. Be strict — reject a +listing whenever it's unclear or ambiguous, rather than including it on the hope it might fit. +False positives cost the candidate real time reviewing bad matches; a missed listing costs +nothing since it stays in the source digest. + +Candidate preferences: +- Target titles: ${preferences.titles.join(", ")} +- Min FTE salary: $${preferences.salary.min_fte.toLocaleString()} +- Min contract hourly: $${preferences.salary.min_contract_hourly}/hr +- Open to contract: ${preferences.open_to_contract} +- Remote OK: ${preferences.locations.remote} +- Hybrid OK: ${preferences.locations.hybrid} +- Local OK: within ${preferences.locations.hub_radius_miles} miles of ${preferences.locations.hub_city}, ${preferences.locations.hub_state} + +Email subject: "${email.subject}" +Email from: ${email.from} +Email body: +${email.body} + +This email may be a job-alert digest listing multiple postings (e.g. from Indeed, TheLadders, or +Lensa), a single-posting alert, or not a job alert at all. For each listing, apply every rule +below — a listing only qualifies if it passes all of them: + +1. Title: the role must closely match one of the target titles above, at the same seniority + tier and same discipline. Reject IC-level or junior/mid titles when the target list is + director/head/VP/principal-tier, and reject listings in an unrelated discipline (e.g. interior + design, motion/multimedia design, marketing) even if the word "design" or "designer" appears + in the title. A near-miss title in the wrong discipline is not a match. +2. Salary: if a salary or rate is stated, it must meet the FTE or contract floor above (whichever + applies). If no salary is stated, only include the listing if everything else is a clear + strong match — do not assume an unstated salary meets the floor. +3. Location: must be explicitly remote, explicitly hybrid, or within the local radius above. + Vague location text ("Virtual / Travel", "Various", unspecified) does not count as remote — + treat it as on-site/unclear and reject unless the listing separately confirms remote/hybrid + eligibility. +4. Not a listing at all: ignore anything that isn't an actual job posting (ads, "don't miss out" + footers, unsubscribe/preference links, unrelated content). + +Return a JSON array only, no other text. Each item: { "company": "", "role": "", "url": "", "salary": "", "notes": "" } +The "notes" field must briefly state why it passed all four rules (e.g. "Director-level UX role, +$190K remote — matches title/salary/location"). Return [] if nothing in this email clearly +qualifies.`; + + const text = (await callAI({ userPrompt: prompt, temperature: 0 })).trim(); + + try { + return JSON.parse(text.replace(/^```json\n?/, "").replace(/\n?```$/, "")); + } catch { + console.warn(` Could not parse AI response as JSON for "${email.subject}":`, text.slice(0, 200)); + return []; + } +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + const required = ["GITHUB_TOKEN", "GITHUB_DATA_REPO", "GITHUB_DATA_BRANCH", "AI_API_KEY"]; + const missing = required.filter((k) => !process.env[k]); + if (missing.length) { + console.error(`Missing required env vars: ${missing.join(", ")}`); + process.exit(1); + } + + const { GITHUB_DATA_REPO, GITHUB_DATA_BRANCH } = process.env; + + const config = loadConfig(); + const { preferences } = config; + const emails = await loadEmails(); + + if (emails.length === 0) { + console.log(JSON.stringify({ processed: 0, added: [], skippedDuplicate: [], skippedNoMatch: [] })); + return; + } + + console.error(`Loading jobs from ${GITHUB_DATA_REPO}...`); + const { data: jobs, sha } = await githubGet(GITHUB_DATA_REPO, GITHUB_DATA_BRANCH, "data/jobs.json", USER_AGENT); + + const existingJobs = Object.values(jobs).filter(Array.isArray).flat(); + const existingUrls = new Set(existingJobs.map((j) => j.url).filter(Boolean)); + const existingKeys = new Set(existingJobs.map((j) => jobKey(j.company ?? "", j.role ?? ""))); + + console.error(`${existingUrls.size} existing job URLs loaded for deduplication.`); + + const today = new Date().toISOString().slice(0, 10); + const added = []; + const skippedDuplicate = []; + const skippedNoMatch = []; + + for (const email of emails) { + console.error(`Email: "${email.subject}" from ${email.from}`); + + let extracted; + try { + extracted = await extractQualifyingJobs(email, preferences); + } catch (err) { + console.error(` AI extraction failed: ${err.message}`); + continue; + } + + if (extracted.length === 0) { + skippedNoMatch.push({ emailId: email.id, subject: email.subject }); + continue; + } + + for (const job of extracted) { + if (!job.url) continue; + const key = jobKey(job.company ?? "", job.role ?? ""); + if (existingUrls.has(job.url) || existingKeys.has(key)) { + skippedDuplicate.push({ company: job.company, role: job.role, url: job.url }); + continue; + } + existingUrls.add(job.url); + existingKeys.add(key); + const pendingJob = { + company: job.company ?? "", + role: job.role ?? "", + url: job.url, + salary: job.salary ?? "", + notes: job.notes ?? "", + scrapeGroup: "remote", + scrapeDate: today, + }; + added.push(pendingJob); + console.error(` + ${pendingJob.company} — ${pendingJob.role}`); + } + } + + if (added.length > 0) { + if (!Array.isArray(jobs.pending)) jobs.pending = []; + jobs.pending.push(...added); + + console.error(`Writing ${added.length} new job(s) to ${GITHUB_DATA_REPO}...`); + await githubPut( + GITHUB_DATA_REPO, + GITHUB_DATA_BRANCH, + "data/jobs.json", + sha, + jobs, + `Add ${added.length} job${added.length === 1 ? "" : "s"} to pending queue via email digest ingestion`, + USER_AGENT + ); + } + + console.log(JSON.stringify({ processed: emails.length, added, skippedDuplicate, skippedNoMatch })); +} + +main().catch((err) => { + console.error("Fatal:", err); + process.exit(1); +}); diff --git a/scripts/job-search.mjs b/scripts/job-search.mjs index 64de453..5870a5d 100644 --- a/scripts/job-search.mjs +++ b/scripts/job-search.mjs @@ -7,18 +7,21 @@ * against existing jobs, and writes new entries to the pending queue in your * GitHub data repo. * - * Works with any OpenAI-compatible AI provider. + * Works with Anthropic natively, or any OpenAI-compatible AI provider. * * Required env vars: * AI_API_KEY — your AI provider API key - * AI_MODEL — model to use (e.g. gemini-2.0-flash, gpt-4o-mini) * GITHUB_TOKEN — PAT with write access to your data repo * GITHUB_DATA_REPO — e.g. your-org/your-private-repo * GITHUB_DATA_BRANCH — e.g. main * * Optional: - * AI_BASE_URL — base URL for OpenAI-compatible endpoint - * (defaults to https://api.openai.com/v1) + * AI_PROVIDER — anthropic (default) | openai | gemini | grok | ollama | custom + * AI_MODEL — model to use (default: claude-sonnet-4-6) + * AI_BASE_URL — base URL for OpenAI-compatible endpoints; ignored + * for AI_PROVIDER=anthropic (native Messages API + * instead). Defaults per provider, falling back to + * https://api.openai.com/v1 * BRAVE_SEARCH_API_KEY — Brave Search API key (recommended for reliable results) * Falls back to DuckDuckGo if not set. */ @@ -26,9 +29,13 @@ import { readFileSync } from "fs"; import { resolve, dirname } from "path"; import { fileURLToPath } from "url"; +import { githubGet, githubPut } from "./lib/github-contents.mjs"; +import { callAI } from "./lib/ai-provider.mjs"; +import { jobKey } from "./lib/job-key.mjs"; const __dirname = dirname(fileURLToPath(import.meta.url)); const CONFIG_PATH = resolve(__dirname, "../data/config.json"); +const USER_AGENT = "deckhandAI/job-search"; // --------------------------------------------------------------------------- // Config @@ -43,39 +50,6 @@ function loadConfig() { } } -// --------------------------------------------------------------------------- -// GitHub API -// --------------------------------------------------------------------------- - -async function githubGet(repo, branch, path) { - const res = await fetch( - `https://api.github.com/repos/${repo}/contents/${path}?ref=${branch}`, - { - headers: { - Authorization: `Bearer ${process.env.GITHUB_TOKEN}`, - "User-Agent": "deckhandAI/job-search", - }, - } - ); - if (!res.ok) throw new Error(`GitHub GET ${path}: ${res.status} ${await res.text()}`); - const { content, sha } = await res.json(); - return { data: JSON.parse(Buffer.from(content, "base64").toString("utf8")), sha }; -} - -async function githubPut(repo, branch, path, sha, data, message) { - const content = Buffer.from(JSON.stringify(data, null, 2)).toString("base64"); - const res = await fetch(`https://api.github.com/repos/${repo}/contents/${path}`, { - method: "PUT", - headers: { - Authorization: `Bearer ${process.env.GITHUB_TOKEN}`, - "Content-Type": "application/json", - "User-Agent": "deckhandAI/job-search", - }, - body: JSON.stringify({ message, content, sha, branch }), - }); - if (!res.ok) throw new Error(`GitHub PUT ${path}: ${res.status} ${await res.text()}`); -} - // --------------------------------------------------------------------------- // Search backends // --------------------------------------------------------------------------- @@ -136,9 +110,10 @@ async function searchDuckDuckGo(query) { // --------------------------------------------------------------------------- async function filterWithAI(results, preferences, query) { - const baseUrl = (process.env.AI_BASE_URL ?? "https://api.openai.com/v1").replace(/\/$/, ""); - - const prompt = `You are helping filter job search results for a candidate. + const prompt = `You are helping filter job search results for a candidate. Be strict — reject a +result whenever it's unclear or ambiguous, rather than including it on the hope it might fit. +False positives cost the candidate real time reviewing bad matches; a missed result costs +nothing since the source search can be rerun. Candidate preferences: - Target titles: ${preferences.titles.join(", ")} @@ -147,34 +122,35 @@ Candidate preferences: - Open to contract: ${preferences.open_to_contract} - Remote OK: ${preferences.locations.remote} - Hybrid OK: ${preferences.locations.hybrid} -- Hub city: ${preferences.locations.hub_city}, ${preferences.locations.hub_state} +- Local OK: within ${preferences.locations.hub_radius_miles} miles of ${preferences.locations.hub_city}, ${preferences.locations.hub_state} Search query used: "${query}" Search results: ${results.map((r, i) => `${i + 1}. ${r.title}\n URL: ${r.url}\n ${r.description}`).join("\n\n")} -From these results, extract any that appear to be active job listings matching the candidate's seniority level and preferences. Ignore results that are clearly not job listings (blog posts, news articles, company homepages). +Apply every rule below to each result — it only qualifies if it passes all of them: + +1. Title: the role must closely match one of the target titles above, at the same seniority + tier and same discipline. Reject IC-level or junior/mid titles when the target list is + director/head/VP/principal-tier, and reject listings in an unrelated discipline even if + "design"/"designer" appears in the title. A near-miss title in the wrong discipline is not a + match. +2. Salary: if a salary or rate is stated, it must meet the FTE or contract floor above (whichever + applies). Search snippets often omit salary — that's fine, but don't assume an unstated + salary meets the floor; only include salary-unstated results when title/location are a clear + strong match. +3. Location: must be explicitly remote, explicitly hybrid, or within the local radius above. + Vague or missing location does not count as remote — reject unless remote/hybrid/local + eligibility is clearly stated. +4. Not a listing at all: ignore results that are clearly not active job listings (blog posts, + news articles, company homepages, expired/closed postings). Return a JSON array only, no other text. Each item: { "company": "", "role": "", "url": "", "salary": "", "notes": "" } -Return [] if nothing qualifies.`; - - const res = await fetch(`${baseUrl}/chat/completions`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${process.env.AI_API_KEY}`, - }, - body: JSON.stringify({ - model: process.env.AI_MODEL, - messages: [{ role: "user", content: prompt }], - temperature: 0, - }), - }); +The "notes" field must briefly state why it passed all four rules. Return [] if nothing clearly +qualifies.`; - if (!res.ok) throw new Error(`AI API: ${res.status} ${await res.text()}`); - const data = await res.json(); - const text = data.choices[0].message.content.trim(); + const text = (await callAI({ userPrompt: prompt, temperature: 0 })).trim(); try { return JSON.parse(text.replace(/^```json\n?/, "").replace(/\n?```$/, "")); @@ -189,7 +165,7 @@ Return [] if nothing qualifies.`; // --------------------------------------------------------------------------- async function main() { - const required = ["GITHUB_TOKEN", "GITHUB_DATA_REPO", "GITHUB_DATA_BRANCH", "AI_API_KEY", "AI_MODEL"]; + const required = ["GITHUB_TOKEN", "GITHUB_DATA_REPO", "GITHUB_DATA_BRANCH", "AI_API_KEY"]; const missing = required.filter((k) => !process.env[k]); if (missing.length) { console.error(`Missing required env vars: ${missing.join(", ")}`); @@ -207,15 +183,11 @@ async function main() { } console.log(`Loading jobs from ${GITHUB_DATA_REPO}...`); - const { data: jobs, sha } = await githubGet(GITHUB_DATA_REPO, GITHUB_DATA_BRANCH, "data/jobs.json"); - - const existingUrls = new Set( - Object.values(jobs) - .filter(Array.isArray) - .flat() - .map((j) => j.url) - .filter(Boolean) - ); + const { data: jobs, sha } = await githubGet(GITHUB_DATA_REPO, GITHUB_DATA_BRANCH, "data/jobs.json", USER_AGENT); + + const existingJobs = Object.values(jobs).filter(Array.isArray).flat(); + const existingUrls = new Set(existingJobs.map((j) => j.url).filter(Boolean)); + const existingKeys = new Set(existingJobs.map((j) => jobKey(j.company ?? "", j.role ?? ""))); console.log(`${existingUrls.size} existing job URLs loaded for deduplication.`); @@ -248,8 +220,11 @@ async function main() { } for (const job of extracted) { - if (!job.url || existingUrls.has(job.url)) continue; + if (!job.url) continue; + const key = jobKey(job.company ?? "", job.role ?? ""); + if (existingUrls.has(job.url) || existingKeys.has(key)) continue; existingUrls.add(job.url); + existingKeys.add(key); newJobs.push({ company: job.company ?? "", role: job.role ?? "", @@ -279,7 +254,8 @@ async function main() { "data/jobs.json", sha, jobs, - `Add ${newJobs.length} job${newJobs.length === 1 ? "" : "s"} to pending queue via WebSearch pass` + `Add ${newJobs.length} job${newJobs.length === 1 ? "" : "s"} to pending queue via WebSearch pass`, + USER_AGENT ); console.log(`Done. ${newJobs.length} new job(s) added to pending queue.`); diff --git a/scripts/lib/ai-provider.mjs b/scripts/lib/ai-provider.mjs new file mode 100644 index 0000000..0781ce7 --- /dev/null +++ b/scripts/lib/ai-provider.mjs @@ -0,0 +1,68 @@ +// Shared AI call helper for standalone scripts (scripts/job-search.mjs, +// scripts/email-job-search.mjs, scripts/backfill-ai-summaries.mjs). Speaks +// Anthropic's Messages API natively when AI_PROVIDER=anthropic (the +// default — matches lib/model.ts's default), and falls back to an +// OpenAI-compatible chat/completions call for openai/gemini/grok/ollama/ +// custom, so the same AI_API_KEY these scripts already require works +// whichever provider it belongs to. +// +// Env vars: AI_PROVIDER (default "anthropic"), AI_MODEL, AI_API_KEY, +// AI_BASE_URL (only used for non-Anthropic providers; falls back to each +// provider's known default, then OpenAI's). + +const BUILTIN_BASE_URLS = { + openai: "https://api.openai.com/v1", + gemini: "https://generativelanguage.googleapis.com/v1beta/openai", + grok: "https://api.x.ai/v1", +}; + +export async function callAI({ systemPrompt, userPrompt, maxTokens = 1024, temperature }) { + const provider = process.env.AI_PROVIDER || "anthropic"; + const model = process.env.AI_MODEL || "claude-sonnet-4-6"; + const apiKey = process.env.AI_API_KEY || ""; + + if (provider === "anthropic") { + const res = await fetch("https://api.anthropic.com/v1/messages", { + method: "POST", + headers: { + "x-api-key": apiKey, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + body: JSON.stringify({ + model, + max_tokens: maxTokens, + ...(temperature !== undefined ? { temperature } : {}), + ...(systemPrompt ? { system: systemPrompt } : {}), + messages: [{ role: "user", content: userPrompt }], + }), + }); + if (!res.ok) throw new Error(`AI provider error ${res.status}: ${await res.text()}`); + const json = await res.json(); + return (json.content ?? []).filter((c) => c.type === "text").map((c) => c.text).join(""); + } + + const baseUrl = (process.env.AI_BASE_URL || BUILTIN_BASE_URLS[provider] || "https://api.openai.com/v1").replace( + /\/$/, + "" + ); + const messages = systemPrompt + ? [{ role: "system", content: systemPrompt }, { role: "user", content: userPrompt }] + : [{ role: "user", content: userPrompt }]; + const res = await fetch(`${baseUrl}/chat/completions`, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model, + max_tokens: maxTokens, + ...(temperature !== undefined ? { temperature } : {}), + messages, + }), + }); + if (!res.ok) throw new Error(`AI provider error ${res.status}: ${await res.text()}`); + const json = await res.json(); + return json.choices?.[0]?.message?.content ?? ""; +} diff --git a/scripts/lib/github-contents.mjs b/scripts/lib/github-contents.mjs new file mode 100644 index 0000000..dbb375b --- /dev/null +++ b/scripts/lib/github-contents.mjs @@ -0,0 +1,34 @@ +// Shared GitHub Contents API read/write helpers for the data-repo JSON files +// (data/jobs.json, data/config.json). Used by any script that reads or +// writes the private data repo directly (scripts/job-search.mjs, +// scripts/backfill-ai-summaries.mjs, scripts/email-job-search.mjs) so the +// GET-then-sha-guarded-PUT contract only lives in one place. +// +// Requires GITHUB_TOKEN in the environment; callers are responsible for +// validating it's set before calling. + +export async function githubGet(repo, branch, path, userAgent) { + const res = await fetch(`https://api.github.com/repos/${repo}/contents/${path}?ref=${branch}`, { + headers: { + Authorization: `Bearer ${process.env.GITHUB_TOKEN}`, + "User-Agent": userAgent, + }, + }); + if (!res.ok) throw new Error(`GitHub GET ${path}: ${res.status} ${await res.text()}`); + const { content, sha } = await res.json(); + return { data: JSON.parse(Buffer.from(content, "base64").toString("utf8")), sha }; +} + +export async function githubPut(repo, branch, path, sha, data, message, userAgent) { + const content = Buffer.from(JSON.stringify(data, null, 2)).toString("base64"); + const res = await fetch(`https://api.github.com/repos/${repo}/contents/${path}`, { + method: "PUT", + headers: { + Authorization: `Bearer ${process.env.GITHUB_TOKEN}`, + "Content-Type": "application/json", + "User-Agent": userAgent, + }, + body: JSON.stringify({ message, content, sha, branch }), + }); + if (!res.ok) throw new Error(`GitHub PUT ${path}: ${res.status} ${await res.text()}`); +} diff --git a/scripts/lib/job-key.mjs b/scripts/lib/job-key.mjs new file mode 100644 index 0000000..89764d4 --- /dev/null +++ b/scripts/lib/job-key.mjs @@ -0,0 +1,9 @@ +// Mirrors jobKey() in lib/jobs.ts — keep in sync. Used by standalone scripts +// (scripts/email-job-search.mjs, scripts/job-search.mjs) as a company+role +// dedup fallback for sources whose links aren't stable identifiers (e.g. +// TheLadders/Lensa email tracking-redirect URLs, which mint a fresh token +// per send, so the same posting re-appearing in a later digest would not +// match on url alone). +export function jobKey(company, role) { + return `${company}::${role}`; +}