Skip to content

Commit 121e487

Browse files
fix: Make CDP extraction provider-aware for all providers
The CDP extraction code was hardcoded for Claude.ai selectors, causing it to hang or fail when used as fallback for ChatGPT and other providers. ## Changes ### Provider-Specific Selector Discovery - Now uses `PROVIDER_SELECTOR_CANDIDATES[provider]` instead of hardcoded Claude selectors - Same selector patterns as Playwright code ensures consistency - Logs which selector pattern matched for debugging ### Provider-Aware Message Extraction - ChatGPT: Uses `[data-message-author-role]` attribute for role detection - Claude: Uses `[data-testid="user-message"]` and `[data-is-streaming]` - Gemini: Uses custom elements `user-query` and `response-container` - Grok: Uses `[data-testid*="message"]` and role attributes - Generic fallback for unknown structures ### Dynamic Title Generation - Title fallback now uses provider name instead of hardcoded "Claude Conversation" This ensures CDP mode works correctly as a fallback for any provider, not just Claude.ai. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 2991865 commit 121e487

2 files changed

Lines changed: 50 additions & 36 deletions

File tree

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "chat-shared-conversation-to-file",
3-
"version": "0.4.1",
3+
"version": "0.4.2",
44
"description": "CLI to download public share links (ChatGPT/Gemini/Grok/Claude) and save clean Markdown + HTML transcripts.",
55
"license": "MIT",
66
"bin": {

src/index.ts

Lines changed: 49 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -2656,65 +2656,79 @@ async function scrape(
26562656

26572657
if (!opts.quiet) console.error(chalk.blue('[3/8] Extracting conversation...'))
26582658

2659-
// CDP mode: Extract content directly using puppeteer and return early
2660-
// Claude.ai selectors
2661-
const claudeSelectors = [
2662-
'[data-testid="user-message"]',
2663-
'[data-is-streaming]',
2664-
'.prose',
2665-
'[class*="message"]'
2666-
]
2659+
// CDP mode: Extract content using puppeteer with provider-specific selectors
2660+
const cdpSelectors = PROVIDER_SELECTOR_CANDIDATES[provider] ?? PROVIDER_SELECTOR_CANDIDATES.chatgpt
26672661

2668-
// Wait for content
2669-
let hasContent = false
2670-
for (const selector of claudeSelectors) {
2662+
// Wait for content with provider-specific selectors
2663+
let workingSelector: string | null = null
2664+
for (const group of cdpSelectors) {
2665+
const combined = group.join(',')
26712666
try {
2672-
await puppeteerPage.waitForSelector(selector, { timeout: 5000 })
2673-
hasContent = true
2667+
await puppeteerPage.waitForSelector(combined, { timeout: 5000 })
2668+
workingSelector = combined
2669+
if (!opts.quiet) console.error(chalk.gray(` Found content with: ${combined.slice(0, 50)}...`))
26742670
break
26752671
} catch {
2676-
// Try next selector
2672+
// Try next selector group
26772673
}
26782674
}
26792675

2680-
if (!hasContent) {
2676+
if (!workingSelector) {
26812677
throw new AppError(
2682-
'Could not find conversation content on Claude.ai.',
2678+
`Could not find conversation content for ${provider}.`,
26832679
'The page may still be loading or the share link may be invalid.'
26842680
)
26852681
}
26862682

2687-
// Extract messages using puppeteer
2688-
const messages = await puppeteerPage.evaluate(() => {
2683+
// Extract messages using puppeteer with provider-aware logic
2684+
const messages = await puppeteerPage.evaluate((prov: string) => {
26892685
const results: { role: string; content: string }[] = []
26902686

2691-
// Try to find user and assistant messages
2692-
const userMsgs = document.querySelectorAll('[data-testid="user-message"]')
2693-
const assistantMsgs = document.querySelectorAll('[data-is-streaming], .prose')
2687+
if (prov === 'chatgpt') {
2688+
// ChatGPT: messages have data-message-author-role attribute
2689+
const msgs = document.querySelectorAll('[data-message-author-role]')
2690+
msgs.forEach(el => {
2691+
const role = el.getAttribute('data-message-author-role') || 'unknown'
2692+
results.push({ role, content: el.innerHTML })
2693+
})
2694+
} else if (prov === 'claude') {
2695+
// Claude: user-message and data-is-streaming
2696+
const userMsgs = document.querySelectorAll('[data-testid="user-message"]')
2697+
const assistantMsgs = document.querySelectorAll('[data-is-streaming], .prose')
2698+
userMsgs.forEach(el => results.push({ role: 'user', content: el.innerHTML }))
2699+
assistantMsgs.forEach(el => results.push({ role: 'assistant', content: el.innerHTML }))
2700+
} else if (prov === 'gemini') {
2701+
// Gemini: user-query and response-container
2702+
const userMsgs = document.querySelectorAll('user-query')
2703+
const assistantMsgs = document.querySelectorAll('response-container')
2704+
userMsgs.forEach(el => results.push({ role: 'user', content: el.innerHTML }))
2705+
assistantMsgs.forEach(el => results.push({ role: 'assistant', content: el.innerHTML }))
2706+
} else if (prov === 'grok') {
2707+
// Grok: similar to ChatGPT with message containers
2708+
const msgs = document.querySelectorAll('[data-testid*="message"], [data-message-author-role]')
2709+
msgs.forEach(el => {
2710+
const role = el.getAttribute('data-message-author-role') ||
2711+
(el.className.includes('user') ? 'user' : 'assistant')
2712+
results.push({ role, content: el.innerHTML })
2713+
})
2714+
}
26942715

2695-
// If specific selectors don't work, try generic message containers
2696-
if (userMsgs.length === 0 && assistantMsgs.length === 0) {
2697-
const allMsgs = document.querySelectorAll('[class*="message"]')
2716+
// Generic fallback if no messages found
2717+
if (results.length === 0) {
2718+
const allMsgs = document.querySelectorAll('[class*="message"], article, [role="article"]')
26982719
allMsgs.forEach(el => {
26992720
const text = el.textContent?.trim() || ''
2700-
if (text) {
2721+
if (text && text.length > 10) {
27012722
results.push({ role: 'unknown', content: el.innerHTML })
27022723
}
27032724
})
2704-
} else {
2705-
userMsgs.forEach(el => {
2706-
results.push({ role: 'user', content: el.innerHTML })
2707-
})
2708-
assistantMsgs.forEach(el => {
2709-
results.push({ role: 'assistant', content: el.innerHTML })
2710-
})
27112725
}
27122726

27132727
return results
2714-
})
2728+
}, provider)
27152729

27162730
// Get page title
2717-
const pageTitle = await puppeteerPage.title() || 'Claude Conversation'
2731+
const pageTitle = await puppeteerPage.title() || `${provider.charAt(0).toUpperCase() + provider.slice(1)} Conversation`
27182732

27192733
// Close puppeteer browser
27202734
await puppeteerBrowser.disconnect()
@@ -2836,7 +2850,7 @@ async function scrape(
28362850
}
28372851

28382852
return {
2839-
title: pageTitle.replace(/\s*[-|].*$/, '').trim() || 'Claude Conversation',
2853+
title: pageTitle.replace(/\s*[-|].*$/, '').trim() || `${provider.charAt(0).toUpperCase() + provider.slice(1)} Conversation`,
28402854
markdown: normalizeLineTerminators(lines.join('\n')),
28412855
retrievedAt: new Date().toISOString()
28422856
}

0 commit comments

Comments
 (0)