Skip to content

Commit 26bee69

Browse files
rochdevrochdev
authored
minimize requests for all green script and retry faster (#8180)
1 parent 7763af7 commit 26bee69

1 file changed

Lines changed: 98 additions & 125 deletions

File tree

‎scripts/all-green.mjs‎

Lines changed: 98 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,11 @@ const {
1313
RETRIES,
1414
} = process.env
1515

16-
const maxRerunFailedJobs = 3
17-
1816
const octokit = new Octokit({ auth: GITHUB_TOKEN })
1917
const owner = 'DataDog'
2018
const repo = 'dd-trace-js'
2119
const ref = context.payload.pull_request?.head.sha || GITHUB_SHA
22-
const params = { owner, repo, ref }
20+
2321
const conclusionEmojis = {
2422
action_required: '🔶',
2523
cancelled: '🚫',
@@ -42,179 +40,154 @@ const conclusionSeverity = {
4240
success: 7,
4341
}
4442

45-
let retries = 0
46-
let hasRerun = false
47-
48-
async function hasCompleted () {
49-
const { data: inProgressRuns } = await octokit.rest.checks.listForRef({
50-
...params,
51-
per_page: 1, // Minimum is 1 but we don't need any pages.
52-
status: 'in_progress',
53-
})
54-
55-
// If there are any in progress runs it means we're not ready to check
56-
// statuses. We will always have minimum 1 for the All Green job.
57-
if (inProgressRuns.total_count > 1) return false
43+
const failureConclusions = new Set(['failure', 'timed_out'])
5844

59-
const { data: queuedRuns } = await octokit.rest.checks.listForRef({
60-
...params,
61-
per_page: 1, // Minimum is 1 but we don't need any pages.
62-
status: 'queued',
63-
})
64-
65-
// Same as above, but jobs that are queued are not even in progress yet.
66-
if (queuedRuns.total_count > 0) return false
67-
68-
return true
69-
}
45+
let retries = 0
46+
const retriedRunIds = new Set()
7047

71-
async function checkCompleted () {
72-
if (!await hasCompleted()) {
73-
retries++
48+
// ETag cache for the workflow-runs poll. GitHub returns 304 Not Modified when
49+
// the response is unchanged, and 304 responses don't count against the rate
50+
// limit. workflow_runs are sorted newest first, so an unchanged first page is
51+
// a reliable proxy for "no changes since last poll".
52+
let runsCache
7453

75-
if (RETRIES && retries > RETRIES) {
76-
throw new Error(`State is still pending after ${RETRIES} retries.`)
54+
async function getRuns () {
55+
try {
56+
const allRuns = []
57+
let etag
58+
for await (const { data, headers } of octokit.paginate.iterator(
59+
octokit.rest.actions.listWorkflowRunsForRepo,
60+
{
61+
owner,
62+
repo,
63+
head_sha: ref,
64+
per_page: 100,
65+
headers: runsCache ? { 'if-none-match': runsCache.etag } : {},
66+
}
67+
)) {
68+
etag ??= headers.etag
69+
allRuns.push(...data)
7770
}
78-
79-
console.log(`Status is still pending, waiting for ${POLLING_INTERVAL} minutes before retrying.`)
80-
await setTimeout(POLLING_INTERVAL * 60_000)
81-
console.log('Retrying.')
82-
await checkCompleted()
71+
// Isolate per trigger so a parallel all-green run on the same SHA doesn't
72+
// see our runs (and we don't see theirs). Filter by event, by PR number
73+
// when on a PR (handles two PRs sharing the same head commit), and drop
74+
// our own All Green run since it stays in_progress while we poll.
75+
const myPR = context.payload.pull_request?.number
76+
const filtered = allRuns.filter(r =>
77+
r.name !== context.workflow &&
78+
r.event === context.eventName &&
79+
(myPR == null || r.pull_requests?.some(pr => pr.number === myPR))
80+
)
81+
runsCache = { etag, runs: filtered }
82+
return filtered
83+
} catch (err) {
84+
if (err.status === 304 && runsCache) return runsCache.runs
85+
throw err
8386
}
8487
}
8588

86-
async function getLatestRuns () {
87-
const checkRuns = await octokit.paginate(
88-
'GET /repos/:owner/:repo/commits/:ref/check-runs',
89-
{
90-
...params,
91-
per_page: 100,
92-
}
93-
)
89+
async function pollUntilDone () {
90+
const runs = await getRuns()
9491

95-
// When a check is re-run, older runs remain with their original conclusions.
96-
// Deduplicate by name and evaluate only the latest run for each check.
97-
const latestByName = new Map()
98-
for (const run of checkRuns) {
99-
const existing = latestByName.get(run.name)
100-
if (!existing || new Date(run.started_at) >= new Date(existing.started_at)) {
101-
latestByName.set(run.name, run)
102-
}
103-
}
104-
105-
return [...latestByName.values()]
106-
}
92+
const toRetry = runs.filter(r =>
93+
r.status === 'completed' &&
94+
failureConclusions.has(r.conclusion) &&
95+
!retriedRunIds.has(r.id)
96+
)
10797

108-
async function rerunFailedWorkflows (failedRuns) {
109-
const failedCountByCheckSuiteId = new Map()
110-
for (const run of failedRuns) {
111-
const id = run.check_suite?.id
112-
if (id !== undefined) {
113-
failedCountByCheckSuiteId.set(id, (failedCountByCheckSuiteId.get(id) ?? 0) + 1)
114-
}
98+
if (toRetry.length > 0) {
99+
await rerunFailedWorkflows(toRetry)
100+
for (const run of toRetry) retriedRunIds.add(run.id)
101+
runsCache = undefined
115102
}
116103

117-
const eligibleSuiteIds = [...failedCountByCheckSuiteId.entries()]
118-
.filter(([, count]) => count <= maxRerunFailedJobs)
119-
.map(([id]) => id)
104+
const pending = runs.filter(r => r.status !== 'completed').length
105+
if (pending === 0 && toRetry.length === 0) return { runs, done: true }
120106

121-
// If a workflow has many jobs failed, it's unlikely to be flakiness to no
122-
// point in re-running.
123-
if (eligibleSuiteIds.length < failedCountByCheckSuiteId.size) {
124-
console.log(
125-
`Skipping rerun for ${failedCountByCheckSuiteId.size - eligibleSuiteIds.length} workflow(s) ` +
126-
`with more than ${maxRerunFailedJobs} failed job(s).`
127-
)
128-
}
107+
retries++
129108

130-
const workflowRunsPerSuite = await Promise.all(
131-
eligibleSuiteIds.map(checkSuiteId =>
132-
octokit.rest.actions.listWorkflowRunsForRepo({ owner, repo, check_suite_id: checkSuiteId })
133-
.then(({ data }) => data.workflow_runs)
134-
)
135-
)
109+
if (RETRIES && retries > RETRIES) return { runs, done: false }
136110

137-
const workflowRuns = workflowRunsPerSuite.flat()
111+
console.log(`Status is still pending, waiting for ${POLLING_INTERVAL} minutes before retrying.`)
112+
await setTimeout(POLLING_INTERVAL * 60_000)
113+
console.log('Retrying.')
114+
return pollUntilDone()
115+
}
138116

117+
async function rerunFailedWorkflows (workflowRuns) {
139118
await Promise.all(
140119
workflowRuns.map(workflowRun => {
141120
console.log(`Rerunning failed jobs for workflow run ${workflowRun.id} (${workflowRun.name}).`)
142121
return octokit.rest.actions.reRunWorkflowFailedJobs({ owner, repo, run_id: workflowRun.id })
143122
})
144123
)
145-
146-
return workflowRuns.length > 0
147124
}
148125

149126
async function checkAllGreen () {
150-
let latestRuns
127+
const { runs, done } = await pollUntilDone()
151128

152-
try {
153-
await checkCompleted()
154-
} finally {
155-
latestRuns = await getLatestRuns()
129+
await printSummary(runs)
130+
131+
if (!done) {
132+
console.log(`State is still pending after ${RETRIES} retries.`)
133+
process.exitCode = 1
134+
return
156135
}
157136

158-
const failedRuns = latestRuns.filter(run =>
159-
run.conclusion === 'failure' || run.conclusion === 'timed_out'
160-
)
137+
const failedRuns = runs.filter(r => failureConclusions.has(r.conclusion))
161138

162139
if (failedRuns.length === 0) {
163-
await printSummary(latestRuns)
164140
console.log('All jobs were successful.')
165-
return
141+
} else {
142+
console.log('One or more jobs failed.')
143+
process.exitCode = 1
166144
}
145+
}
167146

168-
if (!hasRerun) {
169-
hasRerun = true
170-
console.log(`${failedRuns.length} job(s) failed. Rerunning failed workflows...`)
171-
const didRerun = await rerunFailedWorkflows(failedRuns)
172-
if (didRerun) {
173-
retries = 0
174-
console.log(`Waiting for ${POLLING_INTERVAL} minutes before polling for rerun results.`)
175-
await setTimeout(POLLING_INTERVAL * 60_000)
176-
await checkAllGreen()
177-
return
178-
}
179-
}
147+
function formatConclusion (conclusion) {
148+
return conclusion ? `${conclusion} ${conclusionEmojis[conclusion]}` : ' '
149+
}
180150

181-
await printSummary(latestRuns)
182-
throw new Error('One or more jobs failed.')
151+
function bySeverity (a, b) {
152+
return (conclusionSeverity[a.conclusion] ?? 8) - (conclusionSeverity[b.conclusion] ?? 8)
183153
}
184154

185-
async function printSummary (checkRuns) {
186-
const runs = [...checkRuns]
187-
.sort((a, b) => (conclusionSeverity[a.conclusion] ?? 8) - (conclusionSeverity[b.conclusion] ?? 8))
155+
async function printSummary (runs) {
156+
const rows = runs
157+
.sort(bySeverity)
188158
.map(run => ({
189159
name: run.name,
190160
status: run.status,
191-
conclusion: run.conclusion
192-
? `${run.conclusion} ${conclusionEmojis[run.conclusion]}`
193-
: ' ',
194-
started_at: run.started_at,
195-
completed_at: run.completed_at ?? ' ',
161+
conclusion: formatConclusion(run.conclusion),
162+
// workflow_run has no completed_at; updated_at reflects the final state
163+
// change once status === 'completed', otherwise it's an in-flight tick.
164+
started_at: run.run_started_at,
165+
completed_at: run.status === 'completed' ? run.updated_at : ' ',
166+
url: run.html_url,
196167
}))
197168

198-
console.table(runs)
169+
// console.table can't render HTML, so the raw URL goes here as its own
170+
// column. The GitHub Actions summary below renders the name as a link.
171+
console.table(rows)
199172

200173
const header = [
201-
{ data: 'name', header: true },
174+
{ data: 'workflow', header: true },
202175
{ data: 'status', header: true },
203176
{ data: 'conclusion', header: true },
204177
{ data: 'started_at', header: true },
205178
{ data: 'completed_at', header: true },
206179
]
207180

208-
const body = runs.map(run => [
209-
run.name,
210-
run.status,
211-
run.conclusion,
212-
run.started_at,
213-
run.completed_at,
181+
const body = rows.map(row => [
182+
`<a href="${row.url}">${row.name}</a>`,
183+
row.status,
184+
row.conclusion,
185+
row.started_at,
186+
row.completed_at,
214187
])
215188

216189
await summary
217-
.addHeading('Checks Summary')
190+
.addHeading('Workflows Summary')
218191
.addTable([header, ...body])
219192
.write()
220193
}

0 commit comments

Comments
 (0)