[test optimization] Update extraction of runner diagnostic dir using glob pattern matching (#8115)

Author: cbasitodx

packages/dd-trace/src/plugins/util/ci.js

@@ -103,42 +103,129 @@ function getGitHubEventPayload () {
   return JSON.parse(readFileSync(path, 'utf8'))
 }
 
-function getJobIDFromDiagFile (runnerTemp) {
-  if (!runnerTemp || !existsSync(runnerTemp)) { return null }
+const uniq = (items) => [...new Set(items)]
+
+/**
+ * GitHub runner diagnostic logs live under the runner installation directory in `_diag`.
+ * On many runners, we can derive the installation directory from RUNNER_TEMP:
+ *   <runnerRoot>/_work/_temp  ->  <runnerRoot>/_diag
+ *
+ * This is much more robust than relying on hardcoded paths, especially on self-hosted runners
+ * and GHES environments where the runner may be installed under arbitrary directories/users.
+ */
+function getGithubDiagnosticDirsFromEnv (runnerTemp) {
+  const dirs = []
+
+  if (runnerTemp) {
+    // RUNNER_TEMP is typically: <runnerRoot>/_work/_temp
+    const runnerRoot = path.resolve(runnerTemp, '..', '..').replaceAll(path.sep, '/')
+    // Bounded-depth patterns cover every runner layout we've observed
+    // (including cached/<version>/_diag) without assuming a `cached` wrapper
+    // and without walking the whole tree.
+    dirs.push(
+      path.posix.join(runnerRoot, 'actions-runner', '_diag'),
+      `${runnerRoot}/actions-runner/*/_diag`,
+      `${runnerRoot}/actions-runner/*/*/_diag`,
+      path.posix.join(runnerRoot, '_diag'),
+      `${runnerRoot}/*/_diag`,
+      `${runnerRoot}/*/*/_diag`
+    )
+  }
 
-  // RUNNER_TEMP usually looks like:
-  // Linux/mac hosted:   /home/runner/work/_temp
-  // Windows hosted:     C:\actions-runner\_work\_temp
-  // Self-hosted (unix): /opt/actions-runner/_work/_temp
+  return uniq(dirs.filter(Boolean))
+}
 
-  const workDir = path.dirname(runnerTemp) // .../work or .../_work
-  const runnerRoot = path.dirname(workDir) // /home/runner/ (runner root)
+function hasMagicChars (str) {
+  return str.includes('*') || str.includes('?')
+}
 
-  const dirs = [
-    path.join(runnerRoot, 'cached', '_diag'),
-    path.join(runnerRoot, '_diag'),
-    path.join(runnerRoot, 'actions-runner', 'cached', '_diag'),
-    path.join(runnerRoot, 'actions-runner', '_diag'),
-  ]
+// Expands a glob pattern with only `*`/`?` at path-segment boundaries (no `**`)
+// into matching concrete paths using readdirSync — no external dependency needed.
+function expandGlobPattern (pattern) {
+  const parts = pattern.split(/[/\\]/)
+  const wildcardIdx = parts.findIndex(p => hasMagicChars(p))
+  if (wildcardIdx === -1) return [pattern]
 
-  const isWin = process.platform === 'win32'
+  const prefix = parts.slice(0, wildcardIdx).join('/')
+  const results = []
 
-  // Hardcoded fallbacks
-  if (isWin) {
-    dirs.push(
-      'C:/actions-runner/cached/_diag',
-      'C:/actions-runner/_diag',
-    )
-  } else {
-    dirs.push(
-      '/home/runner/actions-runner/cached/_diag',
-      '/home/runner/actions-runner/_diag',
-      '/opt/actions-runner/_diag',
-    )
+  function walk (dir, segIdx) {
+    if (segIdx === parts.length) {
+      results.push(dir)
+      return
+    }
+    const seg = parts[segIdx]
+    if (!hasMagicChars(seg)) {
+      walk(`${dir}/${seg}`, segIdx + 1)
+      return
+    }
+    try {
+      const re = new RegExp(
+        '^' + seg.replaceAll(/[.+^${}()|[\]\\]/g, String.raw`\$&`).replaceAll('*', String.raw`[^/\\]*`).replaceAll('?', String.raw`[^/\\]`) + '$'
+      )
+      for (const entry of readdirSync(dir)) {
+        if (re.test(entry)) {
+          walk(`${dir}/${entry}`, segIdx + 1)
+        }
+      }
+    } catch {
+      // directory doesn't exist or isn't accessible
+    }
   }
 
-  // Remove duplicates
-  const possibleDiagsPaths = [...new Set(dirs)]
+  walk(prefix, wildcardIdx)
+  return results
+}
+
+/**
+ * Expands a mixed list of literal directories and glob patterns into concrete
+ * directories. Literals pass through unchanged (existence is checked later).
+ */
+function expandDiagnosticDirCandidates (candidates) {
+  const expanded = []
+  for (const candidate of candidates) {
+    if (hasMagicChars(candidate)) {
+      expanded.push(...expandGlobPattern(candidate))
+    } else {
+      expanded.push(candidate)
+    }
+  }
+
+  return uniq(expanded)
+}
+
+const githubWellKnownDiagnosticDirsUnix = [
+  '/home/runner/actions-runner/_diag',
+  '/opt/actions-runner/_diag',
+]
+const githubWellKnownDiagnosticDirsWin = [
+  'C:/actions-runner/_diag',
+]
+
+// Glob patterns covering layouts that namespace `_diag` under one or two
+// intermediate directories. This includes both observed SaaS layouts
+// (<runnerRoot>/cached/_diag pre-2.334.0, <runnerRoot>/cached/<version>/_diag
+// since v2.334.0) and hypothetical future layouts that follow the same shape
+// without a `cached` wrapper (e.g. <runnerRoot>/<version>/_diag). Depth is
+// bounded on purpose: `*` matches a single segment, so no filesystem walk.
+const githubWellKnownDiagnosticDirPatternsUnix = [
+  '/home/runner/actions-runner/*/_diag',
+  '/home/runner/actions-runner/*/*/_diag',
+]
+const githubWellKnownDiagnosticDirPatternsWin = ['C:/actions-runner/*/_diag', 'C:/actions-runner/*/*/_diag']
+
+const githubJobIDRegex = /"job":\s*{[\s\S]*?"v"\s*:\s*(\d+)(?:\.0)?/
+
+function getJobIDFromDiagFile () {
+  const runnerTemp = getValueFromEnvSources('RUNNER_TEMP')
+  if (!runnerTemp || !existsSync(runnerTemp)) { return null }
+
+  const isWin = process.platform === 'win32'
+  const patterns = isWin ? githubWellKnownDiagnosticDirPatternsWin : githubWellKnownDiagnosticDirPatternsUnix
+  const literals = isWin ? githubWellKnownDiagnosticDirsWin : githubWellKnownDiagnosticDirsUnix
+  const possibleDiagsPaths = expandDiagnosticDirCandidates([
+    ...getGithubDiagnosticDirsFromEnv(runnerTemp), ...patterns, ...literals,
+  ])
 
   // This will hold the names of the worker log files that (potentially) contain the Job ID
   let workerLogFiles = []
@@ -177,7 +264,7 @@ function getJobIDFromDiagFile (runnerTemp) {
     const filePath = path.posix.join(chosenDiagPath, logFile)
     const content = readFileSync(filePath, 'utf8')
 
-    const match = content.match(/"job":\s*{[\s\S]*?"v"\s*:\s*(\d+)(?:\.0)?/)
+    const match = content.match(githubJobIDRegex)
 
     // match[1] is the captured group with the display name
     if (match && match[1]) { return match[1] }
@@ -188,6 +275,7 @@ function getJobIDFromDiagFile (runnerTemp) {
 
 module.exports = {
   normalizeRef,
+  expandGlobPattern,
   getJobIDFromDiagFile,
   getCIMetadata () {
     const env = getEnvironmentVariables()
@@ -366,7 +454,6 @@ module.exports = {
         GITHUB_RUN_ATTEMPT,
         GITHUB_JOB,
         GITHUB_BASE_REF,
-        RUNNER_TEMP,
         JOB_CHECK_RUN_ID,
       } = env
 
@@ -378,7 +465,7 @@ module.exports = {
       }
 
       // Build the job url extracting the job ID. If extraction fails, job url is constructed as a generalized url
-      const GITHUB_JOB_ID = JOB_CHECK_RUN_ID ?? getJobIDFromDiagFile(RUNNER_TEMP)
+      const GITHUB_JOB_ID = JOB_CHECK_RUN_ID ?? getJobIDFromDiagFile()
       const jobUrl =
         GITHUB_JOB_ID === null
           ? `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}/checks`

…r/_diag/Worker_20240115-102345-12345.log …0/_diag/Worker_20240115-102345-12345.logpackages/dd-trace/test/plugins/util/fixtures/runner/actions-runner/_diag/Worker_20240115-102345-12345.log renamed to packages/dd-trace/test/plugins/util/fixtures/runner/actions-runner/cached/2.334.0/_diag/Worker_20240115-102345-12345.log packages/dd-trace/test/plugins/util/fixtures/runner/actions-runner/_diag/Worker_20240115-102345-12345.log renamed to packages/dd-trace/test/plugins/util/fixtures/runner/actions-runner/cached/2.334.0/_diag/Worker_20240115-102345-12345.log

@@ -0,0 +1,108 @@
+[2024-01-15 10:23:45Z INFO WorkerMessageServer] THIS IS A MOCK DIAGNOSTICS FILE CREATED BY CLAUDE FOR TESTING getJobIDFromDiagFile FUNCTION
+[2024-01-15 10:23:45Z INFO WorkerMessageServer] Worker starting, pid: 12345
+[2024-01-15 10:23:45Z INFO WorkerMessageServer] Connecting to runner process
+[2024-01-15 10:23:45Z INFO WorkerMessageServer] Connection established
+[2024-01-15 10:23:46Z INFO Worker] Processing JobRequest message
+[2024-01-15 10:23:46Z INFO Worker] Received job message:
+{
+  "messageType": "PipelineAgentJobRequest",
+  "plan": {
+    "scopeIdentifier": "3a7e8b2d-1f4c-4e9a-b6d3-9c2f1a8e5d7b",
+    "planType": "Build",
+    "planId": "f2c4a6e8-3b5d-4f7a-9c1e-2d4f6a8c0e2a",
+    "version": "1",
+    "artifactUri": "https://pipelines.actions.githubusercontent.com/abc123XYZ",
+    "requestedForId": "1234567"
+  },
+  "timeline": {
+    "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
+    "changeId": 1,
+    "location": null
+  },
+  "jobId": "d4e5f6a7-b8c9-0123-def4-567890abcdef",
+  "jobDisplayName": "build-and-test",
+  "jobName": "build-and-test",
+  "requestId": 9876543210,
+  "lockedUntil": "2024-01-15T11:23:46.000Z",
+  "resources": {
+    "endpoints": [
+      {
+        "data": {
+          "downloadUrl": "https://codeload.github.com/"
+        },
+        "name": "GITHUB_TOKEN",
+        "url": "https://api.github.com/",
+        "authorization": {
+          "scheme": "OAuth",
+          "parameters": {
+            "accessToken": "***"
+          }
+        },
+        "isShared": false,
+        "isReady": true
+      }
+    ],
+    "files": [],
+    "repositories": [
+      {
+        "alias": "self",
+        "id": "repo-id-abc123",
+        "type": "GitHub",
+        "name": "my-org/my-repo",
+        "url": "https://github.com/my-org/my-repo",
+        "version": "abc123def456",
+        "ref": "refs/heads/main"
+      }
+    ],
+    "containers": []
+  },
+  "variables": {
+    "system.github.job": { "value": "build-and-test", "isSecret": false },
+    "GITHUB_RUN_ID": { "value": "7654321", "isSecret": false },
+    "GITHUB_RUN_NUMBER": { "value": "42", "isSecret": false },
+    "GITHUB_WORKFLOW": { "value": "CI", "isSecret": false }
+  },
+  "steps": [
+    {
+      "type": "Task",
+      "id": "step-id-0001",
+      "name": "actions/checkout",
+      "displayName": "Checkout repository",
+      "enabled": true,
+      "continueOnError": false,
+      "condition": "succeeded()",
+      "timeoutInMinutes": 0,
+      "inputs": {
+        "repository": "my-org/my-repo",
+        "ref": "refs/heads/main",
+        "token": "***"
+      },
+      "environment": {},
+      "retryCountOnTaskFailure": 0
+    }
+  ],
+  "contextData": {},
+  "workspace": { "clean": null },
+  "mask": [
+    { "type": "regex", "value": "\\*\\*\\*" }
+  ],
+  "oidcToken": null,
+  "finaStrategy": null,
+  "job": {
+    "v": 9876543210,
+    "name": "build-and-test",
+    "id": "d4e5f6a7-b8c9-0123-def4-567890abcdef",
+    "attempt": 1,
+    "workflowName": "CI",
+    "headBranch": "refs/heads/main",
+    "headSha": "abc123def456abc123def456abc123def456abc1"
+  }
+}
+[2024-01-15 10:23:46Z INFO Worker] Job assignment accepted, jobId: d4e5f6a7-b8c9-0123-def4-567890abcdef
+[2024-01-15 10:23:46Z INFO JobRunner] Starting job execution
+[2024-01-15 10:23:47Z INFO StepRunner] Running step: Checkout repository
+[2024-01-15 10:24:01Z INFO StepRunner] Step completed: Checkout repository (result: Succeeded)
+[2024-01-15 10:24:01Z INFO JobRunner] All steps completed
+[2024-01-15 10:24:02Z INFO Worker] Job completed, result: Succeeded
+[2024-01-15 10:24:02Z INFO WorkerMessageServer] Sending job result to runner process
+[2024-01-15 10:24:02Z INFO WorkerMessageServer] Worker shutting down

…ty/actions-runner/_diag/Worker_empty.log …er/cached/2.334.0/_diag/Worker_empty.logpackages/dd-trace/test/plugins/util/fixtures/runner_empty/actions-runner/_diag/Worker_empty.log renamed to packages/dd-trace/test/plugins/util/fixtures/runner_empty/actions-runner/cached/2.334.0/_diag/Worker_empty.log packages/dd-trace/test/plugins/util/fixtures/runner_empty/actions-runner/_diag/Worker_empty.log renamed to packages/dd-trace/test/plugins/util/fixtures/runner_empty/actions-runner/cached/2.334.0/_diag/Worker_empty.log

@@ -0,0 +1,2 @@
+This file exists to simulate the structure of github actions runner
+It is here just so github can keep track of the whole directory structure