From c473ac586172e6e59aea62722537fc355f5484f6 Mon Sep 17 00:00:00 2001 From: Nicolas Will Date: Mon, 27 Jul 2026 21:30:54 +0200 Subject: [PATCH 1/4] Bundle codeql-bundle package data into the frozen binary codeql-bundle's cache-aware bundle support reads two package data files at runtime via importlib.resources: codeql_bundle/supported-codeql-bundles.json codeql_bundle/supported-codeql-bundles.schema.json PyInstaller only freezes Python modules, so these were missing from the frozen binary QLT ships and every invocation aborted with an unhandled FileNotFoundError, breaking 'qlt codeql run install --custom-bundle' and '--quick-bundle'. Pass both files to PyInstaller with --add-data. The separator expected by --add-data is os.pathsep, which is ';' on Windows and ':' elsewhere, so it is resolved from [System.IO.Path]::PathSeparator rather than hardcoded -- this script runs on all three platforms. The Test-Path guard keeps the script working for older codeql-bundle releases that do not ship these files, and the final argument list is echoed before the build. Also bump the pinned codeql-bundle version to 0.5.0, which is the release that introduces these files. --- .github/actions/install-qlt-local/action.yml | 2 +- .../internal-build-release-linux64.yml | 2 +- .../internal-build-release-macos64.yml | 2 +- .../internal-build-release-win64.yml | 2 +- developer_guide.md | 2 +- scripts/build_codeql_bundle_dist.ps1 | 21 ++++++++++++++++++- 6 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/actions/install-qlt-local/action.yml b/.github/actions/install-qlt-local/action.yml index 6e42159..efefa10 100644 --- a/.github/actions/install-qlt-local/action.yml +++ b/.github/actions/install-qlt-local/action.yml @@ -60,7 +60,7 @@ runs: pip install -U pyinstaller # run the packaging - ./scripts/build_codeql_bundle_dist.ps1 -Version 0.4.1 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/ + ./scripts/build_codeql_bundle_dist.ps1 -Version 0.5.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/ env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/internal-build-release-linux64.yml b/.github/workflows/internal-build-release-linux64.yml index 840d42a..f3a95d0 100644 --- a/.github/workflows/internal-build-release-linux64.yml +++ b/.github/workflows/internal-build-release-linux64.yml @@ -50,7 +50,7 @@ jobs: pip install -U pyinstaller # run the packaging - ./scripts/build_codeql_bundle_dist.ps1 -Version 0.4.1 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/ + ./scripts/build_codeql_bundle_dist.ps1 -Version 0.5.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/linux-x64/tools/ env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/internal-build-release-macos64.yml b/.github/workflows/internal-build-release-macos64.yml index d85b522..75d013e 100644 --- a/.github/workflows/internal-build-release-macos64.yml +++ b/.github/workflows/internal-build-release-macos64.yml @@ -48,7 +48,7 @@ jobs: pip install -U pyinstaller # run the packaging - ./scripts/build_codeql_bundle_dist.ps1 -Version 0.4.1 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/macos-arm64/tools/ + ./scripts/build_codeql_bundle_dist.ps1 -Version 0.5.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Release/net6.0/publish/macos-arm64/tools/ env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/internal-build-release-win64.yml b/.github/workflows/internal-build-release-win64.yml index b188ee2..4596f93 100644 --- a/.github/workflows/internal-build-release-win64.yml +++ b/.github/workflows/internal-build-release-win64.yml @@ -45,7 +45,7 @@ jobs: pip install -U pyinstaller # run the packaging - .\scripts\build_codeql_bundle_dist.ps1 -Version 0.4.1 -WorkDirectory dist -DestinationDirectory .\src\CodeQLToolkit.Core\bin\Release\net6.0\publish\windows-x64\tools\ + .\scripts\build_codeql_bundle_dist.ps1 -Version 0.5.0 -WorkDirectory dist -DestinationDirectory .\src\CodeQLToolkit.Core\bin\Release\net6.0\publish\windows-x64\tools\ env: GH_TOKEN: ${{ github.token }} diff --git a/developer_guide.md b/developer_guide.md index 51f0826..11fe0f7 100644 --- a/developer_guide.md +++ b/developer_guide.md @@ -15,7 +15,7 @@ Note that we keep recent copies of tools (for local debugging purposes) in the ` **CodeQL Bundle** ``` -./scripts/build_codeql_bundle_dist.ps1 -Version 0.4.1 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Debug/net6.0/tools +./scripts/build_codeql_bundle_dist.ps1 -Version 0.5.0 -WorkDirectory dist -DestinationDirectory ./src/CodeQLToolkit.Core/bin/Debug/net6.0/tools ``` diff --git a/scripts/build_codeql_bundle_dist.ps1 b/scripts/build_codeql_bundle_dist.ps1 index 72beab8..0d3062a 100644 --- a/scripts/build_codeql_bundle_dist.ps1 +++ b/scripts/build_codeql_bundle_dist.ps1 @@ -54,8 +54,27 @@ if ($LASTEXITCODE -ne 0) { # Move into the cli directory Push-Location "codeql_bundle" +# PyInstaller only freezes Python modules, so any package data files read at +# runtime (via importlib.resources) have to be added explicitly. The separator +# expected by --add-data is os.pathsep, which differs per platform, so it is +# resolved here rather than hardcoded. Older codeql-bundle releases do not ship +# these files, hence the Test-Path guard. +$PathSeparator = [System.IO.Path]::PathSeparator +$DataArgs = @() + +foreach ($DataFile in @("supported-codeql-bundles.json", "supported-codeql-bundles.schema.json")) { + if (Test-Path $DataFile) { + $DataArgs += "--add-data" + $DataArgs += "${DataFile}${PathSeparator}codeql_bundle" + } + else { + Write-Host "Note: data file '$DataFile' not present in this release, skipping." + } +} + # Build executable with pyinstaller -pyinstaller -F -n codeql_bundle cli.py +Write-Host "Running: pyinstaller -F -n codeql_bundle $($DataArgs -join ' ') cli.py" +pyinstaller -F -n codeql_bundle @DataArgs cli.py if ($LASTEXITCODE -ne 0) { throw "PyInstaller build failed" } From ae61f4dbdc6570996400ecf8da474366e9bf6ab8 Mon Sep 17 00:00:00 2001 From: Nicolas Will Date: Mon, 27 Jul 2026 21:31:09 +0200 Subject: [PATCH 2/4] Add --cache-dir pass-through for the codeql-bundle tool Expose the codeql-bundle cache directory through QLT, either as a 'CacheDir' key in qlt.conf.json or as 'qlt codeql run install --cache-dir' on the command line, with the command line taking precedence. The directory holds downloaded bundles and published compilation caches, so restoring it across CI runs (e.g. with actions/cache) avoids re-downloading and lets the cache-aware bundle support reuse prebuilt compilation caches. Relative paths are resolved against the caller's working directory before being handed to the bundle tool, which runs with its working directory set to the QLT base path. --- .../CodeQL/Commands/CodeQLCommandFeature.cs | 9 ++++++--- .../CodeQL/Commands/Targets/InstallCommand.cs | 10 ++++++++++ src/CodeQLToolkit.Shared/CodeQL/CodeQLInstallation.cs | 11 +++++++++++ src/CodeQLToolkit.Shared/Utils/QLTConfig.cs | 7 +++++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/CodeQLToolkit.Features/CodeQL/Commands/CodeQLCommandFeature.cs b/src/CodeQLToolkit.Features/CodeQL/Commands/CodeQLCommandFeature.cs index 7ed70f9..9663b3c 100644 --- a/src/CodeQLToolkit.Features/CodeQL/Commands/CodeQLCommandFeature.cs +++ b/src/CodeQLToolkit.Features/CodeQL/Commands/CodeQLCommandFeature.cs @@ -42,14 +42,16 @@ public void Register(Command parentCommand) var customBundleOption = new Option("--custom-bundle", () => false, "Build a custom bundle and compile the bundle.") { IsRequired = true }; var quickBundleOption = new Option("--quick-bundle", () => false, "Build a custom bundle and DO NOT compile the bundle.") { IsRequired = true }; var packsOption = new Option("--packs", "When creating bundles, this specifies the packs to include, Example `pack1 pack2 pack3`. You may specify also as `--pack pack1 --pack2 --pack3`") { IsRequired = false, AllowMultipleArgumentsPerToken = true }; + var cacheDirOption = new Option("--cache-dir", () => null, "Directory the bundle tool uses for downloaded bundles and compilation caches. Restore it across CI runs (e.g. with actions/cache) to avoid re-downloading and to reuse published compilation caches. Overrides the 'CacheDir' value in qlt.conf.json. If unset, the bundle tool's platform-specific default is used.") { IsRequired = false }; installCommand.Add(customBundleOption); installCommand.Add(quickBundleOption); installCommand.Add(packsOption); + installCommand.Add(cacheDirOption); runCommand.Add(installCommand); - installCommand.SetHandler((basePath, automationType, customBundleOption, quickBundleOption, packs) => + installCommand.SetHandler((basePath, automationType, customBundleOption, quickBundleOption, packs, cacheDir) => { Log.G().LogInformation("Executing install command..."); @@ -59,11 +61,12 @@ public void Register(Command parentCommand) AutomationTarget = automationType, CustomBundles = customBundleOption, QuickBundles = quickBundleOption, - Packs = packs + Packs = packs, + CacheDir = cacheDir }.Run(); - }, Globals.BasePathOption, Globals.AutomationTypeOption, customBundleOption, quickBundleOption, packsOption); + }, Globals.BasePathOption, Globals.AutomationTypeOption, customBundleOption, quickBundleOption, packsOption, cacheDirOption); } public int Run() diff --git a/src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs b/src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs index ee7eb07..f076448 100644 --- a/src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs +++ b/src/CodeQLToolkit.Features/CodeQL/Commands/Targets/InstallCommand.cs @@ -15,6 +15,7 @@ public class InstallCommand : CommandTarget public bool CustomBundles { get; set; } public bool QuickBundles { get; set; } public string[] Packs { get; set; } + public string CacheDir { get; set; } void SetEnvironmentVariableMultiTarget(string name, string value) { @@ -50,6 +51,15 @@ public override void Run() // First, check if CodeQL is installed. var installation = CodeQLInstallation.LoadFromConfig(Base); + + // A cache directory supplied on the command line takes precedence + // over any value loaded from `qlt.conf.json`. + if (!string.IsNullOrEmpty(CacheDir)) + { + Log.G().LogInformation($"Overriding bundle cache directory from the command line: {CacheDir}"); + installation.CacheDir = CacheDir; + } + if (CustomBundles || QuickBundles) { installation.EnableCustomCodeQLBundles = true; diff --git a/src/CodeQLToolkit.Shared/CodeQL/CodeQLInstallation.cs b/src/CodeQLToolkit.Shared/CodeQL/CodeQLInstallation.cs index 9f8ba89..d92346d 100644 --- a/src/CodeQLToolkit.Shared/CodeQL/CodeQLInstallation.cs +++ b/src/CodeQLToolkit.Shared/CodeQL/CodeQLInstallation.cs @@ -25,6 +25,10 @@ public class CodeQLInstallation public bool QuickBundle { get; set; } public string Base { get; set; } + // Optional directory passed to the bundle tool via `--cache-dir`, used + // for downloaded bundles and compilation caches. + public string CacheDir { get; set; } + public static CodeQLInstallation LoadFromConfig(string Base) { var c = new QLTConfig() @@ -48,6 +52,7 @@ public static CodeQLInstallation LoadFromConfig(QLTConfig c) CodeQLPackConfiguration = config.CodeQLPackConfiguration, Base = config.Base, CodeScanningConfig = config.CodeQLConfiguration, + CacheDir = config.CacheDir, QLTConfigFilePath = config.QLTConfigFilePath }; } @@ -266,6 +271,12 @@ private void CustomBundleInstall() } } + if (CacheDir != null && CacheDir.Length > 0) + { + Log.G().LogInformation($"Note: Using bundle cache directory `{CacheDir}` ..."); + bundleArgs = $"--cache-dir \"{Path.GetFullPath(CacheDir)}\" {bundleArgs}"; + } + Log.G().LogInformation($"Executing Bundle Tool with Working Directory: `{workingDirectory}`"); Log.G().LogInformation($"Executing Bundle Tool with Arguments: `{bundleArgs}`"); diff --git a/src/CodeQLToolkit.Shared/Utils/QLTConfig.cs b/src/CodeQLToolkit.Shared/Utils/QLTConfig.cs index 48eeb35..9f5a555 100644 --- a/src/CodeQLToolkit.Shared/Utils/QLTConfig.cs +++ b/src/CodeQLToolkit.Shared/Utils/QLTConfig.cs @@ -23,6 +23,13 @@ public class QLTConfig public string CodeQLCLIBundle { get; set; } public string CodeQLConfiguration { get; set; } + // Optional directory passed through to the codeql-bundle tool via + // `--cache-dir`. It holds downloaded bundles and compilation caches, so + // restoring it across CI runs (e.g. with actions/cache) avoids + // re-downloading and speeds up compilation of customized queries. May be + // overridden on the command line with `--cache-dir`. + public string CacheDir { get; set; } + public CodeQLPackConfiguration[] CodeQLPackConfiguration { get; set; } public string CodeQLStandardLibraryIdent From 2ba5bab826ba346fc633262ea4ccf3a9a44ab314 Mon Sep 17 00:00:00 2001 From: Nicolas Will Date: Tue, 28 Jul 2026 16:25:46 +0200 Subject: [PATCH 3/4] Fix the example customization pack layout codeql-bundle only treats a library pack as a customization pack when `Customizations.qll` lives in `/`, with `-` replaced by `_`. For a pack named `qlt/cpp-customizations` that means `qlt/cpp_customizations/Customizations.qll`, but the example kept the module at `qlt/Customizations.qll`. The check is a plain path existence test in ResolvedCodeQLPack, so the pack was silently classified as a plain library pack. Two consequences: - The customization was never imported into `codeql/cpp-all`, so the standard query packs did not see it. Only queries that explicitly imported the module picked it up, which is why the tests passed while the standard security queries ignored the custom flow source. - No standard query pack was recompiled, so there was nothing for the published compilation caches to accelerate. Move the module to the documented location and update the post-bundle test to the corresponding module path. codeql-bundle now reports "Bundling the customization pack qlt/cpp-customizations" and recreates codeql/cpp-examples and codeql/cpp-queries, and both example tests still pass against a stock CodeQL 2.15.5. --- .../src/qlt/{ => cpp_customizations}/Customizations.qll | 0 .../test/post-bundle/FooExternalSourceFunction/Foo.ql | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename example/cpp/customizations/src/qlt/{ => cpp_customizations}/Customizations.qll (100%) diff --git a/example/cpp/customizations/src/qlt/Customizations.qll b/example/cpp/customizations/src/qlt/cpp_customizations/Customizations.qll similarity index 100% rename from example/cpp/customizations/src/qlt/Customizations.qll rename to example/cpp/customizations/src/qlt/cpp_customizations/Customizations.qll diff --git a/example/cpp/customizations/test/post-bundle/FooExternalSourceFunction/Foo.ql b/example/cpp/customizations/test/post-bundle/FooExternalSourceFunction/Foo.ql index 43c8c08..fce5569 100644 --- a/example/cpp/customizations/test/post-bundle/FooExternalSourceFunction/Foo.ql +++ b/example/cpp/customizations/test/post-bundle/FooExternalSourceFunction/Foo.ql @@ -1,5 +1,5 @@ import cpp -import qlt.Customizations +import qlt.cpp_customizations.Customizations import semmle.code.cpp.security.FlowSources select any(RemoteFlowSource s) \ No newline at end of file From 03ef07d96c206aaf63b3906107e668687e9b20ea Mon Sep 17 00:00:00 2001 From: Nicolas Will Date: Tue, 28 Jul 2026 20:26:53 +0200 Subject: [PATCH 4/4] Stop bundled pack sources from shadowing their bundled copies codeql-bundle ships customization packs inside the custom bundle with their dependencies stripped, and adds them as a dependency of the standard library pack. That is how it avoids a cycle between the two. The pack sources stay discoverable through the CodeQL workspace though, and workspace packs take precedence over the ones in the distribution, so the source copy shadows the bundled one and puts the cycle back. Every 'codeql pack install' that reaches the customized standard library then fails with a cyclic dependency error. This never surfaced because the example's customization pack was laid out so that it was classified as an ordinary library pack and never injected. Move the sources of packs marked 'Bundle' aside while installing query packs in bundle mode and restore them afterwards, including on the failure path. Bump the example to 2.26.1 as well. Now that its customization pack is really applied, the standard query packs are genuinely recompiled, and 2.26.1 is covered by the published compilation cache while 2.15.5 is not. --- .../customizations/src/codeql-pack.lock.yml | 16 +-- example/cpp/customizations/src/qlpack.yml | 2 +- .../customizations/test/codeql-pack.lock.yml | 16 +-- example/cpp/stuff/src/codeql-pack.lock.yml | 16 +-- example/cpp/stuff/src/qlpack.yml | 2 +- example/cpp/stuff/test/codeql-pack.lock.yml | 16 +-- example/cpp/stuff2/src/codeql-pack.lock.yml | 16 +-- example/cpp/stuff2/src/qlpack.yml | 2 +- example/cpp/stuff2/test/codeql-pack.lock.yml | 16 +-- example/qlt.conf.json | 10 +- .../Targets/InstallQueryPacksCommandTarget.cs | 98 ++++++++++++++++--- 11 files changed, 100 insertions(+), 110 deletions(-) diff --git a/example/cpp/customizations/src/codeql-pack.lock.yml b/example/cpp/customizations/src/codeql-pack.lock.yml index 4edf97c..5300427 100644 --- a/example/cpp/customizations/src/codeql-pack.lock.yml +++ b/example/cpp/customizations/src/codeql-pack.lock.yml @@ -1,18 +1,4 @@ --- lockVersion: 1.0.0 -dependencies: - codeql/cpp-all: - version: 0.12.2 - codeql/dataflow: - version: 0.1.5 - codeql/rangeanalysis: - version: 0.0.4 - codeql/ssa: - version: 0.2.5 - codeql/tutorial: - version: 0.2.5 - codeql/typetracking: - version: 0.2.5 - codeql/util: - version: 0.2.5 +dependencies: {} compiled: false diff --git a/example/cpp/customizations/src/qlpack.yml b/example/cpp/customizations/src/qlpack.yml index 06c5c2a..f1cbba4 100644 --- a/example/cpp/customizations/src/qlpack.yml +++ b/example/cpp/customizations/src/qlpack.yml @@ -2,4 +2,4 @@ library: true name: qlt/cpp-customizations version: 0.0.1 dependencies: - "codeql/cpp-all": "0.12.2" \ No newline at end of file + "codeql/cpp-all": "12.0.0" \ No newline at end of file diff --git a/example/cpp/customizations/test/codeql-pack.lock.yml b/example/cpp/customizations/test/codeql-pack.lock.yml index 4edf97c..5300427 100644 --- a/example/cpp/customizations/test/codeql-pack.lock.yml +++ b/example/cpp/customizations/test/codeql-pack.lock.yml @@ -1,18 +1,4 @@ --- lockVersion: 1.0.0 -dependencies: - codeql/cpp-all: - version: 0.12.2 - codeql/dataflow: - version: 0.1.5 - codeql/rangeanalysis: - version: 0.0.4 - codeql/ssa: - version: 0.2.5 - codeql/tutorial: - version: 0.2.5 - codeql/typetracking: - version: 0.2.5 - codeql/util: - version: 0.2.5 +dependencies: {} compiled: false diff --git a/example/cpp/stuff/src/codeql-pack.lock.yml b/example/cpp/stuff/src/codeql-pack.lock.yml index 4edf97c..5300427 100644 --- a/example/cpp/stuff/src/codeql-pack.lock.yml +++ b/example/cpp/stuff/src/codeql-pack.lock.yml @@ -1,18 +1,4 @@ --- lockVersion: 1.0.0 -dependencies: - codeql/cpp-all: - version: 0.12.2 - codeql/dataflow: - version: 0.1.5 - codeql/rangeanalysis: - version: 0.0.4 - codeql/ssa: - version: 0.2.5 - codeql/tutorial: - version: 0.2.5 - codeql/typetracking: - version: 0.2.5 - codeql/util: - version: 0.2.5 +dependencies: {} compiled: false diff --git a/example/cpp/stuff/src/qlpack.yml b/example/cpp/stuff/src/qlpack.yml index 51c46db..7662fc2 100644 --- a/example/cpp/stuff/src/qlpack.yml +++ b/example/cpp/stuff/src/qlpack.yml @@ -6,4 +6,4 @@ description: Default description suites: license: dependencies: - codeql/cpp-all: "0.12.2" \ No newline at end of file + codeql/cpp-all: "12.0.0" \ No newline at end of file diff --git a/example/cpp/stuff/test/codeql-pack.lock.yml b/example/cpp/stuff/test/codeql-pack.lock.yml index 4edf97c..5300427 100644 --- a/example/cpp/stuff/test/codeql-pack.lock.yml +++ b/example/cpp/stuff/test/codeql-pack.lock.yml @@ -1,18 +1,4 @@ --- lockVersion: 1.0.0 -dependencies: - codeql/cpp-all: - version: 0.12.2 - codeql/dataflow: - version: 0.1.5 - codeql/rangeanalysis: - version: 0.0.4 - codeql/ssa: - version: 0.2.5 - codeql/tutorial: - version: 0.2.5 - codeql/typetracking: - version: 0.2.5 - codeql/util: - version: 0.2.5 +dependencies: {} compiled: false diff --git a/example/cpp/stuff2/src/codeql-pack.lock.yml b/example/cpp/stuff2/src/codeql-pack.lock.yml index 4edf97c..5300427 100644 --- a/example/cpp/stuff2/src/codeql-pack.lock.yml +++ b/example/cpp/stuff2/src/codeql-pack.lock.yml @@ -1,18 +1,4 @@ --- lockVersion: 1.0.0 -dependencies: - codeql/cpp-all: - version: 0.12.2 - codeql/dataflow: - version: 0.1.5 - codeql/rangeanalysis: - version: 0.0.4 - codeql/ssa: - version: 0.2.5 - codeql/tutorial: - version: 0.2.5 - codeql/typetracking: - version: 0.2.5 - codeql/util: - version: 0.2.5 +dependencies: {} compiled: false diff --git a/example/cpp/stuff2/src/qlpack.yml b/example/cpp/stuff2/src/qlpack.yml index 2dad82b..9a09525 100644 --- a/example/cpp/stuff2/src/qlpack.yml +++ b/example/cpp/stuff2/src/qlpack.yml @@ -6,4 +6,4 @@ description: Default description suites: license: dependencies: - codeql/cpp-all: "0.12.2" \ No newline at end of file + codeql/cpp-all: "12.0.0" \ No newline at end of file diff --git a/example/cpp/stuff2/test/codeql-pack.lock.yml b/example/cpp/stuff2/test/codeql-pack.lock.yml index 4edf97c..5300427 100644 --- a/example/cpp/stuff2/test/codeql-pack.lock.yml +++ b/example/cpp/stuff2/test/codeql-pack.lock.yml @@ -1,18 +1,4 @@ --- lockVersion: 1.0.0 -dependencies: - codeql/cpp-all: - version: 0.12.2 - codeql/dataflow: - version: 0.1.5 - codeql/rangeanalysis: - version: 0.0.4 - codeql/ssa: - version: 0.2.5 - codeql/tutorial: - version: 0.2.5 - codeql/typetracking: - version: 0.2.5 - codeql/util: - version: 0.2.5 +dependencies: {} compiled: false diff --git a/example/qlt.conf.json b/example/qlt.conf.json index 9beeefd..b3170cf 100644 --- a/example/qlt.conf.json +++ b/example/qlt.conf.json @@ -1,9 +1,9 @@ { - "CodeQLCLI": "2.15.5", - "CodeQLStandardLibrary": "codeql-cli/v2.15.5", - "CodeQLCLIBundle": "codeql-bundle-v2.15.5", + "CodeQLCLI": "2.26.1", + "CodeQLStandardLibrary": "codeql-cli/v2.26.1", + "CodeQLCLIBundle": "codeql-bundle-v2.26.1", "EnableCustomCodeQLBundles": true, - "CodeQLStandardLibraryIdent": "codeql-cli_v2.15.5", + "CodeQLStandardLibraryIdent": "codeql-cli_v2.26.1", "CodeQLPackConfiguration" : [ { "Name": "qlt/cpp-customizations", @@ -15,4 +15,4 @@ "ReferencesBundle" : true } ] -} \ No newline at end of file +} diff --git a/src/CodeQLToolkit.Features/Query/Commands/Targets/InstallQueryPacksCommandTarget.cs b/src/CodeQLToolkit.Features/Query/Commands/Targets/InstallQueryPacksCommandTarget.cs index b38d2bc..a8cb73e 100644 --- a/src/CodeQLToolkit.Features/Query/Commands/Targets/InstallQueryPacksCommandTarget.cs +++ b/src/CodeQLToolkit.Features/Query/Commands/Targets/InstallQueryPacksCommandTarget.cs @@ -12,12 +12,19 @@ namespace CodeQLToolkit.Features.Query.Commands.Targets { public class InstallQueryPacksCommandTarget : CommandTarget { + // Suffix used to temporarily hide the sources of packs that are shipped inside + // the custom bundle. See HideBundledPackSources for why this is necessary. + private const string BundledPackSuffix = ".qlt-bundled"; + public override void Run() { Log.G().LogInformation("Finding all qlpacks..."); // starting at the base path, find all qlpacks and install them + RestoreBundledPackSources(Directory.GetFiles(Base, "qlpack.yml" + BundledPackSuffix, SearchOption.AllDirectories)); + string[] files = Directory.GetFiles(Base, "qlpack.yml", SearchOption.AllDirectories); + string[] allFiles = files; Log.G().LogInformation($"Got {files.Length} packs..."); @@ -33,6 +40,8 @@ public override void Run() // filter the packs that are part of a custom bundle if we are using bundles. + string[] bundledPackFiles = Array.Empty(); + if(UseBundle) { // load the config @@ -58,33 +67,98 @@ public override void Run() { Log.G().LogInformation($"Pack {CodeQLPackReader.read(file).Name} in {file} will installed because it is not part of the bundle..."); } + + // Packs that were bundled are shipped inside the custom bundle with their + // dependencies stripped, which is how the bundle tool avoids a cycle between + // the standard library pack and the customization pack. Their sources are still + // discoverable through the CodeQL workspace though, and workspace packs take + // precedence over the ones in the distribution. Left in place they shadow the + // bundled copies and reintroduce that cycle, so dependency resolution fails for + // every pack that depends on the customized standard library. + bundledPackFiles = allFiles.Where(f => + config.CodeQLPackConfiguration.Any(p => CodeQLPackReader.read(f).Name == p.Name && p.Bundle == true) + ).ToArray(); } - foreach ( string file in files ) + string failedPack = null; + var hiddenPacks = new List(); + + try { - Log.G().LogInformation($"Installing qlpack {file}..."); + hiddenPacks = HideBundledPackSources(bundledPackFiles); - using(Process process = new Process()) + foreach ( string file in files ) { - process.StartInfo.FileName = installation.CodeQLToolBinary; - process.StartInfo.UseShellExecute = false; - process.StartInfo.RedirectStandardOutput = false; - process.StartInfo.Arguments = $"pack install {file}"; - process.Start(); + Log.G().LogInformation($"Installing qlpack {file}..."); - process.WaitForExit(); - - if(process.ExitCode !=0) + using(Process process = new Process()) { - DieWithError($"Failed to install query pack {file}."); + process.StartInfo.FileName = installation.CodeQLToolBinary; + process.StartInfo.UseShellExecute = false; + process.StartInfo.RedirectStandardOutput = false; + process.StartInfo.Arguments = $"pack install {file}"; + process.Start(); + + process.WaitForExit(); + + if(process.ExitCode !=0) + { + // Do not exit here. DieWithError terminates the process, which would + // leave the hidden pack sources behind in the user's working tree. + failedPack = file; + break; + } } + } + } + finally + { + RestoreBundledPackSources(hiddenPacks.ToArray()); + } + if (failedPack != null) + { + DieWithError($"Failed to install query pack {failedPack}."); } Log.G().LogInformation($"Installed {files.Length} packs."); } + + // Moves the given 'qlpack.yml' files aside so that CodeQL's workspace discovery no longer + // sees them, and returns the paths that were hidden so they can be restored afterwards. + private static List HideBundledPackSources(string[] packFiles) + { + var hidden = new List(); + + foreach (var packFile in packFiles) + { + var hiddenPath = packFile + BundledPackSuffix; + + Log.G().LogInformation($"Temporarily hiding {packFile} because the pack is provided by the custom bundle..."); + + File.Move(packFile, hiddenPath); + hidden.Add(hiddenPath); + } + + return hidden; + } + + // Restores files previously moved aside by HideBundledPackSources. Also used on start-up to + // recover from an interrupted run that never got to restore them. + private static void RestoreBundledPackSources(string[] hiddenPaths) + { + foreach (var hiddenPath in hiddenPaths) + { + var packFile = hiddenPath.Substring(0, hiddenPath.Length - BundledPackSuffix.Length); + + if (File.Exists(hiddenPath) && !File.Exists(packFile)) + { + File.Move(hiddenPath, packFile); + } + } + } } }