diff --git a/docs/commands/extract.md b/docs/commands/extract.md index de214343..9a54bbeb 100644 --- a/docs/commands/extract.md +++ b/docs/commands/extract.md @@ -94,7 +94,9 @@ For local development, `az login` is the simplest option. For CI/CD pipelines, u By default, `apiops extract` exports **all** resources from the APIM instance (34 resource types including APIs, products, backends, named values, tags, policies, and more). -To extract only specific resources, pass a YAML filter file with `--filter`. Filter entries support exact names and wildcard patterns (`*` for any characters, `?` for a single character): +To extract only specific resources, pass a YAML filter file with `--filter`. Filter entries support exact names, wildcard patterns (`*` for any characters, `?` for a single character), and `!`-prefixed **exclusions** (e.g. `'!prod-legacy'` — see [`filtering-resources.md`](../guides/filtering-resources.md#excluding-resources-with-) for details): + +> **Always quote `!`-prefixed entries** — unquoted `- !prod-legacy` is parsed by YAML as a tag and fails to load. ```yaml # configuration.extractor.yaml @@ -102,6 +104,7 @@ apis: - echo-api - petstore-api - 'prod-*' # Wildcard: all APIs starting with prod- + - '!prod-legacy-*' # Exclusion: skip legacy prod APIs products: - starter backends: diff --git a/docs/guides/filtering-resources.md b/docs/guides/filtering-resources.md index 4beb4e7d..544a47ca 100644 --- a/docs/guides/filtering-resources.md +++ b/docs/guides/filtering-resources.md @@ -86,6 +86,7 @@ namedValues: - Names are matched case-insensitively against APIM resource names - Wildcard patterns are supported — `*` matches any characters, `?` matches a single character (see below) - Exact names and wildcard patterns can be mixed in the same array +- Entries beginning with `!` are **exclusions** — see [Excluding resources with `!`](#excluding-resources-with-) below - An empty file extracts everything (same as no filter) - An empty array (`[]`) excludes ALL resources of that type @@ -125,6 +126,54 @@ Wildcard matching is case-insensitive, just like exact matching. Special charact --- +## Excluding resources with `!` + +Any filter entry whose first character is `!` is treated as an **exclusion**. Exclusions are applied *after* inclusions for the same list, so you can write patterns like "include everything matching this shape, except these specific ones." + +Semantics: + +- `!` must be the **first character** of the entry to count as negation. `foo!bar` is a literal name. +- The rest of the entry is a normal filter value — exact name or wildcard pattern, matched case-insensitively. +- A resource is included iff at least one inclusion matches it **and** no exclusion matches it. +- A list containing only exclusions is treated as "include everything, then subtract" — equivalent to prepending an implicit `*`. +- Exclusions respect the same semantics as inclusions: they match API root names (stripping revision suffixes), and excluding a parent (Api, Product, Gateway, Workspace) cascades to its children. + +> **Always quote `!`-prefixed entries in YAML.** An unquoted leading `!` (e.g. `- !prod-legacy-billing`) is parsed by YAML as a **tag** and produces an "unknown tag" error before the filter code ever sees the value. Wrap the entry in single or double quotes, exactly like the examples in this guide: `- '!prod-legacy-billing'`. + +### Examples + +```yaml +# Include all prod-* APIs except one specific legacy API and any deprecated variants +apis: + - 'prod-*' + - '!prod-legacy-billing' + - '!prod-*-deprecated' + +# Include every backend except the shared infra ones +backends: + - '*' + - '!shared-monitoring' + - '!shared-*-infra' + +# Include every named value except Key Vault-backed ones (pure-exclusion list) +namedValues: + - '!keyvault-*' +``` + +Exclusions work anywhere a string list is accepted, including sub-filter fields inside `apiSubFilters` and `workspaceSubFilters`: + +```yaml +apis: + - 'my-api' +apiSubFilters: + my-api: + operations: + - 'get-*' + - '!get-internal-*' # keep all get-* operations except internal ones +``` + +--- + ## Nested Sub-Resource Filtering ### API sub-resource filters diff --git a/src/services/filter-service.ts b/src/services/filter-service.ts index d412cc06..e3a82962 100644 --- a/src/services/filter-service.ts +++ b/src/services/filter-service.ts @@ -238,6 +238,19 @@ export function wildcardMatch(pattern: string, text: string): boolean { return wildcardToRegex(pattern).test(text); } +/** + * Check whether a single filter entry matches a resource name. + * Handles both exact (case-insensitive) and wildcard matching, and + * matches against the API root name (revision-suffix stripped) as well. + */ +function entryMatches(entry: string, lowerName: string, lowerRoot: string): boolean { + if (isWildcardPattern(entry)) { + return wildcardMatch(entry, lowerName) || wildcardMatch(entry, lowerRoot); + } + const lowerEntry = entry.toLowerCase(); + return lowerName === lowerEntry || lowerRoot === lowerEntry; +} + /** * Match a resource name against a filter allowlist. * @@ -246,6 +259,16 @@ export function wildcardMatch(pattern: string, text: string): boolean { * - non-empty array → case-insensitive exact match or wildcard pattern match * * Wildcard patterns use `*` (zero or more characters) and `?` (single character). + * + * Negation: entries beginning with `!` are treated as exclusions. Exclusions + * are evaluated after inclusions for the same list: + * - If the list contains only exclusions, an implicit `*` include is assumed + * ("include everything, then subtract"). + * - Otherwise a resource is included iff at least one inclusion matches + * AND no exclusion matches. + * `!` must be the first character to be interpreted as negation; `foo!bar` + * is a literal name. `!` cannot appear in a valid APIM resource name, so a + * leading `!` is unambiguous. */ function matchesFilter(name: string, allowlist: string[] | undefined): boolean { if (allowlist === undefined) { @@ -256,17 +279,31 @@ function matchesFilter(name: string, allowlist: string[] | undefined): boolean { return false; } + const includes: string[] = []; + const excludes: string[] = []; + for (const entry of allowlist) { + if (entry.startsWith('!')) { + excludes.push(entry.slice(1)); + } else { + includes.push(entry); + } + } + const lowerName = name.toLowerCase(); // For APIs, also match by root name (strip revision suffix) const lowerRoot = extractRootApiName(lowerName); - return allowlist.some((allowed) => { - if (isWildcardPattern(allowed)) { - return wildcardMatch(allowed, lowerName) || wildcardMatch(allowed, lowerRoot); - } - const lowerAllowed = allowed.toLowerCase(); - return lowerName === lowerAllowed || lowerRoot === lowerAllowed; - }); + // Pure-exclusion list is treated as "include-all, then subtract". + const included = + includes.length === 0 + ? true + : includes.some((entry) => entryMatches(entry, lowerName, lowerRoot)); + + if (!included) { + return false; + } + + return !excludes.some((entry) => entryMatches(entry, lowerName, lowerRoot)); } /** diff --git a/src/templates/configs/filter-config.yaml b/src/templates/configs/filter-config.yaml index 7a935ebd..1a9b3324 100644 --- a/src/templates/configs/filter-config.yaml +++ b/src/templates/configs/filter-config.yaml @@ -116,3 +116,12 @@ # - Use ? to match a single character: api-v? matches api-v1, api-v2 # - Exact names and wildcard patterns can be mixed in the same list # - All matching is case-insensitive +# - Prefix an entry with `!` to EXCLUDE it (e.g. '!prod-legacy-*'). A list +# containing only `!` entries means "include everything, then subtract." +# IMPORTANT: always QUOTE `!`-prefixed entries in YAML — an unquoted +# leading `!` is parsed as a YAML tag and fails to load. +# Example: +# apis: +# - 'prod-*' +# - '!prod-legacy-billing' +# - '!prod-*-deprecated' diff --git a/src/templates/copilot/configure-filter-prompt.md b/src/templates/copilot/configure-filter-prompt.md index 95a1c5f9..a08c73ec 100644 --- a/src/templates/copilot/configure-filter-prompt.md +++ b/src/templates/copilot/configure-filter-prompt.md @@ -47,7 +47,7 @@ Walk through the resource types **one type at a time**. For each type, ask the u - **Extract ALL** — include every resource of this type. Leave this type **out** of the filter (APIOps extracts everything by default). - **Extract NONE** — exclude all resources of this type. Add the type with an empty array: `tags: []`. -- **Extract SOME** — include only specific resources. The user provides which names (or wildcard patterns) to include. Matching is case-insensitive and supports `*` and `?` wildcards. +- **Extract SOME** — include only specific resources. The user provides which names (or wildcard patterns) to include. Matching is case-insensitive and supports `*` and `?` wildcards. Entries can also be prefixed with `!` to **exclude** a name or pattern (e.g. `'!prod-legacy-*'`); a list containing only `!` entries means "include everything, then subtract." **Always quote `!`-prefixed entries in YAML** — an unquoted leading `!` is parsed as a YAML tag and fails to load. **Single-resource-type cadence for Step 1:** diff --git a/tests/unit/services/filter-service.test.ts b/tests/unit/services/filter-service.test.ts index c6c38ca5..be4ce7c1 100644 --- a/tests/unit/services/filter-service.test.ts +++ b/tests/unit/services/filter-service.test.ts @@ -583,4 +583,137 @@ describe('filter-service', () => { expect(shouldIncludeResource(excluded, filter)).toBe(false); }); }); + + describe('negation (`!` prefix) exclusions', () => { + it('should treat a leading `!` as an exclusion in an otherwise-inclusive list', () => { + const filter: FilterConfig = { apis: ['prod-*', '!prod-legacy-billing'] }; + const included: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['prod-users'], + }; + const excluded: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['prod-legacy-billing'], + }; + expect(shouldIncludeResource(included, filter)).toBe(true); + expect(shouldIncludeResource(excluded, filter)).toBe(false); + }); + + it('should support wildcard exclusions', () => { + const filter: FilterConfig = { apis: ['prod-*', '!prod-*-deprecated'] }; + const included: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['prod-users'], + }; + const excluded: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['prod-users-deprecated'], + }; + expect(shouldIncludeResource(included, filter)).toBe(true); + expect(shouldIncludeResource(excluded, filter)).toBe(false); + }); + + it('should treat a pure-exclusion list as "include all, then subtract"', () => { + const filter: FilterConfig = { apis: ['!prod-legacy-billing'] }; + const kept: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['prod-users'], + }; + const dropped: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['prod-legacy-billing'], + }; + expect(shouldIncludeResource(kept, filter)).toBe(true); + expect(shouldIncludeResource(dropped, filter)).toBe(false); + }); + + it('should treat a list with only wildcard exclusions as include-all-minus', () => { + const filter: FilterConfig = { namedValues: ['!keyvault-*'] }; + const kept: ResourceDescriptor = { + type: ResourceType.NamedValue, + nameParts: ['api-token'], + }; + const dropped: ResourceDescriptor = { + type: ResourceType.NamedValue, + nameParts: ['keyvault-secret'], + }; + expect(shouldIncludeResource(kept, filter)).toBe(true); + expect(shouldIncludeResource(dropped, filter)).toBe(false); + }); + + it('should match exclusions case-insensitively', () => { + const filter: FilterConfig = { apis: ['*', '!Prod-Legacy-Billing'] }; + const excluded: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['prod-legacy-billing'], + }; + expect(shouldIncludeResource(excluded, filter)).toBe(false); + }); + + it('should exclude API revisions when the root name matches an exclusion', () => { + const filter: FilterConfig = { apis: ['prod-*', '!prod-legacy-billing'] }; + const revision: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['prod-legacy-billing;rev=2'], + }; + expect(shouldIncludeResource(revision, filter)).toBe(false); + }); + + it('should cascade parent exclusions to child resources', () => { + const filter: FilterConfig = { apis: ['prod-*', '!prod-legacy-billing'] }; + const policyOfExcludedApi: ResourceDescriptor = { + type: ResourceType.ApiPolicy, + nameParts: ['prod-legacy-billing'], + }; + const policyOfIncludedApi: ResourceDescriptor = { + type: ResourceType.ApiPolicy, + nameParts: ['prod-users'], + }; + const opOfExcludedApi: ResourceDescriptor = { + type: ResourceType.ApiOperation, + nameParts: ['prod-legacy-billing', 'get-invoices'], + }; + expect(shouldIncludeResource(policyOfExcludedApi, filter)).toBe(false); + expect(shouldIncludeResource(policyOfIncludedApi, filter)).toBe(true); + expect(shouldIncludeResource(opOfExcludedApi, filter)).toBe(false); + }); + + it('should treat `!` only as negation when it is the first character', () => { + // "foo!bar" is a literal name, not an exclusion of "bar" that starts with "foo". + const filter: FilterConfig = { apis: ['foo!bar'] }; + const literal: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['foo!bar'], + }; + const other: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['bar'], + }; + // APIM resource names can't actually contain `!`, but the matcher must + // still treat non-leading `!` as a literal character rather than negation. + expect(shouldIncludeResource(literal, filter)).toBe(true); + expect(shouldIncludeResource(other, filter)).toBe(false); + }); + + it('should apply negation inside apiSubFilters', () => { + const filter: FilterConfig = { + apis: ['my-api'], + apiSubFilters: { + 'my-api': { + operations: ['get-*', '!get-internal-*'], + }, + }, + }; + const included: ResourceDescriptor = { + type: ResourceType.ApiOperation, + nameParts: ['my-api', 'get-users'], + }; + const excluded: ResourceDescriptor = { + type: ResourceType.ApiOperation, + nameParts: ['my-api', 'get-internal-metrics'], + }; + expect(shouldIncludeResource(included, filter)).toBe(true); + expect(shouldIncludeResource(excluded, filter)).toBe(false); + }); + }); });