From 1e58737f78e1653f17cdcf34b7bb17aca1666af9 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Mon, 27 Jul 2026 07:35:51 +0000 Subject: [PATCH] feat: add @pgsql/transform (SQL transformation, classification, qualification, closure) --- .github/workflows/run-tests.yaml | 1 + AGENTS.md | 1 + README.md | 1 + packages/transform/.gitignore | 29 + packages/transform/README.md | 72 + .../__fixtures__/input/kitchen-sink.sql | 353 ++++ .../__fixtures__/input/plpgsql-constructs.sql | 157 ++ .../__fixtures__/output/kitchen-sink.sql | 272 +++ .../output/plpgsql-constructs.sql | 145 ++ .../transform/__tests__/categorize.test.ts | 100 ++ packages/transform/__tests__/facts.test.ts | 145 ++ .../__tests__/fixture-closure.test.ts | 127 ++ packages/transform/__tests__/qualify.test.ts | 307 ++++ .../__tests__/round-trip-core.test.ts | 145 ++ .../transform/__tests__/transform.test.ts | 1218 ++++++++++++++ packages/transform/jest.config.js | 10 + packages/transform/package.json | 52 + .../transform/scripts/hunt-plpgsql-bugs.js | 73 + packages/transform/scripts/make-fixtures.ts | 58 + packages/transform/scripts/scan-corpus.js | 131 ++ packages/transform/src/categorize.ts | 81 + packages/transform/src/facts.ts | 378 +++++ packages/transform/src/fixture-closure.ts | 286 ++++ packages/transform/src/index.ts | 75 + packages/transform/src/qualify.ts | 405 +++++ packages/transform/src/round-trip-core.ts | 147 ++ packages/transform/src/round-trip.ts | 158 ++ packages/transform/src/transform.ts | 1457 +++++++++++++++++ packages/transform/tsconfig.esm.json | 9 + packages/transform/tsconfig.json | 9 + pnpm-lock.yaml | 17 + 31 files changed, 6419 insertions(+) create mode 100644 packages/transform/.gitignore create mode 100644 packages/transform/README.md create mode 100644 packages/transform/__fixtures__/input/kitchen-sink.sql create mode 100644 packages/transform/__fixtures__/input/plpgsql-constructs.sql create mode 100644 packages/transform/__fixtures__/output/kitchen-sink.sql create mode 100644 packages/transform/__fixtures__/output/plpgsql-constructs.sql create mode 100644 packages/transform/__tests__/categorize.test.ts create mode 100644 packages/transform/__tests__/facts.test.ts create mode 100644 packages/transform/__tests__/fixture-closure.test.ts create mode 100644 packages/transform/__tests__/qualify.test.ts create mode 100644 packages/transform/__tests__/round-trip-core.test.ts create mode 100644 packages/transform/__tests__/transform.test.ts create mode 100644 packages/transform/jest.config.js create mode 100644 packages/transform/package.json create mode 100644 packages/transform/scripts/hunt-plpgsql-bugs.js create mode 100644 packages/transform/scripts/make-fixtures.ts create mode 100644 packages/transform/scripts/scan-corpus.js create mode 100644 packages/transform/src/categorize.ts create mode 100644 packages/transform/src/facts.ts create mode 100644 packages/transform/src/fixture-closure.ts create mode 100644 packages/transform/src/index.ts create mode 100644 packages/transform/src/qualify.ts create mode 100644 packages/transform/src/round-trip-core.ts create mode 100644 packages/transform/src/round-trip.ts create mode 100644 packages/transform/src/transform.ts create mode 100644 packages/transform/tsconfig.esm.json create mode 100644 packages/transform/tsconfig.json diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index ecf56daf..498a691e 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -23,6 +23,7 @@ jobs: - pg-proto-parser - '@pgsql/quotes' - '@pgsql/transform-ast' + - '@pgsql/transform' steps: - name: checkout uses: actions/checkout@v3 diff --git a/AGENTS.md b/AGENTS.md index e588cfc6..938a2b02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,6 +17,7 @@ A pnpm monorepo for PostgreSQL AST parsing, deparsing, and code generation. All | `@pgsql/utils` | `packages/utils` | Type-safe AST node creation utilities | | `@pgsql/traverse` | `packages/traverse` | Visitor-pattern AST traversal | | `@pgsql/transform-ast` | `packages/transform-ast` | Multi-version AST transformer (PG 13-17) | +| `@pgsql/transform` | `packages/transform` | SQL schema transformation, statement classification, qualification, dependency closure | | `@pgsql/quotes` | `packages/quotes` | SQL identifier/string quoting and keyword classification | | `@pgsql/cli` | `packages/pgsql-cli` | CLI tool for parse/deparse operations | | `pg-proto-parser` | `packages/proto-parser` | Generate TypeScript from PostgreSQL protobuf definitions | diff --git a/README.md b/README.md index 99f5498e..64330fb1 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,7 @@ walk(ast, visitor); | [**@pgsql/utils**](./packages/utils) | Type-safe AST node creation utilities | • Programmatic AST construction
• Runtime Schema
• Seamless integration with types | | [**pg-proto-parser**](./packages/proto-parser) | PostgreSQL protobuf parser and code generator | • Generate TypeScript interfaces from protobuf
• Create enum mappings and utilities
• AST helper generation | | [**@pgsql/transform-ast**](./packages/transform-ast) | Multi-version PostgreSQL AST transformer | • Transform ASTs between PostgreSQL versions (13→17)
• Single source of truth deparser pipeline
• Backward compatibility for legacy SQL | +| [**@pgsql/transform**](./packages/transform) | SQL transformation & classification | • Schema-name rewriting (incl. PL/pgSQL bodies)
• Per-statement AST facts (`classifyStatements`)
• Transitive dependency closure & qualification | | [**@pgsql/traverse**](./packages/traverse) | PostgreSQL AST traversal utilities | • Visitor pattern for traversing PostgreSQL AST nodes
• NodePath context with parent/path information
• Runtime schema-based precise traversal | ## 🛠️ Development diff --git a/packages/transform/.gitignore b/packages/transform/.gitignore new file mode 100644 index 00000000..06439b12 --- /dev/null +++ b/packages/transform/.gitignore @@ -0,0 +1,29 @@ + + +# Dependencies +node_modules/ + +# Build output +dist/ +*.tsbuildinfo + +# IDE +.vscode/ +.idea/ + +# OS +.DS_Store +Thumbs.db + +# Logs +*.log +npm-debug.log* + +# Coverage +coverage/ +.nyc_output/ + +# Temp files +*.tmp +*.temp +.cache/ \ No newline at end of file diff --git a/packages/transform/README.md b/packages/transform/README.md new file mode 100644 index 00000000..67f270da --- /dev/null +++ b/packages/transform/README.md @@ -0,0 +1,72 @@ +# @pgsql/transform + +

+ +

+ +

+ + + + + +

+ +AST-based SQL transformation, qualification, classification, and dependency-closure analysis for PostgreSQL. Works on plain SQL **and inside PL/pgSQL function bodies** (via `plpgsql-parser` hydration) — no regexes. + +> Looking for the PG13→17/18 version-upgrade AST transformer previously published under this name? It now lives at [`@pgsql/transform-ast`](../transform-ast). + +## Installation + +```bash +npm install @pgsql/transform +``` + +The parser runs on a WASM build of the real PostgreSQL parser; call `loadModule()` from `plpgsql-parser` once before using any synchronous API. + +## Features + +### `transformSql` — schema-name rewriting + +Rewrite schema names everywhere they can appear (DDL, DML, function bodies, trigger definitions, grants, policies, comments, type casts, string-embedded types inside PL/pgSQL, ...): + +```typescript +import { loadModule } from 'plpgsql-parser'; +import { transformSql } from '@pgsql/transform'; + +await loadModule(); +const mapping = new Map([['my-schema', 'my_schema']]); +const { sql } = transformSql(inputSql, mapping); +``` + +### `classifyStatements` — per-statement AST facts + +```typescript +import { classifyStatements } from '@pgsql/transform'; + +const facts = classifyStatements(sql); +// per statement: kind (schema|table|view|index|type|function|trigger|policy|grant|...), +// creates, references (incl. inside PL/pgSQL bodies), referencedSchemas, roles, +// fkTargets, securityRelevant, securityDefiner, dynamicSql +``` + +### `qualifyUnqualified` — add schema qualification + +Qualify unqualified object references against an inventory of known objects, with multi-schema routing support. + +### `resolveFixtureClosure` — transitive dependency closure + +Given a set of changes (name + SQL + optional declared dependencies), compute the transitive closure of a selection: forward producers of every referenced object/schema/role, plus attached fixtures (policies/grants/RLS targeting closure members), with explicit unresolved-reference reporting. + +### `categorizeChange` / `buildCategoryOf` — change categorization + +Profile-driven categorization of migration changes (e.g. schema / functionality / security / fixtures) from their AST facts. + +### Round-trip validation + +`normalizeTree` / `cleanTree` / `validateRoundTrip` — dependency-free AST normalization and mutation-aware parse→deparse→re-parse validation. + +## Scripts + +- `npm run fixtures` — regenerate `__fixtures__/output/` golden files from `__fixtures__/input/` +- `npm run scan-corpus [dir...]` — audit node-type coverage over a SQL corpus diff --git a/packages/transform/__fixtures__/input/kitchen-sink.sql b/packages/transform/__fixtures__/input/kitchen-sink.sql new file mode 100644 index 00000000..cf4e9c42 --- /dev/null +++ b/packages/transform/__fixtures__/input/kitchen-sink.sql @@ -0,0 +1,353 @@ +-- Deploy schemas/my-schema/procedures/kitchen_sink to pg +-- requires: schemas/my-schema/schema +-- requires: schemas/other-schema/schema +-- made with mass-deploy + +-- Kitchen sink fixture: every combinatorial way of having schemas +-- in qualified and unqualified positions across all node types. + +BEGIN; + +-- ============================================================================= +-- 1. CREATE SCHEMA +-- ============================================================================= +CREATE SCHEMA "my-schema"; +CREATE SCHEMA "other-schema"; + +-- ============================================================================= +-- 2. GRANT / REVOKE ON SCHEMA +-- ============================================================================= +GRANT USAGE ON SCHEMA "my-schema" TO authenticated_role; +GRANT CREATE ON SCHEMA "other-schema" TO admin_role; +REVOKE ALL ON SCHEMA "my-schema" FROM public; + +-- ============================================================================= +-- 3. ALTER DEFAULT PRIVILEGES IN SCHEMA +-- ============================================================================= +ALTER DEFAULT PRIVILEGES IN SCHEMA "my-schema" GRANT SELECT ON TABLES TO authenticated_role; +ALTER DEFAULT PRIVILEGES IN SCHEMA "other-schema" GRANT EXECUTE ON FUNCTIONS TO admin_role; + +-- ============================================================================= +-- 4. SET search_path +-- ============================================================================= +SET search_path TO "my-schema", public; + +-- ============================================================================= +-- 5. CREATE TABLE (CreateStmt) with schema-qualified names +-- Tests: CreateStmt.relation, ColumnDef.typeName, Constraint FK pktable +-- ============================================================================= +CREATE TABLE "my-schema".users ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + org_id bigint, + username text NOT NULL, + email text NOT NULL, + status "my-schema".user_status_type DEFAULT 'active', + metadata jsonb DEFAULT '{}'::jsonb, + created_at timestamptz DEFAULT now(), + updated_at timestamptz DEFAULT now() +); + +CREATE TABLE "other-schema".audit_log ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + table_name text NOT NULL, + record_id bigint NOT NULL, + action text NOT NULL, + changed_by bigint REFERENCES "my-schema".users(id), + changed_at timestamptz DEFAULT now() +); + +-- ============================================================================= +-- 6. CREATE TYPE / ENUM / DOMAIN +-- Tests: CreateEnumStmt, CreateDomainStmt, CompositeTypeStmt +-- ============================================================================= +CREATE TYPE "my-schema".user_status_type AS ENUM ('active', 'inactive', 'suspended'); +CREATE TYPE "other-schema".severity_level AS ENUM ('low', 'medium', 'high', 'critical'); + +CREATE DOMAIN "my-schema".positive_int AS integer CHECK (VALUE > 0); +CREATE DOMAIN "other-schema".email_address AS text CHECK (VALUE ~* '^[^@]+@[^@]+\.[^@]+$'); + +CREATE TYPE "my-schema".address_type AS ( + street text, + city text, + zip text +); + +-- ============================================================================= +-- 7. ALTER TYPE / ENUM / DOMAIN +-- ============================================================================= +ALTER TYPE "my-schema".user_status_type ADD VALUE 'pending'; +ALTER DOMAIN "my-schema".positive_int SET NOT NULL; + +-- ============================================================================= +-- 8. INSERT INTO (InsertStmt) +-- ============================================================================= +INSERT INTO "my-schema".users (username, email) VALUES ('admin', 'admin@example.com'); +INSERT INTO "other-schema".audit_log (table_name, record_id, action, changed_by) + VALUES ('users', 1, 'INSERT', 1); + +-- ============================================================================= +-- 9. SELECT with schema-qualified references (RangeVar, TypeCast) +-- ============================================================================= +SELECT + u.id, + u.username, + u.email, + u.status::"my-schema".user_status_type, + u.metadata::text, + NULL::"my-schema".address_type +FROM "my-schema".users u +WHERE u.status = 'active' +ORDER BY u.created_at; + +SELECT + a.id, + a.action, + u.username +FROM "other-schema".audit_log a +JOIN "my-schema".users u ON u.id = a.changed_by; + +-- ============================================================================= +-- 10. UPDATE / DELETE with schema-qualified tables +-- Tests: UpdateStmt.relation, DeleteStmt.relation +-- ============================================================================= +UPDATE "my-schema".users SET status = 'inactive' WHERE username = 'admin'; +DELETE FROM "other-schema".audit_log WHERE action = 'INSERT'; + +-- ============================================================================= +-- 11. CREATE INDEX (IndexStmt) +-- Tests: IndexStmt.relation +-- ============================================================================= +CREATE INDEX idx_users_email ON "my-schema".users (email); +CREATE INDEX idx_audit_action ON "other-schema".audit_log (action); + +-- ============================================================================= +-- 12. DROP SCHEMA +-- ============================================================================= +DROP SCHEMA IF EXISTS "my-schema" CASCADE; + +-- ============================================================================= +-- 13. COMMENT ON (CommentStmt) - various object types +-- ============================================================================= +COMMENT ON TABLE "my-schema".users IS 'Main users table'; +COMMENT ON COLUMN "my-schema".users.email IS 'User email address'; +COMMENT ON FUNCTION "my-schema".get_user_by_id(bigint) IS 'Fetch user by primary key'; + +-- ============================================================================= +-- 14. CREATE TRIGGER (CreateTrigStmt) +-- Tests: CreateTrigStmt.relation, CreateTrigStmt.funcname +-- ============================================================================= +CREATE TRIGGER trg_audit_insert + AFTER INSERT ON "my-schema".users + FOR EACH ROW + EXECUTE FUNCTION "other-schema".log_change(); + +CREATE TRIGGER trg_audit_update + AFTER UPDATE ON "my-schema".users + FOR EACH ROW + EXECUTE PROCEDURE "my-schema".on_user_update(); + +-- ============================================================================= +-- 15. ALTER TABLE SET SCHEMA (AlterObjectSchemaStmt) +-- ============================================================================= +ALTER TABLE "my-schema".users SET SCHEMA "other-schema"; + +-- ============================================================================= +-- 16. Schema-qualified function calls in expressions (FuncCall) +-- ============================================================================= +SELECT "my-schema".get_user_by_id(1); +SELECT "other-schema".compute_severity('high'); + +-- ============================================================================= +-- 17. CREATE FUNCTION — comprehensive PL/pgSQL function +-- This single function tests ALL PL/pgSQL-specific schema patterns: +-- - Schema-qualified function name +-- - Schema-qualified return type +-- - DECLARE with schema-qualified variable types (both schemas) +-- - SELECT FROM schema-qualified table +-- - Cross-schema references in body +-- ============================================================================= +CREATE FUNCTION "my-schema".kitchen_sink_fn( + user_id bigint +) RETURNS "my-schema".users AS $$ +DECLARE + v_user "my-schema".users; + v_status "my-schema".user_status_type; + v_addr "my-schema".address_type; + v_severity "other-schema".severity_level; + v_audit_id bigint; +BEGIN + SELECT * INTO v_user + FROM "my-schema".users + WHERE id = user_id; + + v_status := v_user.status; + + INSERT INTO "other-schema".audit_log (user_id) + VALUES (v_user.id) + RETURNING id INTO v_audit_id; + + RETURN v_user; +END; +$$ LANGUAGE plpgsql STABLE; + +-- ============================================================================= +-- 17b. Second PL/pgSQL function in the same file (multi-function support) +-- - Trigger function: conditional RETURN NEW, no explicit final return +-- - Verifies each function keeps its own body and no implicit RETURN; +-- is emitted (invalid in trigger functions) +-- ============================================================================= +CREATE FUNCTION "other-schema".tg_audit_users() RETURNS trigger AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + INSERT INTO "other-schema".audit_log (user_id) + VALUES (NEW.id); + RETURN NEW; + END IF; +END; +$$ LANGUAGE plpgsql; + +-- ============================================================================= +-- 17c. Third PL/pgSQL function: cursor/exception/RAISE edge cases +-- - SCROLL cursor declaration (option must stay on the declaration) +-- - MOVE/FETCH with explicit counts and FIRST/LAST directions +-- - SQLSTATE exception condition +-- - Bare RAISE re-throw +-- - Array element assignment target +-- ============================================================================= +CREATE FUNCTION "my-schema".cursor_edge_cases() RETURNS int AS $$ +DECLARE + scan_cursor SCROLL CURSOR FOR + SELECT id FROM "my-schema".users ORDER BY id; + ids int[] := ARRAY[0, 0, 0]; + current_id int; +BEGIN + OPEN scan_cursor; + MOVE FORWARD 2 FROM scan_cursor; + FETCH PRIOR FROM scan_cursor INTO current_id; + MOVE LAST IN scan_cursor; + CLOSE scan_cursor; + ids[1] := current_id; + RETURN ids[1]; +EXCEPTION + WHEN unique_violation OR SQLSTATE '23503' THEN + RAISE; +END; +$$ LANGUAGE plpgsql; + +-- ============================================================================= +-- 18. CREATE VIEW (ViewStmt) +-- Tests: ViewStmt.view +-- ============================================================================= +CREATE VIEW "my-schema".active_users AS + SELECT id, username, email FROM "my-schema".users WHERE status = 'active'; + +-- ============================================================================= +-- 19. ALTER TABLE ADD COLUMN (AlterTableStmt) +-- Tests: AlterTableStmt.relation +-- ============================================================================= +ALTER TABLE "my-schema".users ADD COLUMN bio text; +ALTER TABLE "other-schema".audit_log ADD COLUMN ip_address inet; + +-- ============================================================================= +-- 20. CREATE SEQUENCE / ALTER SEQUENCE (CreateSeqStmt, AlterSeqStmt) +-- Tests: CreateSeqStmt.sequence, AlterSeqStmt.sequence, AlterSeqStmt OWNED BY +-- ============================================================================= +CREATE SEQUENCE "my-schema".users_id_seq; +ALTER SEQUENCE "my-schema".users_id_seq OWNED BY "my-schema".users.id; + +-- ============================================================================= +-- 21. CREATE POLICY (CreatePolicyStmt) +-- Tests: CreatePolicyStmt.table +-- ============================================================================= +CREATE POLICY users_select_policy ON "my-schema".users FOR SELECT USING (true); +CREATE POLICY audit_insert_policy ON "other-schema".audit_log FOR INSERT WITH CHECK (true); + +-- ============================================================================= +-- 22. CREATE RULE (RuleStmt) +-- Tests: RuleStmt.relation +-- ============================================================================= +CREATE RULE notify_insert AS ON INSERT TO "my-schema".users DO NOTHING; + +-- ============================================================================= +-- 23. CLUSTER (ClusterStmt) +-- Tests: ClusterStmt.relation +-- ============================================================================= +CLUSTER "my-schema".users USING idx_users_email; + +-- ============================================================================= +-- 24. VACUUM / ANALYZE (VacuumRelation) +-- Tests: VacuumRelation.relation +-- ============================================================================= +VACUUM "my-schema".users; +ANALYZE "other-schema".audit_log; + +-- ============================================================================= +-- 25. Verify function calls (string literal patterns) +-- ============================================================================= +SELECT verify_schema('my-schema'); +SELECT verify_table('my-schema.users'); +SELECT verify_function('my-schema.get_user_by_id'); +SELECT verify_trigger('my-schema.trg_audit_insert'); +SELECT verify_type('my-schema.user_status_type'); +SELECT verify_domain('my-schema.positive_int'); + +-- ============================================================================= +-- 26. JSON string values with schema names +-- ============================================================================= +SELECT '{"schema":"my-schema"}'::jsonb; +SELECT '{"source":"other-schema"}'::jsonb; + +-- ============================================================================= +-- 27. MERGE (MergeStmt / MergeWhenClause) +-- Tests: MergeStmt.relation, sourceRelation, mergeWhenClauses +-- ============================================================================= +MERGE INTO "my-schema".users u +USING "other-schema".audit_log s +ON u.id = s.id +WHEN MATCHED THEN + UPDATE SET email = s.action +WHEN NOT MATCHED THEN + INSERT (id, email) VALUES (s.id, s.action); + +-- ============================================================================= +-- 28. SECURITY LABEL (SecLabelStmt) +-- Tests: SecLabelStmt.object qualified names +-- ============================================================================= +SECURITY LABEL FOR anon ON COLUMN "my-schema".users.email IS 'MASKED WITH FUNCTION anon.fake_email()'; + +-- ============================================================================= +-- 29. GRANT / REVOKE ... IN SCHEMA (GrantStmt ACL_TARGET_ALL_IN_SCHEMA) +-- ============================================================================= +GRANT SELECT ON ALL TABLES IN SCHEMA "my-schema" TO PUBLIC; +REVOKE ALL ON ALL SEQUENCES IN SCHEMA "other-schema" FROM PUBLIC; + +-- ============================================================================= +-- 30. ALTER ... SET SCHEMA (AlterObjectSchemaStmt) +-- ============================================================================= +ALTER TABLE "my-schema".users SET SCHEMA "other-schema"; +ALTER TYPE "my-schema".user_status_type SET SCHEMA "other-schema"; + +-- ============================================================================= +-- 31. DO block (DoStmt body transformation) +-- ============================================================================= +DO $$ +BEGIN + PERFORM "my-schema".get_user_by_id('00000000-0000-0000-0000-000000000000'::uuid); +END; +$$; + +-- ============================================================================= +-- 32. Array-typed composite DECLARE variables (array bounds preservation) +-- ============================================================================= +CREATE FUNCTION "my-schema".cache_users() +RETURNS void AS $$ +DECLARE + cached "my-schema".users[]; + current_user_row "my-schema".users; +BEGIN + SELECT * INTO current_user_row FROM "my-schema".users LIMIT 1; + cached := array_fill(current_user_row, ARRAY[1]); +END; +$$ LANGUAGE plpgsql; + +COMMIT; diff --git a/packages/transform/__fixtures__/input/plpgsql-constructs.sql b/packages/transform/__fixtures__/input/plpgsql-constructs.sql new file mode 100644 index 00000000..b106ea62 --- /dev/null +++ b/packages/transform/__fixtures__/input/plpgsql-constructs.sql @@ -0,0 +1,157 @@ +-- Deploy schemas/my-schema/procedures/plpgsql_constructs to pg +-- requires: schemas/my-schema/schema +-- made with mass-deploy + +-- PL/pgSQL construct fixture: every PLpgSQL_stmt_* node type emitted by +-- generate:constructive (per scripts/scan-corpus.js), so the schema transform +-- and round-trip validation cover the real shapes we produce. +-- +-- To add a new case: append SQL here (or add a new file in __fixtures__/input/) +-- and run `pnpm fixtures` to regenerate the golden output. + +BEGIN; + +-- 1. Searched CASE (PLpgSQL_stmt_case / PLpgSQL_case_when) +CREATE FUNCTION "my-schema".case_fn(n int) RETURNS text AS $$ +BEGIN + CASE + WHEN n > 0 THEN + RETURN "my-schema".label_pos(); + WHEN n < 0 THEN + PERFORM "my-schema".note_neg(); + RETURN 'neg'; + ELSE + RETURN 'zero'; + END CASE; +END; +$$ LANGUAGE plpgsql; + +-- 2. Simple CASE with a test expression +CREATE FUNCTION "my-schema".simple_case(n int) RETURNS text AS $$ +BEGIN + CASE n + WHEN 1 THEN RETURN "my-schema".one(); + WHEN 2 THEN RETURN 'two'; + ELSE RETURN 'many'; + END CASE; +END; +$$ LANGUAGE plpgsql; + +-- 3. WHILE loop (PLpgSQL_stmt_while) +CREATE FUNCTION "my-schema".while_fn() RETURNS int AS $$ +DECLARE + i int := 0; + total int := 0; +BEGIN + WHILE i < (SELECT count(*) FROM "my-schema".users) LOOP + total := total + 1; + i := i + 1; + END LOOP; + RETURN total; +END; +$$ LANGUAGE plpgsql; + +-- 4. Bare LOOP with EXIT WHEN (PLpgSQL_stmt_loop / PLpgSQL_stmt_exit) +CREATE FUNCTION "my-schema".loop_fn() RETURNS int AS $$ +DECLARE + i int := 0; +BEGIN + LOOP + i := i + 1; + EXIT WHEN i >= (SELECT count(*) FROM "my-schema".users); + END LOOP; + RETURN i; +END; +$$ LANGUAGE plpgsql; + +-- 5. FOREACH ... IN ARRAY (PLpgSQL_stmt_foreach_a) +CREATE FUNCTION "my-schema".foreach_fn(arr int[]) RETURNS int AS $$ +DECLARE + x int; + total int := 0; +BEGIN + FOREACH x IN ARRAY arr LOOP + total := total + x; + PERFORM "my-schema".track(x); + END LOOP; + RETURN total; +END; +$$ LANGUAGE plpgsql; + +-- 6. CONTINUE inside a loop (PLpgSQL_stmt_exit, is_exit = false) +CREATE FUNCTION "my-schema".continue_fn() RETURNS int AS $$ +DECLARE + i int := 0; + total int := 0; +BEGIN + FOR i IN 1..10 LOOP + CONTINUE WHEN i % 2 = 0; + total := total + (SELECT count(*) FROM "my-schema".users WHERE id = i); + END LOOP; + RETURN total; +END; +$$ LANGUAGE plpgsql; + +-- 7. RETURN NEXT bare / OUT-parameter form (PLpgSQL_stmt_return_next). +-- This is the only RETURN NEXT shape the generated corpus emits. +CREATE FUNCTION "my-schema".rn_out(OUT x int, OUT y int) RETURNS SETOF record AS $$ +BEGIN + FOR x IN SELECT g FROM "my-schema".gs LOOP + y := x * 2; + RETURN NEXT; + END LOOP; +END; +$$ LANGUAGE plpgsql; + +-- 8. Nested EXCEPTION block with GET STACKED DIAGNOSTICS +CREATE FUNCTION "my-schema".diag_fn() RETURNS text AS $$ +DECLARE + msg text; +BEGIN + PERFORM "my-schema".might_fail(); + RETURN 'ok'; +EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS msg = MESSAGE_TEXT; + PERFORM "my-schema".log_err(msg); + RETURN msg; +END; +$$ LANGUAGE plpgsql; + +-- 9. CALL of a schema-qualified procedure (CallStmt), both inside a +-- PL/pgSQL body (PLpgSQL_stmt_call) and at the top level +CREATE PROCEDURE "my-schema".do_work(n int) AS $$ +BEGIN + PERFORM "my-schema".track(n); +END; +$$ LANGUAGE plpgsql; + +CREATE FUNCTION "my-schema".call_fn() RETURNS void AS $$ +BEGIN + CALL "my-schema".do_work(1); +END; +$$ LANGUAGE plpgsql; + +CALL "my-schema".do_work(2); + +-- 10. Bound cursor with explicit arguments (deparser fix: pgsql-parser #306) +CREATE FUNCTION "my-schema".cursor_args_fn() RETURNS void AS $$ +DECLARE + c CURSOR (key int) FOR SELECT * FROM "my-schema".users WHERE id = key; + r record; +BEGIN + OPEN c(42); + FETCH c INTO r; + CLOSE c; +END; +$$ LANGUAGE plpgsql; + +-- 11. RAISE with a SQLSTATE condition code (deparser fix: pgsql-parser #306) +CREATE FUNCTION "my-schema".raise_sqlstate_fn() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM "my-schema".users) THEN + RAISE SQLSTATE '22012'; + END IF; +END; +$$ LANGUAGE plpgsql; + +COMMIT; diff --git a/packages/transform/__fixtures__/output/kitchen-sink.sql b/packages/transform/__fixtures__/output/kitchen-sink.sql new file mode 100644 index 00000000..2e62f79d --- /dev/null +++ b/packages/transform/__fixtures__/output/kitchen-sink.sql @@ -0,0 +1,272 @@ +-- Deploy schemas/my_schema/procedures/kitchen_sink to pg +-- requires: schemas/my_schema/schema +-- requires: schemas/other_schema/schema +-- made with mass-deploy + +-- Kitchen sink fixture: every combinatorial way of having schemas +-- in qualified and unqualified positions across all node types. + +BEGIN; + +CREATE SCHEMA my_schema; + +CREATE SCHEMA other_schema; + +GRANT USAGE ON SCHEMA my_schema TO authenticated_role; + +GRANT CREATE ON SCHEMA other_schema TO admin_role; + +REVOKE ALL ON SCHEMA my_schema FROM PUBLIC RESTRICT; + +ALTER DEFAULT PRIVILEGES IN SCHEMA my_schema + GRANT SELECT ON TABLES TO authenticated_role; + +ALTER DEFAULT PRIVILEGES IN SCHEMA other_schema + GRANT EXECUTE ON FUNCTIONS TO admin_role; + +SET search_path TO my_schema, public; + +CREATE TABLE my_schema.users ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + org_id bigint, + username text NOT NULL, + email text NOT NULL, + status my_schema.user_status_type DEFAULT 'active', + metadata jsonb DEFAULT '{}'::jsonb, + created_at timestamptz DEFAULT now(), + updated_at timestamptz DEFAULT now() +); + +CREATE TABLE other_schema.audit_log ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + table_name text NOT NULL, + record_id bigint NOT NULL, + action text NOT NULL, + changed_by bigint REFERENCES my_schema.users (id), + changed_at timestamptz DEFAULT now() +); + +CREATE TYPE my_schema.user_status_type AS ENUM ('active', 'inactive', 'suspended'); + +CREATE TYPE other_schema.severity_level AS ENUM ('low', 'medium', 'high', 'critical'); + +CREATE DOMAIN my_schema.positive_int AS int + CHECK (value > 0); + +CREATE DOMAIN other_schema.email_address AS text + CHECK (value ~* E'^[^@]+@[^@]+\\.[^@]+$'); + +CREATE TYPE my_schema.address_type AS (street text, city text, zip text); + +ALTER TYPE my_schema.user_status_type ADD VALUE 'pending'; + +ALTER DOMAIN my_schema.positive_int SET NOT NULL; + +INSERT INTO my_schema.users ( + username, + email +) VALUES + ('admin', 'admin@example.com'); + +INSERT INTO other_schema.audit_log ( + table_name, + record_id, + action, + changed_by +) VALUES + ('users', 1, 'INSERT', 1); + +SELECT + u.id, + u.username, + u.email, + CAST(u.status AS my_schema.user_status_type), + u.metadata::text, + CAST(NULL AS my_schema.address_type) +FROM my_schema.users AS u +WHERE + u.status = 'active' +ORDER BY + u.created_at; + +SELECT + a.id, + a.action, + u.username +FROM other_schema.audit_log AS a +JOIN my_schema.users AS u ON u.id = a.changed_by; + +UPDATE my_schema.users SET status = 'inactive' WHERE username = 'admin'; + +DELETE FROM other_schema.audit_log WHERE action = 'INSERT'; + +CREATE INDEX idx_users_email ON my_schema.users (email); + +CREATE INDEX idx_audit_action ON other_schema.audit_log (action); + +DROP SCHEMA IF EXISTS my_schema CASCADE; + +COMMENT ON TABLE my_schema.users IS 'Main users table'; + +COMMENT ON COLUMN my_schema.users.email IS 'User email address'; + +COMMENT ON FUNCTION my_schema.get_user_by_id(bigint) IS 'Fetch user by primary key'; + +CREATE TRIGGER trg_audit_insert + AFTER INSERT + ON my_schema.users + FOR EACH ROW + EXECUTE PROCEDURE other_schema.log_change(); + +CREATE TRIGGER trg_audit_update + AFTER UPDATE + ON my_schema.users + FOR EACH ROW + EXECUTE PROCEDURE my_schema.on_user_update(); + +ALTER TABLE my_schema.users SET SCHEMA other_schema; + +SELECT my_schema.get_user_by_id(1); + +SELECT other_schema.compute_severity('high'); + +CREATE FUNCTION my_schema.kitchen_sink_fn( + user_id bigint +) RETURNS my_schema.users AS $$DECLARE + v_user my_schema.users; + v_status my_schema.user_status_type; + v_addr my_schema.address_type; + v_severity other_schema.severity_level; + v_audit_id bigint; +BEGIN + SELECT * INTO v_user + FROM my_schema.users + WHERE + id = user_id; + v_status := v_user.status; + INSERT INTO other_schema.audit_log ( + user_id + ) VALUES + (v_user.id) RETURNING id INTO v_audit_id; + RETURN v_user; +END$$ LANGUAGE plpgsql STABLE; + +CREATE FUNCTION other_schema.tg_audit_users() RETURNS trigger AS $$BEGIN + IF tg_op = 'INSERT' THEN + INSERT INTO other_schema.audit_log ( + user_id + ) VALUES + (new.id); + RETURN NEW; + END IF; +END$$ LANGUAGE plpgsql; + +CREATE FUNCTION my_schema.cursor_edge_cases() RETURNS int AS $$DECLARE + scan_cursor SCROLL CURSOR FOR SELECT id + FROM my_schema.users + ORDER BY + id; + ids int[] := ARRAY[0, 0, 0]; + current_id int; +BEGIN + OPEN scan_cursor; + MOVE FORWARD 2 FROM scan_cursor; + FETCH PRIOR FROM scan_cursor INTO current_id; + MOVE LAST FROM scan_cursor; + CLOSE scan_cursor; + ids[1] := current_id; + RETURN (ids)[1]; +EXCEPTION + WHEN unique_violation OR SQLSTATE '23503' THEN + RAISE; +END$$ LANGUAGE plpgsql; + +CREATE VIEW my_schema.active_users AS SELECT + id, + username, + email +FROM my_schema.users +WHERE + status = 'active'; + +ALTER TABLE my_schema.users + ADD COLUMN bio text; + +ALTER TABLE other_schema.audit_log + ADD COLUMN ip_address inet; + +CREATE SEQUENCE my_schema.users_id_seq; + +ALTER SEQUENCE my_schema.users_id_seq OWNED BY my_schema.users.id; + +CREATE POLICY users_select_policy + ON my_schema.users + AS PERMISSIVE + FOR SELECT + TO PUBLIC + USING ( + true + ); + +CREATE POLICY audit_insert_policy + ON other_schema.audit_log + AS PERMISSIVE + FOR INSERT + TO PUBLIC + WITH CHECK ( + true + ); + +CREATE RULE notify_insert AS ON INSERT TO my_schema.users DO NOTHING; + +CLUSTER my_schema.users USING idx_users_email; + +VACUUM my_schema.users; + +ANALYZE other_schema.audit_log; + +SELECT verify_schema('my_schema'); + +SELECT verify_table('my_schema.users'); + +SELECT verify_function('my_schema.get_user_by_id'); + +SELECT verify_trigger('my_schema.trg_audit_insert'); + +SELECT verify_type('my_schema.user_status_type'); + +SELECT verify_domain('my_schema.positive_int'); + +SELECT '{"schema":"my_schema"}'::jsonb; + +SELECT '{"source":"other_schema"}'::jsonb; + +MERGE INTO my_schema.users AS u USING other_schema.audit_log AS s ON u.id = s.id WHEN MATCHED THEN UPDATE SET email = s.action WHEN NOT MATCHED THEN INSERT (id, email) VALUES (s.id, s.action); + +SECURITY LABEL FOR anon ON COLUMN my_schema.users.email IS 'MASKED WITH FUNCTION anon.fake_email()'; + +GRANT SELECT ON ALL TABLES IN SCHEMA my_schema TO PUBLIC; + +REVOKE ALL ON ALL SEQUENCES IN SCHEMA other_schema FROM PUBLIC RESTRICT; + +ALTER TABLE my_schema.users SET SCHEMA other_schema; + +ALTER TYPE my_schema.user_status_type SET SCHEMA other_schema; + +DO $$ +BEGIN + PERFORM "my_schema".get_user_by_id('00000000-0000-0000-0000-000000000000'::uuid); +END; +$$; + +CREATE FUNCTION my_schema.cache_users() RETURNS void AS $$DECLARE + cached my_schema.users[]; + current_user_row my_schema.users; +BEGIN + SELECT * INTO current_user_row + FROM my_schema.users + LIMIT 1; + cached := array_fill(current_user_row, ARRAY[1]); +END$$ LANGUAGE plpgsql; + +COMMIT; \ No newline at end of file diff --git a/packages/transform/__fixtures__/output/plpgsql-constructs.sql b/packages/transform/__fixtures__/output/plpgsql-constructs.sql new file mode 100644 index 00000000..b97d3606 --- /dev/null +++ b/packages/transform/__fixtures__/output/plpgsql-constructs.sql @@ -0,0 +1,145 @@ +-- Deploy schemas/my_schema/procedures/plpgsql_constructs to pg +-- requires: schemas/my_schema/schema +-- made with mass-deploy + +-- PL/pgSQL construct fixture: every PLpgSQL_stmt_* node type emitted by +-- generate:constructive (per scripts/scan-corpus.js), so the schema transform +-- and round-trip validation cover the real shapes we produce. +-- +-- To add a new case: append SQL here (or add a new file in __fixtures__/input/) +-- and run `pnpm fixtures` to regenerate the golden output. + +BEGIN; + +CREATE FUNCTION my_schema.case_fn( + n int +) RETURNS text AS $$BEGIN + CASE + WHEN n > 0 THEN + RETURN my_schema.label_pos(); + WHEN n < 0 THEN + PERFORM my_schema.note_neg(); + RETURN 'neg'; + ELSE + RETURN 'zero'; + END CASE; +END$$ LANGUAGE plpgsql; + +CREATE FUNCTION my_schema.simple_case( + n int +) RETURNS text AS $$BEGIN + CASE n + WHEN 1 THEN + RETURN my_schema.one(); + WHEN 2 THEN + RETURN 'two'; + ELSE + RETURN 'many'; + END CASE; +END$$ LANGUAGE plpgsql; + +CREATE FUNCTION my_schema.while_fn() RETURNS int AS $$DECLARE + i int := 0; + total int := 0; +BEGIN + WHILE i < ((SELECT count(*) + FROM my_schema.users)) LOOP + total := total + 1; + i := i + 1; + END LOOP; + RETURN total; +END$$ LANGUAGE plpgsql; + +CREATE FUNCTION my_schema.loop_fn() RETURNS int AS $$DECLARE + i int := 0; +BEGIN + LOOP + i := i + 1; + EXIT WHEN i >= ((SELECT count(*) + FROM my_schema.users)); + END LOOP; + RETURN i; +END$$ LANGUAGE plpgsql; + +CREATE FUNCTION my_schema.foreach_fn( + arr int[] +) RETURNS int AS $$DECLARE + x int; + total int := 0; +BEGIN + FOREACH x IN ARRAY arr LOOP + total := total + x; + PERFORM my_schema.track(x); + END LOOP; + RETURN total; +END$$ LANGUAGE plpgsql; + +CREATE FUNCTION my_schema.continue_fn() RETURNS int AS $$DECLARE + i int := 0; + total int := 0; +BEGIN + FOR i IN 1..10 LOOP + CONTINUE WHEN (i % 2) = 0; + total := total + ((SELECT count(*) + FROM my_schema.users + WHERE + id = i)); + END LOOP; + RETURN total; +END$$ LANGUAGE plpgsql; + +CREATE FUNCTION my_schema.rn_out( + OUT x int, + OUT y int +) RETURNS SETOF record AS $$BEGIN + FOR x IN SELECT g + FROM my_schema.gs LOOP + y := x * 2; + RETURN NEXT; + END LOOP; +END$$ LANGUAGE plpgsql; + +CREATE FUNCTION my_schema.diag_fn() RETURNS text AS $$DECLARE + msg text; +BEGIN + PERFORM my_schema.might_fail(); + RETURN 'ok'; +EXCEPTION + WHEN others THEN + GET STACKED DIAGNOSTICS msg = MESSAGE_TEXT; + PERFORM my_schema.log_err(msg); + RETURN msg; +END$$ LANGUAGE plpgsql; + +CREATE PROCEDURE my_schema.do_work( + n int +) AS $$BEGIN + PERFORM my_schema.track(n); +END$$ LANGUAGE plpgsql; + +CREATE FUNCTION my_schema.call_fn() RETURNS void AS $$BEGIN + CALL my_schema.do_work(1); +END$$ LANGUAGE plpgsql; + +CALL my_schema.do_work(2); + +CREATE FUNCTION my_schema.cursor_args_fn() RETURNS void AS $$DECLARE + c CURSOR (key int) FOR SELECT * + FROM my_schema.users + WHERE + id = key; + r record; +BEGIN + OPEN c (42); + FETCH FROM c INTO r; + CLOSE c; +END$$ LANGUAGE plpgsql; + +CREATE FUNCTION my_schema.raise_sqlstate_fn() RETURNS void AS $$BEGIN + IF NOT (EXISTS (SELECT 1 + FROM my_schema.users)) THEN + RAISE EXCEPTION SQLSTATE '22012'; + END IF; +END$$ LANGUAGE plpgsql; + +COMMIT; \ No newline at end of file diff --git a/packages/transform/__tests__/categorize.test.ts b/packages/transform/__tests__/categorize.test.ts new file mode 100644 index 00000000..e7dbeb35 --- /dev/null +++ b/packages/transform/__tests__/categorize.test.ts @@ -0,0 +1,100 @@ +import { loadModule } from 'plpgsql-parser'; + +import { buildCategoryOf, categorizeChange, CategoryProfile, TIER_PROFILE } from '../src/categorize'; + +beforeAll(async () => { + await loadModule(); +}); + +describe('categorizeChange (tier profile)', () => { + it('classifies base DDL as schema', () => { + expect(categorizeChange('CREATE SCHEMA app_public;', 'schemas/app/schema')).toBe('schema'); + expect( + categorizeChange('CREATE TABLE app_public.users (id uuid PRIMARY KEY);', 'schemas/app/tables/users') + ).toBe('schema'); + expect( + categorizeChange('CREATE INDEX users_org_idx ON app_public.users (org_id);', 'schemas/app/indexes/users_org') + ).toBe('schema'); + }); + + it('classifies functions and triggers as functionality', () => { + expect( + categorizeChange( + 'CREATE FUNCTION app_public.touch() RETURNS trigger AS $$ BEGIN RETURN NEW; END; $$ LANGUAGE plpgsql;', + 'schemas/app/procedures/touch' + ) + ).toBe('functionality'); + expect( + categorizeChange( + 'CREATE TRIGGER touch_users BEFORE UPDATE ON app_public.users FOR EACH ROW EXECUTE FUNCTION app_public.touch();', + 'schemas/app/triggers/touch_users' + ) + ).toBe('functionality'); + }); + + it('classifies policies, grants and RLS as security (wins over everything)', () => { + expect( + categorizeChange('GRANT SELECT ON app_public.users TO authenticated;', 'schemas/app/grants/users') + ).toBe('security'); + expect( + categorizeChange( + 'CREATE POLICY users_select ON app_public.users FOR SELECT TO authenticated USING (true);', + 'schemas/app/policies/users_select' + ) + ).toBe('security'); + // A change that both creates a table AND enables RLS is pulled into security. + expect( + categorizeChange( + 'CREATE TABLE app_public.secrets (id uuid PRIMARY KEY); ALTER TABLE app_public.secrets ENABLE ROW LEVEL SECURITY;', + 'schemas/app/tables/secrets' + ) + ).toBe('security'); + }); + + it('classifies seed DML as fixtures', () => { + expect( + categorizeChange("INSERT INTO app_public.settings (key, value) VALUES ('theme', 'dark');", 'seeds/settings') + ).toBe('fixtures'); + }); + + it('defaults empty/unclassifiable SQL to schema', () => { + expect(categorizeChange('-- just a comment\n', 'schemas/app/noop')).toBe('schema'); + }); +}); + +describe('buildCategoryOf', () => { + const changeSql: Record = { + 'schemas/app/schema': 'CREATE SCHEMA app_public;', + 'schemas/app/tables/users': 'CREATE TABLE app_public.users (id uuid PRIMARY KEY);', + 'schemas/app/procedures/touch': + 'CREATE FUNCTION app_public.touch() RETURNS trigger AS $$ BEGIN RETURN NEW; END; $$ LANGUAGE plpgsql;', + 'schemas/app/grants/users': 'GRANT SELECT ON app_public.users TO authenticated;', + }; + + it('produces a categoryOf function keyed by change name', () => { + const categoryOf = buildCategoryOf(changeSql); + expect(categoryOf('schemas/app/schema')).toBe('schema'); + expect(categoryOf('schemas/app/tables/users')).toBe('schema'); + expect(categoryOf('schemas/app/procedures/touch')).toBe('functionality'); + expect(categoryOf('schemas/app/grants/users')).toBe('security'); + }); + + it('returns undefined for changes it never saw (core falls back to folder key)', () => { + const categoryOf = buildCategoryOf(changeSql); + expect(categoryOf('schemas/other/unknown')).toBeUndefined(); + }); + + it('honors a custom profile', () => { + // A per-schema profile: the category is the top-level schema segment. + const bySchema: CategoryProfile = { + name: 'by-schema', + categorize: (_facts, changeName) => changeName.split('/')[1] ?? 'root', + }; + const categoryOf = buildCategoryOf(changeSql, bySchema); + expect(categoryOf('schemas/app/schema')).toBe('app'); + }); + + it('exposes the default profile name', () => { + expect(TIER_PROFILE.name).toBe('tier'); + }); +}); diff --git a/packages/transform/__tests__/facts.test.ts b/packages/transform/__tests__/facts.test.ts new file mode 100644 index 00000000..16919bc4 --- /dev/null +++ b/packages/transform/__tests__/facts.test.ts @@ -0,0 +1,145 @@ +import { loadModule } from 'plpgsql-parser'; + +import { classifyStatements } from '../src/facts'; + +beforeAll(async () => { + await loadModule(); +}); + +describe('classifyStatements', () => { + it('classifies schema, table, view, index and type DDL', () => { + const facts = classifyStatements(` + CREATE SCHEMA app_public; + CREATE TABLE app_public.users (id uuid PRIMARY KEY, org_id uuid REFERENCES app_public.orgs (id)); + CREATE VIEW app_public.active_users AS SELECT * FROM app_public.users; + CREATE INDEX users_org_idx ON app_public.users (org_id); + CREATE TYPE app_public.status AS ENUM ('on', 'off'); + `); + + expect(facts.map(f => f.kind)).toEqual(['schema', 'table', 'view', 'index', 'type']); + expect(facts[0].creates).toEqual([{ schema: null, name: 'app_public' }]); + expect(facts[1].creates).toEqual([{ schema: 'app_public', name: 'users' }]); + expect(facts[1].fkTargets).toEqual([{ schema: 'app_public', name: 'orgs' }]); + expect(facts[2].references).toContainEqual({ schema: 'app_public', name: 'users' }); + expect(facts[4].creates).toEqual([{ schema: 'app_public', name: 'status' }]); + }); + + it('flags security statements: policies, grants, RLS enable', () => { + const facts = classifyStatements(` + CREATE POLICY users_select ON app_public.users FOR SELECT TO authenticated USING (org_id = app_private.current_org_id()); + GRANT SELECT ON app_public.users TO authenticated, administrator; + ALTER TABLE app_public.users ENABLE ROW LEVEL SECURITY; + ALTER TABLE app_public.users FORCE ROW LEVEL SECURITY; + `); + + expect(facts.map(f => f.kind)).toEqual(['policy', 'grant', 'rls_enable', 'rls_enable']); + expect(facts.every(f => f.securityRelevant)).toBe(true); + expect(facts[0].roles).toEqual(['authenticated']); + expect(facts[0].references).toContainEqual({ schema: 'app_private', name: 'current_org_id' }); + expect(facts[1].roles).toEqual(['authenticated', 'administrator']); + }); + + it('captures the guarded table of a policy via the generic walker', () => { + const facts = classifyStatements(` + CREATE POLICY users_select ON app_public.users FOR SELECT USING (true); + `); + + expect(facts[0].creates).toEqual([{ schema: 'app_public', name: 'users.users_select' }]); + expect(facts[0].references).toContainEqual({ schema: 'app_public', name: 'users' }); + }); + + it('classifies FK constraints added via ALTER TABLE', () => { + const facts = classifyStatements(` + ALTER TABLE app_public.users ADD CONSTRAINT users_org_fkey FOREIGN KEY (org_id) REFERENCES billing.orgs (id); + ALTER TABLE app_public.users ADD CONSTRAINT users_email_uniq UNIQUE (email); + `); + + expect(facts[0].kind).toBe('fk_constraint'); + expect(facts[0].fkTargets).toEqual([{ schema: 'billing', name: 'orgs' }]); + expect(facts[0].referencedSchemas).toContain('billing'); + expect(facts[1].kind).toBe('constraint'); + }); + + it('extracts references from PL/pgSQL function bodies', () => { + const facts = classifyStatements(` + CREATE FUNCTION app_public.quota_gate_tg() RETURNS trigger AS $$ + BEGIN + IF (SELECT count(*) FROM billing.usage_log WHERE org_id = NEW.org_id) > 100 THEN + RAISE EXCEPTION 'quota exceeded'; + END IF; + PERFORM store.release_usage(NEW.id); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + `); + + expect(facts).toHaveLength(1); + expect(facts[0].kind).toBe('function'); + expect(facts[0].creates).toEqual([{ schema: 'app_public', name: 'quota_gate_tg' }]); + expect(facts[0].references).toContainEqual({ schema: 'billing', name: 'usage_log' }); + expect(facts[0].references).toContainEqual({ schema: 'store', name: 'release_usage' }); + expect(facts[0].referencedSchemas).toEqual(expect.arrayContaining(['billing', 'store'])); + }); + + it('separates body-only references as late-binding bodyReferences', () => { + const facts = classifyStatements(` + CREATE FUNCTION app_public.quota_gate(org app_types.org_ref) RETURNS boolean + LANGUAGE plpgsql AS $$ + BEGIN + RETURN (SELECT count(*) FROM billing.usage_log) < 100; + END; + $$; + `); + + expect(facts[0].references).toContainEqual({ schema: 'billing', name: 'usage_log' }); + expect(facts[0].bodyReferences).toEqual([{ schema: 'billing', name: 'usage_log' }]); + // signature type is not body-only + expect(facts[0].bodyReferences).not.toContainEqual({ schema: 'app_types', name: 'org_ref' }); + expect(facts[0].references).toContainEqual({ schema: 'app_types', name: 'org_ref' }); + }); + + it('classifies triggers with their function reference', () => { + const facts = classifyStatements(` + CREATE TRIGGER timestamps_tg BEFORE INSERT ON app_public.users + FOR EACH ROW EXECUTE PROCEDURE app_private.tg_timestamps(); + `); + + expect(facts[0].kind).toBe('trigger'); + expect(facts[0].creates).toEqual([{ schema: 'app_public', name: 'users.timestamps_tg' }]); + expect(facts[0].references).toContainEqual({ schema: 'app_private', name: 'tg_timestamps' }); + }); + + it('classifies seed DML with its target', () => { + const facts = classifyStatements(` + INSERT INTO app_public.db_presets (name) VALUES ('default'); + `); + + expect(facts[0].kind).toBe('seed_dml'); + expect(facts[0].creates).toEqual([{ schema: 'app_public', name: 'db_presets' }]); + }); + + it('ignores pg_catalog and unqualified references', () => { + const facts = classifyStatements(` + CREATE TABLE app_public.t (id int4, n numeric); + SELECT count(*) FROM unqualified_table; + `); + + expect(facts[0].references).toEqual([]); + expect(facts[1].references).toEqual([]); + }); + + it('does not modify input and is safe to call repeatedly', () => { + const sql = `CREATE POLICY p ON a.t USING (x = b.f());`; + const first = classifyStatements(sql); + const second = classifyStatements(sql); + expect(second).toEqual(first); + }); + + it('collects ownership changes as security-relevant with roles', () => { + const facts = classifyStatements(` + ALTER TABLE app_public.users OWNER TO administrator; + `); + expect(facts[0].securityRelevant).toBe(true); + expect(facts[0].roles).toEqual(['administrator']); + }); +}); diff --git a/packages/transform/__tests__/fixture-closure.test.ts b/packages/transform/__tests__/fixture-closure.test.ts new file mode 100644 index 00000000..3a344f26 --- /dev/null +++ b/packages/transform/__tests__/fixture-closure.test.ts @@ -0,0 +1,127 @@ +import { loadModule } from 'plpgsql-parser'; + +import { ClosureInputChange, resolveFixtureClosure } from '../src/fixture-closure'; + +beforeAll(async () => { + await loadModule(); +}); + +// A small module: two schemas, tables, a policy + grant + RLS (security), +// a seed insert (fixtures), and a trigger function (functionality). +const CHANGES: ClosureInputChange[] = [ + { name: 'schemas/app_public/schema', sql: 'CREATE SCHEMA app_public;' }, + { name: 'schemas/app_private/schema', sql: 'CREATE SCHEMA app_private;' }, + { + name: 'schemas/app_public/tables/orgs', + sql: 'CREATE TABLE app_public.orgs (id uuid PRIMARY KEY);', + dependencies: ['schemas/app_public/schema'] + }, + { + name: 'schemas/app_public/tables/users', + sql: 'CREATE TABLE app_public.users (id uuid PRIMARY KEY, org_id uuid REFERENCES app_public.orgs (id));', + dependencies: ['schemas/app_public/schema'] + }, + { + name: 'schemas/app_private/tables/sprt_org_members', + sql: 'CREATE TABLE app_private.sprt_org_members (user_id uuid, org_id uuid REFERENCES app_public.orgs (id));', + dependencies: ['schemas/app_private/schema'] + }, + { + name: 'schemas/app_public/policies/users_select', + sql: `CREATE POLICY users_select ON app_public.users FOR SELECT TO authenticated + USING (org_id IN (SELECT org_id FROM app_private.sprt_org_members));` + }, + { + name: 'schemas/app_public/grants/users_grant', + sql: 'GRANT SELECT ON app_public.users TO authenticated;' + }, + { + name: 'schemas/app_public/rls/users_enable', + sql: 'ALTER TABLE app_public.users ENABLE ROW LEVEL SECURITY;' + }, + { + name: 'schemas/app_public/seed/default_org', + sql: "INSERT INTO app_public.orgs (id) VALUES ('00000000-0000-0000-0000-000000000000');" + }, + // an entirely unrelated table (same schema) that must NOT be pulled in + { + name: 'schemas/other/tables/widgets', + sql: 'CREATE TABLE app_public.widgets (id uuid PRIMARY KEY);', + dependencies: ['schemas/app_public/schema'] + }, + // a grant on the unrelated table — shares the schema but must NOT attach to users + { + name: 'schemas/other/grants/widgets_grant', + sql: 'GRANT SELECT ON app_public.widgets TO authenticated;' + } +]; + +describe('resolveFixtureClosure', () => { + it('pulls prerequisites and attached fixtures for a selected table', () => { + const closure = resolveFixtureClosure(CHANGES, ['schemas/app_public/tables/users']); + + // prerequisites: schema + FK target table + expect(closure.order).toContain('schemas/app_public/schema'); + expect(closure.order).toContain('schemas/app_public/tables/orgs'); + + // attached fixtures: policy, grant, RLS enable, and the seed on orgs + expect(closure.fixtures).toEqual( + expect.arrayContaining([ + 'schemas/app_public/policies/users_select', + 'schemas/app_public/grants/users_grant', + 'schemas/app_public/rls/users_enable', + 'schemas/app_public/seed/default_org' + ]) + ); + + // the policy references the SPRT table -> its prerequisite is pulled too + expect(closure.order).toContain('schemas/app_private/tables/sprt_org_members'); + expect(closure.order).toContain('schemas/app_private/schema'); + + // unrelated table and its grant stay out despite sharing the schema + expect(closure.order).not.toContain('schemas/other/tables/widgets'); + expect(closure.order).not.toContain('schemas/other/grants/widgets_grant'); + }); + + it('is deterministic and preserves plan order', () => { + const a = resolveFixtureClosure(CHANGES, ['schemas/app_public/tables/users']); + const b = resolveFixtureClosure(CHANGES, ['schemas/app_public/tables/users']); + expect(a).toEqual(b); + + const planOrder = CHANGES.map(c => c.name); + const positions = a.order.map(n => planOrder.indexOf(n)); + expect(positions).toEqual([...positions].sort((x, y) => x - y)); + }); + + it('reports required roles and surfaces unresolved external roles', () => { + const closure = resolveFixtureClosure(CHANGES, ['schemas/app_public/tables/users']); + // `authenticated` is referenced by policy/grant but created by no change + expect(closure.roles).toContain('authenticated'); + expect(closure.unresolved.roles).toContain('authenticated'); + }); + + it('labels each closure member with a category and reason', () => { + const closure = resolveFixtureClosure(CHANGES, ['schemas/app_public/tables/users']); + const users = closure.changes.find(c => c.name === 'schemas/app_public/tables/users')!; + const policy = closure.changes.find(c => c.name === 'schemas/app_public/policies/users_select')!; + const orgs = closure.changes.find(c => c.name === 'schemas/app_public/tables/orgs')!; + + expect(users.reason).toBe('selected'); + expect(users.category).toBe('schema'); + expect(policy.reason).toBe('fixture'); + expect(policy.category).toBe('security'); + expect(orgs.reason).toBe('prerequisite'); + }); + + it('can skip prerequisites while still pulling attached fixtures', () => { + const closure = resolveFixtureClosure(CHANGES, ['schemas/app_public/tables/users'], { + includePrerequisites: false + }); + // still pulls the fixtures attached to users + expect(closure.fixtures).toEqual( + expect.arrayContaining(['schemas/app_public/grants/users_grant']) + ); + // but does not walk references to the FK target as a prerequisite + expect(closure.order).not.toContain('schemas/app_public/tables/orgs'); + }); +}); diff --git a/packages/transform/__tests__/qualify.test.ts b/packages/transform/__tests__/qualify.test.ts new file mode 100644 index 00000000..0405f971 --- /dev/null +++ b/packages/transform/__tests__/qualify.test.ts @@ -0,0 +1,307 @@ +import { loadModule } from 'plpgsql-parser'; + +import { + collectCreatedObjects, + mergeInventories, + qualifyUnqualified +} from '../src/qualify'; +import { transformSql } from '../src/transform'; + +beforeAll(async () => { + await loadModule(); +}); + +const HANDWRITTEN = `CREATE TABLE users ( + id uuid PRIMARY KEY, + status user_status NOT NULL DEFAULT 'active' +); +CREATE TYPE user_status AS ENUM ('active', 'disabled'); +CREATE TABLE widgets ( + id serial PRIMARY KEY, + user_id uuid REFERENCES users (id) +); +CREATE FUNCTION get_user(uid uuid) RETURNS users AS $$ + SELECT * FROM users WHERE id = uid +$$ LANGUAGE sql; +GRANT SELECT ON users TO authenticated;`; + +describe('collectCreatedObjects', () => { + it('buckets unqualified creations by namespace', () => { + const inv = collectCreatedObjects(HANDWRITTEN); + expect([...inv.relations].sort()).toEqual(['users', 'widgets']); + expect([...inv.functions]).toEqual(['get_user']); + expect([...inv.types]).toEqual(['user_status']); + }); + + it('skips creations that are already qualified', () => { + const inv = collectCreatedObjects('CREATE TABLE app.users (id int);'); + expect(inv.relations.size).toBe(0); + }); + + it('collects sequences as relations', () => { + const inv = collectCreatedObjects('CREATE SEQUENCE user_id_seq;'); + expect([...inv.relations]).toEqual(['user_id_seq']); + }); + + it('merges inventories across scripts', () => { + const merged = mergeInventories([ + collectCreatedObjects('CREATE TABLE a (id int);'), + collectCreatedObjects('CREATE TABLE b (id int);') + ]); + expect([...merged.relations].sort()).toEqual(['a', 'b']); + }); +}); + +describe('qualifyUnqualified', () => { + it('pins unqualified creations and references to the schema', () => { + const { sql, result } = qualifyUnqualified(HANDWRITTEN, { schema: 'public' }); + + expect(sql).toContain('CREATE TABLE public.users'); + expect(sql).toContain('CREATE TABLE public.widgets'); + expect(sql).toContain('REFERENCES public.users'); + expect(sql).toContain('CREATE TYPE public.user_status'); + expect(sql).toContain('CREATE FUNCTION public.get_user'); + expect(sql).toContain('RETURNS public.users'); + expect(sql).toContain('FROM public.users'); + expect(sql).toContain('GRANT SELECT ON public.users'); + expect(sql).toContain('status public.user_status'); + + expect(result.qualified.get('users')).toBeGreaterThanOrEqual(4); + expect(result.qualified.get('get_user')).toBe(1); + }); + + it('leaves builtins, columns, and unknown names untouched', () => { + const { sql } = qualifyUnqualified( + `CREATE TABLE logs (id int, at timestamptz DEFAULT now()); +SELECT count(*) FROM logs WHERE at < now(); +SELECT * FROM external_table;`, + { schema: 'public' } + ); + expect(sql).toContain('now()'); + expect(sql).not.toContain('public.now'); + expect(sql).not.toContain('public.count'); + expect(sql).toContain('FROM public.logs'); + expect(sql).toContain('FROM external_table'); + expect(sql).not.toContain('public.external_table'); + }); + + it('does not qualify CTE references that shadow an inventory relation', () => { + const { sql } = qualifyUnqualified( + `CREATE TABLE users (id int); +WITH users AS (SELECT 1 AS id) SELECT * FROM users;`, + { schema: 'public' } + ); + expect(sql).toContain('CREATE TABLE public.users'); + expect(sql).toMatch(/users AS \(SELECT 1 AS id\)/); + expect(sql).toContain('FROM users;'); + }); + + it('qualifies references inside plpgsql bodies', () => { + const { sql } = qualifyUnqualified( + `CREATE TABLE users (id uuid); +CREATE FUNCTION touch() RETURNS trigger AS $$ +BEGIN + UPDATE users SET id = NEW.id; + RETURN NEW; +END; +$$ LANGUAGE plpgsql;`, + { schema: 'public' } + ); + expect(sql).toContain('UPDATE public.users'); + }); + + it('accepts an external inventory for cross-script references', () => { + const inventory = collectCreatedObjects('CREATE TABLE users (id int);'); + const { sql } = qualifyUnqualified('SELECT * FROM users;', { + schema: 'public', + inventory + }); + expect(sql).toContain('FROM public.users'); + }); + + it('injects CREATE SCHEMA IF NOT EXISTS when requested and absent', () => { + const { sql } = qualifyUnqualified('CREATE TABLE users (id int);', { + schema: 'myapp', + injectCreateSchema: true + }); + expect(sql.startsWith('CREATE SCHEMA IF NOT EXISTS myapp;')).toBe(true); + expect(sql).toContain('CREATE TABLE myapp.users'); + }); + + it('does not inject CREATE SCHEMA when the content already creates it', () => { + const { sql } = qualifyUnqualified( + 'CREATE SCHEMA myapp;\nCREATE TABLE users (id int);', + { schema: 'myapp', injectCreateSchema: true } + ); + expect(sql.match(/CREATE SCHEMA/g)).toHaveLength(1); + }); +}); + +describe('qualifyUnqualified targets (multi-schema routing)', () => { + const MULTI = `CREATE TYPE widget AS ENUM ('gear', 'gadget'); +CREATE TABLE users (id uuid PRIMARY KEY); +CREATE TABLE products ( + id uuid PRIMARY KEY, + owner uuid REFERENCES users (id), + kind widget +); +CREATE FUNCTION get_products(uid uuid) RETURNS SETOF products AS $$ + SELECT * FROM products WHERE owner = uid +$$ LANGUAGE sql; +GRANT SELECT ON users TO authenticated;`; + + it('routes tables, types, and functions to different schemas in one pass', () => { + const { sql, result } = qualifyUnqualified(MULTI, { + targets: { + auth: { relations: ['users'] }, + shop: { relations: ['products'], functions: ['get_products'] }, + shared: { types: ['widget'] } + } + }); + + expect(sql).toContain('CREATE TABLE auth.users'); + expect(sql).toContain('CREATE TABLE shop.products'); + expect(sql).toContain('CREATE TYPE shared.widget'); + expect(sql).toContain('REFERENCES auth.users'); + expect(sql).toContain('kind shared.widget'); + expect(sql).toContain('CREATE FUNCTION shop.get_products'); + expect(sql).toContain('RETURNS SETOF shop.products'); + expect(sql).toContain('FROM shop.products'); + expect(sql).toContain('GRANT SELECT ON auth.users'); + + expect(result.routed.get('users')).toBe('auth'); + expect(result.routed.get('products')).toBe('shop'); + expect(result.routed.get('widget')).toBe('shared'); + expect(result.routed.get('get_products')).toBe('shop'); + }); + + it('leaves unrouted objects untouched', () => { + const { sql } = qualifyUnqualified(MULTI, { + targets: { auth: { relations: ['users'] } } + }); + expect(sql).toContain('CREATE TABLE auth.users'); + expect(sql).toContain('CREATE TABLE products'); + expect(sql).toContain('CREATE TYPE widget'); + expect(sql).toContain('kind widget'); + }); + + it('disambiguates a relation and a type sharing a schema route by namespace', () => { + const { sql } = qualifyUnqualified( + `CREATE TYPE status AS ENUM ('on', 'off'); +CREATE TABLE devices (id int, state status);`, + { + targets: { + hw: { relations: ['devices'] }, + shared: { types: ['status'] } + } + } + ); + expect(sql).toContain('CREATE TABLE hw.devices'); + expect(sql).toContain('CREATE TYPE shared.status'); + expect(sql).toContain('state shared.status'); + }); + + it('injects CREATE SCHEMA IF NOT EXISTS for every missing target schema', () => { + const { sql } = qualifyUnqualified(MULTI, { + targets: { + auth: { relations: ['users'] }, + shop: { relations: ['products'], functions: ['get_products'] }, + shared: { types: ['widget'] } + }, + injectCreateSchema: true + }); + expect(sql).toContain('CREATE SCHEMA IF NOT EXISTS auth;'); + expect(sql).toContain('CREATE SCHEMA IF NOT EXISTS shop;'); + expect(sql).toContain('CREATE SCHEMA IF NOT EXISTS shared;'); + }); + + it('throws on conflicting routes for the same name and namespace', () => { + expect(() => + qualifyUnqualified('CREATE TABLE users (id int);', { + targets: { + a: { relations: ['users'] }, + b: { relations: ['users'] } + } + }) + ).toThrow(/conflicting targets for relation "users"/); + }); + + it('rejects passing both schema and targets', () => { + expect(() => + qualifyUnqualified('SELECT 1;', { + schema: 'public', + targets: { a: { relations: ['x'] } } + }) + ).toThrow(/either `schema` or `targets`/); + }); + + it('rejects neither schema nor targets', () => { + expect(() => qualifyUnqualified('SELECT 1;', {})).toThrow( + /one of `schema` or `targets` is required/ + ); + }); + + it('composes with transformSql for a subsequent rename', () => { + const { content } = transformSql( + qualifyUnqualified(MULTI, { + targets: { auth: { relations: ['users'] }, shop: { relations: ['products'] } } + }).sql, + new Map([['auth', 'tenant_auth']]) + ); + expect(content).toContain('CREATE TABLE tenant_auth.users'); + expect(content).toContain('REFERENCES tenant_auth.users'); + expect(content).toContain('CREATE TABLE shop.products'); + }); +}); + +describe('transformSql qualifyUnqualified integration', () => { + it('ingests handwritten public SQL into a named schema', () => { + const { content, result } = transformSql( + HANDWRITTEN, + new Map([['public', 'myapp']]), + { qualifyUnqualified: { schema: 'public' } } + ); + + expect(result.errors).toEqual([]); + expect(content).toContain('CREATE TABLE myapp.users'); + expect(content).toContain('CREATE TABLE myapp.widgets'); + expect(content).toContain('REFERENCES myapp.users'); + expect(content).toContain('CREATE FUNCTION myapp.get_user'); + expect(content).toContain('RETURNS myapp.users'); + expect(content).toContain('FROM myapp.users'); + expect(content).toContain('GRANT SELECT ON myapp.users'); + expect(content).not.toContain('public.'); + }); + + it('preserves pgpm headers around a qualified body', () => { + const withHeader = `-- Deploy schemas/public/tables/users to pg\n\nCREATE TABLE users (id int);`; + const { content } = transformSql( + withHeader, + new Map([['public', 'myapp']]), + { qualifyUnqualified: { schema: 'public' } } + ); + expect(content).toContain('-- Deploy schemas/myapp/tables/users to pg'); + expect(content).toContain('CREATE TABLE myapp.users'); + }); + + it('descopes a named schema onto public with assumeSchemasExist', () => { + const { content, result } = transformSql( + `CREATE SCHEMA myapp;\nCREATE TABLE myapp.users (id int);\nCREATE FUNCTION myapp.get_user() RETURNS myapp.users AS $$ SELECT * FROM myapp.users $$ LANGUAGE sql;`, + new Map([['myapp', 'public']]), + { assumeSchemasExist: ['public'] } + ); + expect(result.errors).toEqual([]); + expect(content).toContain('CREATE SCHEMA IF NOT EXISTS public'); + expect(content).toContain('CREATE TABLE public.users'); + expect(content).toContain('FROM public.users'); + expect(content).not.toContain('myapp'); + }); + + it('defaults to current behavior when the options are unset', () => { + const { content } = transformSql(HANDWRITTEN, new Map([['public', 'myapp']])); + // Only explicitly qualified references are rewritten; unqualified stay. + expect(content).toContain('CREATE TABLE users'); + expect(content).toContain('CREATE TABLE widgets'); + expect(content).toContain('FROM users'); + }); +}); diff --git a/packages/transform/__tests__/round-trip-core.test.ts b/packages/transform/__tests__/round-trip-core.test.ts new file mode 100644 index 00000000..9f6b3aa2 --- /dev/null +++ b/packages/transform/__tests__/round-trip-core.test.ts @@ -0,0 +1,145 @@ +import { + CLEAN_TREE_VOLATILE_KEYS, + cleanTree, + firstDifference, + normalizeTree, + trimDefElemBody, +} from '../src/round-trip-core'; + +describe('normalizeTree', () => { + it('drops volatile keys recursively', () => { + const tree = { + A: { sval: 'x', location: 5, inner: { location: 9, keep: 1 } }, + list: [{ location: 1, v: 2 }], + }; + const out = normalizeTree(tree, { volatileKeys: new Set(['location']) }); + expect(out).toEqual({ + A: { sval: 'x', inner: { keep: 1 } }, + list: [{ v: 2 }], + }); + }); + + it('sorts object keys when sortKeys is set', () => { + const out = normalizeTree({ b: 1, a: 2, c: 3 }, { sortKeys: true }); + expect(Object.keys(out)).toEqual(['a', 'b', 'c']); + }); + + it('preserves key order by default', () => { + const out = normalizeTree({ b: 1, a: 2 }); + expect(Object.keys(out)).toEqual(['b', 'a']); + }); + + it('filters array items via dropArrayItem', () => { + const out = normalizeTree([{ drop: true }, { keep: 1 }, { drop: true }], { + dropArrayItem: (i: any) => i?.drop === true, + }); + expect(out).toEqual([{ keep: 1 }]); + }); + + it('applies per-key handlers with a recurse callback', () => { + const out = normalizeTree( + { Wrap: { location: 1, sval: ' body ' } }, + { + volatileKeys: new Set(['location']), + keyHandlers: { + Wrap: (v: any, recurse) => ({ ...recurse(v), tag: 'seen' }), + }, + } + ); + expect(out).toEqual({ Wrap: { sval: ' body ', tag: 'seen' } }); + }); + + it('returns primitives untouched and clones Dates', () => { + expect(normalizeTree(42)).toBe(42); + expect(normalizeTree('s')).toBe('s'); + expect(normalizeTree(null)).toBe(null); + const d = new Date(1234567890); + const out = normalizeTree(d); + expect(out).toEqual(d); + expect(out).not.toBe(d); + }); +}); + +describe('firstDifference', () => { + it('returns null for deeply equal trees', () => { + expect(firstDifference({ a: [1, 2] }, { a: [1, 2] })).toBeNull(); + }); + + it('reports the path of a scalar difference', () => { + expect(firstDifference({ a: { b: 1 } }, { a: { b: 2 } })).toBe( + '$.a.b (1 vs 2)' + ); + }); + + it('reports array length differences', () => { + expect(firstDifference([1], [1, 2])).toBe('$ (array length 1 vs 2)'); + }); +}); + +describe('cleanTree preset (upstream-compatible)', () => { + it('exposes the upstream volatile key set', () => { + expect([...CLEAN_TREE_VOLATILE_KEYS].sort()).toEqual([ + 'list_end', + 'list_start', + 'location', + 'rexpr_list_end', + 'rexpr_list_start', + 'stmt_len', + 'stmt_location', + ]); + }); + + it('strips stmt_len / stmt_location / location', () => { + const tree = { + stmts: [ + { + stmt_len: 10, + stmt_location: 0, + stmt: { SelectStmt: { location: 7, op: 'SETOP_NONE' } }, + }, + ], + }; + expect(cleanTree(tree)).toEqual({ + stmts: [{ stmt: { SelectStmt: { op: 'SETOP_NONE' } } }], + }); + }); + + it('trims DefElem "as" body for the array-arg (function) shape', () => { + const tree = { + DefElem: { defname: 'as', arg: [{ String: { sval: ' body \n' } }] }, + }; + expect(cleanTree(tree)).toEqual({ + DefElem: { defname: 'as', arg: [{ String: { sval: 'body' } }] }, + }); + }); + + it('trims DefElem "as" body for the List.items shape', () => { + const tree = { + DefElem: { + defname: 'as', + arg: { List: { items: [{ String: { sval: ' x ' } }] } }, + }, + }; + expect((cleanTree(tree) as any).DefElem.arg.List.items[0].String.sval).toBe( + 'x' + ); + }); + + it('trims DefElem "as" body for the plain String (DO block) shape', () => { + const tree = { DefElem: { defname: 'as', arg: { String: { sval: ' do ' } } } }; + expect((cleanTree(tree) as any).DefElem.arg.String.sval).toBe('do'); + }); + + it('leaves non-"as" DefElem bodies untouched', () => { + const tree = { DefElem: { defname: 'volatile', arg: { String: { sval: ' keep ' } } } }; + expect((cleanTree(tree) as any).DefElem.arg.String.sval).toBe(' keep '); + }); +}); + +describe('trimDefElemBody', () => { + it('is a no-op for non-"as" defnames', () => { + const d = { defname: 'strict', arg: { String: { sval: ' x ' } } }; + trimDefElemBody(d); + expect(d.arg.String.sval).toBe(' x '); + }); +}); diff --git a/packages/transform/__tests__/transform.test.ts b/packages/transform/__tests__/transform.test.ts new file mode 100644 index 00000000..db2906aa --- /dev/null +++ b/packages/transform/__tests__/transform.test.ts @@ -0,0 +1,1218 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { loadModule } from 'plpgsql-parser'; + +import { + captureAstsFromSql, + escapeRegexp, + extractPgpmHeader, + SchemaTransformPass, + SchemaTransformResult, + shouldTransformSchema, + transformComments, + transformJsonStringValues, + transformNameList, + transformSchemaName, + transformSql, + TransformSqlOptions, + transformSqlStatement, + transformVerifyCalls, + validateNoUntransformedSchemas, + validateRoundTrip, +} from '../src'; + +// Initialize the WASM module before any tests run +beforeAll(async () => { + await loadModule(); +}); + +/** + * Helper: create a standard schema mapping for tests. + * Maps hyphenated schema names to underscored equivalents. + */ +function makeMapping(...pairs: [string, string][]): Map { + return new Map(pairs); +} + +/** Default mapping used by most tests */ +const DEFAULT_MAPPING = makeMapping( + ['my-schema', 'my_schema'], + ['other-schema', 'other_schema'] +); + +function freshResult(): SchemaTransformResult { + return { + schemasFound: new Set(), + schemasTransformed: new Map(), + errors: [], + }; +} + +// ============================================================================= +// Unit tests for helper functions +// ============================================================================= + +describe('should_transform_schema', () => { + it('returns true for schemas in the mapping', () => { + expect(shouldTransformSchema('my-schema', DEFAULT_MAPPING)).toBe(true); + }); + + it('returns false for schemas not in the mapping', () => { + expect(shouldTransformSchema('unknown', DEFAULT_MAPPING)).toBe(false); + }); + + it('returns false for undefined', () => { + expect(shouldTransformSchema(undefined, DEFAULT_MAPPING)).toBe(false); + }); +}); + +describe('transform_schema_name', () => { + it('transforms a mapped schema name', () => { + expect(transformSchemaName('my-schema', DEFAULT_MAPPING)).toBe('my_schema'); + }); + + it('returns the original for unmapped names', () => { + expect(transformSchemaName('unknown', DEFAULT_MAPPING)).toBe('unknown'); + }); + + it('returns undefined for undefined input', () => { + expect(transformSchemaName(undefined, DEFAULT_MAPPING)).toBeUndefined(); + }); +}); + +describe('transform_name_list', () => { + it('transforms the first element when it is a mapped schema', () => { + const names = [ + { String: { sval: 'my-schema' } }, + { String: { sval: 'my_func' } } + ]; + const result = freshResult(); + transformNameList(names, DEFAULT_MAPPING, result); + expect(names[0].String.sval).toBe('my_schema'); + expect(result.schemasTransformed.get('my-schema')).toBe('my_schema'); + }); + + it('does not transform single-element lists (unqualified)', () => { + const names = [{ String: { sval: 'my-schema' } }]; + const result = freshResult(); + transformNameList(names, DEFAULT_MAPPING, result); + expect(names[0].String.sval).toBe('my-schema'); + }); + + it('skips undefined or empty lists', () => { + const result = freshResult(); + transformNameList(undefined, DEFAULT_MAPPING, result); + transformNameList([], DEFAULT_MAPPING, result); + expect(result.schemasTransformed.size).toBe(0); + }); +}); + +describe('escape_regexp', () => { + it('escapes special regex characters', () => { + expect(escapeRegexp('my-schema.public')).toBe('my-schema\\.public'); + }); +}); + +// ============================================================================= +// Unit tests for regex-based transforms +// ============================================================================= + +describe('transform_comments', () => { + it('transforms schema names in Deploy/requires comment headers', () => { + const content = [ + '-- Deploy schemas/my-schema/tables/users/table', + '-- requires: schemas/my-schema/schema', + '-- requires: schemas/other-schema/tables/posts/table', + ].join('\n'); + const result = freshResult(); + const out = transformComments(content, DEFAULT_MAPPING, result); + expect(out).toContain('schemas/my_schema/tables/users/table'); + expect(out).toContain('schemas/my_schema/schema'); + expect(out).toContain('schemas/other_schema/tables/posts/table'); + }); + + it('does not transform non-header comments', () => { + const content = '-- This is a regular comment with schemas/my-schema/stuff'; + const result = freshResult(); + const out = transformComments(content, DEFAULT_MAPPING, result); + expect(out).toBe(content); + }); +}); + +describe('transform_verify_calls', () => { + it('transforms schema names inside verify_function calls', () => { + const content = "SELECT verify_function('my-schema.do_something');"; + const result = freshResult(); + const out = transformVerifyCalls(content, DEFAULT_MAPPING, result); + expect(out).toContain("verify_function('my_schema.do_something')"); + }); + + it('transforms verify_table, verify_trigger, etc.', () => { + const content = "SELECT verify_table('my-schema.users');"; + const result = freshResult(); + const out = transformVerifyCalls(content, DEFAULT_MAPPING, result); + expect(out).toContain("verify_table('my_schema.users')"); + }); + + it('transforms verify_schema with just the schema name', () => { + const content = "SELECT verify_schema('my-schema');"; + const result = freshResult(); + const out = transformVerifyCalls(content, DEFAULT_MAPPING, result); + expect(out).toContain("verify_schema('my_schema')"); + }); +}); + +describe('transform_json_string_values', () => { + it('transforms schema names in JSON value contexts', () => { + const content = `'{"authenticate_schema":"my-schema"}'`; + const result = freshResult(); + const out = transformJsonStringValues(content, DEFAULT_MAPPING, result); + expect(out).toContain(':"my_schema"'); + }); + + it('does not transform non-JSON contexts', () => { + const content = 'SELECT "my-schema".users'; + const result = freshResult(); + const out = transformJsonStringValues(content, DEFAULT_MAPPING, result); + expect(out).toBe(content); + }); +}); + +describe('extract_pgpm_header', () => { + it('separates header from SQL body', () => { + const content = [ + '-- Deploy schemas/my-schema/tables/users/table', + '-- made with <3 @ constructive.io', + '', + '', + 'CREATE TABLE "my-schema".users (id uuid);' + ].join('\n'); + const { header, body } = extractPgpmHeader(content); + expect(header).toContain('-- Deploy'); + expect(header).toContain('-- made with'); + expect(body).toContain('CREATE TABLE'); + expect(body).not.toContain('-- Deploy'); + }); + + it('lifts the psql \\echo extension guard into the header', () => { + const content = [ + '\\echo Use "CREATE EXTENSION my-module" to load this file. \\quit', + '', + 'CREATE TABLE "my-schema".users (id uuid);' + ].join('\n'); + const { header, body } = extractPgpmHeader(content); + expect(header).toContain('\\echo'); + expect(body).toContain('CREATE TABLE'); + expect(body).not.toContain('\\echo'); + }); +}); + +describe('schema references in strings and parameter types', () => { + it('transforms schema-qualified refs inside string constants', () => { + const sql = `SELECT pg_advisory_xact_lock(hashtextextended('my-schema.some_fn', 0));`; + const { sql: out } = transformSqlStatement(sql, DEFAULT_MAPPING); + expect(out).toContain(`'my_schema.some_fn'`); + expect(out).not.toContain('my-schema'); + }); + + it('transforms schema-qualified refs inside COMMENT text', () => { + const sql = `COMMENT ON TABLE "my-schema".users IS 'See my-schema.audit_log for history.';`; + const { sql: out } = transformSqlStatement(sql, DEFAULT_MAPPING); + expect(out).toContain('my_schema.audit_log'); + expect(out).not.toContain('my-schema'); + }); + + it('does not rewrite hyphenated prose words that are not mapped schemas', () => { + const sql = `COMMENT ON TABLE "my-schema".users IS 'An allow-list. When set, applies server-side.';`; + const { sql: out } = transformSqlStatement(sql, DEFAULT_MAPPING); + expect(out).toContain('allow-list. When'); + expect(out).toContain('server-side.'); + }); + + it('transforms schema-qualified composite types in function parameters', () => { + const sql = `CREATE FUNCTION f(IN obj "my-schema".my_type) RETURNS void AS $$ SELECT 1; $$ LANGUAGE sql;`; + const { sql: out } = transformSqlStatement(sql, DEFAULT_MAPPING); + expect(out).toContain('my_schema.my_type'); + expect(out).not.toContain('"my-schema".my_type'); + }); + + it('transforms schema-qualified names in DROP statements', () => { + const cases = [ + 'DROP TABLE "my-schema".users;', + 'DROP INDEX "my-schema".users_idx;', + 'DROP TRIGGER my_tg ON "my-schema".users;', + 'DROP POLICY my_policy ON "my-schema".users;', + 'DROP FUNCTION "my-schema".my_fn();', + 'DROP TYPE "my-schema".my_type;', + 'DROP SCHEMA "my-schema";' + ]; + for (const sql of cases) { + const { sql: out } = transformSqlStatement(sql, DEFAULT_MAPPING); + expect(out).toContain('my_schema'); + expect(out).not.toContain('my-schema'); + } + }); + + it('transforms schema-qualified refs inside LANGUAGE sql function bodies', () => { + const sql = [ + 'CREATE FUNCTION "my-schema".user_ids() RETURNS uuid[] AS $_PGFN_$', + 'SELECT array_agg(u.id)', + 'FROM "my-schema".users AS u', + '$_PGFN_$ LANGUAGE sql STABLE;' + ].join('\n'); + const { sql: out } = transformSqlStatement(sql, DEFAULT_MAPPING); + expect(out).toContain('"my_schema".users'); + expect(out).not.toContain('my-schema'); + }); + + it('transforms schema-qualified refs inside RAISE messages', () => { + const sql = [ + 'CREATE FUNCTION "my-schema".fail_fn() RETURNS void AS $$', + 'BEGIN', + ` RAISE EXCEPTION 'missing secrets in my-schema.secrets: %', 'x';`, + 'END;', + '$$ LANGUAGE plpgsql;' + ].join('\n'); + const { sql: out } = transformSqlStatement(sql, DEFAULT_MAPPING); + expect(out).toContain('my_schema.secrets'); + expect(out).not.toContain('my-schema'); + }); +}); + +// ============================================================================= +// Unit tests for validate_no_untransformed_schemas +// ============================================================================= + +describe('validate_no_untransformed_schemas', () => { + it('does not throw when all schemas are transformed', () => { + const content = 'SELECT * FROM "my_schema".users'; + expect(() => validateNoUntransformedSchemas(content, DEFAULT_MAPPING)).not.toThrow(); + }); + + it('throws when a schema-qualified reference remains', () => { + const content = 'SELECT * FROM "my-schema".users'; + expect(() => validateNoUntransformedSchemas(content, DEFAULT_MAPPING)).toThrow( + /AST transformation incomplete/ + ); + }); + + it('throws for standalone schema names in SQL contexts', () => { + const content = 'GRANT USAGE ON SCHEMA "my-schema" TO app_user'; + expect(() => validateNoUntransformedSchemas(content, DEFAULT_MAPPING)).toThrow( + /AST transformation incomplete/ + ); + }); + + it('does not throw for empty mapping', () => { + const content = 'SELECT * FROM "my-schema".users'; + expect(() => validateNoUntransformedSchemas(content, new Map())).not.toThrow(); + }); +}); + +// ============================================================================= +// SQL AST Visitor Tests — one test per node type +// +// Note: The deparser outputs unquoted identifiers when they are valid SQL +// identifiers (e.g., my_schema not "my_schema"). Hyphenated names require +// quoting ("my-schema") but underscored names do not. +// ============================================================================= + +describe('SQL AST visitors', () => { + // --- Already-handled nodes (existing visitors) --- + + describe('RangeVar (table references)', () => { + it('transforms schema in SELECT FROM', () => { + const { sql } = transformSqlStatement( + 'SELECT * FROM "my-schema".users;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + + it('transforms schema in INSERT INTO', () => { + const { sql } = transformSqlStatement( + 'INSERT INTO "my-schema".users (id) VALUES (1);', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + + it('transforms schema in CREATE TABLE', () => { + const { sql } = transformSqlStatement( + 'CREATE TABLE "my-schema".users (id uuid PRIMARY KEY);', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + + it('tracks the old name correctly in schemas_transformed', () => { + const { result } = transformSqlStatement( + 'SELECT * FROM "my-schema".users;', + DEFAULT_MAPPING + ); + expect(result.schemasTransformed.get('my-schema')).toBe('my_schema'); + }); + }); + + describe('CreateSchemaStmt', () => { + it('transforms CREATE SCHEMA', () => { + const { sql } = transformSqlStatement( + 'CREATE SCHEMA "my-schema";', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('FuncCall', () => { + it('transforms schema-qualified function calls', () => { + const { sql } = transformSqlStatement( + 'SELECT "my-schema".do_something();', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('TypeName', () => { + it('transforms schema-qualified type casts', () => { + const { sql } = transformSqlStatement( + 'SELECT NULL::"my-schema".my_type;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('ColumnRef', () => { + it('transforms schema-qualified column references', () => { + const { sql } = transformSqlStatement( + 'SELECT "my-schema".users.id FROM "my-schema".users;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('GrantStmt', () => { + it('transforms GRANT USAGE ON SCHEMA', () => { + const { sql } = transformSqlStatement( + 'GRANT USAGE ON SCHEMA "my-schema" TO app_user;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema'); + expect(sql).not.toContain('my-schema'); + }); + + it('transforms REVOKE ON SCHEMA', () => { + const { sql } = transformSqlStatement( + 'REVOKE ALL ON SCHEMA "my-schema" FROM PUBLIC;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('VariableSetStmt', () => { + it('transforms SET search_path', () => { + const { sql } = transformSqlStatement( + 'SET search_path TO "my-schema", public;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('AlterDefaultPrivilegesStmt', () => { + it('transforms ALTER DEFAULT PRIVILEGES IN SCHEMA', () => { + const { sql } = transformSqlStatement( + 'ALTER DEFAULT PRIVILEGES IN SCHEMA "my-schema" GRANT SELECT ON TABLES TO app_user;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('DropStmt (DROP SCHEMA)', () => { + it('transforms DROP SCHEMA', () => { + const { sql } = transformSqlStatement( + 'DROP SCHEMA IF EXISTS "my-schema";', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('CreateFunctionStmt', () => { + it('transforms schema-qualified function name in CREATE FUNCTION', () => { + const { sql } = transformSqlStatement( + `CREATE FUNCTION "my-schema".my_func() RETURNS void AS $$ BEGIN NULL; END; $$ LANGUAGE plpgsql;`, + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + // --- New node handlers --- + + describe('CreateTrigStmt', () => { + it('transforms schema-qualified trigger function name and relation', () => { + const { sql } = transformSqlStatement( + 'CREATE TRIGGER my_tg BEFORE INSERT ON "my-schema".users FOR EACH ROW EXECUTE PROCEDURE "other-schema".my_tg_func();', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).toContain('other_schema.'); + expect(sql).not.toContain('my-schema'); + expect(sql).not.toContain('other-schema'); + }); + + it('transforms EXECUTE FUNCTION variant', () => { + const { sql } = transformSqlStatement( + 'CREATE TRIGGER my_tg AFTER UPDATE ON "my-schema".posts FOR EACH ROW EXECUTE FUNCTION "my-schema".posts_update_tg();', + DEFAULT_MAPPING + ); + expect(sql).not.toContain('my-schema'); + expect(sql).toContain('my_schema.'); + }); + }); + + describe('CommentStmt', () => { + it('transforms COMMENT ON TABLE', () => { + const { sql } = transformSqlStatement( + `COMMENT ON TABLE "my-schema".users IS 'User accounts';`, + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + + it('transforms COMMENT ON COLUMN', () => { + const { sql } = transformSqlStatement( + `COMMENT ON COLUMN "my-schema".users.email IS 'User email address';`, + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('CreateFunctionStmt with body references', () => { + it('transforms schema references inside PL/pgSQL function body', () => { + const sqlIn = ` +CREATE FUNCTION "my-schema".lookup_user(id uuid) +RETURNS SETOF "my-schema".users AS $$ +BEGIN + RETURN QUERY SELECT * FROM "my-schema".users WHERE id = id; +END; +$$ LANGUAGE plpgsql STABLE;`; + const { sql } = transformSqlStatement(sqlIn, DEFAULT_MAPPING); + expect(sql).not.toContain('my-schema'); + // All references should be transformed + const occurrences = (sql.match(/my_schema\./g) || []).length; + expect(occurrences).toBeGreaterThanOrEqual(2); + }); + }); + + describe('AlterObjectSchemaStmt', () => { + it('transforms ALTER TABLE SET SCHEMA', () => { + const { sql } = transformSqlStatement( + 'ALTER TABLE "my-schema".users SET SCHEMA "other-schema";', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).toContain('other_schema'); + expect(sql).not.toContain('my-schema'); + expect(sql).not.toContain('other-schema'); + }); + }); + + // --- New node handlers (round 2) --- + + describe('ViewStmt', () => { + it('transforms schema in CREATE VIEW', () => { + const { sql } = transformSqlStatement( + 'CREATE VIEW "my-schema".active_users AS SELECT id FROM "my-schema".users;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.active_users'); + expect(sql).toContain('my_schema.users'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('AlterTableStmt', () => { + it('transforms schema in ALTER TABLE ADD COLUMN', () => { + const { sql } = transformSqlStatement( + 'ALTER TABLE "my-schema".users ADD COLUMN bio text;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('CreateSeqStmt', () => { + it('transforms schema in CREATE SEQUENCE', () => { + const { sql } = transformSqlStatement( + 'CREATE SEQUENCE "my-schema".users_id_seq;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('AlterSeqStmt', () => { + it('transforms schema in ALTER SEQUENCE', () => { + const { sql } = transformSqlStatement( + 'ALTER SEQUENCE "my-schema".users_id_seq RESTART WITH 1;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + + it('transforms schema in ALTER SEQUENCE OWNED BY', () => { + const { sql } = transformSqlStatement( + 'ALTER SEQUENCE "my-schema".users_id_seq OWNED BY "my-schema".users.id;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + // Both the sequence name and the OWNED BY target should be transformed + const occurrences = (sql.match(/my_schema\./g) || []).length; + expect(occurrences).toBeGreaterThanOrEqual(2); + }); + }); + + describe('CreatePolicyStmt', () => { + it('transforms schema in CREATE POLICY', () => { + const { sql } = transformSqlStatement( + 'CREATE POLICY sel_policy ON "my-schema".users FOR SELECT USING (true);', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('RuleStmt', () => { + it('transforms schema in CREATE RULE', () => { + const { sql } = transformSqlStatement( + 'CREATE RULE notify_insert AS ON INSERT TO "my-schema".users DO NOTHING;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('CopyStmt', () => { + it('transforms schema in COPY (AST-level)', () => { + // Note: the deparser changes STDOUT to STDIN, but the AST transformation + // correctly transforms the schema name. + const { sql } = transformSqlStatement( + 'COPY "my-schema".users TO STDOUT;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('ClusterStmt', () => { + it('transforms schema in CLUSTER', () => { + const { sql } = transformSqlStatement( + 'CLUSTER "my-schema".users USING users_pkey;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('VacuumRelation', () => { + it('transforms schema in VACUUM', () => { + const { sql } = transformSqlStatement( + 'VACUUM "my-schema".users;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + + it('transforms schema in ANALYZE', () => { + const { sql } = transformSqlStatement( + 'ANALYZE "my-schema".users;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + describe('RefreshMatViewStmt', () => { + it('transforms schema in REFRESH MATERIALIZED VIEW (AST-level)', () => { + // Note: the deparser has a bug where it drops the relation name, + // but the AST transformation correctly transforms the schema name. + const { result } = transformSqlStatement( + 'REFRESH MATERIALIZED VIEW "my-schema".user_stats;', + DEFAULT_MAPPING + ); + expect(result.schemasTransformed.get('my-schema')).toBe('my_schema'); + }); + }); + + describe('CreateTableAsStmt', () => { + it('transforms schema in CREATE MATERIALIZED VIEW', () => { + const { sql } = transformSqlStatement( + 'CREATE MATERIALIZED VIEW "my-schema".user_stats AS SELECT count(*) FROM "my-schema".users;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).not.toContain('my-schema'); + }); + }); + + // --- Multiple schemas in one statement --- + + describe('multiple schemas in one statement', () => { + it('transforms multiple different schemas', () => { + const { sql } = transformSqlStatement( + 'SELECT * FROM "my-schema".users u JOIN "other-schema".posts p ON u.id = p.user_id;', + DEFAULT_MAPPING + ); + expect(sql).toContain('my_schema.'); + expect(sql).toContain('other_schema.'); + expect(sql).not.toContain('my-schema'); + expect(sql).not.toContain('other-schema'); + }); + }); +}); + +// ============================================================================= +// Full transform_sql tests (with headers + regex passes) +// ============================================================================= + +describe('transform_sql (full pipeline)', () => { + it('transforms a complete SQL file with header and body', () => { + const input = [ + '-- Deploy schemas/my-schema/tables/users/table', + '-- made with <3 @ constructive.io', + '', + '-- requires: schemas/my-schema/schema', + '', + '', + 'CREATE TABLE "my-schema".users (id uuid PRIMARY KEY);', + ].join('\n'); + + const { content } = transformSql(input, DEFAULT_MAPPING); + expect(content).toContain('schemas/my_schema/tables/users/table'); + expect(content).toContain('schemas/my_schema/schema'); + expect(content).toContain('my_schema.'); + expect(content).not.toContain('my-schema'); + }); + + it('does NOT transform verify calls or JSON values by default', () => { + // Use a schema name that is NOT in the mapping so the validator won't fire. + // This proves that transform_sql itself no longer includes the regex passes. + const mapping = makeMapping(['other-schema', 'other_schema']); + const input = [ + '-- Deploy schemas/other-schema/tables/users/table', + '', + '', + "SELECT verify_table('my-schema.users');", + "SELECT setup('{\"target_schema\":\"my-schema\"}');", + ].join('\n'); + + const { content } = transformSql(input, mapping); + // verify calls and JSON values are opaque string literals — not transformed without a pre_pass + expect(content).toContain("verify_table('my-schema.users')"); + expect(content).toContain(':"my-schema"'); + }); + + it('transforms verify calls when passed as a pre_pass', () => { + const input = [ + '-- Verify schemas/my-schema/tables/users/table', + '', + '', + "SELECT verify_table('my-schema.users');", + ].join('\n'); + + const opts: TransformSqlOptions = { + prePasses: [transformVerifyCalls], + }; + const { content } = transformSql(input, DEFAULT_MAPPING, opts); + expect(content).toContain("verify_table('my_schema.users')"); + }); + + it('transforms JSON string values when passed as a pre_pass', () => { + const input = [ + '-- Deploy schemas/my-schema/procedures/setup', + '', + '', + `SELECT setup('{"target_schema":"my-schema"}');`, + ].join('\n'); + + const opts: TransformSqlOptions = { + prePasses: [transformJsonStringValues], + }; + const { content } = transformSql(input, DEFAULT_MAPPING, opts); + expect(content).toContain(':"my_schema"'); + expect(content).not.toContain(':"my-schema"'); + }); + + it('supports legacy signature with result as third arg', () => { + const input = [ + '-- Deploy schemas/my-schema/tables/users/table', + '', + '', + 'CREATE TABLE "my-schema".users (id uuid PRIMARY KEY);', + ].join('\n'); + + const r = freshResult(); + const { content, result } = transformSql(input, DEFAULT_MAPPING, r); + expect(content).toContain('my_schema.'); + expect(result).toBe(r); + expect(result.schemasTransformed.get('my-schema')).toBe('my_schema'); + }); + + it('runs post_passes after AST transformation', () => { + const input = [ + '-- Deploy schemas/my-schema/tables/users/table', + '', + '', + 'CREATE TABLE "my-schema".users (id uuid PRIMARY KEY);', + ].join('\n'); + + const appendComment: SchemaTransformPass = (content) => + content + '\n-- post-pass was here'; + + const opts: TransformSqlOptions = { postPasses: [appendComment] }; + const { content } = transformSql(input, DEFAULT_MAPPING, opts); + expect(content).toContain('-- post-pass was here'); + expect(content).toContain('my_schema.'); + }); + + it('chains multiple pre_passes in order', () => { + const input = [ + '-- Deploy schemas/my-schema/tables/users/table', + '', + '', + `INSERT INTO "my-schema".config (data) VALUES ('{"schema":"my-schema"}');`, + "SELECT verify_table('my-schema.config');", + ].join('\n'); + + const opts: TransformSqlOptions = { + prePasses: [transformVerifyCalls, transformJsonStringValues], + }; + const { content } = transformSql(input, DEFAULT_MAPPING, opts); + expect(content).toContain("verify_table('my_schema.config')"); + expect(content).toContain(':"my_schema"'); + expect(content).toContain('my_schema.config'); + }); + + it('returns unchanged content when mapping is empty', () => { + const input = 'SELECT * FROM "my-schema".users;'; + const { content } = transformSql(input, new Map()); + expect(content).toBe(input); + }); +}); + +// ============================================================================= +// Edge cases and regression tests +// ============================================================================= + +describe('edge cases', () => { + it('handles schemas without hyphens that are in the mapping', () => { + const mapping = makeMapping(['old_schema', 'new_schema']); + const { sql } = transformSqlStatement( + 'SELECT * FROM old_schema.users;', + mapping + ); + expect(sql).toContain('new_schema'); + expect(sql).not.toContain('old_schema'); + }); + + it('does not transform schemas that are not in the mapping', () => { + const { sql } = transformSqlStatement( + 'SELECT * FROM public.users;', + DEFAULT_MAPPING + ); + expect(sql).toContain('public'); + }); + + it('handles empty mapping gracefully', () => { + const { sql } = transformSqlStatement('SELECT 1;', new Map()); + expect(sql).toContain('SELECT'); + }); +}); + +// ============================================================================= +// Fixture-based golden tests +// +// Every .sql file in __fixtures__/input/ is auto-enumerated: transformed with +// the default mapping, asserted against the golden __fixtures__/output/, +// checked for leftover hyphenated schema refs, determinism, and full AST +// round-trip validation (AST1 === AST2, incl. PL/pgSQL bodies). +// +// To add a new fixture: drop a .sql file in __fixtures__/input/ and run +// `pnpm fixtures` to generate its golden output. +// ============================================================================= + +const FIXTURES_DIR = path.resolve(__dirname, '..', '__fixtures__'); +const FIXTURE_INPUT_DIR = path.join(FIXTURES_DIR, 'input'); +const FIXTURE_OUTPUT_DIR = path.join(FIXTURES_DIR, 'output'); +const FIXTURE_FILES = fs + .readdirSync(FIXTURE_INPUT_DIR) + .filter((f) => f.endsWith('.sql')) + .sort(); + +describe.each(FIXTURE_FILES)('fixture: %s', (file) => { + const FIXTURE_OPTS: TransformSqlOptions = { + prePasses: [transformVerifyCalls, transformJsonStringValues], + }; + const input = fs.readFileSync(path.join(FIXTURE_INPUT_DIR, file), 'utf8'); + + it('transforms the input to match the golden output', () => { + const expectedOutput = fs.readFileSync( + path.join(FIXTURE_OUTPUT_DIR, file), + 'utf8' + ); + const { content, result } = transformSql(input, DEFAULT_MAPPING, FIXTURE_OPTS); + expect(content).toBe(expectedOutput); + expect(result.errors).toEqual([]); + }); + + it('records the schema transformations it performed', () => { + const { result } = transformSql(input, DEFAULT_MAPPING, FIXTURE_OPTS); + for (const [schema, renamed] of result.schemasTransformed) { + expect(renamed).toBe(DEFAULT_MAPPING.get(schema)); + } + expect(result.schemasTransformed.size).toBeGreaterThan(0); + }); + + it('leaves no untransformed hyphenated schema names in the output', () => { + const { content } = transformSql(input, DEFAULT_MAPPING, FIXTURE_OPTS); + // Ignore comment lines, which may document the fixture purpose + const sqlOnly = content + .split('\n') + .filter((line) => !line.trim().startsWith('--')) + .join('\n'); + + expect(sqlOnly).not.toContain('"my-schema"'); + expect(sqlOnly).not.toContain('"other-schema"'); + expect(sqlOnly).not.toMatch(/\bmy-schema\b/); + expect(sqlOnly).not.toMatch(/\bother-schema\b/); + }); + + it('produces deterministic output on repeated transforms', () => { + const first = transformSql(input, DEFAULT_MAPPING, FIXTURE_OPTS); + const second = transformSql(input, DEFAULT_MAPPING, FIXTURE_OPTS); + expect(first.content).toBe(second.content); + }); + + it('passes round-trip validation (AST1 === AST2)', () => { + const { content } = transformSql(input, DEFAULT_MAPPING, { + ...FIXTURE_OPTS, + roundTrip: true, + }); + expect(content.length).toBeGreaterThan(0); + }); +}); + +describe('kitchen-sink fixture', () => { + it('finds both schemas', () => { + const input = fs.readFileSync( + path.join(FIXTURE_INPUT_DIR, 'kitchen-sink.sql'), + 'utf8' + ); + const { result } = transformSql(input, DEFAULT_MAPPING, { + prePasses: [transformVerifyCalls, transformJsonStringValues], + }); + expect(result.schemasFound).toContain('my-schema'); + expect(result.schemasFound).toContain('other-schema'); + }); +}); + +describe('plpgsql deparser round-trip regressions', () => { + it('preserves INTO after DML RETURNING inside plpgsql functions', () => { + const sql = `CREATE FUNCTION "my-schema".create_store() RETURNS uuid AS $$ +DECLARE + store_id uuid; +BEGIN + INSERT INTO "my-schema".platform_infra_store (scope_id, name) + VALUES ('028752cb-510b-1438-2f39-64534bd1cbd7'::uuid, 'infra') + RETURNING id INTO store_id; + RETURN store_id; +END; +$$ LANGUAGE plpgsql;`; + + const { sql: transformed } = transformSqlStatement(sql, DEFAULT_MAPPING); + expect(transformed).toContain('my_schema.'); + expect(transformed).toMatch(/RETURNING\s+id\s+INTO\s+store_id/); + }); + + it('does not emit an implicit trailing RETURN in trigger functions', () => { + const sql = `CREATE FUNCTION "my-schema".tg_guard() RETURNS trigger AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + RETURN NEW; + END IF; +END; +$$ LANGUAGE plpgsql;`; + + const { sql: transformed } = transformSqlStatement(sql, DEFAULT_MAPPING); + expect(transformed).toContain('my_schema.'); + expect(transformed).not.toMatch(/RETURN;/); + }); + + it('preserves explicit trailing RETURN in void functions', () => { + const sql = `CREATE FUNCTION "my-schema".notify_only() RETURNS void AS $$ +BEGIN + RAISE NOTICE 'hi'; + RETURN; +END; +$$ LANGUAGE plpgsql;`; + + const { sql: transformed } = transformSqlStatement(sql, DEFAULT_MAPPING); + expect(transformed).toMatch(/RETURN;/); + }); +}); + +describe('adversarial statement coverage', () => { + const noLeftover = (sql: string) => { + const { sql: out } = transformSqlStatement(sql, DEFAULT_MAPPING); + expect(out).not.toMatch(/(?:"my-schema"|"other-schema"|\bmy-schema(?=\.)|\bother-schema(?=\.))/); + return out; + }; + + it('transforms GRANT/REVOKE ... ON ALL ... IN SCHEMA', () => { + noLeftover(`GRANT SELECT ON ALL TABLES IN SCHEMA "my-schema" TO PUBLIC;`); + noLeftover(`GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA "my-schema" TO app_user;`); + noLeftover(`REVOKE ALL ON ALL SEQUENCES IN SCHEMA "my-schema" FROM PUBLIC;`); + }); + + it('transforms ALTER ... SET SCHEMA for types and functions', () => { + noLeftover(`ALTER TYPE "my-schema".ty SET SCHEMA "other-schema";`); + noLeftover(`ALTER FUNCTION "my-schema".fn() SET SCHEMA "other-schema";`); + noLeftover(`ALTER TABLE "my-schema".users SET SCHEMA "other-schema";`); + }); + + it('transforms ALTER SCHEMA ... RENAME TO and ALTER TABLE ... RENAME', () => { + noLeftover(`ALTER SCHEMA "my-schema" RENAME TO renamed_schema;`); + noLeftover(`ALTER TABLE "my-schema".users RENAME TO users2;`); + }); + + it('transforms COMMENT ON SCHEMA', () => { + const out = noLeftover(`COMMENT ON SCHEMA "my-schema" IS 'the my-schema.stuff schema';`); + expect(out).toContain('my_schema.stuff'); + }); + + it('transforms ALTER FUNCTION ... SET search_path', () => { + noLeftover(`ALTER FUNCTION "my-schema".f() SET search_path = "my-schema", public;`); + }); + + it('transforms DO block bodies', () => { + const out = noLeftover(`DO $$ +BEGIN + PERFORM "my-schema".fn(); + INSERT INTO "my-schema".users (id) VALUES (1); +END; +$$;`); + expect(out).toContain('my_schema'); + }); + + it('transforms CREATE CAST types and functions', () => { + noLeftover(`CREATE CAST ("my-schema".ty1 AS "my-schema".ty2) WITH FUNCTION "my-schema".cast_fn("my-schema".ty1);`); + }); + + it('transforms CREATE EVENT TRIGGER function references', () => { + noLeftover(`CREATE EVENT TRIGGER my_evt ON ddl_command_end EXECUTE FUNCTION "my-schema".evt_fn();`); + }); + + it('transforms index opclass references', () => { + noLeftover(`CREATE INDEX idx2 ON "my-schema".users USING gin (name "other-schema".my_opclass);`); + }); + + it('transforms SECURITY LABEL object references', () => { + noLeftover(`SECURITY LABEL FOR anon ON COLUMN "my-schema".users.name IS 'MASKED WITH FUNCTION anon.fake_name()';`); + }); + + it('transforms statements inside plpgsql EXCEPTION handler bodies', () => { + const out = noLeftover(`CREATE FUNCTION "my-schema".exf() RETURNS void AS $$ +BEGIN + NULL; +EXCEPTION WHEN OTHERS THEN + PERFORM "my-schema".log_err(); +END; +$$ LANGUAGE plpgsql;`); + expect(out).toContain('my_schema.log_err'); + }); + + it('preserves array bounds on schema-qualified DECLARE types', () => { + const out = noLeftover(`CREATE FUNCTION "my-schema".arr_fn() RETURNS void AS $$ +DECLARE + items "my-schema".mytype[]; + item "my-schema".mytype; + rows "my-schema".users%rowtype; +BEGIN + items := array_fill(item, ARRAY[1]); +END; +$$ LANGUAGE plpgsql;`); + expect(out).toContain('my_schema.mytype[];'); + expect(out).toContain('item my_schema.mytype;'); + expect(out).toMatch(/my_schema\.users%ROWTYPE/i); + }); + + it('transforms MERGE statements including when clauses', () => { + const out = noLeftover(`MERGE INTO "my-schema".users u USING "other-schema".t s ON u.id = s.id WHEN MATCHED THEN UPDATE SET name = s.name WHEN NOT MATCHED THEN INSERT (id) VALUES (s.id);`); + expect(out).toContain('MERGE INTO my_schema.users'); + expect(out).toContain('other_schema.t'); + }); + + it('transforms qualified aggregates and operators', () => { + noLeftover(`CREATE AGGREGATE "my-schema".agg2(int) (SFUNC = "my-schema".step_fn, STYPE = int);`); + noLeftover(`CREATE OPERATOR "my-schema".=== (LEFTARG = int, RIGHTARG = int, FUNCTION = "my-schema".op_fn);`); + }); +}); + +// ============================================================================= +// Generated PL/pgSQL construct coverage +// +// A scan of the generated corpus (application/, services/, pgpm-modules/; +// see scripts/scan-corpus.js) enumerated every PL/pgSQL statement node type we +// emit. Coverage for those constructs lives in the dedicated fixture +// (__fixtures__/input/plpgsql-constructs.sql: CASE, simple CASE, WHILE, +// bare LOOP + EXIT, FOREACH, CONTINUE, RETURN NEXT, EXCEPTION + GET STACKED +// DIAGNOSTICS), which the auto-enumerated fixture harness runs through the +// golden-output, no-leftover, determinism, and round-trip checks. +// +// This block only tracks a shape that CANNOT live in the golden fixture because +// it is silently corrupted upstream: +// +// KNOWN UPSTREAM LIMITATION (tracked, not a transform/deparser bug): +// libpg-query's PL/pgSQL parser emits `RETURN NEXT ` as a bare node with +// no `expr`/`retvarno`, so the returned expression is dropped on deparse +// (`RETURN NEXT r` -> `RETURN NEXT`). Plain `RETURN ` is unaffected. Our +// generated corpus only emits the bare OUT-parameter form (in the fixture), so +// generation is not affected. `it.failing` tracks the gap: this test will START +// failing (alerting us) if a future libpg-query build fixes the parser, at which +// point it should become a normal passing test. +// ============================================================================= + +describe('generated PL/pgSQL construct coverage', () => { + // Deparser bug fixed upstream in pgsql-parser #306 (plpgsql-deparser 0.7.13) + it('bound cursor keeps its explicit argument list', () => { + const { content } = transformSql( + `CREATE FUNCTION "my-schema".cur_args() RETURNS void AS $$ +DECLARE + c CURSOR (key int) FOR SELECT * FROM "my-schema".users WHERE id = key; + r record; +BEGIN + OPEN c(42); + FETCH c INTO r; + CLOSE c; +END; +$$ LANGUAGE plpgsql;`, + DEFAULT_MAPPING, + { roundTrip: true } + ); + expect(content).toMatch(/c CURSOR \(key int\) FOR/); + }); + + // Deparser bug fixed upstream in pgsql-parser #306 (plpgsql-deparser 0.7.13) + it('RAISE SQLSTATE keeps the SQLSTATE literal form', () => { + const { content } = transformSql( + `CREATE FUNCTION "my-schema".raise_state() RETURNS void AS $$ +BEGIN + RAISE SQLSTATE '22012'; +END; +$$ LANGUAGE plpgsql;`, + DEFAULT_MAPPING, + { roundTrip: true } + ); + expect(content).toMatch(/SQLSTATE '22012'/); + }); + + // Fixed upstream: libpg-query 18.1.2 serializes ALIAS declarations as an + // `aliases` array on PLpgSQL_function ({name, varno, lineno}) and + // plpgsql-deparser 18.1.2 re-emits `arg ALIAS FOR $1;` from that metadata. + it('ALIAS FOR $1 declarations survive the round trip', () => { + const { content } = transformSql( + `CREATE FUNCTION "my-schema".alias_fn(int) RETURNS int AS $$ +DECLARE + arg ALIAS FOR $1; +BEGIN + RETURN arg + 1; +END; +$$ LANGUAGE plpgsql;`, + DEFAULT_MAPPING, + { roundTrip: true } + ); + expect(content).toMatch(/ALIAS FOR \$1/i); + }); + + // Fixed upstream: @libpg-query/parser 17.8.0 serializes retvarno and + // plpgsql-deparser 0.8.0 emits the variable when it isn't an OUT parameter. + it('RETURN NEXT preserves the returned expression', () => { + const { content } = transformSql( + `CREATE FUNCTION my_schema.rn_expr() RETURNS SETOF int AS $$ +DECLARE + r int; +BEGIN + FOR r IN SELECT g FROM generate_series(1, 3) g LOOP + RETURN NEXT r; + END LOOP; +END; +$$ LANGUAGE plpgsql;`, + makeMapping(['other-schema', 'other_schema']) + ); + expect(content).toMatch(/RETURN NEXT r\b/); + }); +}); + +// ============================================================================= +// Round-trip validation +// ============================================================================= + +describe('round-trip validation', () => { + it('catches dropped array bounds in a DECLARE section', () => { + const good = `CREATE FUNCTION my_schema.f() RETURNS void AS $$DECLARE x my_schema.users[]; BEGIN NULL; END;$$ LANGUAGE plpgsql;`; + const bad = good.replace('users[]', 'users'); + const before = captureAstsFromSql(good); + expect(() => validateRoundTrip(before, bad)).toThrow( + /PL\/pgSQL AST mismatch/ + ); + expect(() => validateRoundTrip(before, good)).not.toThrow(); + }); + + it('catches SQL-level corruption (changed identifier)', () => { + const good = `CREATE TABLE my_schema.users (id int);`; + const bad = `CREATE TABLE my_schema.user_z (id int);`; + const before = captureAstsFromSql(good); + expect(() => validateRoundTrip(before, bad)).toThrow( + /SQL AST mismatch/ + ); + }); + + it('catches a dropped statement', () => { + const good = `CREATE TABLE my_schema.a (id int); CREATE TABLE my_schema.b (id int);`; + const bad = `CREATE TABLE my_schema.a (id int);`; + const before = captureAstsFromSql(good); + expect(() => validateRoundTrip(before, bad)).toThrow(/round-trip/); + }); +}); diff --git a/packages/transform/jest.config.js b/packages/transform/jest.config.js new file mode 100644 index 00000000..299504d2 --- /dev/null +++ b/packages/transform/jest.config.js @@ -0,0 +1,10 @@ +module.exports = { + forceExit: true, + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': 'ts-jest' + }, + testMatch: ['**/__tests__/**/*.test.ts'], + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + testTimeout: 30000 +}; diff --git a/packages/transform/package.json b/packages/transform/package.json new file mode 100644 index 00000000..9477a8a5 --- /dev/null +++ b/packages/transform/package.json @@ -0,0 +1,52 @@ +{ + "name": "@pgsql/transform", + "version": "18.2.0", + "author": "Constructive ", + "description": "AST-based SQL transformation, qualification, classification and closure analysis for PostgreSQL", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "homepage": "https://github.com/constructive-io/pgsql-parser", + "license": "MIT", + "publishConfig": { + "access": "public", + "directory": "dist" + }, + "repository": { + "type": "git", + "url": "https://github.com/constructive-io/pgsql-parser" + }, + "bugs": { + "url": "https://github.com/constructive-io/pgsql-parser/issues" + }, + "scripts": { + "copy": "makage assets", + "clean": "makage clean dist", + "prepublishOnly": "npm run build", + "build": "npm run clean && tsc && tsc -p tsconfig.esm.json && npm run copy", + "build:dev": "npm run clean && tsc --declarationMap && tsc -p tsconfig.esm.json && npm run copy", + "lint": "eslint . --fix", + "test": "jest", + "test:watch": "jest --watch", + "fixtures": "ts-node scripts/make-fixtures.ts", + "scan-corpus": "node scripts/scan-corpus.js" + }, + "dependencies": { + "@pgsql/quotes": "workspace:*", + "@pgsql/traverse": "workspace:*", + "plpgsql-parser": "workspace:*" + }, + "devDependencies": { + "makage": "^0.1.8" + }, + "keywords": [ + "sql", + "postgres", + "postgresql", + "pg", + "ast", + "transform", + "classify", + "plpgsql" + ] +} diff --git a/packages/transform/scripts/hunt-plpgsql-bugs.js b/packages/transform/scripts/hunt-plpgsql-bugs.js new file mode 100644 index 00000000..34b4de63 --- /dev/null +++ b/packages/transform/scripts/hunt-plpgsql-bugs.js @@ -0,0 +1,73 @@ +/* eslint-disable */ +// Bug-hunt sweep: run esoteric PL/pgSQL shapes through transformSql with +// roundTrip:true, and also check that key tokens survive in the output. +const { loadModule } = require('plpgsql-parser'); +const { transformSql } = require('./dist'); + +const M = new Map([['my-schema', 'my_schema'], ['other-schema', 'other_schema']]); + +const fn = (body, decl = '', ret = 'void', args = '') => + `CREATE FUNCTION "my-schema".t(${args}) RETURNS ${ret} AS $$\n${decl ? 'DECLARE\n' + decl + '\n' : ''}BEGIN\n${body}\nEND;\n$$ LANGUAGE plpgsql;`; + +const cases = [ + ['labeled block + EXIT label', `CREATE FUNCTION "my-schema".t() RETURNS int AS $$\n<>\nDECLARE i int := 0;\nBEGIN\n <>\n LOOP\n i := i + 1;\n EXIT myloop WHEN i > 3;\n END LOOP myloop;\n RETURN i;\nEND outer;\n$$ LANGUAGE plpgsql;`, ['<>', 'EXIT myloop']], + ['CONTINUE with label', `CREATE FUNCTION "my-schema".t() RETURNS int AS $$\nDECLARE i int; j int; total int := 0;\nBEGIN\n <>\n FOR i IN 1..3 LOOP\n FOR j IN 1..3 LOOP\n CONTINUE outerloop WHEN j = 2;\n total := total + 1;\n END LOOP;\n END LOOP;\n RETURN total;\nEND;\n$$ LANGUAGE plpgsql;`, ['CONTINUE outerloop']], + ['FOR REVERSE with BY', fn(`FOR i IN REVERSE 10..1 BY 2 LOOP\n PERFORM "my-schema".track(i);\nEND LOOP;`, 'i int;'), ['REVERSE', 'BY 2']], + ['FOR ... IN EXECUTE dynamic query', fn(`FOR r IN EXECUTE 'SELECT * FROM "my-schema".users' LOOP\n PERFORM "my-schema".track(r.id);\nEND LOOP;`, 'r record;'), ['EXECUTE']], + ['FOR ... IN EXECUTE ... USING', fn(`FOR r IN EXECUTE 'SELECT * FROM my_tbl WHERE id = $1' USING 42 LOOP\n PERFORM "my-schema".track(r.id);\nEND LOOP;`, 'r record;'), ['USING 42']], + ['EXECUTE INTO STRICT USING', fn(`EXECUTE 'SELECT name FROM tbl WHERE id = $1' INTO STRICT nm USING 7;\nPERFORM "my-schema".log_it(nm);`, 'nm text;'), ['INTO STRICT', 'USING 7']], + ['SELECT INTO STRICT', fn(`SELECT name INTO STRICT nm FROM "my-schema".users WHERE id = 1;`, 'nm text;'), ['STRICT']], + ['OPEN cursor FOR EXECUTE', fn(`OPEN c FOR EXECUTE 'SELECT * FROM "my-schema".users';\nCLOSE c;`, 'c refcursor;'), ['OPEN c FOR EXECUTE']], + ['bound cursor with args', `CREATE FUNCTION "my-schema".t() RETURNS void AS $$\nDECLARE\n c CURSOR (key int) FOR SELECT * FROM "my-schema".users WHERE id = key;\n r record;\nBEGIN\n OPEN c(42);\n FETCH c INTO r;\n CLOSE c;\nEND;\n$$ LANGUAGE plpgsql;`, ['CURSOR', 'OPEN c(42)', 'FETCH']], + ['WHERE CURRENT OF', `CREATE FUNCTION "my-schema".t() RETURNS void AS $$\nDECLARE\n c CURSOR FOR SELECT * FROM "my-schema".users FOR UPDATE;\nBEGIN\n OPEN c;\n MOVE c;\n UPDATE "my-schema".users SET name = 'x' WHERE CURRENT OF c;\n CLOSE c;\nEND;\n$$ LANGUAGE plpgsql;`, ['CURRENT OF']], + ['GET DIAGNOSTICS ROW_COUNT', fn(`UPDATE "my-schema".users SET name = 'x';\nGET DIAGNOSTICS n = ROW_COUNT;\nPERFORM "my-schema".track(n);`, 'n int;'), ['GET DIAGNOSTICS', 'ROW_COUNT']], + ['RAISE USING ERRCODE/DETAIL/HINT', fn(`RAISE EXCEPTION 'boom %', 1 USING ERRCODE = 'P0001', DETAIL = 'd', HINT = 'h';`), ['USING', 'ERRCODE', 'HINT']], + ['RAISE SQLSTATE form', fn(`RAISE SQLSTATE '22012';`), ['22012']], + ['ASSERT with message', fn(`ASSERT (SELECT count(*) FROM "my-schema".users) > 0, 'no users';`), ['ASSERT']], + ['ALIAS FOR $1', `CREATE FUNCTION "my-schema".t(int) RETURNS int AS $$\nDECLARE\n arg ALIAS FOR $1;\nBEGIN\n RETURN arg + 1;\nEND;\n$$ LANGUAGE plpgsql;`, ['ALIAS FOR']], + ['CONSTANT with DEFAULT', fn(`PERFORM "my-schema".track(lim);`, `lim CONSTANT int DEFAULT 10;`), ['CONSTANT']], + ['%TYPE and %ROWTYPE', fn(`SELECT id INTO uid FROM "my-schema".users LIMIT 1;\nSELECT * INTO urow FROM "my-schema".users LIMIT 1;`, `uid "my-schema".users.id%TYPE;\nurow "my-schema".users%ROWTYPE;`), ['%TYPE', '%ROWTYPE']], + ['NOT NULL variable', fn(`PERFORM "my-schema".track(x);`, `x int NOT NULL := 5;`), ['NOT NULL']], + ['RETURN QUERY', fn(`RETURN QUERY SELECT id FROM "my-schema".users;`, '', 'SETOF uuid'), ['RETURN QUERY']], + ['RETURN QUERY EXECUTE USING', fn(`RETURN QUERY EXECUTE 'SELECT id FROM "my-schema".users WHERE id = $1' USING 1;`, '', 'SETOF uuid'), ['RETURN QUERY EXECUTE', 'USING 1']], + ['CASE without ELSE (case_not_found)', fn(`CASE n WHEN 1 THEN PERFORM "my-schema".one(); END CASE;`, 'n int := 1;'), ['END CASE']], + ['nested exception + RAISE re-raise', fn(`BEGIN\n PERFORM "my-schema".might_fail();\nEXCEPTION WHEN division_by_zero OR OTHERS THEN\n RAISE;\nEND;`), ['RAISE']], + ['SQLSTATE condition in handler', fn(`BEGIN\n PERFORM "my-schema".might_fail();\nEXCEPTION WHEN SQLSTATE '22P02' THEN\n PERFORM "my-schema".log_it(SQLERRM);\nEND;`), ['22P02', 'SQLERRM']], + ['FOREACH SLICE', fn(`FOREACH sl SLICE 1 IN ARRAY arr LOOP\n PERFORM "my-schema".track(sl[1]);\nEND LOOP;`, 'arr int[][] := ARRAY[[1,2],[3,4]];\nsl int[];'), ['SLICE 1']], + ['array element assignment', fn(`arr[2] := 5;\narr[1][1] := 6;`, 'arr int[] := ARRAY[1,2,3];'), ['arr[2]']], + ['record field assignment', fn(`r.name := 'x';\nPERFORM "my-schema".log_it(r.name);`, 'r "my-schema".users%ROWTYPE;'), ['r.name']], + ['IF / ELSIF / ELSE chain', fn(`IF n = 1 THEN\n PERFORM "my-schema".one();\nELSIF n = 2 THEN\n PERFORM "my-schema".two();\nELSIF n = 3 THEN\n NULL;\nELSE\n PERFORM "my-schema".many();\nEND IF;`, 'n int := 2;'), ['ELSIF']], + ['COMMIT/ROLLBACK in procedure', `CREATE PROCEDURE "my-schema".p() AS $$\nBEGIN\n INSERT INTO "my-schema".users DEFAULT VALUES;\n COMMIT;\n INSERT INTO "my-schema".users DEFAULT VALUES;\n ROLLBACK;\nEND;\n$$ LANGUAGE plpgsql;`, ['COMMIT', 'ROLLBACK']], + ['CALL another procedure', fn(`CALL "my-schema".p(1, 'x');`), ['CALL']], + ['ON CONFLICT inside plpgsql', fn(`INSERT INTO "my-schema".users (id, name) VALUES (1, 'a')\nON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name;`), ['ON CONFLICT']], + ['CTE inside plpgsql', fn(`WITH x AS (SELECT id FROM "my-schema".users)\nSELECT count(*) INTO n FROM x;`, 'n int;'), ['WITH x AS']], + ['GET STACKED DIAGNOSTICS multi', fn(`BEGIN\n PERFORM "my-schema".might_fail();\nEXCEPTION WHEN OTHERS THEN\n GET STACKED DIAGNOSTICS msg = MESSAGE_TEXT, det = PG_EXCEPTION_DETAIL, ctx = PG_EXCEPTION_CONTEXT;\n PERFORM "my-schema".log_it(msg || det || ctx);\nEND;`, 'msg text; det text; ctx text;'), ['PG_EXCEPTION_DETAIL', 'PG_EXCEPTION_CONTEXT']], + ['EXIT out of block (not loop)', `CREATE FUNCTION "my-schema".t() RETURNS int AS $$\n<>\nBEGIN\n IF true THEN\n EXIT blk;\n END IF;\n PERFORM "my-schema".track(1);\n RETURN 1;\nEND;\n$$ LANGUAGE plpgsql;`, ['EXIT blk']], + ['RETURN QUERY with window fn', fn(`RETURN QUERY\nSELECT id, row_number() OVER (PARTITION BY name ORDER BY id) FROM "my-schema".users;`, '', 'TABLE(id uuid, rn bigint)'), ['OVER (PARTITION BY']], + ['dynamic EXECUTE with format()', fn(`EXECUTE format('SELECT count(*) FROM %I.%I', 'my_schema', 'users') INTO n;`, 'n int;'), ['format(']], + ['WHILE with complex condition', fn(`WHILE n < 10 AND NOT done LOOP\n n := n + 1;\n done := n = (SELECT count(*) FROM "my-schema".users);\nEND LOOP;`, 'n int := 0; done boolean := false;'), ['WHILE']], + ['trigger fn with TG_ vars', `CREATE FUNCTION "my-schema".tg() RETURNS trigger AS $$\nBEGIN\n IF TG_OP = 'UPDATE' AND OLD.name IS DISTINCT FROM NEW.name THEN\n NEW.name := lower(NEW.name);\n END IF;\n RETURN NEW;\nEND;\n$$ LANGUAGE plpgsql;`, ['TG_OP', 'IS DISTINCT FROM']], +]; + +async function main() { + await loadModule(); + let fails = 0; + for (const [name, sql, tokens] of cases) { + try { + const { content } = transformSql(sql, M, { roundTrip: true }); + const missing = (tokens || []).filter((t) => !content.includes(t)); + if (missing.length) { + fails++; + console.log(`TOKEN-LOSS: ${name} — missing: ${missing.join(' | ')}`); + console.log('---- output ----\n' + content + '\n----------------'); + } else { + console.log(`ok: ${name}`); + } + } catch (e) { + fails++; + console.log(`ROUND-TRIP FAIL: ${name}\n ${String(e.message).split('\n')[0]}`); + } + } + console.log(`\n${fails} failures / ${cases.length} cases`); +} +main(); diff --git a/packages/transform/scripts/make-fixtures.ts b/packages/transform/scripts/make-fixtures.ts new file mode 100644 index 00000000..d92e605d --- /dev/null +++ b/packages/transform/scripts/make-fixtures.ts @@ -0,0 +1,58 @@ +/** + * Regenerate the expected output fixtures from the input fixtures. + * + * Usage (from packages/transform): + * pnpm fixtures + * + * For each file in __fixtures__/input/, runs transform_sql with the default + * my-schema/other-schema mapping (matching __tests__/transform.test.ts) and + * writes the result to __fixtures__/output/. + * + * Review the diff of __fixtures__/output/ before committing — the output is + * the golden file the kitchen-sink test asserts against. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { loadModule } from 'plpgsql-parser'; + +import { + transformJsonStringValues, + transformSql, + TransformSqlOptions, + transformVerifyCalls, +} from '../src'; + +const FIXTURES_DIR = path.resolve(__dirname, '..', '__fixtures__'); +const INPUT_DIR = path.join(FIXTURES_DIR, 'input'); +const OUTPUT_DIR = path.join(FIXTURES_DIR, 'output'); + +const DEFAULT_MAPPING = new Map([ + ['my-schema', 'my_schema'], + ['other-schema', 'other_schema'], +]); + +async function main() { + await loadModule(); + + const opts: TransformSqlOptions = { + prePasses: [transformVerifyCalls, transformJsonStringValues], + }; + + for (const file of fs.readdirSync(INPUT_DIR)) { + if (!file.endsWith('.sql')) continue; + const input = fs.readFileSync(path.join(INPUT_DIR, file), 'utf8'); + const { content, result } = transformSql(input, DEFAULT_MAPPING, opts); + if (result.errors.length > 0) { + console.error(`Errors transforming ${file}:`, result.errors); + process.exitCode = 1; + continue; + } + fs.writeFileSync(path.join(OUTPUT_DIR, file), content); + console.log(`Wrote __fixtures__/output/${file}`); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/packages/transform/scripts/scan-corpus.js b/packages/transform/scripts/scan-corpus.js new file mode 100644 index 00000000..f4dd287a --- /dev/null +++ b/packages/transform/scripts/scan-corpus.js @@ -0,0 +1,131 @@ +/* eslint-disable */ +// Coverage audit tool: enumerate every SQL statement node type and PL/pgSQL +// node type present in a generated corpus, with an example file + frequency for +// each. Use it to confirm the transform visitor + round-trip tests +// cover every shape the codegen actually emits before it reaches production. +// +// Usage (from repo root): +// node packages/transform/scripts/scan-corpus.js \ +// application services pgpm-modules +// +// Writes a machine-readable summary to /tmp/corpus-scan.json. Files that are +// not valid standalone SQL (e.g. compiled sql/--.sql bundles that +// contain psql meta-commands) are counted under parseErrors and skipped; they +// are not transformer inputs. +const fs = require('fs'); +const path = require('path'); +const { parseSync, transformSync, loadModule } = require('plpgsql-parser'); + +const roots = process.argv.slice(2); +if (roots.length === 0) { + console.error('usage: node scan-corpus.js [dir...]'); + process.exit(1); +} + +function findSql(dir, out) { + let entries; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const e of entries) { + const p = path.join(dir, e.name); + if (e.isDirectory()) findSql(p, out); + else if (e.isFile() && e.name.endsWith('.sql')) out.push(p); + } +} + +const sqlTypes = new Map(); // nodeType -> { count, example } +const plTypes = new Map(); +let parseErrors = 0; +let hydrateErrors = 0; + +function tally(map, key, file) { + const e = map.get(key) || { count: 0, example: file }; + e.count++; + map.set(key, e); +} + +function tallyPlNode(node, file) { + if (!node || typeof node !== 'object') return; + if (Array.isArray(node)) { + for (const n of node) tallyPlNode(n, file); + return; + } + for (const key of Object.keys(node)) { + if (key.startsWith('PLpgSQL_')) tally(plTypes, key, file); + tallyPlNode(node[key], file); + } +} + +function dump(title, map) { + console.log(`\n=== ${title} (${map.size} distinct) ===`); + for (const [k, v] of [...map.entries()].sort((a, b) => b[1].count - a[1].count)) { + console.log(`${String(v.count).padStart(8)} ${k} e.g. ${v.example.replace(process.env.HOME, '~')}`); + } +} + +async function main() { + await loadModule(); + const files = []; + for (const r of roots) findSql(r, files); + files.sort(); + console.error(`scanning ${files.length} sql files...`); + + let done = 0; + for (const file of files) { + const sql = fs.readFileSync(file, 'utf-8'); + let parsed; + try { + parsed = parseSync(sql); + } catch { + parseErrors++; + continue; + } + const stmts = (parsed.sql && parsed.sql.stmts) || parsed.stmts || []; + for (const s of stmts) { + if (s.stmt) tally(sqlTypes, Object.keys(s.stmt)[0], file); + } + if (/\bfunction\b|\bprocedure\b|\bdo\b/i.test(sql)) { + try { + transformSync( + sql, + (ctx) => { + for (const fn of ctx.functions || []) { + if (fn.plpgsql && fn.plpgsql.hydrated) tallyPlNode(fn.plpgsql.hydrated, file); + } + }, + { hydrate: true } + ); + } catch { + hydrateErrors++; + } + } + if (++done % 5000 === 0) console.error(` ${done}/${files.length}`); + } + + dump('SQL statement node types', sqlTypes); + dump('PL/pgSQL node types', plTypes); + console.log(`\nparseErrors=${parseErrors} hydrateErrors=${hydrateErrors} files=${files.length}`); + + fs.writeFileSync( + '/tmp/corpus-scan.json', + JSON.stringify( + { + sqlTypes: Object.fromEntries(sqlTypes), + plTypes: Object.fromEntries(plTypes), + parseErrors, + hydrateErrors, + files: files.length, + }, + null, + 2 + ) + ); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/transform/src/categorize.ts b/packages/transform/src/categorize.ts new file mode 100644 index 00000000..e2ddc810 --- /dev/null +++ b/packages/transform/src/categorize.ts @@ -0,0 +1,81 @@ +import { classifyStatements, StatementFacts } from './facts'; + +/** + * A category is a chunk-seam key a change is assigned to. Free-form string; the + * built-in {@link TIER_PROFILE} uses `'security' | 'functionality' | 'fixtures' + * | 'schema'`. + * + * This module is the classifier-driven counterpart to `@pgpmjs/core`'s + * `boundary: 'category'` rebundle mode: core stays classifier-agnostic and only + * consumes a `categoryOf(changeName)` seam function, while the rules that decide + * *what* each change is (the profile) live here, next to the AST facts. + */ +export type ChangeCategory = string; + +/** + * Decides a change's category from the AST facts of its (deploy) statements. + * A profile is a pure function of the facts — swap it to carve a monolith along + * different seams (e.g. per-role admin/users modules) without touching core. + */ +export interface CategoryProfile { + /** Identifier for the profile (for logging / provenance). */ + name: string; + /** Map a change's statement facts to its category. */ + categorize: (facts: StatementFacts[], changeName: string) => ChangeCategory; +} + +/** + * Default tier profile: isolates the security surface and procedural code out + * of the schema base. Precedence, highest first — a change is placed in the + * most specialized tier any of its statements reaches: + * + * security any securityRelevant statement (policy/grant/RLS/role/owner) + * functionality functions and triggers + * fixtures seed DML (INSERT/UPDATE/DELETE) + * schema schemas, tables, views, indexes, types, constraints (base) + * + * Ordering/dependencies across the resulting chunks are still resolved by + * core's rebundle graph, so this only decides *naming/grouping*, never deploy + * order. + */ +export const TIER_PROFILE: CategoryProfile = { + name: 'tier', + categorize(facts): ChangeCategory { + if (facts.length === 0) return 'schema'; + if (facts.some(f => f.securityRelevant)) return 'security'; + if (facts.some(f => f.kind === 'function' || f.kind === 'trigger')) return 'functionality'; + if (facts.some(f => f.kind === 'seed_dml')) return 'fixtures'; + return 'schema'; + }, +}; + +/** + * Classify a single change's deploy SQL into its category under `profile` + * (defaults to {@link TIER_PROFILE}). + */ +export function categorizeChange( + sql: string, + changeName: string, + profile: CategoryProfile = TIER_PROFILE +): ChangeCategory { + return profile.categorize(classifyStatements(sql), changeName); +} + +/** + * Build a `categoryOf(changeName)` seam function from a map of change name to + * its deploy SQL — ready to pass straight to `@pgpmjs/core`'s rebundle + * `boundary: 'category'` / `categoryOf` options. + * + * Changes absent from `changeSql` return `undefined`, letting core fall back to + * its folder key for anything the classifier did not see. + */ +export function buildCategoryOf( + changeSql: Record, + profile: CategoryProfile = TIER_PROFILE +): (changeName: string) => ChangeCategory | undefined { + const categories = new Map(); + for (const [name, sql] of Object.entries(changeSql)) { + categories.set(name, categorizeChange(sql, name, profile)); + } + return (changeName: string): ChangeCategory | undefined => categories.get(changeName); +} diff --git a/packages/transform/src/facts.ts b/packages/transform/src/facts.ts new file mode 100644 index 00000000..ab787d2e --- /dev/null +++ b/packages/transform/src/facts.ts @@ -0,0 +1,378 @@ +import { walk as walkSql } from '@pgsql/traverse'; +import { transformSync, walk as walkPlpgsql } from 'plpgsql-parser'; + +/** + * A (possibly schema-qualified) object name extracted from a statement. + */ +export interface QualifiedName { + schema: string | null; + name: string; +} + +/** + * Coarse statement classification used for tier/package sorting. + */ +export type StatementKind = + | 'schema' + | 'table' + | 'view' + | 'index' + | 'type' + | 'function' + | 'trigger' + | 'policy' + | 'grant' + | 'rls_enable' + | 'fk_constraint' + | 'constraint' + | 'comment' + | 'seed_dml' + | 'other'; + +/** + * AST-derived facts about a single top-level SQL statement. + * + * Read-only: classification never modifies the statement. Facts are the + * substrate for classifier-driven slicing (schema / functionality / security + * tiers) — replacing path/name-glob decisions with computed properties like + * "this trigger function references billing". + */ +export interface StatementFacts { + /** Coarse category of the statement. */ + kind: StatementKind; + /** The raw parser node tag (e.g. `CreateStmt`, `CreatePolicyStmt`). */ + nodeTag: string; + /** Objects this statement creates or directly targets. */ + creates: QualifiedName[]; + /** + * Schema-qualified objects this statement references — tables, functions + * and types reached anywhere in the statement, including PL/pgSQL bodies. + * Unqualified references are omitted (they resolve via search_path and + * carry no cross-schema information). + */ + references: QualifiedName[]; + /** + * The subset of `references` reached only inside a PL/pgSQL body. These + * are late-binding: Postgres resolves them at call time, not at CREATE + * time, so they do not constrain deploy order (and legitimately form + * recursion cycles between functions). + */ + bodyReferences: QualifiedName[]; + /** Distinct schemas reached by `references`. */ + referencedSchemas: string[]; + /** Role names granted to, owning, or bound by this statement. */ + roles: string[]; + /** Foreign-key target tables (from column/table FK constraints). */ + fkTargets: QualifiedName[]; + /** + * Whether the statement is part of the security surface: policies, grants, + * RLS enable/force, security labels, ownership. + */ + securityRelevant: boolean; + /** For functions: declared with SECURITY DEFINER. */ + securityDefiner: boolean; + /** + * For functions: the body executes dynamic SQL (EXECUTE / EXECUTE ... USING + * / FOR ... IN EXECUTE). Analogous to `eval` — references inside the + * dynamic string are invisible to the AST, so edges from this statement + * are incomplete and slicing should treat it conservatively. + */ + dynamicSql: boolean; +} + +const SECURITY_TAGS = new Set([ + 'CreatePolicyStmt', + 'AlterPolicyStmt', + 'GrantStmt', + 'GrantRoleStmt', + 'AlterDefaultPrivilegesStmt', + 'SecLabelStmt', + 'AlterOwnerStmt', + 'CreateRoleStmt', + 'AlterRoleStmt' +]); + +const KIND_BY_TAG: Record = { + CreateSchemaStmt: 'schema', + CreateStmt: 'table', + ViewStmt: 'view', + IndexStmt: 'index', + CompositeTypeStmt: 'type', + CreateEnumStmt: 'type', + CreateDomainStmt: 'type', + CreateRangeStmt: 'type', + DefineStmt: 'type', + CreateFunctionStmt: 'function', + CreateTrigStmt: 'trigger', + CreateEventTrigStmt: 'trigger', + CreatePolicyStmt: 'policy', + AlterPolicyStmt: 'policy', + GrantStmt: 'grant', + GrantRoleStmt: 'grant', + AlterDefaultPrivilegesStmt: 'grant', + CommentStmt: 'comment', + InsertStmt: 'seed_dml', + UpdateStmt: 'seed_dml', + DeleteStmt: 'seed_dml' +}; + +function qn(schema: string | null | undefined, name: string): QualifiedName { + return { schema: schema ?? null, name }; +} + +function nameListToQualified(names: any[] | undefined): QualifiedName | null { + if (!Array.isArray(names) || names.length === 0) return null; + const parts = names + .map((n: any) => n?.String?.sval) + .filter((s: any) => typeof s === 'string'); + if (parts.length === 0) return null; + if (parts.length === 1) return qn(null, parts[0]); + return qn(parts[parts.length - 2], parts[parts.length - 1]); +} + +function isCatalogSchema(schema: string | null): boolean { + return schema === 'pg_catalog' || schema === 'information_schema'; +} + +function pushRef(refs: QualifiedName[], ref: QualifiedName | null): void { + if (!ref || !ref.schema || isCatalogSchema(ref.schema)) return; + if (refs.some(r => r.schema === ref.schema && r.name === ref.name)) return; + refs.push(ref); +} + +function collectRoles(node: any, roles: string[]): void { + const push = (role: string | undefined) => { + if (typeof role === 'string' && role.length > 0 && !roles.includes(role)) { + roles.push(role); + } + }; + if (Array.isArray(node?.grantees)) { + for (const g of node.grantees) push(g?.RoleSpec?.rolename); + } + if (Array.isArray(node?.roles)) { + for (const r of node.roles) push(r?.RoleSpec?.rolename); + } + push(node?.newowner?.rolename); + push(node?.role?.rolename); +} + +/** + * Create a read-only visitor that accumulates references, roles and FK + * targets into the provided facts object. + */ +function createFactsVisitor(facts: StatementFacts, bodyRefs?: QualifiedName[]) { + const push = (ref: QualifiedName | null) => { + pushRef(facts.references, ref); + if (bodyRefs) pushRef(bodyRefs, ref); + }; + return { + RangeVar: (path: any) => { + const node = path.node; + if (node.schemaname) { + push(qn(node.schemaname, node.relname)); + } + }, + FuncCall: (path: any) => { + push(nameListToQualified(path.node.funcname)); + }, + TypeName: (path: any) => { + push(nameListToQualified(path.node.names)); + }, + ColumnRef: (path: any) => { + // schema.table.column references carry cross-schema information + const fields = path.node.fields; + if (Array.isArray(fields) && fields.length >= 3) { + const parts = fields + .map((f: any) => f?.String?.sval) + .filter((s: any) => typeof s === 'string'); + if (parts.length >= 3) { + push(qn(parts[0], parts[1])); + } + } + }, + Constraint: (path: any) => { + const node = path.node; + if (node.contype === 'CONSTR_FOREIGN' && node.pktable) { + const target = qn(node.pktable.schemaname ?? null, node.pktable.relname); + if (!facts.fkTargets.some(t => t.schema === target.schema && t.name === target.name)) { + facts.fkTargets.push(target); + } + if (target.schema) pushRef(facts.references, target); + } + } + }; +} + +function classifyOne(nodeTag: string, node: any): StatementFacts { + const facts: StatementFacts = { + kind: KIND_BY_TAG[nodeTag] ?? 'other', + nodeTag, + creates: [], + references: [], + referencedSchemas: [], + roles: [], + fkTargets: [], + bodyReferences: [], + securityRelevant: SECURITY_TAGS.has(nodeTag), + securityDefiner: false, + dynamicSql: false + }; + + switch (nodeTag) { + case 'CreateSchemaStmt': + facts.creates.push(qn(null, node.schemaname)); + break; + case 'CreateStmt': + case 'ViewStmt': { + const rel = nodeTag === 'ViewStmt' ? node.view : node.relation; + if (rel) facts.creates.push(qn(rel.schemaname ?? null, rel.relname)); + break; + } + case 'IndexStmt': + if (node.relation) { + facts.creates.push(qn(node.relation.schemaname ?? null, node.idxname ?? node.relation.relname)); + } + break; + case 'CreateFunctionStmt': { + const name = nameListToQualified(node.funcname); + if (name) facts.creates.push(name); + for (const opt of node.options ?? []) { + const def = opt?.DefElem; + if (def?.defname === 'security' && def?.arg?.Boolean?.boolval === true) { + facts.securityDefiner = true; + } + } + break; + } + case 'CreateTrigStmt': + if (node.relation) { + // Trigger names are only unique per table; qualify with the table. + facts.creates.push( + qn(node.relation.schemaname ?? null, `${node.relation.relname}.${node.trigname}`) + ); + } + pushRef(facts.references, nameListToQualified(node.funcname)); + break; + case 'CreatePolicyStmt': + case 'AlterPolicyStmt': + if (node.table) { + // Policy names are only unique per table; qualify with the table. + facts.creates.push( + qn(node.table.schemaname ?? null, `${node.table.relname}.${node.policy_name}`) + ); + // The guarded table lives on `node.table`; the generic RangeVar walker + // does not descend into it, so capture the reference explicitly. + if (node.table.schemaname) { + pushRef(facts.references, qn(node.table.schemaname, node.table.relname)); + } + } + break; + case 'CreateSeqStmt': + if (node.sequence) { + facts.creates.push(qn(node.sequence.schemaname ?? null, node.sequence.relname)); + } + break; + case 'CompositeTypeStmt': + if (node.typevar) { + facts.creates.push(qn(node.typevar.schemaname ?? null, node.typevar.relname)); + } + break; + case 'CreateEnumStmt': + case 'CreateDomainStmt': + case 'CreateRangeStmt': { + const name = nameListToQualified(node.typeName ?? node.domainname); + if (name) facts.creates.push(name); + break; + } + case 'AlterTableStmt': { + if (node.relation) { + facts.creates.push(qn(node.relation.schemaname ?? null, node.relation.relname)); + } + const cmds: any[] = node.cmds ?? []; + for (const cmd of cmds) { + const subtype = cmd?.AlterTableCmd?.subtype; + if (subtype === 'AT_EnableRowSecurity' || subtype === 'AT_ForceRowSecurity' || + subtype === 'AT_DisableRowSecurity' || subtype === 'AT_NoForceRowSecurity') { + facts.kind = 'rls_enable'; + facts.securityRelevant = true; + } else if (subtype === 'AT_AddConstraint') { + const contype = cmd?.AlterTableCmd?.def?.Constraint?.contype; + facts.kind = contype === 'CONSTR_FOREIGN' ? 'fk_constraint' : 'constraint'; + } else if (subtype === 'AT_ChangeOwner') { + facts.securityRelevant = true; + const owner = cmd?.AlterTableCmd?.newowner?.rolename; + if (owner && !facts.roles.includes(owner)) facts.roles.push(owner); + } + } + break; + } + case 'InsertStmt': + case 'UpdateStmt': + case 'DeleteStmt': + if (node.relation) { + facts.creates.push(qn(node.relation.schemaname ?? null, node.relation.relname)); + } + break; + default: + break; + } + + collectRoles(node, facts.roles); + return facts; +} + +/** + * Classify each top-level statement in a SQL script into {@link StatementFacts}. + * + * Uses the same parser stack as the schema transformer (pgsql AST walk plus + * hydrated PL/pgSQL body walk), so references inside function bodies are + * included. The input SQL is never modified. + */ +export function classifyStatements(sql: string): StatementFacts[] { + const allFacts: StatementFacts[] = []; + + transformSync(sql, (ctx) => { + const stmts: any[] = ctx.sql?.stmts ?? []; + for (const stmt of stmts) { + const stmtNode = stmt?.stmt; + const nodeTag = stmtNode ? Object.keys(stmtNode)[0] : 'other'; + const node = stmtNode?.[nodeTag] ?? {}; + const facts = classifyOne(nodeTag, node); + + if (stmtNode) { + walkSql(stmtNode, createFactsVisitor(facts)); + } + allFacts.push(facts); + } + + for (const fn of ctx.functions ?? []) { + const facts = allFacts[fn.stmtIndex]; + if (!facts || !fn.plpgsql?.hydrated) continue; + // Refs already found outside the body (signature types, defaults) + // constrain deploy order and are not body-only. + const outer = new Set( + facts.references.map(r => `${r.schema ?? '?'}.${r.name}`) + ); + walkPlpgsql(fn.plpgsql.hydrated, { + PLpgSQL_stmt_dynexecute: () => { facts.dynamicSql = true; }, + PLpgSQL_stmt_dynfors: () => { facts.dynamicSql = true; } + }, { + walkSqlExpressions: true, + sqlVisitor: createFactsVisitor(facts, facts.bodyReferences) + }); + facts.bodyReferences = facts.bodyReferences.filter( + r => !outer.has(`${r.schema ?? '?'}.${r.name}`) + ); + } + }, { hydrate: true }); + + for (const facts of allFacts) { + const selfRef = (r: QualifiedName) => + facts.creates.some(c => c.schema === r.schema && c.name === r.name); + facts.references = facts.references.filter(r => !selfRef(r)); + facts.bodyReferences = facts.bodyReferences.filter(r => !selfRef(r)); + facts.referencedSchemas = [...new Set(facts.references.map(r => r.schema).filter(Boolean))] as string[]; + } + + return allFacts; +} diff --git a/packages/transform/src/fixture-closure.ts b/packages/transform/src/fixture-closure.ts new file mode 100644 index 00000000..d3679544 --- /dev/null +++ b/packages/transform/src/fixture-closure.ts @@ -0,0 +1,286 @@ +import { + CategoryProfile, + ChangeCategory, + TIER_PROFILE +} from './categorize'; +import { classifyStatements, QualifiedName, StatementFacts } from './facts'; + +/** + * A change plus its deploy SQL. Optional plan `dependencies` (from `pgpm.plan`) + * are followed as authoritative prerequisites in addition to AST-derived edges. + */ +export interface ClosureInputChange { + name: string; + sql: string; + dependencies?: string[]; +} + +/** Why a change ended up in the closure. */ +export type ClosureReason = 'selected' | 'prerequisite' | 'fixture'; + +/** + * A change included in the closure, with its category and the reason it was pulled. + */ +export interface ClosureChange { + name: string; + category: ChangeCategory; + reason: ClosureReason; +} + +/** + * The fixture closure of a schema slice. + * + * Everything the selected changes need to actually *work*: schema/functionality + * prerequisites they depend on, plus the fixtures that attach to them (policies, + * grants, RLS enables, role bindings, seed rows / SPRT-style support changes). + * References that nothing in the input produces are surfaced in `unresolved` + * rather than silently dropped. + */ +export interface FixtureClosure { + /** Full closure (selection + prerequisites + fixtures) in input/plan order. */ + order: string[]; + /** Per-change detail in the same order as {@link order}. */ + changes: ClosureChange[]; + /** Closure members categorized as security/fixtures (attached fixtures). */ + fixtures: string[]; + /** Non-selected schema/functionality members pulled in as prerequisites. */ + prerequisites: string[]; + /** Roles required across the closure. */ + roles: string[]; + /** Requirements no input change produces — explicit, never dropped. */ + unresolved: { + objects: QualifiedName[]; + schemas: string[]; + roles: string[]; + }; +} + +export interface ResolveFixtureClosureOptions { + /** Category profile used to label changes (default {@link TIER_PROFILE}). */ + profile?: CategoryProfile; + /** + * When false, do not follow AST references from selected changes to their + * schema/functionality producers (only pull attached fixtures + plan deps). + * Default true. + */ + includePrerequisites?: boolean; +} + +function objectKey(schema: string | null, name: string): string { + return `${schema ?? ''}\u0000${name}`; +} + +interface ChangeAnalysis { + name: string; + category: ChangeCategory; + facts: StatementFacts[]; + /** Declared plan dependencies (authoritative prerequisites). */ + dependencies: string[]; + /** Objects this change creates (for producer indexing). */ + creates: QualifiedName[]; + /** Schemas this change creates. */ + createsSchemas: string[]; + /** Roles this change creates (CreateRoleStmt). */ + createsRoles: string[]; + /** Schema-qualified objects this change requires. */ + needsObjects: QualifiedName[]; + /** Schemas this change requires. */ + needsSchemas: string[]; + /** Roles this change requires (excludes roles it creates itself). */ + needsRoles: string[]; + /** True when categorized as an attached fixture (security/fixtures). */ + isFixture: boolean; +} + +function analyze(change: ClosureInputChange, profile: CategoryProfile): ChangeAnalysis { + const facts = classifyStatements(change.sql); + const category = profile.categorize(facts, change.name); + + const creates: QualifiedName[] = []; + const createsSchemas: string[] = []; + const createsRoles: string[] = []; + const needsObjects: QualifiedName[] = []; + const needsSchemas = new Set(); + const needsRoles = new Set(); + + for (const f of facts) { + for (const c of f.creates) { + if (f.kind === 'schema') createsSchemas.push(c.name); + else { + creates.push(c); + if (c.schema) needsSchemas.add(c.schema); + } + } + if (f.nodeTag === 'CreateRoleStmt') createsRoles.push(...f.roles); + + for (const r of [...f.references, ...f.fkTargets]) { + if (r.schema) needsObjects.push(r); + } + for (const s of f.referencedSchemas) needsSchemas.add(s); + } + + const created = new Set(createsRoles); + for (const f of facts) { + if (f.nodeTag === 'CreateRoleStmt') continue; + for (const role of f.roles) if (!created.has(role)) needsRoles.add(role); + } + for (const s of createsSchemas) needsSchemas.delete(s); + + return { + name: change.name, + category, + facts, + dependencies: change.dependencies ?? [], + creates, + createsSchemas, + createsRoles, + needsObjects, + needsSchemas: [...needsSchemas], + needsRoles: [...needsRoles], + isFixture: category === 'security' || category === 'fixtures' + }; +} + +/** + * Resolve the fixture closure of a schema slice. + * + * Pure and deterministic — no I/O — and runs to a fixpoint in both directions: + * - forward: a selected change pulls in the producers of the objects/schemas/ + * roles it references, plus its declared plan dependencies (prerequisites); + * - reverse: any fixture (security/fixtures-tier change) that attaches to an + * object already in the closure is pulled in (attached fixtures), and then + * resolved for its own prerequisites. + * + * Output order follows the input (plan) order, so it stays deploy-safe. + */ +export function resolveFixtureClosure( + allChanges: ClosureInputChange[], + selection: string[], + options: ResolveFixtureClosureOptions = {} +): FixtureClosure { + const profile = options.profile ?? TIER_PROFILE; + const includePrerequisites = options.includePrerequisites ?? true; + + const analyses = allChanges.map(c => analyze(c, profile)); + const byName = new Map(analyses.map(a => [a.name, a])); + + const producerByObject = new Map(); + const producerBySchema = new Map(); + const producerByRole = new Map(); + for (const a of analyses) { + for (const c of a.creates) { + const key = objectKey(c.schema, c.name); + if (!producerByObject.has(key)) producerByObject.set(key, a.name); + } + for (const s of a.createsSchemas) { + if (!producerBySchema.has(s)) producerBySchema.set(s, a.name); + } + for (const r of a.createsRoles) { + if (!producerByRole.has(r)) producerByRole.set(r, a.name); + } + } + + const reason = new Map(); + for (const name of selection) { + if (byName.has(name)) reason.set(name, 'selected'); + } + + const producedObjectKeys = new Set(); + const producedSchemas = new Set(); + const refreshProduced = (): void => { + producedObjectKeys.clear(); + producedSchemas.clear(); + for (const name of reason.keys()) { + const a = byName.get(name)!; + for (const c of a.creates) producedObjectKeys.add(objectKey(c.schema, c.name)); + for (const s of a.createsSchemas) producedSchemas.add(s); + } + }; + + let changed = true; + while (changed) { + changed = false; + + // forward: pull producers of what current members need + for (const name of [...reason.keys()]) { + const a = byName.get(name)!; + const pull = (producer: string | undefined, r: ClosureReason): void => { + if (!producer || reason.has(producer)) return; + const pa = byName.get(producer)!; + reason.set(producer, pa.isFixture ? 'fixture' : r); + changed = true; + }; + for (const dep of a.dependencies) { + pull(byName.has(dep) ? dep : undefined, 'prerequisite'); + } + if (!includePrerequisites) continue; + for (const o of a.needsObjects) pull(producerByObject.get(objectKey(o.schema, o.name)), 'prerequisite'); + for (const s of a.needsSchemas) pull(producerBySchema.get(s), 'prerequisite'); + for (const role of a.needsRoles) pull(producerByRole.get(role), 'prerequisite'); + } + + // reverse: pull attached fixtures that target current members + refreshProduced(); + for (const a of analyses) { + if (reason.has(a.name) || !a.isFixture) continue; + const objectAttaches = + a.needsObjects.some(o => producedObjectKeys.has(objectKey(o.schema, o.name))) || + a.creates.some(c => producedObjectKeys.has(objectKey(c.schema, c.name))); + // Schema-scoped fixtures (e.g. ALTER DEFAULT PRIVILEGES IN SCHEMA x) name no + // specific object; attach them by schema. Object-scoped fixtures attach only + // via their object refs, so a shared schema does not drag in every fixture. + const schemaScoped = a.needsObjects.length === 0 && a.creates.length === 0; + const schemaAttaches = + schemaScoped && + (a.needsSchemas.some(s => producedSchemas.has(s)) || + a.createsSchemas.some(s => producedSchemas.has(s))); + if (objectAttaches || schemaAttaches) { + reason.set(a.name, 'fixture'); + changed = true; + } + } + } + + const order = analyses.filter(a => reason.has(a.name)).map(a => a.name); + const changes: ClosureChange[] = order.map(name => { + const a = byName.get(name)!; + return { name, category: a.category, reason: reason.get(name)! }; + }); + + const roles = new Set(); + const unresolvedObjects: QualifiedName[] = []; + const unresolvedObjectKeys = new Set(); + const unresolvedSchemas = new Set(); + const unresolvedRoles = new Set(); + + for (const name of order) { + const a = byName.get(name)!; + for (const role of a.needsRoles) { + roles.add(role); + if (!producerByRole.has(role)) unresolvedRoles.add(role); + } + for (const o of a.needsObjects) { + const key = objectKey(o.schema, o.name); + if (!producerByObject.has(key) && !unresolvedObjectKeys.has(key)) { + unresolvedObjectKeys.add(key); + unresolvedObjects.push(o); + } + } + for (const s of a.needsSchemas) { + if (!producerBySchema.has(s)) unresolvedSchemas.add(s); + } + } + + return { + order, + changes, + fixtures: changes.filter(c => c.reason === 'fixture').map(c => c.name), + prerequisites: changes.filter(c => c.reason === 'prerequisite').map(c => c.name), + roles: [...roles], + unresolved: { + objects: unresolvedObjects, + schemas: [...unresolvedSchemas], + roles: [...unresolvedRoles] + } + }; +} diff --git a/packages/transform/src/index.ts b/packages/transform/src/index.ts new file mode 100644 index 00000000..cd447917 --- /dev/null +++ b/packages/transform/src/index.ts @@ -0,0 +1,75 @@ +export type { + CategoryProfile, + ChangeCategory, +} from './categorize'; +export { + buildCategoryOf, + categorizeChange, + TIER_PROFILE, +} from './categorize'; +export type { + QualifiedName, + StatementFacts, + StatementKind, +} from './facts'; +export { classifyStatements } from './facts'; +export type { + ClosureChange, + ClosureInputChange, + ClosureReason, + FixtureClosure, + ResolveFixtureClosureOptions, +} from './fixture-closure'; +export { resolveFixtureClosure } from './fixture-closure'; +export type { + ObjectInventory, + QualifyResult, + QualifyRoutes, + QualifyTargetSelector, + QualifyUnqualifiedOptions, +} from './qualify'; +export { + collectCreatedObjects, + createQualifyVisitor, + mergeInventories, + qualifyUnqualified, +} from './qualify'; +export type { CapturedAsts } from './round-trip'; +export { + captureAstsFromSql, + captureTransformAsts, + firstDifference, + normalizeParseTree, + validateRoundTrip, +} from './round-trip'; +export type { NormalizeTreeOptions } from './round-trip-core'; +export { + CLEAN_TREE_VOLATILE_KEYS, + cleanTree, + normalizeTree, + trimDefElemBody, +} from './round-trip-core'; +export type { + SchemaTransformPass, + SchemaTransformResult, + TransformSqlOptions, +} from './transform'; +export { + createPlpgsqlVisitor, + createSqlVisitor, + escapeRegexp, + extractPgpmHeader, + shouldTransformSchema, + transformComments, + transformJsonStringValues, + transformNameList, + transformPlpgsqlTypeAst, + transformPlpgsqlTypeString, + transformRelation, + transformSchemaName, + transformSql, + transformSqlStatement, + transformVerifyCalls, + validateNoUntransformedSchemas, + walkPlpgsqlForSchemas, +} from './transform'; diff --git a/packages/transform/src/qualify.ts b/packages/transform/src/qualify.ts new file mode 100644 index 00000000..1f7c5e50 --- /dev/null +++ b/packages/transform/src/qualify.ts @@ -0,0 +1,405 @@ +/** + * Opt-in qualification of unqualified object references. + * + * Handwritten SQL is typically unqualified — objects implicitly land in (and + * resolve against) the `public` schema. The core schema transform only + * rewrites *explicitly* qualified references, so ingesting such code into + * named schemas needs a prior pass that pins every unqualified reference to a + * schema first. This module provides that pass. + * + * Safety model: only routed names are qualified — either everything in an + * {@link ObjectInventory} to a single schema, or per-object routes via + * `targets`. Builtin functions (`now()`, `count(...)`), builtin types, and + * column references are never touched because they are not routed. Names + * defined by a CTE in the same statement are excluded even when they collide + * with a routed relation. + */ + +import { walk as walkSql } from '@pgsql/traverse'; +import { Deparser, parseSql, transformSync, walk as walkPlpgsql } from 'plpgsql-parser'; + +import { classifyStatements } from './facts'; + +/** Named objects eligible for qualification, bucketed by namespace. */ +export interface ObjectInventory { + /** Tables, views, sequences — anything a RangeVar can reference. */ + relations: Set; + /** Function and procedure names. */ + functions: Set; + /** Enum/domain/range/composite type names. */ + types: Set; +} + +/** The objects a target schema should receive, by namespace. */ +export interface QualifyTargetSelector { + relations?: Iterable; + functions?: Iterable; + types?: Iterable; +} + +export interface QualifyUnqualifiedOptions { + /** + * Single-target form: schema to pin unqualified references to (e.g. + * `'public'`). Mutually exclusive with `targets`. + */ + schema?: string; + /** + * Names eligible for qualification (single-target form only). Defaults to + * the objects created by the content itself ({@link collectCreatedObjects}). + * When qualifying scripts that reference objects created elsewhere (e.g. + * the scripts of a whole module), collect the inventory across all scripts + * and pass it here. + */ + inventory?: ObjectInventory; + /** + * Multi-target form: route specific objects to specific schemas in one + * pass. Mutually exclusive with `schema`/`inventory`. + * + * ```ts + * qualifyUnqualified(sql, { + * targets: { + * auth: { relations: ['users'] }, + * shop: { relations: ['products'], functions: ['get_products'] }, + * shared: { types: ['widget'] } + * } + * }); + * ``` + * + * Routing a name to two schemas in the same namespace is a conflict and + * throws. + */ + targets?: Record; + /** + * Prepend `CREATE SCHEMA IF NOT EXISTS ;` for every target schema + * the content does not already create. Useful when ingesting standalone + * handwritten SQL whose target schemas may not exist. + */ + injectCreateSchema?: boolean; +} + +export interface QualifyResult { + /** Count of references qualified, keyed by object name. */ + qualified: Map; + /** Schema each qualified object was routed to. */ + routed: Map; +} + +/** Resolved name → schema routing, by namespace. */ +export interface QualifyRoutes { + relations: Map; + functions: Map; + types: Map; +} + +const RELATION_TAGS = new Set([ + 'CreateStmt', + 'ViewStmt', + 'CreateSeqStmt', + 'CreateTableAsStmt' +]); +const TYPE_TAGS = new Set([ + 'CreateEnumStmt', + 'CreateDomainStmt', + 'CreateRangeStmt', + 'CompositeTypeStmt' +]); + +/** + * Collect the objects a SQL script creates, bucketed for qualification. + * Only unqualified (schema-less) creations are collected — an explicitly + * qualified creation already declares where it lives. + */ +export function collectCreatedObjects(sql: string): ObjectInventory { + const inventory: ObjectInventory = { + relations: new Set(), + functions: new Set(), + types: new Set() + }; + for (const facts of classifyStatements(sql)) { + for (const created of facts.creates) { + if (created.schema) continue; + if (RELATION_TAGS.has(facts.nodeTag)) { + inventory.relations.add(created.name); + } else if (facts.nodeTag === 'CreateFunctionStmt') { + inventory.functions.add(created.name); + } else if (TYPE_TAGS.has(facts.nodeTag)) { + inventory.types.add(created.name); + } + } + } + return inventory; +} + +/** Merge inventories collected from multiple scripts. */ +export function mergeInventories(inventories: ObjectInventory[]): ObjectInventory { + const merged: ObjectInventory = { + relations: new Set(), + functions: new Set(), + types: new Set() + }; + for (const inv of inventories) { + for (const name of inv.relations) merged.relations.add(name); + for (const name of inv.functions) merged.functions.add(name); + for (const name of inv.types) merged.types.add(name); + } + return merged; +} + +/** Resolve the routing table from the option forms. */ +function resolveRoutes(sql: string, options: QualifyUnqualifiedOptions): QualifyRoutes { + if (options.targets && options.schema) { + throw new Error('qualifyUnqualified: pass either `schema` or `targets`, not both'); + } + const routes: QualifyRoutes = { + relations: new Map(), + functions: new Map(), + types: new Map() + }; + const route = (bucket: Map, namespace: string, name: string, schema: string): void => { + const existing = bucket.get(name); + if (existing && existing !== schema) { + throw new Error( + `qualifyUnqualified: conflicting targets for ${namespace} "${name}": ${existing} vs ${schema}` + ); + } + bucket.set(name, schema); + }; + + if (options.targets) { + if (options.inventory) { + throw new Error('qualifyUnqualified: `inventory` only applies to the `schema` form'); + } + for (const [schema, selector] of Object.entries(options.targets)) { + for (const name of selector.relations ?? []) route(routes.relations, 'relation', name, schema); + for (const name of selector.functions ?? []) route(routes.functions, 'function', name, schema); + for (const name of selector.types ?? []) route(routes.types, 'type', name, schema); + } + return routes; + } + + if (!options.schema) { + throw new Error('qualifyUnqualified: one of `schema` or `targets` is required'); + } + const inventory = options.inventory ?? collectCreatedObjects(sql); + for (const name of inventory.relations) routes.relations.set(name, options.schema); + for (const name of inventory.functions) routes.functions.set(name, options.schema); + for (const name of inventory.types) routes.types.set(name, options.schema); + return routes; +} + +function collectCteNames(stmt: any): Set { + const names = new Set(); + walkSql(stmt, { + CommonTableExpr: (path: any) => { + if (path.node?.ctename) names.add(path.node.ctename); + } + }); + return names; +} + +/** + * Qualify unqualified references inside a LANGUAGE sql function body string. + * (`AS $$...$$` is a String under the 'as' DefElem; only PL/pgSQL bodies are + * exposed as a hydrated AST, so sql-language bodies need their own parse.) + */ +function qualifySqlBodyString( + body: string, + routes: QualifyRoutes, + result: QualifyResult +): string { + const parseResult = parseSql(body); + const stmts: any[] = parseResult?.stmts ?? []; + if (stmts.length === 0) return body; + const pieces: string[] = []; + for (const stmt of stmts) { + if (!stmt?.stmt) continue; + const visitor = createQualifyVisitor( + { routes, cteNames: collectCteNames(stmt.stmt) }, + result + ); + walkSql(stmt.stmt, visitor); + pieces.push(Deparser.deparse(stmt.stmt)); + } + return pieces.join(';\n'); +} + +function schemaStringNode(schema: string): any { + return { String: { sval: schema } }; +} + +/** + * Create a SQL AST visitor that qualifies unqualified references against a + * routing table. Composable with the walkers used by the core transform. + */ +export function createQualifyVisitor( + options: { routes: QualifyRoutes; cteNames?: Set }, + result: QualifyResult +) { + const { routes } = options; + const cteNames = options.cteNames ?? new Set(); + const typeSchemaOf = (name: string): string | undefined => + routes.types.get(name) ?? routes.relations.get(name); + + const record = (name: string, schema: string): void => { + result.qualified.set(name, (result.qualified.get(name) ?? 0) + 1); + result.routed.set(name, schema); + }; + + const qualifyNameList = ( + names: any[] | undefined, + schemaOf: (name: string) => string | undefined + ): void => { + if (!names || names.length !== 1) return; + const name = names[0]?.String?.sval; + if (!name) return; + const schema = schemaOf(name); + if (schema) { + names.unshift(schemaStringNode(schema)); + record(name, schema); + } + }; + + const functionSchemaOf = (name: string): string | undefined => routes.functions.get(name); + + return { + RangeVar: (path: any) => { + const node = path.node; + if (!node.schemaname && node.relname && !cteNames.has(node.relname)) { + const schema = routes.relations.get(node.relname); + if (schema) { + node.schemaname = schema; + record(node.relname, schema); + } + } + }, + + FuncCall: (path: any) => { + qualifyNameList(path.node.funcname, functionSchemaOf); + }, + + CallStmt: (path: any) => { + // The walker does not auto-recurse into CallStmt.funccall. + if (path.node.funccall) { + qualifyNameList(path.node.funccall.funcname, functionSchemaOf); + } + }, + + TypeName: (path: any) => { + qualifyNameList(path.node.names, typeSchemaOf); + }, + + CreateFunctionStmt: (path: any) => { + const node = path.node; + qualifyNameList(node.funcname, functionSchemaOf); + // LANGUAGE sql bodies are opaque strings to the walker; parse and + // qualify them separately (plpgsql bodies ride the hydrated AST walk). + const isPlpgsql = (node.options ?? []).some( + (opt: any) => + opt?.DefElem?.defname === 'language' && + opt.DefElem.arg?.String?.sval === 'plpgsql' + ); + if (!isPlpgsql && Array.isArray(node.options)) { + for (const opt of node.options) { + if (opt?.DefElem?.defname === 'as' && opt.DefElem.arg?.List?.items) { + for (const item of opt.DefElem.arg.List.items) { + if (typeof item?.String?.sval === 'string') { + try { + item.String.sval = qualifySqlBodyString(item.String.sval, routes, result); + } catch { + // Not parseable standalone (e.g. C symbol names) — leave as-is. + } + } + } + } + } + } + }, + + CreateTrigStmt: (path: any) => { + qualifyNameList(path.node.funcname, functionSchemaOf); + }, + + ObjectWithArgs: (path: any) => { + qualifyNameList(path.node.objname, functionSchemaOf); + }, + + CreateEnumStmt: (path: any) => { + qualifyNameList(path.node.typeName, name => routes.types.get(name)); + }, + + CreateRangeStmt: (path: any) => { + qualifyNameList(path.node.typeName, name => routes.types.get(name)); + }, + + CreateDomainStmt: (path: any) => { + qualifyNameList(path.node.domainname, name => routes.types.get(name)); + } + }; +} + +/** + * Qualify unqualified references in a SQL string. + * + * Two forms: + * - `{ schema, inventory? }` — pin every inventoried name to one schema. + * - `{ targets }` — route specific objects to specific schemas in one pass. + * + * Runs as a standalone AST pass (parse → qualify → deparse). To combine with + * a schema rename, run this first (typically with `schema: 'public'`) and + * then map via the core transform — or use the `qualifyUnqualified` option on + * `transformSql`, which does exactly that. + */ +export function qualifyUnqualified( + sql: string, + options: QualifyUnqualifiedOptions +): { sql: string; result: QualifyResult } { + const result: QualifyResult = { qualified: new Map(), routed: new Map() }; + const routes = resolveRoutes(sql, options); + + let qualified = transformSync(sql, (ctx) => { + const stmts: any[] = ctx.sql?.stmts ?? []; + + for (const stmt of stmts) { + if (stmt?.stmt) { + const visitor = createQualifyVisitor( + { routes, cteNames: collectCteNames(stmt.stmt) }, + result + ); + walkSql(stmt.stmt, visitor); + } + } + + const bodyVisitor = createQualifyVisitor({ routes }, result); + for (const fn of ctx.functions) { + if (fn.plpgsql?.hydrated) { + walkPlpgsql(fn.plpgsql.hydrated, {}, { + walkSqlExpressions: true, + sqlVisitor: bodyVisitor + }); + } + } + }, { hydrate: true, pretty: true }); + + if (options.injectCreateSchema) { + const targetSchemas = new Set([ + ...routes.relations.values(), + ...routes.functions.values(), + ...routes.types.values() + ]); + const created = new Set(); + for (const facts of classifyStatements(qualified)) { + if (facts.nodeTag === 'CreateSchemaStmt') { + for (const createdSchema of facts.creates) created.add(createdSchema.name); + } + } + const missing = [...targetSchemas].filter(schema => !created.has(schema)).sort(); + if (missing.length > 0) { + const prelude = missing + .map(schema => `CREATE SCHEMA IF NOT EXISTS ${schema};`) + .join('\n'); + qualified = `${prelude}\n\n${qualified}`; + } + } + + return { sql: qualified, result }; +} diff --git a/packages/transform/src/round-trip-core.ts b/packages/transform/src/round-trip-core.ts new file mode 100644 index 00000000..88360043 --- /dev/null +++ b/packages/transform/src/round-trip-core.ts @@ -0,0 +1,147 @@ +/** + * Shared, dependency-free core for round-trip AST validation. + * + * This module is intentionally self-contained — it imports no parser or + * deparser — so it can be lifted verbatim into an upstream package (e.g. + * `@pgsql/round-trip`) and shared between: + * + * - pgsql-parser's test fixtures (`cleanTree` / `expectParseDeparse`), which + * parse → deparse → re-parse and assert the normalized ASTs are equal, and + * - downstream mutation-aware validators (this package's + * round-trip check), which compare a *mutated* AST against the AST obtained + * by re-parsing the deparsed output. + * + * The upstream `cleanTree` is expressed here as a preset over the generic + * `normalizeTree` engine; the downstream fortified normalizer layers extra + * volatile keys, lazy-datum filtering, and body-sentinel handling on top of + * the same engine (see round-trip.ts). + */ + +export interface NormalizeTreeOptions { + /** Object keys dropped entirely (volatile / non-semantic fields). */ + volatileKeys?: ReadonlySet; + /** Sort object keys so comparison is insensitive to key order. Default false. */ + sortKeys?: boolean; + /** Array items for which this returns true are removed before recursing. */ + dropArrayItem?: (item: unknown) => boolean; + /** + * Per-key transform hooks keyed by object-property name. The hook receives + * the value at that key plus a `recurse` callback (which re-enters + * `normalizeTree` with the same options) and returns the replacement value. + */ + keyHandlers?: Record< + string, + (value: any, recurse: (node: any) => any) => any + >; +} + +/** + * Deep-clone an AST while dropping volatile keys, optionally sorting keys, + * filtering array items, and applying per-key transform hooks. Primitives are + * returned as-is; Dates are cloned (parity with the original upstream + * `transform` helper). + */ +export function normalizeTree(node: any, options: NormalizeTreeOptions = {}): any { + const { volatileKeys, sortKeys, dropArrayItem, keyHandlers } = options; + const recurse = (n: any) => normalizeTree(n, options); + + if (Array.isArray(node)) { + const items = dropArrayItem ? node.filter((i) => !dropArrayItem(i)) : node; + return items.map(recurse); + } + + if (node instanceof Date) { + return new Date(node.getTime()); + } + + if (node && typeof node === 'object') { + const out: Record = {}; + const keys = sortKeys ? Object.keys(node).sort() : Object.keys(node); + for (const key of keys) { + if (volatileKeys?.has(key)) continue; + const handler = keyHandlers?.[key]; + out[key] = handler ? handler(node[key], recurse) : recurse(node[key]); + } + return out; + } + + return node; +} + +/** + * Return the JSON path of the first structural difference between two trees, + * or null if they are deeply equal. Useful for actionable mismatch messages. + */ +export function firstDifference(a: any, b: any, path = '$'): string | null { + if (a === b) return null; + if (Array.isArray(a) && Array.isArray(b)) { + if (a.length !== b.length) { + return `${path} (array length ${a.length} vs ${b.length})`; + } + for (let i = 0; i < a.length; i++) { + const diff = firstDifference(a[i], b[i], `${path}[${i}]`); + if (diff) return diff; + } + return null; + } + if (a && b && typeof a === 'object' && typeof b === 'object') { + const keys = new Set([...Object.keys(a), ...Object.keys(b)]); + for (const key of keys) { + const diff = firstDifference(a[key], b[key], `${path}.${key}`); + if (diff) return diff; + } + return null; + } + return `${path} (${JSON.stringify(a)} vs ${JSON.stringify(b)})`; +} + +// --------------------------------------------------------------------------- +// Upstream-compatible cleanTree preset +// --------------------------------------------------------------------------- + +/** Position/length fields stripped by the upstream `cleanTree`. */ +export const CLEAN_TREE_VOLATILE_KEYS: ReadonlySet = new Set([ + 'stmt_len', + 'stmt_location', + 'location', + 'rexpr_list_start', + 'rexpr_list_end', + 'list_start', + 'list_end', +]); + +/** + * Trim the function/DO body string carried by a `DefElem` with defname "as". + * Mirrors the in-place trimming in pgsql-parser's `cleanTree` so the body + * text (whose surrounding whitespace legitimately shifts on deparse) does not + * cause spurious AST mismatches. Mutates the passed DefElem in place. + */ +export function trimDefElemBody(defElem: any): void { + if (!defElem || defElem.defname !== 'as') return; + const arg = defElem.arg; + if (Array.isArray(arg) && arg.length && arg[0]?.String) { + arg[0].String.sval = arg[0].String.sval.trim(); + } else if (arg?.List?.items?.length && arg.List.items[0]?.String) { + arg.List.items[0].String.sval = arg.List.items[0].String.sval.trim(); + } else if (arg?.String) { + arg.String.sval = arg.String.sval.trim(); + } +} + +/** + * Upstream-compatible normalization: strips `stmt_len` / `stmt_location` / + * `location` and trims `DefElem` "as" body strings. Equivalent to + * pgsql-parser's deparser `cleanTree`, provided here so upstream can adopt the + * shared core without behavior change. + */ +export function cleanTree(tree: any): any { + return normalizeTree(tree, { + volatileKeys: CLEAN_TREE_VOLATILE_KEYS, + keyHandlers: { + DefElem: (defElem: any, recurse: (n: any) => any) => { + trimDefElemBody(defElem); + return recurse(defElem); + }, + }, + }); +} diff --git a/packages/transform/src/round-trip.ts b/packages/transform/src/round-trip.ts new file mode 100644 index 00000000..bee34a03 --- /dev/null +++ b/packages/transform/src/round-trip.ts @@ -0,0 +1,158 @@ +/** + * Round-trip validation for the schema transform. + * + * After the transform deparses its mutated AST to SQL text, the emitted text + * must parse back to an AST that is structurally identical to the AST we + * deparsed (AST1 === AST2 after normalization). Any mismatch means the + * deparser dropped or mangled something (e.g. array bounds on a DECLAREd + * type), even though the output may still be syntactically valid SQL. + * + * Two comparisons are performed per file: + * - the SQL-level parse tree (with volatile fields and function body strings + * normalized — PL/pgSQL bodies are compared separately) + * - each PL/pgSQL function body, compared in dehydrated form so both sides + * go through the same hydrate -> dehydrate rendering + */ + +import { dehydratePlpgsqlAst,transformSync } from 'plpgsql-parser'; + +import { + CLEAN_TREE_VOLATILE_KEYS, + firstDifference, + normalizeTree, +} from './round-trip-core'; + +export { firstDifference }; + +/** + * Fields that legitimately differ between parses. Extends the upstream + * `cleanTree` volatile set (location/stmt_location/stmt_len) with: + * - lineno: source positions in PL/pgSQL nodes + * - dno/varno/recparentno: PL/pgSQL datum numbers — compiler bookkeeping + * assigned in parse-encounter order, which shifts when the deparser + * emits an equivalent-but-reordered statement (e.g. INTO placement). + * Structural identity is still checked via names/fieldnames. + */ +const VOLATILE_KEYS = new Set([ + ...CLEAN_TREE_VOLATILE_KEYS, + 'lineno', + 'dno', + 'varno', + 'recparentno', +]); + +/** + * PLpgSQL_recfield datums are created lazily by the PL/pgSQL compiler as + * record.field references are encountered while parsing expressions, so + * their presence/order in the datums array depends on expression text + * layout, not semantics. + */ +function isRecfieldDatum(node: unknown): boolean { + return !!node && typeof node === 'object' && 'PLpgSQL_recfield' in node; +} + +/** + * Fortified, mutation-aware normalizer built on the shared round-trip core. + * Beyond the upstream `cleanTree` (position stripping + body trimming), it + * additionally sorts keys, drops lazily-created recfield datums, and replaces + * DefElem "as" bodies with a sentinel (PL/pgSQL body fidelity is checked + * separately via the dehydrated PL/pgSQL ASTs). + */ +export function normalizeParseTree(node: any): any { + return normalizeTree(node, { + volatileKeys: VOLATILE_KEYS, + sortKeys: true, + dropArrayItem: isRecfieldDatum, + keyHandlers: { + DefElem: (defElem: any, recurse: (n: any) => any) => + defElem?.defname === 'as' + ? { ...recurse(defElem), arg: '' } + : recurse(defElem), + }, + }); +} + +export interface CapturedAsts { + sqlAst: any; + plpgsqlAsts: any[]; +} + +/** + * Capture the normalized SQL parse tree and dehydrated PL/pgSQL function + * ASTs from a transform context (as produced by plpgsql-parser's + * transformSync callback). Call this AFTER mutating the context. + */ +export function captureTransformAsts(ctx: any): CapturedAsts { + const plpgsqlAsts: any[] = []; + for (const fn of ctx.functions) { + if (fn.plpgsql?.hydrated) { + plpgsqlAsts.push( + normalizeParseTree( + dehydratePlpgsqlAst(structuredClone(fn.plpgsql.hydrated)) + ) + ); + } + } + return { + sqlAst: normalizeParseTree(ctx.sql), + plpgsqlAsts, + }; +} + +/** + * Parse already-transformed SQL text and capture its ASTs the same way + * capture_transform_asts does for the pre-deparse context, so the two + * sides are directly comparable. + */ +export function captureAstsFromSql(sql: string): CapturedAsts { + let captured: CapturedAsts | undefined; + transformSync( + sql, + (ctx: any) => { + captured = captureTransformAsts(ctx); + }, + { hydrate: true, pretty: true } + ); + if (!captured) { + throw new Error('round-trip: failed to capture ASTs from transformed SQL'); + } + return captured; +} + +/** + * Assert that the transformed AST (captured before deparse) matches the AST + * obtained by re-parsing the emitted SQL. Throws with the first differing + * path on mismatch. + */ +export function validateRoundTrip( + before: CapturedAsts, + transformedSql: string +): void { + const after = captureAstsFromSql(transformedSql); + + const sqlDiff = firstDifference(before.sqlAst, after.sqlAst); + if (sqlDiff) { + throw new Error( + `round-trip validation failed: SQL AST mismatch after deparse at ${sqlDiff}` + ); + } + + if (before.plpgsqlAsts.length !== after.plpgsqlAsts.length) { + throw new Error( + `round-trip validation failed: PL/pgSQL function count mismatch ` + + `(${before.plpgsqlAsts.length} before deparse vs ${after.plpgsqlAsts.length} after re-parse)` + ); + } + + for (let i = 0; i < before.plpgsqlAsts.length; i++) { + const diff = firstDifference( + before.plpgsqlAsts[i], + after.plpgsqlAsts[i] + ); + if (diff) { + throw new Error( + `round-trip validation failed: PL/pgSQL AST mismatch in function #${i} at ${diff}` + ); + } + } +} diff --git a/packages/transform/src/transform.ts b/packages/transform/src/transform.ts new file mode 100644 index 00000000..e94f5720 --- /dev/null +++ b/packages/transform/src/transform.ts @@ -0,0 +1,1457 @@ +/** + * AST-based Schema Name Transformer (Core Logic) + * + * Pure transformation functions for converting schema names in SQL strings. + * No file I/O — takes SQL strings in and returns transformed SQL strings out. + * + * The transformer handles: + * - Schema-qualified identifiers in RangeVar nodes (schema.table) + * - Schema names in CreateSchemaStmt nodes + * - Schema-qualified function names in FuncCall and CreateFunctionStmt nodes + * - Schema-qualified type names in TypeName nodes + * - Schema names in GrantStmt objects array + * - Schema names in VariableSetStmt (SET search_path) + * - Schema names in AlterDefaultPrivilegesStmt + * - Schema names in DropStmt (DROP SCHEMA) + * - Schema-qualified trigger function names in CreateTrigStmt + * - Schema-qualified object references in CommentStmt + * - Schema-qualified names in DefineStmt (CREATE TYPE, CREATE AGGREGATE) + * - Schema-qualified names in CreateDomainStmt, CreateEnumStmt, AlterEnumStmt + * - Schema-qualified names in AlterDomainStmt, AlterTypeStmt + * - Schema-qualified names in ObjectWithArgs (ALTER FUNCTION, etc.) + * - Schema name in AlterObjectSchemaStmt (ALTER ... SET SCHEMA) + * - Schema names inside PL/pgSQL function bodies (hydrated AST) + * - Comment headers, verify function calls, JSON string values (regex fallback) + */ + +import { QuoteUtils } from '@pgsql/quotes'; +import { walk as walkSql } from '@pgsql/traverse'; +import { Deparser,parseSql, transformSync, walk as walkPlpgsql } from 'plpgsql-parser'; + +import type { QualifyUnqualifiedOptions } from './qualify'; +import { qualifyUnqualified } from './qualify'; +import type { CapturedAsts } from './round-trip'; +import { captureTransformAsts, validateRoundTrip } from './round-trip'; + +export interface SchemaTransformResult { + schemasFound: Set; + schemasTransformed: Map; + errors: Array<{ file: string; error: string }>; +} + +/** + * A pluggable string-level transform pass. + * + * Extension passes run on the raw SQL text before or after the core AST + * transformation. They exist for content that is opaque to the SQL parser + * (string literals, comments, JSON values, etc.). + * + * Each pass receives the current content string, the schema mapping, and the + * shared result tracker, and must return the (possibly transformed) content. + */ +export type SchemaTransformPass = ( + content: string, + schemaMapping: Map, + result: SchemaTransformResult +) => string; + +/** + * Options for transform_sql. + */ +export interface TransformSqlOptions { + /** + * Extension passes that run BEFORE the core AST transformation. + * Use these for app-specific string-level transforms that must happen + * before the parser sees the content (e.g. verify calls, JSON values). + */ + prePasses?: SchemaTransformPass[]; + + /** + * Extension passes that run AFTER the core AST transformation. + * Use these for app-specific transforms on the deparsed output. + */ + postPasses?: SchemaTransformPass[]; + + /** + * Validate that the emitted SQL re-parses to an AST structurally identical + * to the transformed AST that was deparsed (catches deparser fidelity bugs + * such as dropped array bounds). Adds a second parse per file. + */ + roundTrip?: boolean; + + /** + * Qualify unqualified object references BEFORE the schema mapping runs + * (an extra AST pass, opt-in). Pin unqualified references to a schema + * (typically `'public'`) so a mapping on that schema moves them too — + * the ingestion path for handwritten, unqualified SQL. Only names in the + * inventory (defaulting to objects the content itself creates) are + * qualified. See {@link qualifyUnqualified}. + */ + qualifyUnqualified?: QualifyUnqualifiedOptions; + + /** + * Schema names (post-mapping) whose `CREATE SCHEMA` statements should be + * emitted as `CREATE SCHEMA IF NOT EXISTS`. Use when mapping into a schema + * that always exists — e.g. mapping a named schema onto `public`. + */ + assumeSchemasExist?: string[]; +} + +/** + * Create a fresh result object + */ +function createResult(): SchemaTransformResult { + return { + schemasFound: new Set(), + schemasTransformed: new Map(), + errors: [], + }; +} + +/** + * Transform a schema name if it exists in the mapping + */ +export function transformSchemaName( + schemaName: string | undefined, + schemaMapping: Map +): string | undefined { + if (!schemaName) return schemaName; + return schemaMapping.get(schemaName) ?? schemaName; +} + +/** + * Check if a schema name should be transformed + */ +export function shouldTransformSchema( + schemaName: string | undefined, + schemaMapping: Map +): boolean { + if (!schemaName) return false; + return schemaMapping.has(schemaName); +} + +/** + * Transform schema names in a String node array (used for funcname, names, etc.) + * These arrays contain String nodes like { String: { sval: 'schema_name' } } + */ +export function transformNameList( + names: any[] | undefined, + schemaMapping: Map, + result: SchemaTransformResult +): void { + if (!names || names.length < 2) return; + + // For schema-qualified names, the first element is the schema + const first = names[0]; + if (first?.String?.sval) { + const schemaName = first.String.sval; + if (shouldTransformSchema(schemaName, schemaMapping)) { + result.schemasFound.add(schemaName); + const newName = schemaMapping.get(schemaName); + if (newName) { + first.String.sval = newName; + result.schemasTransformed.set(schemaName, newName); + } + } + } +} + +/** + * Transform a bare schema-name field on a node (e.g. { String: { sval } } + * entries in GrantStmt.objects, RenameStmt.subname, CommentStmt.object). + * Returns the (possibly new) schema name. + */ +export function transformSchemaNameField( + container: any, + field: string, + schemaMapping: Map, + result: SchemaTransformResult +): void { + const schemaName = container?.[field]; + if (typeof schemaName !== 'string') return; + if (shouldTransformSchema(schemaName, schemaMapping)) { + result.schemasFound.add(schemaName); + const newName = schemaMapping.get(schemaName); + if (newName) { + container[field] = newName; + result.schemasTransformed.set(schemaName, newName); + } + } +} + +/** + * Transform a RangeVar-like relation object (has schemaname/relname fields). + * Many statement nodes embed a relation that the walker does NOT auto-recurse into. + */ +export function transformRelation( + relation: any, + schemaMapping: Map, + result: SchemaTransformResult +): void { + if (relation?.schemaname && shouldTransformSchema(relation.schemaname, schemaMapping)) { + const oldName = relation.schemaname; + result.schemasFound.add(oldName); + const newName = schemaMapping.get(oldName); + if (newName) { + relation.schemaname = newName; + result.schemasTransformed.set(oldName, newName); + } + } +} + +/** + * Transform schema-qualified references embedded in a plain string + * (e.g. advisory-lock keys like 'schema-name.fn_name', COMMENT text, + * RAISE messages). Only occurrences of a mapped schema name immediately + * followed by a dot are rewritten. + */ +export function transformSchemaRefsInString( + str: string, + schemaMapping: Map, + result: SchemaTransformResult +): string { + let out = str; + for (const [oldSchema, newSchema] of schemaMapping) { + const pattern = new RegExp(`(?, + result: SchemaTransformResult, + visitorOptions?: { assumeSchemasExist?: Set } +) { + const assumeSchemasExist = visitorOptions?.assumeSchemasExist; + return { + // Transform RangeVar nodes (table references) + RangeVar: (path: any) => { + const node = path.node; + if (node.schemaname && shouldTransformSchema(node.schemaname, schemaMapping)) { + const oldName = node.schemaname; + result.schemasFound.add(oldName); + const newName = schemaMapping.get(oldName); + if (newName) { + node.schemaname = newName; + result.schemasTransformed.set(oldName, newName); + } + } + }, + + // Transform CreateSchemaStmt nodes + CreateSchemaStmt: (path: any) => { + const node = path.node; + if (node.schemaname && shouldTransformSchema(node.schemaname, schemaMapping)) { + const oldName = node.schemaname; + result.schemasFound.add(oldName); + const newName = schemaMapping.get(oldName); + if (newName) { + node.schemaname = newName; + result.schemasTransformed.set(oldName, newName); + } + } + if (node.schemaname && assumeSchemasExist?.has(node.schemaname)) { + node.if_not_exists = true; + } + }, + + // Transform FuncCall nodes (function calls with schema-qualified names) + FuncCall: (path: any) => { + const node = path.node; + transformNameList(node.funcname, schemaMapping, result); + }, + + // Transform CallStmt (CALL schema.procedure(...)). + // Note: The walker does NOT auto-recurse into CallStmt.funccall, + // so the FuncCall visitor above never sees it. + CallStmt: (path: any) => { + const node = path.node; + if (node.funccall?.funcname) { + transformNameList(node.funccall.funcname, schemaMapping, result); + } + }, + + // Transform TypeName nodes (type references with schema-qualified names) + TypeName: (path: any) => { + const node = path.node; + transformNameList(node.names, schemaMapping, result); + }, + + // Transform ColumnRef nodes (column references with schema-qualified names) + ColumnRef: (path: any) => { + const node = path.node; + transformNameList(node.fields, schemaMapping, result); + }, + + // Transform GrantStmt objects (GRANT ON SCHEMA schema; + // GRANT ... ON ALL TABLES/FUNCTIONS/SEQUENCES IN SCHEMA schema) + GrantStmt: (path: any) => { + const node = path.node; + // For schema-name targets, objects contains String nodes with schema names + if ((node.objtype === 'OBJECT_SCHEMA' || node.targtype === 'ACL_TARGET_ALL_IN_SCHEMA') && node.objects) { + for (const obj of node.objects) { + if (obj?.String) { + transformSchemaNameField(obj.String, 'sval', schemaMapping, result); + } + } + } + }, + + // Transform VariableSetStmt (SET search_path) + VariableSetStmt: (path: any) => { + const node = path.node; + if (node.name === 'search_path' && node.args) { + for (const arg of node.args) { + // search_path args can be String nodes or A_Const with sval + if (arg?.String?.sval) { + const schemaName = arg.String.sval; + if (shouldTransformSchema(schemaName, schemaMapping)) { + result.schemasFound.add(schemaName); + const newName = schemaMapping.get(schemaName); + if (newName) { + arg.String.sval = newName; + result.schemasTransformed.set(schemaName, newName); + } + } + } else if (arg?.A_Const?.sval?.sval) { + const schemaName = arg.A_Const.sval.sval; + if (shouldTransformSchema(schemaName, schemaMapping)) { + result.schemasFound.add(schemaName); + const newName = schemaMapping.get(schemaName); + if (newName) { + arg.A_Const.sval.sval = newName; + result.schemasTransformed.set(schemaName, newName); + } + } + } + } + } + }, + + // Transform AlterDefaultPrivilegesStmt (ALTER DEFAULT PRIVILEGES IN SCHEMA) + AlterDefaultPrivilegesStmt: (path: any) => { + const node = path.node; + if (node.options) { + for (const opt of node.options) { + if (opt?.DefElem?.defname === 'schemas' && opt.DefElem.arg?.List?.items) { + for (const item of opt.DefElem.arg.List.items) { + if (item?.String?.sval) { + const schemaName = item.String.sval; + if (shouldTransformSchema(schemaName, schemaMapping)) { + result.schemasFound.add(schemaName); + const newName = schemaMapping.get(schemaName); + if (newName) { + item.String.sval = newName; + result.schemasTransformed.set(schemaName, newName); + } + } + } + } + } + } + } + }, + + // Transform DropStmt (DROP SCHEMA name; DROP TABLE/INDEX/POLICY/TRIGGER/ + // TYPE/FUNCTION schema.obj). The walker does NOT auto-recurse into + // DropStmt.objects. + DropStmt: (path: any) => { + const node = path.node; + if (node.removeType !== 'OBJECT_SCHEMA' && Array.isArray(node.objects)) { + for (const obj of node.objects) { + if (obj?.List?.items) { + transformNameList(obj.List.items, schemaMapping, result); + } else if (obj?.ObjectWithArgs?.objname) { + transformNameList(obj.ObjectWithArgs.objname, schemaMapping, result); + } else if (obj?.TypeName?.names) { + transformNameList(obj.TypeName.names, schemaMapping, result); + } + } + return; + } + if (node.removeType === 'OBJECT_SCHEMA' && node.objects) { + for (const obj of node.objects) { + // DROP SCHEMA objects can be List of String nodes + if (obj?.List?.items) { + for (const item of obj.List.items) { + if (item?.String?.sval) { + const schemaName = item.String.sval; + if (shouldTransformSchema(schemaName, schemaMapping)) { + result.schemasFound.add(schemaName); + const newName = schemaMapping.get(schemaName); + if (newName) { + item.String.sval = newName; + result.schemasTransformed.set(schemaName, newName); + } + } + } + } + } else if (obj?.String?.sval) { + const schemaName = obj.String.sval; + if (shouldTransformSchema(schemaName, schemaMapping)) { + result.schemasFound.add(schemaName); + const newName = schemaMapping.get(schemaName); + if (newName) { + obj.String.sval = newName; + result.schemasTransformed.set(schemaName, newName); + } + } + } + } + } + }, + + // Transform InsertStmt (INSERT INTO schema.table) + // Note: The walker does NOT auto-recurse into InsertStmt.relation. + InsertStmt: (path: any) => { + transformRelation(path.node.relation, schemaMapping, result); + }, + + // Transform CreateStmt (CREATE TABLE schema.table) + // Note: The walker does NOT auto-recurse into CreateStmt.relation. + CreateStmt: (path: any) => { + transformRelation(path.node.relation, schemaMapping, result); + }, + + // Transform UpdateStmt (UPDATE schema.table SET ...) + // Note: The walker does NOT auto-recurse into UpdateStmt.relation. + UpdateStmt: (path: any) => { + transformRelation(path.node.relation, schemaMapping, result); + }, + + // Transform DeleteStmt (DELETE FROM schema.table) + // Note: The walker does NOT auto-recurse into DeleteStmt.relation. + DeleteStmt: (path: any) => { + transformRelation(path.node.relation, schemaMapping, result); + }, + + // Transform MergeStmt (MERGE INTO schema.table USING ...) + // Note: The walker does NOT auto-recurse into MergeStmt.relation. + MergeStmt: (path: any) => { + transformRelation(path.node.relation, schemaMapping, result); + }, + + // Transform IndexStmt (CREATE INDEX ... ON schema.table) + // Note: The walker does NOT auto-recurse into IndexStmt.relation. + IndexStmt: (path: any) => { + transformRelation(path.node.relation, schemaMapping, result); + }, + + // Transform Constraint (REFERENCES schema.table for foreign keys) + // Note: The walker does NOT auto-recurse into Constraint.pktable. + Constraint: (path: any) => { + const node = path.node; + if (node.contype === 'CONSTR_FOREIGN' && node.pktable) { + transformRelation(node.pktable, schemaMapping, result); + } + }, + + // Transform CompositeTypeStmt (CREATE TYPE schema.name AS (...)) + // Note: The walker does NOT auto-recurse into CompositeTypeStmt.typevar. + CompositeTypeStmt: (path: any) => { + transformRelation(path.node.typevar, schemaMapping, result); + }, + + // Transform ColumnDef (column definitions with schema-qualified types) + // Note: The walker does NOT auto-recurse into ColumnDef.typeName. + ColumnDef: (path: any) => { + const node = path.node; + if (node.typeName?.names) { + transformNameList(node.typeName.names, schemaMapping, result); + } + }, + + // Transform ViewStmt (CREATE VIEW schema.viewname AS ...) + // Note: The walker does NOT auto-recurse into ViewStmt.view. + ViewStmt: (path: any) => { + transformRelation(path.node.view, schemaMapping, result); + }, + + // Transform AlterTableStmt (ALTER TABLE schema.table ...) + // Note: The walker does NOT auto-recurse into AlterTableStmt.relation. + AlterTableStmt: (path: any) => { + transformRelation(path.node.relation, schemaMapping, result); + }, + + // Transform CreateSeqStmt (CREATE SEQUENCE schema.seqname) + // Note: The walker does NOT auto-recurse into CreateSeqStmt.sequence. + CreateSeqStmt: (path: any) => { + transformRelation(path.node.sequence, schemaMapping, result); + }, + + // Transform AlterSeqStmt (ALTER SEQUENCE schema.seqname [OWNED BY schema.table.col]) + // Note: The walker does NOT auto-recurse into AlterSeqStmt.sequence. + // Also handles OWNED BY clause where the schema name is in DefElem.arg.List.items. + AlterSeqStmt: (path: any) => { + const node = path.node; + transformRelation(node.sequence, schemaMapping, result); + // Handle OWNED BY option: schema name is in the name list + if (node.options) { + for (const opt of node.options) { + if (opt?.DefElem?.defname === 'owned_by' && opt.DefElem?.arg?.List?.items) { + transformNameList(opt.DefElem.arg.List.items, schemaMapping, result); + } + } + } + }, + + // Transform CreatePolicyStmt (CREATE POLICY ... ON schema.table) + // Note: The walker does NOT auto-recurse into CreatePolicyStmt.table. + CreatePolicyStmt: (path: any) => { + transformRelation(path.node.table, schemaMapping, result); + }, + + // Transform RuleStmt (CREATE RULE ... ON schema.table) + // Note: The walker does NOT auto-recurse into RuleStmt.relation. + RuleStmt: (path: any) => { + transformRelation(path.node.relation, schemaMapping, result); + }, + + // Transform CopyStmt (COPY schema.table ...) + // Note: The walker does NOT auto-recurse into CopyStmt.relation. + CopyStmt: (path: any) => { + transformRelation(path.node.relation, schemaMapping, result); + }, + + // Transform ClusterStmt (CLUSTER schema.table USING index) + // Note: The walker does NOT auto-recurse into ClusterStmt.relation. + ClusterStmt: (path: any) => { + transformRelation(path.node.relation, schemaMapping, result); + }, + + // Transform VacuumRelation (VACUUM/ANALYZE schema.table) + // Note: The walker visits VacuumRelation but does NOT recurse into its relation. + VacuumRelation: (path: any) => { + transformRelation(path.node.relation, schemaMapping, result); + }, + + // Transform RefreshMatViewStmt (REFRESH MATERIALIZED VIEW schema.view) + // Note: The walker does NOT auto-recurse into RefreshMatViewStmt.relation. + RefreshMatViewStmt: (path: any) => { + transformRelation(path.node.relation, schemaMapping, result); + }, + + // Transform CreateTableAsStmt (CREATE MATERIALIZED VIEW schema.viewname AS ...) + // Note: The walker does NOT auto-recurse into CreateTableAsStmt.into.rel. + CreateTableAsStmt: (path: any) => { + transformRelation(path.node.into?.rel, schemaMapping, result); + }, + + // Transform TypeCast nodes (type casts like NULL::schema.type) + // Note: The walker visits TypeCast but does NOT recurse into the TypeName node. + TypeCast: (path: any) => { + const node = path.node; + if (node.typeName?.names) { + transformNameList(node.typeName.names, schemaMapping, result); + } + }, + + // Transform CreateFunctionStmt (CREATE FUNCTION schema.funcname) + // Also handles RETURNS [SETOF] schema.type via returnType.names + CreateFunctionStmt: (path: any) => { + const node = path.node; + transformNameList(node.funcname, schemaMapping, result); + // Transform the return type (e.g., RETURNS SETOF schema.tablename) + if (node.returnType?.names) { + transformNameList(node.returnType.names, schemaMapping, result); + } + // Transform parameter types (the walker does NOT auto-recurse into + // FunctionParameter.argType TypeName nodes) + if (Array.isArray(node.parameters)) { + for (const param of node.parameters) { + if (param?.FunctionParameter?.argType?.names) { + transformNameList(param.FunctionParameter.argType.names, schemaMapping, result); + } + } + } + // Transform schema-qualified references inside the function body + // (AS $$...$$ is a String under the 'as' DefElem, which the walker + // does not parse as SQL). LANGUAGE plpgsql bodies are additionally + // rebuilt from the PL/pgSQL AST by transformSync; for LANGUAGE sql + // functions this string is the deparsed body. + if (Array.isArray(node.options)) { + for (const opt of node.options) { + if (opt?.DefElem?.defname === 'as' && opt.DefElem.arg?.List?.items) { + for (const item of opt.DefElem.arg.List.items) { + if (typeof item?.String?.sval === 'string' && item.String.sval.includes('.')) { + item.String.sval = transformSchemaRefsInString(item.String.sval, schemaMapping, result); + } + } + } + } + } + }, + + // Transform CreateTrigStmt (CREATE TRIGGER ... EXECUTE PROCEDURE schema.func) + // Note: The walker does NOT auto-recurse into CreateTrigStmt.relation, + // so we must explicitly transform the RangeVar here. + CreateTrigStmt: (path: any) => { + const node = path.node; + transformNameList(node.funcname, schemaMapping, result); + transformRelation(node.relation, schemaMapping, result); + }, + + // Transform CommentStmt (COMMENT ON TABLE/COLUMN/FUNCTION schema.obj, + // COMMENT ON SCHEMA schema) + CommentStmt: (path: any) => { + const node = path.node; + // The object field structure varies by objtype. + // For tables/views/etc, object is { List: { items: [{ String: ... }, ...] } } + // For functions, object is { ObjectWithArgs: { objname: [...] } } + // For schemas, object is a bare { String: { sval } } + if (node.object) { + // Handle List-wrapped objects (COMMENT ON TABLE schema.tbl, COMMENT ON COLUMN schema.tbl.col) + if (node.object?.List?.items) { + transformNameList(node.object.List.items, schemaMapping, result); + } + // Handle ObjectWithArgs-style objects (COMMENT ON FUNCTION schema.func(args)) + else if (node.object?.ObjectWithArgs?.objname) { + transformNameList(node.object.ObjectWithArgs.objname, schemaMapping, result); + } + // Handle bare schema names (COMMENT ON SCHEMA schema) + else if (node.object?.String && node.objtype === 'OBJECT_SCHEMA') { + transformSchemaNameField(node.object.String, 'sval', schemaMapping, result); + } + } + // Transform schema-qualified references inside the comment text itself + if (typeof node.comment === 'string') { + node.comment = transformSchemaRefsInString(node.comment, schemaMapping, result); + } + }, + + // Transform schema-qualified references embedded in string constants + // (e.g. advisory-lock keys: hashtextextended('schema.fn_name', 0)) + A_Const: (path: any) => { + const node = path.node; + if (typeof node.sval?.sval === 'string' && node.sval.sval.includes('.')) { + node.sval.sval = transformSchemaRefsInString(node.sval.sval, schemaMapping, result); + } + }, + + // Transform DefineStmt (CREATE TYPE schema.name, CREATE AGGREGATE schema.agg) + DefineStmt: (path: any) => { + const node = path.node; + transformNameList(node.defnames, schemaMapping, result); + }, + + // Transform CreateDomainStmt (CREATE DOMAIN schema.domname) + CreateDomainStmt: (path: any) => { + const node = path.node; + transformNameList(node.domainname, schemaMapping, result); + }, + + // Transform CreateEnumStmt (CREATE TYPE schema.enumname AS ENUM) + CreateEnumStmt: (path: any) => { + const node = path.node; + transformNameList(node.typeName, schemaMapping, result); + }, + + // Transform AlterEnumStmt (ALTER TYPE schema.enumname ADD VALUE) + AlterEnumStmt: (path: any) => { + const node = path.node; + transformNameList(node.typeName, schemaMapping, result); + }, + + // Transform AlterDomainStmt (ALTER DOMAIN schema.domname) + AlterDomainStmt: (path: any) => { + const node = path.node; + transformNameList(node.typeName, schemaMapping, result); + }, + + // Transform AlterTypeStmt (ALTER TYPE schema.typename) + // Note: composite type alters use RangeVar (handled by RangeVar visitor), + // but some ALTER TYPE statements use typeName as a name list + AlterTypeStmt: (path: any) => { + const node = path.node; + transformNameList(node.typeName, schemaMapping, result); + }, + + // Transform ObjectWithArgs (used in ALTER FUNCTION, DROP FUNCTION with args, etc.) + ObjectWithArgs: (path: any) => { + const node = path.node; + transformNameList(node.objname, schemaMapping, result); + }, + + // Transform AlterObjectSchemaStmt (ALTER ... SET SCHEMA newschema) + // The walker does NOT auto-recurse into AlterObjectSchemaStmt.relation + // or .object, so we must transform relation, object names, and newschema. + AlterObjectSchemaStmt: (path: any) => { + const node = path.node; + transformRelation(node.relation, schemaMapping, result); + // Non-relation objects (ALTER TYPE/FUNCTION ... SET SCHEMA) + if (node.object?.List?.items) { + transformNameList(node.object.List.items, schemaMapping, result); + } else if (node.object?.ObjectWithArgs?.objname) { + transformNameList(node.object.ObjectWithArgs.objname, schemaMapping, result); + } else if (node.object?.TypeName?.names) { + transformNameList(node.object.TypeName.names, schemaMapping, result); + } + // Transform the newschema (destination schema) + transformSchemaNameField(node, 'newschema', schemaMapping, result); + }, + + // Transform RenameStmt (ALTER SCHEMA old RENAME TO new; + // ALTER TABLE schema.tbl RENAME ...). The walker does NOT auto-recurse + // into RenameStmt.relation or .object. + RenameStmt: (path: any) => { + const node = path.node; + transformRelation(node.relation, schemaMapping, result); + if (node.renameType === 'OBJECT_SCHEMA') { + // subname is the old schema name; newname is the target + transformSchemaNameField(node, 'subname', schemaMapping, result); + transformSchemaNameField(node, 'newname', schemaMapping, result); + } + if (node.object?.List?.items) { + transformNameList(node.object.List.items, schemaMapping, result); + } else if (node.object?.ObjectWithArgs?.objname) { + transformNameList(node.object.ObjectWithArgs.objname, schemaMapping, result); + } + }, + + // Transform AlterFunctionStmt (ALTER FUNCTION schema.fn(...) SET ...) + AlterFunctionStmt: (path: any) => { + const node = path.node; + if (node.func?.objname) { + transformNameList(node.func.objname, schemaMapping, result); + } + }, + + // Transform AlterOwnerStmt (ALTER ... OWNER TO role) + AlterOwnerStmt: (path: any) => { + const node = path.node; + transformRelation(node.relation, schemaMapping, result); + if (node.object?.List?.items) { + transformNameList(node.object.List.items, schemaMapping, result); + } else if (node.object?.ObjectWithArgs?.objname) { + transformNameList(node.object.ObjectWithArgs.objname, schemaMapping, result); + } else if (node.object?.String && node.objectType === 'OBJECT_SCHEMA') { + transformSchemaNameField(node.object.String, 'sval', schemaMapping, result); + } + }, + + // Transform CreateCastStmt (CREATE CAST (src AS tgt) WITH FUNCTION fn) + CreateCastStmt: (path: any) => { + const node = path.node; + if (node.sourcetype?.names) { + transformNameList(node.sourcetype.names, schemaMapping, result); + } + if (node.targettype?.names) { + transformNameList(node.targettype.names, schemaMapping, result); + } + if (node.func?.objname) { + transformNameList(node.func.objname, schemaMapping, result); + } + if (Array.isArray(node.func?.objargs)) { + for (const arg of node.func.objargs) { + if (arg?.TypeName?.names) { + transformNameList(arg.TypeName.names, schemaMapping, result); + } else if (arg?.names) { + transformNameList(arg.names, schemaMapping, result); + } + } + } + if (Array.isArray(node.func?.objfuncargs)) { + for (const param of node.func.objfuncargs) { + if (param?.FunctionParameter?.argType?.names) { + transformNameList(param.FunctionParameter.argType.names, schemaMapping, result); + } + } + } + }, + + // Transform CreateEventTrigStmt (CREATE EVENT TRIGGER ... EXECUTE FUNCTION schema.fn()) + CreateEventTrigStmt: (path: any) => { + const node = path.node; + transformNameList(node.funcname, schemaMapping, result); + }, + + // Transform IndexElem opclass (CREATE INDEX ... (col schema.opclass)) + IndexElem: (path: any) => { + const node = path.node; + transformNameList(node.opclass, schemaMapping, result); + }, + + // Transform SecLabelStmt object (SECURITY LABEL ... ON COLUMN schema.tbl.col) + SecLabelStmt: (path: any) => { + const node = path.node; + if (node.object?.List?.items) { + transformNameList(node.object.List.items, schemaMapping, result); + } else if (node.object?.ObjectWithArgs?.objname) { + transformNameList(node.object.ObjectWithArgs.objname, schemaMapping, result); + } + }, + + // Transform DO block bodies. The body is an opaque string under the 'as' + // DefElem; schema-qualified references are rewritten with the strict + // schema-followed-by-dot pattern. + DoStmt: (path: any) => { + const node = path.node; + if (Array.isArray(node.args)) { + for (const arg of node.args) { + if (arg?.DefElem?.defname === 'as' && typeof arg.DefElem.arg?.String?.sval === 'string') { + arg.DefElem.arg.String.sval = transformSchemaRefsInString( + arg.DefElem.arg.String.sval, schemaMapping, result + ); + } + } + } + }, + }; +} + +/** + * Validate that no untransformed schema names remain in the output. + * Checks both schema-qualified references (schema.object) and standalone + * schema name contexts (ON SCHEMA, IN SCHEMA, CREATE SCHEMA, etc.). + * + * Throws an error if any schema names from the mapping are found in the output, + * indicating that the AST visitor is missing a handler for that node type. + */ +export function validateNoUntransformedSchemas( + content: string, + schemaMapping: Map +): void { + if (schemaMapping.size === 0) { + return; + } + + for (const [oldSchema, newSchema] of schemaMapping) { + const escapedSchema = escapeRegexp(oldSchema); + + // Pattern 1: quoted or unquoted schema name followed by dot (schema-qualified) + const dotPattern = new RegExp(`(?:"${escapedSchema}"|\\b${escapedSchema})(?=\\.)`, 'g'); + + // Pattern 2: standalone schema name in known SQL contexts + // Note: We don't use trailing \b because it fails after closing quotes + // (both '"' and whitespace are non-word characters, so no boundary exists). + const standalonePattern = new RegExp( + `(?:ON\\s+SCHEMA\\s+|IN\\s+SCHEMA\\s+|CREATE\\s+SCHEMA\\s+|DROP\\s+SCHEMA\\s+(?:IF\\s+EXISTS\\s+)?|SET\\s+SCHEMA\\s+)` + + `(?:"${escapedSchema}"|\\b${escapedSchema}\\b)`, + 'gi' + ); + + const dotMatches = content.match(dotPattern); + const standaloneMatches = content.match(standalonePattern); + const totalMatches = (dotMatches?.length || 0) + (standaloneMatches?.length || 0); + + if (totalMatches > 0) { + const lines = content.split('\n'); + const locations: string[] = []; + + const combinedPattern = new RegExp( + `(?:"${escapedSchema}"|\\b${escapedSchema})(?=\\.)|` + + `(?:ON\\s+SCHEMA\\s+|IN\\s+SCHEMA\\s+|CREATE\\s+SCHEMA\\s+|DROP\\s+SCHEMA\\s+(?:IF\\s+EXISTS\\s+)?|SET\\s+SCHEMA\\s+)` + + `(?:"${escapedSchema}"|\\b${escapedSchema}\\b)`, + 'gi' + ); + + for (let i = 0; i < lines.length; i++) { + if (combinedPattern.test(lines[i])) { + locations.push(` Line ${i + 1}: ${lines[i].trim()}`); + } + combinedPattern.lastIndex = 0; + } + + throw new Error( + `AST transformation incomplete: found ${totalMatches} untransformed schema name(s) "${oldSchema}" ` + + `that should have been transformed to "${newSchema}". ` + + `This indicates a missing visitor handler in create_sql_visitor or walk_plpgsql_for_schemas.\n` + + `Locations:\n${locations.join('\n')}` + ); + } + } +} + +/** + * Create a PL/pgSQL visitor that transforms schema names in PL/pgSQL-specific nodes. + */ +export function createPlpgsqlVisitor( + schemaMapping: Map, + result: SchemaTransformResult +) { + return { + PLpgSQL_type: (path: any) => { + const node = path.node; + if (node.typname) { + for (const [oldSchema, newSchema] of schemaMapping.entries()) { + if (node.typname.startsWith(oldSchema + '.')) { + const typeName = node.typname.substring(oldSchema.length + 1); + result.schemasFound.add(oldSchema); + node.typname = newSchema + '.' + typeName; + result.schemasTransformed.set(oldSchema, newSchema); + break; + } + } + } + }, + + PLpgSQL_var: (path: any) => { + const node = path.node; + if (node.refname) { + for (const [oldSchema, newSchema] of schemaMapping.entries()) { + if (node.refname.startsWith(oldSchema + '.')) { + const rest = node.refname.substring(oldSchema.length + 1); + result.schemasFound.add(oldSchema); + node.refname = newSchema + '.' + rest; + result.schemasTransformed.set(oldSchema, newSchema); + break; + } + } + } + }, + }; +} + +/** + * Transform a PLpgSQL_type typname using proper AST parsing. + */ +export function transformPlpgsqlTypeAst( + typname: string, + schemaMapping: Map, + result: SchemaTransformResult +): string { + let suffix = ''; + let baseTypname = typname; + + const rowtypeMatch = typname.match(/(%rowtype|%type)$/i); + if (rowtypeMatch) { + suffix = rowtypeMatch[1]; + baseTypname = typname.substring(0, typname.length - suffix.length); + } + + let needsTransform = false; + for (const oldSchema of schemaMapping.keys()) { + if (baseTypname.startsWith(oldSchema + '.') || baseTypname.startsWith('"' + oldSchema + '".')) { + needsTransform = true; + break; + } + } + + if (!needsTransform) { + return typname; + } + + try { + const sql = `SELECT NULL::${baseTypname}`; + const parseResult = parseSql(sql); + + if (!parseResult?.stmts?.[0]?.stmt) { + return transformPlpgsqlTypeString(typname, schemaMapping, result); + } + + const sqlVisitor = { + TypeName: (path: any) => { + const typeNode = path.node; + if (typeNode.names && Array.isArray(typeNode.names)) { + transformNameList(typeNode.names, schemaMapping, result); + } + } + }; + + walkSql(parseResult.stmts[0].stmt, sqlVisitor); + + const deparsed = Deparser.deparse(parseResult.stmts[0].stmt); + + const match = deparsed.match(/SELECT\s+NULL::(.+)/i); + if (match) { + const transformedTypname = match[1].trim().replace(/;$/, ''); + return transformedTypname + suffix; + } + + return transformPlpgsqlTypeString(typname, schemaMapping, result); + } catch { + return transformPlpgsqlTypeString(typname, schemaMapping, result); + } +} + +/** + * Fallback string-based transformation for PLpgSQL_type typname. + * Uses @pgsql/quotes QuoteUtils for proper identifier quoting. + */ +export function transformPlpgsqlTypeString( + typname: string, + schemaMapping: Map, + result: SchemaTransformResult +): string { + for (const [oldSchema, newSchema] of schemaMapping.entries()) { + // Unquoted schema: old_schema.rest + if (typname.startsWith(oldSchema + '.')) { + const rest = typname.substring(oldSchema.length + 1); + result.schemasFound.add(oldSchema); + result.schemasTransformed.set(oldSchema, newSchema); + return QuoteUtils.quoteIdentifier(newSchema) + '.' + rest; + } + // Quoted schema: "old_schema".rest + if (typname.startsWith('"' + oldSchema + '".')) { + const rest = typname.substring(oldSchema.length + 3); + result.schemasFound.add(oldSchema); + result.schemasTransformed.set(oldSchema, newSchema); + return QuoteUtils.quoteIdentifier(newSchema) + '.' + rest; + } + } + return typname; +} + +/** + * Recursively walk the PL/pgSQL AST to transform schema names. + */ +export function walkPlpgsqlForSchemas( + node: any, + schemaMapping: Map, + result: SchemaTransformResult +): void { + if (node === null || node === undefined || typeof node !== 'object') { + return; + } + + if (Array.isArray(node)) { + for (let i = 0; i < node.length; i++) { + walkPlpgsqlForSchemas(node[i], schemaMapping, result); + } + return; + } + + // Fallback for SQL expressions the plpgsql walker does not reach with the + // SQL visitor (e.g. statements inside EXCEPTION handler bodies). The + // deparser renders exprs from their query string, so rewriting mapped + // schema-followed-by-dot references here is safe and idempotent. + if ('PLpgSQL_expr' in node) { + const expr = node.PLpgSQL_expr; + if (typeof expr.query === 'string' && expr.query.includes('.')) { + expr.query = transformSchemaRefsInString(expr.query, schemaMapping, result); + } else if (expr.query && typeof expr.query === 'object' && expr.query.kind === 'sql-stmt') { + // Hydrated exprs carry { original, parseResult }; the deparser renders + // from parseResult, so visit its statements with the SQL visitor. + const sqlVisitor = createSqlVisitor(schemaMapping, result); + if (Array.isArray(expr.query.parseResult?.stmts)) { + for (const stmt of expr.query.parseResult.stmts) { + if (stmt?.stmt) { + walkSql(stmt.stmt, sqlVisitor); + } + } + } + if (typeof expr.query.original === 'string' && expr.query.original.includes('.')) { + expr.query.original = transformSchemaRefsInString(expr.query.original, schemaMapping, result); + } + } + } + + // Transform schema-qualified references inside RAISE message strings + if ('PLpgSQL_stmt_raise' in node) { + const raise = node.PLpgSQL_stmt_raise; + if (typeof raise.message === 'string' && raise.message.includes('.')) { + raise.message = transformSchemaRefsInString(raise.message, schemaMapping, result); + } + } + + if ('PLpgSQL_type' in node) { + const plType = node.PLpgSQL_type; + if (plType.typname) { + if (typeof plType.typname === 'object' && plType.typname.kind === 'type-name') { + // Transform schema names directly in the typeNameNode.names array. + // We cannot use walkSql here because the raw typeNameNode is not + // wrapped in the expected AST envelope that the traverse walker needs. + if (plType.typname.typeNameNode?.names) { + transformNameList(plType.typname.typeNameNode.names, schemaMapping, result); + } + // The deparser can fall back to the 'original' string to render + // DECLARE types, so update it as well. Rewrite only the schema + // prefix within the string so array bounds ([]) and %rowtype/%type + // suffixes are preserved. + if (typeof plType.typname.original === 'string') { + plType.typname.original = transformPlpgsqlTypeString( + plType.typname.original, + schemaMapping, + result + ); + } + } else if (typeof plType.typname === 'string') { + plType.typname = transformPlpgsqlTypeAst( + plType.typname, + schemaMapping, + result + ); + } + } + } + + for (const value of Object.values(node)) { + walkPlpgsqlForSchemas(value, schemaMapping, result); + } +} + +/** + * Escape a string for use in a regular expression + */ +export function escapeRegexp(str: string): string { + return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * Extract pgpm header comments from the beginning of SQL content. + */ +export function extractPgpmHeader(content: string): { header: string; body: string } { + const lines = content.split('\n'); + let headerEndIndex = 0; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const trimmed = line.trim(); + + if (trimmed === '') { + headerEndIndex = i + 1; + continue; + } + + if (trimmed === '--') { + headerEndIndex = i + 1; + continue; + } + + if (trimmed.startsWith('-- Deploy') || + trimmed.startsWith('-- Revert') || + trimmed.startsWith('-- Verify') || + trimmed.startsWith('-- requires:') || + trimmed.startsWith('-- made with')) { + headerEndIndex = i + 1; + continue; + } + + if (trimmed.startsWith('--')) { + headerEndIndex = i + 1; + continue; + } + + // psql meta-command guard emitted by pgpm package exports, e.g. + // \echo Use "CREATE EXTENSION foo" to load this file. \quit + if (trimmed.startsWith('\\echo')) { + headerEndIndex = i + 1; + continue; + } + + break; + } + + const headerLines = lines.slice(0, headerEndIndex); + const bodyLines = lines.slice(headerEndIndex); + + return { + header: headerLines.join('\n') + (headerLines.length > 0 ? '\n' : ''), + body: bodyLines.join('\n') + }; +} + +/** + * Transform comment headers (-- Deploy:, -- requires:, etc.) + * These are not part of the SQL AST, so we use regexp for these. + */ +export function transformComments( + content: string, + schemaMapping: Map, + result: SchemaTransformResult +): string { + const commentPattern = /^(-- (?:Deploy|requires|Revert|Verify):?\s*)(.*)$/gm; + + const schemas = Array.from(schemaMapping.keys()).sort((a, b) => b.length - a.length); + + return content.replace(commentPattern, (match, prefix, pathPart) => { + let newPath = pathPart; + + for (const schema of schemas) { + const newName = schemaMapping.get(schema); + if (!newName) continue; + + result.schemasFound.add(schema); + + const pathPattern = new RegExp(`(schemas/)${escapeRegexp(schema)}(/|$)`, 'g'); + const before = newPath; + newPath = newPath.replace(pathPattern, `$1${newName}$2`); + + if (newPath !== before) { + result.schemasTransformed.set(schema, newName); + } + } + + return prefix + newPath; + }); +} + +/** + * Transform verify function calls that use string literals. + * These are inside SQL strings and not part of the main AST. + */ +export function transformVerifyCalls( + content: string, + schemaMapping: Map, + result: SchemaTransformResult +): string { + const schemas = Array.from(schemaMapping.keys()).sort((a, b) => b.length - a.length); + + let newContent = content; + + for (const schema of schemas) { + const newName = schemaMapping.get(schema); + if (!newName) continue; + + const escapedSchema = escapeRegexp(schema); + + const verifyPattern = new RegExp( + `(verify_(?:function|table|trigger|type|domain|view|index|constraint|schema|policy|table_grant|function_grant|sequence_grant|type_grant)\\s*\\(\\s*')${escapedSchema}(\\.|'\\s*\\))`, + 'gi' + ); + + const before = newContent; + newContent = newContent.replace(verifyPattern, `$1${newName}$2`); + + if (newContent !== before) { + result.schemasFound.add(schema); + result.schemasTransformed.set(schema, newName); + } + } + + return newContent; +} + +/** + * Transform schema names inside JSON/JSONB string values. + */ +export function transformJsonStringValues( + content: string, + schemaMapping: Map, + result: SchemaTransformResult +): string { + const schemas = Array.from(schemaMapping.keys()).sort((a, b) => b.length - a.length); + + let newContent = content; + + for (const schema of schemas) { + const newName = schemaMapping.get(schema); + if (!newName) continue; + + const escapedSchema = escapeRegexp(schema); + + const jsonValuePattern = new RegExp( + `(:")${escapedSchema}(")`, + 'g' + ); + + const before = newContent; + newContent = newContent.replace(jsonValuePattern, `$1${newName}$2`); + + if (newContent !== before) { + result.schemasFound.add(schema); + result.schemasTransformed.set(schema, newName); + } + } + + return newContent; +} + +/** + * Transform a single SQL string using full AST-based transformation. + * + * This is the main entry point for transforming SQL content. It: + * 1. Runs any user-supplied pre-passes (string-level) + * 2. Extracts pgpm header comments and transforms them + * 3. Runs full AST transformation on the SQL body + * 4. Runs any user-supplied post-passes (string-level) + * 5. Validates no untransformed schema names remain + * 6. Returns the combined result + * + * App-specific string-level transforms (verify calls, JSON values, etc.) + * are NOT included by default — pass them via `options.pre_passes` or + * `options.post_passes`. The built-in passes `transform_verify_calls` + * and `transform_json_string_values` are exported for convenience. + */ +export function transformSql( + content: string, + schemaMapping: Map, + options?: TransformSqlOptions | SchemaTransformResult, + result?: SchemaTransformResult +): { content: string; result: SchemaTransformResult } { + // Support legacy signature: transform_sql(content, mapping, result?) + let opts: TransformSqlOptions = {}; + let r: SchemaTransformResult; + if (options && 'schemasFound' in options) { + // Called with legacy signature: (content, mapping, result) + r = options as SchemaTransformResult; + } else { + opts = (options as TransformSqlOptions) || {}; + r = result || createResult(); + } + + if (schemaMapping.size === 0) { + return { content, result: r }; + } + + let newContent = content; + + // Run pre-passes (app-specific string-level transforms) + if (opts.prePasses) { + for (const pass of opts.prePasses) { + newContent = pass(newContent, schemaMapping, r); + } + } + + // Main pass: AST transformation with header handling + newContent = transformSqlContentAst(newContent, schemaMapping, r, opts); + + // Run post-passes (app-specific string-level transforms) + if (opts.postPasses) { + for (const pass of opts.postPasses) { + newContent = pass(newContent, schemaMapping, r); + } + } + + return { content: newContent, result: r }; +} + +/** + * Transform a single SQL statement string using AST. + * Unlike transform_sql, this does NOT handle headers, verify calls, or JSON values. + * Use this for testing individual SQL statements. + */ +export function transformSqlStatement( + sql: string, + schemaMapping: Map, + result?: SchemaTransformResult +): { sql: string; result: SchemaTransformResult } { + const r = result || createResult(); + + if (schemaMapping.size === 0) { + return { sql, result: r }; + } + + try { + const transformed = transformSync(sql, (ctx) => { + const sqlVisitor = createSqlVisitor(schemaMapping, r); + + if (ctx.sql?.stmts) { + for (const stmt of ctx.sql.stmts) { + if (stmt?.stmt) { + walkSql(stmt.stmt, sqlVisitor); + } + } + } + + for (const fn of ctx.functions) { + if (fn.plpgsql?.hydrated) { + walkPlpgsql(fn.plpgsql.hydrated, {}, { + walkSqlExpressions: true, + sqlVisitor: sqlVisitor + }); + + walkPlpgsqlForSchemas(fn.plpgsql.hydrated, schemaMapping, r); + } + } + }, { hydrate: true, pretty: true }); + + return { sql: transformed, result: r }; + } catch (error) { + const errMsg = error instanceof Error ? error.message : String(error); + r.errors.push({ file: 'inline', error: errMsg }); + throw error; + } +} + +/** + * Internal: Transform SQL content with header extraction and AST transformation. + */ +function transformSqlContentAst( + content: string, + schemaMapping: Map, + result: SchemaTransformResult, + options?: TransformSqlOptions +): string { + if (schemaMapping.size === 0) { + return content; + } + + const roundTrip = options?.roundTrip; + const assumeSchemasExist = options?.assumeSchemasExist?.length + ? new Set(options.assumeSchemasExist) + : undefined; + + const { header, body } = extractPgpmHeader(content); + + let transformedHeader = header; + if (header.length > 0) { + transformedHeader = transformComments(header, schemaMapping, result); + } + + let transformedBody = body; + + if (options?.qualifyUnqualified && transformedBody.trim().length > 0) { + try { + transformedBody = qualifyUnqualified(transformedBody, options.qualifyUnqualified).sql; + } catch (error) { + const errMsg = error instanceof Error ? error.message : String(error); + result.errors.push({ file: 'inline', error: errMsg }); + throw error; + } + } + + if (transformedBody.trim().length > 0) { + let before: CapturedAsts | undefined; + try { + transformedBody = transformSync(transformedBody, (ctx) => { + const sqlVisitor = createSqlVisitor(schemaMapping, result, { assumeSchemasExist }); + + if (ctx.sql?.stmts) { + for (const stmt of ctx.sql.stmts) { + if (stmt?.stmt) { + walkSql(stmt.stmt, sqlVisitor); + } + } + } + + for (const fn of ctx.functions) { + if (fn.plpgsql?.hydrated) { + walkPlpgsql(fn.plpgsql.hydrated, {}, { + walkSqlExpressions: true, + sqlVisitor: sqlVisitor + }); + + walkPlpgsqlForSchemas(fn.plpgsql.hydrated, schemaMapping, result); + } + } + + if (roundTrip) { + before = captureTransformAsts(ctx); + } + }, { hydrate: true, pretty: true }); + } catch (error) { + const errMsg = error instanceof Error ? error.message : String(error); + result.errors.push({ file: 'inline', error: errMsg }); + throw error; + } + + validateNoUntransformedSchemas(transformedBody, schemaMapping); + + if (roundTrip && before) { + try { + validateRoundTrip(before, transformedBody); + } catch (error) { + const errMsg = error instanceof Error ? error.message : String(error); + result.errors.push({ file: 'inline', error: errMsg }); + throw error; + } + } + } + + return transformedHeader + transformedBody; +} diff --git a/packages/transform/tsconfig.esm.json b/packages/transform/tsconfig.esm.json new file mode 100644 index 00000000..800d7506 --- /dev/null +++ b/packages/transform/tsconfig.esm.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "dist/esm", + "module": "es2022", + "rootDir": "src/", + "declaration": false + } +} diff --git a/packages/transform/tsconfig.json b/packages/transform/tsconfig.json new file mode 100644 index 00000000..1a9d5696 --- /dev/null +++ b/packages/transform/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src/" + }, + "include": ["src/**/*.ts"], + "exclude": ["dist", "node_modules", "**/*.spec.*", "**/*.test.*"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 676e6aa9..cc9befe1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -389,6 +389,23 @@ importers: version: 0.1.8 publishDirectory: dist + packages/transform: + dependencies: + "@pgsql/quotes": + specifier: workspace:* + version: link:../quotes/dist + "@pgsql/traverse": + specifier: workspace:* + version: link:../traverse/dist + plpgsql-parser: + specifier: workspace:* + version: link:../plpgsql-parser/dist + devDependencies: + makage: + specifier: ^0.1.8 + version: 0.1.8 + publishDirectory: dist + packages/transform-ast: devDependencies: "@pgsql/parser":