diff --git a/opsqueue/migrations/20260717143000_submission_metadata_pk_by_key.down.sql b/opsqueue/migrations/20260717143000_submission_metadata_pk_by_key.down.sql new file mode 100644 index 00000000..cd4aa037 --- /dev/null +++ b/opsqueue/migrations/20260717143000_submission_metadata_pk_by_key.down.sql @@ -0,0 +1,25 @@ +CREATE TABLE submissions_metadata_old +( + submission_id INTEGER NOT NULL, + metadata_key TEXT NOT NULL, + metadata_value INTEGER NOT NULL, + + PRIMARY KEY (submission_id, metadata_key, metadata_value) +) WITHOUT ROWID, STRICT; + +INSERT INTO submissions_metadata_old (submission_id, metadata_key, metadata_value) +SELECT + submission_id + , metadata_key + , metadata_value +FROM submissions_metadata; + +DROP INDEX lookup_submission_by_metadata; +DROP TABLE submissions_metadata; +ALTER TABLE submissions_metadata_old RENAME TO submissions_metadata; + +CREATE INDEX lookup_submission_by_metadata ON submissions_metadata ( + metadata_key + , metadata_value + , submission_id +); diff --git a/opsqueue/migrations/20260717143000_submission_metadata_pk_by_key.up.sql b/opsqueue/migrations/20260717143000_submission_metadata_pk_by_key.up.sql new file mode 100644 index 00000000..fa29bb96 --- /dev/null +++ b/opsqueue/migrations/20260717143000_submission_metadata_pk_by_key.up.sql @@ -0,0 +1,29 @@ +CREATE TABLE submissions_metadata_new +( + submission_id INTEGER NOT NULL, + metadata_key TEXT NOT NULL, + metadata_value INTEGER NOT NULL, + + -- Align with StrategicMetadataMap: one value per key per submission. + PRIMARY KEY (submission_id, metadata_key) +) WITHOUT ROWID, STRICT; + +-- If legacy duplicates exist for the same (submission_id, metadata_key), +-- keep one deterministic value. +INSERT INTO submissions_metadata_new (submission_id, metadata_key, metadata_value) +SELECT + submission_id + , metadata_key + , MAX(metadata_value) AS metadata_value +FROM submissions_metadata +GROUP BY submission_id, metadata_key; + +DROP INDEX lookup_submission_by_metadata; +DROP TABLE submissions_metadata; +ALTER TABLE submissions_metadata_new RENAME TO submissions_metadata; + +CREATE INDEX lookup_submission_by_metadata ON submissions_metadata ( + metadata_key + , metadata_value + , submission_id +); diff --git a/opsqueue/src/common/submission.rs b/opsqueue/src/common/submission.rs index 1ede03ec..9dbb0521 100644 --- a/opsqueue/src/common/submission.rs +++ b/opsqueue/src/common/submission.rs @@ -1157,7 +1157,7 @@ pub mod test { insta::assert_snapshot!(explained, @" 8, 0, SEARCH s0 USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=?) 16, 0, SEARCH submissions USING COVERING INDEX sqlite_autoindex_submissions_1 (id=?) - 21, 0, SEARCH s1 USING PRIMARY KEY (submission_id=? AND metadata_key=? AND metadata_value=?) + 21, 0, SEARCH s1 USING PRIMARY KEY (submission_id=? AND metadata_key=?) "); } diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index 573475d1..5819d024 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -290,20 +290,20 @@ pub mod test { let explained = explain(qb, &mut conn).await; assert_streaming_query(qb, &explained); - insta::assert_snapshot!(explained, @r" + insta::assert_snapshot!(explained, @" 1, 0, COMPOUND QUERY 2, 1, LEFT-MOST SUBQUERY 5, 2, SCAN chunks 8, 2, CORRELATED SCALAR SUBQUERY 4 - 12, 8, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 18, 8, LIST SUBQUERY 2 - 20, 18, SCAN json_each VIRTUAL TABLE INDEX 0: - 63, 1, UNION ALL - 66, 63, SCAN chunks - 69, 63, CORRELATED SCALAR SUBQUERY 6 - 73, 69, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 79, 69, LIST SUBQUERY 2 - 81, 79, SCAN json_each VIRTUAL TABLE INDEX 0: + 12, 8, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 22, 8, LIST SUBQUERY 2 + 24, 22, SCAN json_each VIRTUAL TABLE INDEX 0: + 58, 1, UNION ALL + 61, 58, SCAN chunks + 64, 58, CORRELATED SCALAR SUBQUERY 6 + 68, 64, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 78, 64, LIST SUBQUERY 2 + 80, 78, SCAN json_each VIRTUAL TABLE INDEX 0: "); } @@ -376,20 +376,20 @@ pub mod test { let explained = explain(qb, &mut conn).await; assert_streaming_query(qb, &explained); - insta::assert_snapshot!(explained, @r" + insta::assert_snapshot!(explained, @" 1, 0, COMPOUND QUERY 2, 1, LEFT-MOST SUBQUERY 5, 2, SCAN chunks 8, 2, CORRELATED SCALAR SUBQUERY 4 - 12, 8, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 18, 8, LIST SUBQUERY 2 - 20, 18, SCAN json_each VIRTUAL TABLE INDEX 0: - 63, 1, UNION ALL - 66, 63, SCAN chunks - 69, 63, CORRELATED SCALAR SUBQUERY 6 - 73, 69, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 79, 69, LIST SUBQUERY 2 - 81, 79, SCAN json_each VIRTUAL TABLE INDEX 0: + 12, 8, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 22, 8, LIST SUBQUERY 2 + 24, 22, SCAN json_each VIRTUAL TABLE INDEX 0: + 58, 1, UNION ALL + 61, 58, SCAN chunks + 64, 58, CORRELATED SCALAR SUBQUERY 6 + 68, 64, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 78, 64, LIST SUBQUERY 2 + 80, 78, SCAN json_each VIRTUAL TABLE INDEX 0: "); } @@ -469,36 +469,36 @@ pub mod test { let explained = explain(qb, &mut conn).await; assert_streaming_query(qb, &explained); - insta::assert_snapshot!(explained, @r" + insta::assert_snapshot!(explained, @" 1, 0, COMPOUND QUERY 2, 1, LEFT-MOST SUBQUERY 3, 2, COMPOUND QUERY 4, 3, LEFT-MOST SUBQUERY 7, 4, SEARCH chunks USING INDEX random_chunks_order (random_order>?) 16, 4, CORRELATED SCALAR SUBQUERY 5 - 20, 16, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 26, 16, LIST SUBQUERY 3 - 28, 26, SCAN json_each VIRTUAL TABLE INDEX 0: - 63, 3, UNION ALL - 66, 63, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 137, 125, CORRELATED SCALAR SUBQUERY 7 - 141, 137, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 147, 137, LIST SUBQUERY 3 - 149, 147, SCAN json_each VIRTUAL TABLE INDEX 0: - 184, 124, UNION ALL - 187, 184, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 127, 115, CORRELATED SCALAR SUBQUERY 7 + 131, 127, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 141, 127, LIST SUBQUERY 3 + 143, 141, SCAN json_each VIRTUAL TABLE INDEX 0: + 169, 114, UNION ALL + 172, 169, SEARCH chunks USING INDEX random_chunks_order (random_order?) 18, 6, CORRELATED SCALAR SUBQUERY 5 - 22, 18, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 28, 18, LIST SUBQUERY 3 - 30, 28, SCAN json_each VIRTUAL TABLE INDEX 0: - 57, 6, CORRELATED SCALAR SUBQUERY 11 - 61, 57, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 67, 57, LIST SUBQUERY 9 - 69, 67, SCAN json_each VIRTUAL TABLE INDEX 0: - 104, 5, UNION ALL - 107, 104, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 217, 205, CORRELATED SCALAR SUBQUERY 7 - 221, 217, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 227, 217, LIST SUBQUERY 3 - 229, 227, SCAN json_each VIRTUAL TABLE INDEX 0: - 256, 205, CORRELATED SCALAR SUBQUERY 11 - 260, 256, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 266, 256, LIST SUBQUERY 9 - 268, 266, SCAN json_each VIRTUAL TABLE INDEX 0: - 303, 204, UNION ALL - 306, 303, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 418, 406, CORRELATED SCALAR SUBQUERY 5 - 422, 418, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 428, 418, LIST SUBQUERY 3 - 430, 428, SCAN json_each VIRTUAL TABLE INDEX 0: - 457, 406, CORRELATED SCALAR SUBQUERY 13 - 461, 457, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 467, 457, LIST SUBQUERY 9 - 469, 467, SCAN json_each VIRTUAL TABLE INDEX 0: - 504, 405, UNION ALL - 507, 504, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 617, 605, CORRELATED SCALAR SUBQUERY 7 - 621, 617, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 627, 617, LIST SUBQUERY 3 - 629, 627, SCAN json_each VIRTUAL TABLE INDEX 0: - 656, 605, CORRELATED SCALAR SUBQUERY 13 - 660, 656, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=? AND submission_id=?) - 666, 656, LIST SUBQUERY 9 - 668, 666, SCAN json_each VIRTUAL TABLE INDEX 0: - 703, 604, UNION ALL - 706, 703, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 197, 185, CORRELATED SCALAR SUBQUERY 7 + 201, 197, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 211, 197, LIST SUBQUERY 3 + 213, 211, SCAN json_each VIRTUAL TABLE INDEX 0: + 231, 185, CORRELATED SCALAR SUBQUERY 11 + 235, 231, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 245, 231, LIST SUBQUERY 9 + 247, 245, SCAN json_each VIRTUAL TABLE INDEX 0: + 273, 184, UNION ALL + 276, 273, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 378, 366, CORRELATED SCALAR SUBQUERY 5 + 382, 378, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 392, 378, LIST SUBQUERY 3 + 394, 392, SCAN json_each VIRTUAL TABLE INDEX 0: + 412, 366, CORRELATED SCALAR SUBQUERY 13 + 416, 412, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 426, 412, LIST SUBQUERY 9 + 428, 426, SCAN json_each VIRTUAL TABLE INDEX 0: + 454, 365, UNION ALL + 457, 454, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 557, 545, CORRELATED SCALAR SUBQUERY 7 + 561, 557, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 571, 557, LIST SUBQUERY 3 + 573, 571, SCAN json_each VIRTUAL TABLE INDEX 0: + 591, 545, CORRELATED SCALAR SUBQUERY 13 + 595, 591, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 605, 591, LIST SUBQUERY 9 + 607, 605, SCAN json_each VIRTUAL TABLE INDEX 0: + 633, 544, UNION ALL + 636, 633, SEARCH chunks USING INDEX random_chunks_order (random_order