From 2b042647fef5f8bce64d8d9d0dc5bb7debc780af Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 15 Jul 2026 17:07:12 +0200 Subject: [PATCH 1/4] :sparkles: new KVStore --- doc/api/index.md | 1 + doc/api/kvstore.md | 551 ++++++++++++ lib/internal/bootstrap/realm.js | 2 + lib/internal/errors.js | 6 + lib/internal/kvstore/constants.js | 59 ++ lib/internal/kvstore/keys.js | 423 +++++++++ lib/internal/kvstore/statements.js | 192 +++++ lib/internal/kvstore/utils.js | 59 ++ lib/internal/process/pre_execution.js | 10 + lib/kvstore.js | 410 +++++++++ src/node_options.cc | 5 + src/node_options.h | 1 + test/common/index.js | 9 + test/parallel/test-kvstore-key-codec.js | 319 +++++++ test/parallel/test-kvstore-module-flags.js | 38 + test/parallel/test-kvstore.js | 955 +++++++++++++++++++++ test/parallel/test-process-get-builtin.mjs | 5 +- 17 files changed, 3044 insertions(+), 1 deletion(-) create mode 100644 doc/api/kvstore.md create mode 100644 lib/internal/kvstore/constants.js create mode 100644 lib/internal/kvstore/keys.js create mode 100644 lib/internal/kvstore/statements.js create mode 100644 lib/internal/kvstore/utils.js create mode 100644 lib/kvstore.js create mode 100644 test/parallel/test-kvstore-key-codec.js create mode 100644 test/parallel/test-kvstore-module-flags.js create mode 100644 test/parallel/test-kvstore.js diff --git a/doc/api/index.md b/doc/api/index.md index 24c38d0f3a70c8..2cf9528a4ddade 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -36,6 +36,7 @@ * [Inspector](inspector.md) * [Internationalization](intl.md) * [Iterable Streams API](stream_iter.md) +* [KV store](kvstore.md) * [Modules: CommonJS modules](modules.md) * [Modules: ECMAScript modules](esm.md) * [Modules: `node:module` API](module.md) diff --git a/doc/api/kvstore.md b/doc/api/kvstore.md new file mode 100644 index 00000000000000..93103ae3488666 --- /dev/null +++ b/doc/api/kvstore.md @@ -0,0 +1,551 @@ +# KV store + + + + + +> Stability: 1 - Experimental. + + + +The `node:kvstore` module facilitates working with a KV store database. +To access it: + +```mjs +import { openKv } from 'node:kvstore'; +``` + +```cjs +const { openKv } = require('node:kvstore'); +``` + +This module is only available under the `node:` scheme. The current +implementation is backed by the `node:sqlite` module, and every API it +exposes is synchronous (matching `node:sqlite`'s own `DatabaseSync`). + +The following example shows the basic usage of the `node:kvstore` module to +open an in-memory key-value database, write data to the database, and then +read the data back. + +```mjs +import { openKv } from 'node:kvstore'; +const kv = openKv(); + +// set few values +kv.set('testing-key', { text: 'Hello' }); +kv.set('other-key', { text: 'World' }); + +// fetch multiple keys +const entries = kv.getMany(['testing-key', 'other-key']); +// Log the entries for the given keys +console.log(entries); +// Prints: [{key: ['testing-key'], value: {text: "Hello"}}, {key: ['other-key'], value: {text: "World"}}] + +// get the list of keys within the store as an iterator +for (const key of kv.keys({ prefix: [] })) { + // Log the list of keys in the store + console.log(key); + // Prints: { key: ['testing-key'] } + // Prints: { key: ['other-key'] } +} +``` + +```cjs +'use strict'; +const { openKv } = require('node:kvstore'); +const kv = openKv(); + +// set few values +kv.set('testing-key', { text: 'Hello' }); +kv.set('other-key', { text: 'World' }); + +// fetch multiple keys +const entries = kv.getMany(['testing-key', 'other-key']); +// Log the entries for the given keys +console.log(entries); +// Prints: [{key: ['testing-key'], value: {text: "Hello"}}, {key: ['other-key'], value: {text: "World"}}] + +// get the list of keys within the store as an iterator +for (const key of kv.keys({ prefix: [] })) { + // Log the list of keys in the store + console.log(key); + // Prints: { key: ['testing-key'] } + // Prints: { key: ['other-key'] } +} +``` + +## The `KeyValue` type + +A key can be either a `string` or an array of `string`, `boolean`, `number`, +or `bigint` values. The `string` notation is just a shorthand for `[string]`. + +Composite keys sort in a fixed, well-defined order — both across segments of +the same type and across types: `string < number < bigint < boolean`. This +makes range and prefix queries ("give me everything between key A and key +B") behave the way you'd expect from any ordered key-value store. + +Segment-level notes: + +* **Strings** sort by UTF-8 byte order. Any string is valid except the + literal empty string `''` — whitespace-only strings (e.g. `' '`) are fine. +* **Numbers** are IEEE-754 doubles and sort numerically (not + lexicographically): `-300 < -5 < 0 < 5 < 300`. `NaN` is **not** a valid key + segment (it can't be compared equal to itself) and throws. + `Infinity`/`-Infinity` are valid and sort at the extreme ends of the number + range. `-0` is normalized to `0` — they encode and compare identically. +* **Bigints** sort numerically with arbitrary precision, up to a magnitude of + 255 bytes (about 2040 bits — far beyond any realistic id or timestamp). A + larger bigint throws `ERR_KVSTORE_BIGINT_TOO_LARGE` (see [Errors][] below). +* **Booleans** sort `false < true`. + +A key's encoded form must not exceed 1024 bytes; oversized keys throw. + +## Type conversion between JavaScript and stored values + +Values passed to [`kv.set(key, value)`][] are serialized with Node's +`v8.serialize` (the same algorithm used by `worker_threads.postMessage`). The +following round-trip losslessly: + +* primitives, including `undefined` and `BigInt` +* `Date`, `RegExp`, `Map`, `Set`, typed arrays, `ArrayBuffer`, `DataView` +* sparse arrays and circular references + +What's **not** preserved: + +* functions (cannot be cloned) +* class identity — instances round-trip as plain objects/Maps/Sets +* DOM-only types (irrelevant in Node) + +The on-disk format is V8-internal: stable across Node versions, but not +portable to other runtimes (Bun, Deno, browsers). Pick a wire format like +CBOR if cross-runtime portability matters. + +## Class: `KVStore` + + + +This class represents a single connection to a KV store. All APIs exposed by +this class execute synchronously. Instances are created with +[`kvstore.openKv([options])`][], not with `new` — `KVStore` is exported only +so callers can perform `instanceof` checks and reference the type. + +**Blocking I/O.** Every method that touches the database ([`kv.get(key)`][], +[`kv.set(key, value)`][], [`kv.delete(key)`][], [`kv.clear()`][], +[`kv.keys(selector)`][], [`kv.getMany(keys)`][]) blocks the event loop for the +duration of the underlying `node:sqlite` call — there is no async variant, +matching `node:sqlite`'s own `DatabaseSync`. For an in-memory store this is +sub-microsecond per call and rarely noticeable. For a file-backed store, each +write also waits on a disk `fsync` (mitigated, not eliminated, by the +always-on `WAL` mode described under [`kvstore.openKv([options])`][] below): +on a representative local SSD, `kv.set()` on a file-backed store measured +around 20 microseconds per call (~47,000 ops/sec) — small in isolation, but +on a single-threaded event loop every concurrent request pays that latency +serially. Avoid file-backed `KVStore` calls directly inside a hot request +path serving concurrent traffic; prefer batching writes, moving them off the +request path, or using a worker thread if write volume is high. + +### `kv.clear()` + + + +Deletes every key in the database. + +### `kv.close()` + + + +Closes the connection to the store. If the connection is already closed, +this method does nothing. + +### `kv.delete(key)` + + + +* `key` {KeyValue} The key to delete. +* Returns: {boolean} `true` if an entry was deleted, `false` otherwise. + +Removes the entry associated with `key`, if one exists. + +### `kv.get(key)` + + + +* `key` {KeyValue} +* Returns: {Object} + * `key` {KeyValue} + * `value` {any} `null` if no value exists for the given key. + +Retrieves the value associated with `key`. + +**Note:** a stored value that genuinely *is* `null` is indistinguishable +from an absent key in 1.0 — both return `value: null`. Disambiguating the +two (e.g. via a returned `versionstamp`) is tracked for a 1.x release; +there's no workaround today beyond not storing `null` as a value where the +distinction matters to your application. + +```mjs +import { openKv } from 'node:kvstore'; +const kv = openKv(); + +kv.set('hello', { text: 'World' }); + +console.log(kv.get('hello')); +// Prints: {key: ['hello'], value: {text: "World"}} +``` + +```cjs +const { openKv } = require('node:kvstore'); +const kv = openKv(); + +kv.set('hello', { text: 'World' }); + +console.log(kv.get('hello')); +// Prints: {key: ['hello'], value: {text: "World"}} +``` + +### `kv.getMany(keys)` + + + +* `keys` {KeyValue\[]} An array of keys to retrieve from the store. +* Returns: {Object\[]} An array the same length as `keys`, with entries in + the same order as `keys`. Each entry has the same shape as + [`kv.get(key)`][]'s return value. + +Retrieves the values associated with each of `keys`. If no value exists for +a given key, its entry has a `null` value (same caveat as `kv.get()` above). + +### `kv.keys(selector)` + + + +* `selector` {Object} Either a prefix-based selector or a range-based + selector: + * `prefix` {KeyValue} Matches keys nested under `prefix`, never the prefix + key itself. An empty prefix (`[]`) matches every key in the store — this + is the recipe for a full scan; there is no separate "list everything" + method, precisely so that scanning the whole store is always an + explicit, visible choice at the call site. + * `start` {KeyValue} Inclusive lower bound. Can be combined with `prefix` + as an offset (but not together with `end` both falling outside the + prefix's key space). + * `end` {KeyValue} Inclusive upper bound. Same combination rules as + `start`. +* Returns: {Iterable} An iterator of `{ key: KeyValue }` objects. + +Retrieves the keys matching `selector`. + +```mjs +import { openKv } from 'node:kvstore'; +const kv = openKv(); + +kv.set('testing-key', { text: 'Hello' }); +kv.set('other-key', { text: 'World' }); + +// get the list of keys within the store as an iterator +for (const key of kv.keys({ prefix: [] })) { + // Log the list of keys in the store + console.log(key); + // Prints: { key: ['testing-key'] } + // Prints: { key: ['other-key'] } +} +``` + +```cjs +const { openKv } = require('node:kvstore'); +const kv = openKv(); + +kv.set('testing-key', { text: 'Hello' }); +kv.set('other-key', { text: 'World' }); + +// get the list of keys within the store as an iterator +for (const key of kv.keys({ prefix: [] })) { + // Log the list of keys in the store + console.log(key); + // Prints: { key: ['testing-key'] } + // Prints: { key: ['other-key'] } +} +``` + +### `kv.publish(topic, payload)` + + + +* `topic` {string} A non-empty topic name. +* `payload` {any} Any value, delivered to topic subscribers as-is. + +Publishes `payload` to `topic`. Topics live in their own namespace, +independent of the key store — see [Topic plane][] below. + +```mjs +kv.publish('events.signup', { userId: 42 }); +``` + +### `kv.set(key, value)` + + + +* `key` {KeyValue} The key to set. +* `value` {any} Any structured-cloneable value to store. See + [Type conversion between JavaScript and stored values][]. +* Returns: `undefined` + +Sets the value associated with `key`, overwriting any existing value at that +key. Throws on error; otherwise returns nothing — there's no `{ ok }` +envelope, since under SQLite UPSERT semantics a no-op write of an identical +value would also report zero changed rows, and `{ ok: false }` would +misleadingly read as "the write failed." + +```mjs +import { openKv } from 'node:kvstore'; +const kv = openKv(); + +kv.set('testing-key', { text: 'Hello' }); +// ... +kv.set('testing-key', { text: 'World' }); + +console.log(kv.get('testing-key')); +// Prints { key: ['testing-key'], value: {text: 'World'}} +``` + +```cjs +const { openKv } = require('node:kvstore'); +const kv = openKv(); + +kv.set('testing-key', { text: 'Hello' }); +// ... +kv.set('testing-key', { text: 'World' }); + +console.log(kv.get('testing-key')); +// Prints { key: ['testing-key'], value: {text: 'World'}} +``` + +### `kv.watch(selector[, options])` + + + +* `selector` {Object} Exactly one of: + * `key` {KeyValue} Fires on mutations to that exact key (and on `clear`). + Also accepts `events` (see below). + * `keys` {KeyValue\[]} Fires on mutations to any of the listed exact keys + (and on `clear`). Also accepts `events`. + * `prefix` {KeyValue} Fires on mutations to any key under the prefix (and + on `clear`). An empty prefix (`[]`) watches every key-plane mutation in + the store. Also accepts `events`. + * `topic` {string} Fires on [`kv.publish(topic, payload)`][] calls. + * `events` {string\[]} For the `key`/`keys`/`prefix` forms, restricts + delivery to a subset of `'set'`, `'delete'`, `'clear'`. +* `options` {Object} + * `highWaterMark` {number} Per-watcher buffer size. Must be `>= 1` (throws + otherwise). If the buffer is full, intervening events are dropped and a + single `{type:'lag', dropped}` event is delivered once space frees up. + **Default:** `1024`. +* Returns: {stream.Readable} An object-mode `Readable`, also usable as an + `AsyncIterable`, that emits one event per push. Call `stream.destroy()` to + unsubscribe. + +Watches mutations on the store, or messages on a custom topic. + +**Scope.** Like the topic plane (see [Topic plane][] below), `key`/`keys`/ +`prefix` watchers only observe mutations made through the *same* `KVStore` +instance, in the same process. A file-backed store mutated by another +process, or by a second `openKv({ path })` call in the same process pointed +at the same file, produces no watch events on this instance — `node:sqlite` +gives this module no cross-connection change feed to build on. If you need +writes from another process or instance to be observable, poll +[`kv.keys(selector)`][] instead. + +Key / keys / prefix watchers emit one of: + +```js +{ type: 'set', key: KeyValue, value: any } +{ type: 'delete', key: KeyValue } +{ type: 'clear' } +{ type: 'lag', dropped: number } // see Backpressure +``` + +Topic watchers emit the published payload directly (no envelope). + +```mjs +const sub = kv.watch({ prefix: ['user'] }); +for await (const event of sub) { + // { type: 'set', key: ['user', 42], value: {...} } | ... +} +``` + +> **Migration from 0.x.** The legacy positional form `kv.watch(keys[])` has +> been replaced. Use `kv.watch({ keys: [...] })` instead, and note that the +> stream now emits one event per change (not a positional `entries[]` array +> with `null` placeholders). + +#### Topic plane + +`kv.publish()`/`kv.watch({ topic })` is a separate plane from key-value +mutations, with its own guarantees: + +* **Scope.** Topics live only in the same Node process as the `KVStore` + instance — they do not cross process boundaries, do not persist to disk, + and do not cross machines. Restarting the process, or opening a second + `KVStore` pointed at the same file, gets no history. +* **Isolation from the key plane.** `publish()` never touches the underlying + table; key-plane `set`/`delete`/`clear` never appear on topic streams; + topic names share no namespace with keys (a topic named `'user'` and a key + `['user']` are unrelated). +* **Ordering.** Within a single topic, payloads delivered to one watcher + arrive FIFO (publish order). There is no ordering guarantee *across* + different topics. +* **Delivery.** Best-effort, in-process, at-most-once. No retries, no + persistence — a payload published while no watcher is attached to that + topic is simply gone. +* **Lifecycle.** Topic streams end when [`kv.close()`][] is called, exactly + like key-plane watchers. + +#### Backpressure + +Watchers must consume events in a timely manner. Each watcher has its own +independent buffer — a slow watcher never affects any other watcher or the +publisher. + +* **Drop policy.** When a watcher's buffer reaches `highWaterMark`, the + *new* incoming event is dropped (the buffer keeps what it already has, + rather than evicting older events). +* **`lag` event.** Exactly one `{type:'lag', dropped}` event is emitted per + drop episode, once the buffer has drained back below `highWaterMark`, and + always *before* the next non-`lag` event is delivered. `dropped` counts + every event lost since the previous `lag` event (or since the stream + started) and is always `> 0`. +* **`close()` during a drop window.** [`kv.close()`][] ends every + outstanding stream immediately. Any pending `lag` event for an unflushed + drop window is **not** emitted — the stream simply ends. + +## `kvstore.openKv([options])` + + + +* `options` {Object} Configuration options for the store connection. + * `path` {string} On-disk database file. When omitted, the store is + ephemeral (in-memory) — `openKv()` with no arguments has no disk side + effect. +* Returns: {KVStore} + +Creates a new [`KVStore`][] instance and establishes a connection to it. This +is the documented entry point for obtaining a `KVStore`; all APIs it exposes +execute synchronously (unless stated otherwise). + +```mjs +import { openKv } from 'node:kvstore'; + +const ephemeral = openKv(); +const persistent = openKv({ path: './my-store.sqlite' }); +``` + +```cjs +'use strict'; +const { openKv } = require('node:kvstore'); + +const ephemeral = openKv(); +const persistent = openKv({ path: './my-store.sqlite' }); +``` + +**Storage.** A file-backed store runs with `PRAGMA journal_mode=WAL` and +`PRAGMA synchronous=NORMAL` always on (there is no option to disable this). +This means two sidecar files, `-wal` and `-shm`, exist alongside +the database file while it's open. They're removed automatically on a clean +[`kv.close()`][]. If you copy or back up the database file while a store has +it open, copy the sidecars too (or checkpoint first) — copying just the +`.sqlite` file mid-write is not safe. + +**Security: only open files you trust.** `path` must point at a database +this module (or a trusted process) created. A stored value is read back with +[`v8.deserialize()`][] (see [Type conversion between JavaScript and stored +values][]), which — like `JSON.parse()`, but for the V8-internal wire format +— executes as soon as any value in the file is read, including via +[`kv.keys(selector)`][] range/prefix scans. Opening a `path` containing +attacker-controlled bytes means running `v8.deserialize()` over data you +don't control; treat an untrusted `.sqlite` file the same way you'd treat any +other untrusted serialized-object input, and don't accept one from a +different trust boundary (e.g. a file uploaded by a user) without validating +it out-of-band first. + +## Errors + +Every error thrown by this module is a plain `Error`, `TypeError`, or +`RangeError` instance carrying a stable `.code` string, so callers can branch +on failure kind without parsing `.message`: + +| `.code` | Base | Thrown when | +| ------------------------------- | ------------ | --------------------------------------------------------------------------------------------------------- | +| `ERR_INVALID_ARG_TYPE` | `TypeError` | A key segment, argument, or selector field has the wrong JavaScript type. | +| `ERR_INVALID_ARG_VALUE` | `TypeError` | A key segment or argument has the right type but an invalid value (e.g. `NaN`, empty string, or an oversized encoded key). | +| `ERR_OUT_OF_RANGE` | `RangeError` | A numeric argument is outside its valid range (e.g. `highWaterMark < 1`). | +| `ERR_KVSTORE_CLOSED` | `Error` | Any method called after [`kv.close()`][]. | +| `ERR_KVSTORE_BIGINT_TOO_LARGE` | `RangeError` | A bigint key segment's magnitude exceeds 255 bytes. | +| `ERR_INVALID_STATE` | `Error` | Stored data is corrupt (cannot be decoded as a valid key or value). | + +```mjs +import { openKv } from 'node:kvstore'; + +const kv = openKv(); +kv.close(); +try { + kv.get('x'); +} catch (err) { + if (err.code === 'ERR_KVSTORE_CLOSED') { + console.log(err instanceof Error); // true + } +} +``` + +```cjs +'use strict'; +const { openKv } = require('node:kvstore'); + +const kv = openKv(); +kv.close(); +try { + kv.get('x'); +} catch (err) { + if (err.code === 'ERR_KVSTORE_CLOSED') { + console.log(err instanceof Error); // true + } +} +``` + +[Errors]: #errors +[Topic plane]: #topic-plane +[Type conversion between JavaScript and stored values]: #type-conversion-between-javascript-and-stored-values +[`KVStore`]: #class-kvstore +[`kv.clear()`]: #kvclear +[`kv.close()`]: #kvclose +[`kv.delete(key)`]: #kvdeletekey +[`kv.get(key)`]: #kvgetkey +[`kv.getMany(keys)`]: #kvgetmanykeys +[`kv.keys(selector)`]: #kvkeysselector +[`kv.publish(topic, payload)`]: #kvpublishtopic-payload +[`kv.set(key, value)`]: #kvsetkey-value +[`v8.deserialize()`]: v8.md#v8deserializebuffer +[`kvstore.openKv([options])`]: #kvstoreopenkvoptions diff --git a/lib/internal/bootstrap/realm.js b/lib/internal/bootstrap/realm.js index 8a4d179806aa53..2c18cf1b5fbf03 100644 --- a/lib/internal/bootstrap/realm.js +++ b/lib/internal/bootstrap/realm.js @@ -126,6 +126,7 @@ const legacyWrapperList = new SafeSet([ const schemelessBlockList = new SafeSet([ 'dtls', 'ffi', + 'kvstore', 'sea', 'sqlite', 'quic', @@ -137,6 +138,7 @@ const schemelessBlockList = new SafeSet([ const experimentalModuleList = new SafeSet([ 'dtls', 'ffi', + 'kvstore', 'quic', 'sqlite', 'stream/iter', diff --git a/lib/internal/errors.js b/lib/internal/errors.js index 03d27332c894d8..2a2a576924ac37 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -1596,6 +1596,12 @@ E('ERR_IPC_SYNC_FORK', 'IPC cannot be used with synchronous forks', Error); E('ERR_IP_BLOCKED', function(ip) { return `IP(${ip}) is blocked by net.BlockList`; }, Error); +E( + 'ERR_KVSTORE_BIGINT_TOO_LARGE', + 'bigint magnitude exceeds %s bytes and cannot be encoded as a key segment', + RangeError, +); +E('ERR_KVSTORE_CLOSED', 'Database is closed', Error); E( 'ERR_LOADER_CHAIN_INCOMPLETE', '"%s" did not call the next hook in its chain and did not' + diff --git a/lib/internal/kvstore/constants.js b/lib/internal/kvstore/constants.js new file mode 100644 index 00000000000000..ed7ec9a560bcf6 --- /dev/null +++ b/lib/internal/kvstore/constants.js @@ -0,0 +1,59 @@ +// Auto-generated by scripts/bundle.mjs — do not edit; edit src/constants.mjs in NoNoSQLite. +'use strict'; + + +// Key segment type tags. The tag byte is the first byte of every encoded +// segment, so its numeric value fixes the cross-type sort order for +// composite keys under BLOB memcmp comparison: string < number < bigint < boolean. +// 0x00 is intentionally unused as a tag — it is reserved as the "lower than +// any real segment" sentinel used only when computing prefix scan bounds +// (see `prefixBounds` in keys.mjs), and must never collide with a real tag. +const TAG_STRING = 0x01; +const TAG_NUMBER = 0x02; +const TAG_BIGINT = 0x03; +const TAG_BOOLEAN = 0x04; + +// Sentinel appended to compute a prefix scan's inclusive lower bound; never +// part of a stored key. Guaranteed smaller than every real tag above, so +// `prefix + PREFIX_LOW_MARKER` sorts after an exact match on `prefix` alone +// but before any real segment that extends it. +const PREFIX_LOW_MARKER = "\x00"; + +// String segment framing. The payload is escaped so that a literal +// occurrence of either byte below is unambiguous, then closed with an +// unescaped STRING_TERMINATOR. Both values are smaller than any unescaped +// pass-through byte (0x02-0xFF), which is what keeps escaped low-value bytes +// sorting correctly relative to unescaped ones — the framing has to be +// order-preserving, not just parseable. +const STRING_TERMINATOR = "\x00"; +const STRING_ESCAPE = "\x01"; + +// Arbitrary-precision bigint segments carry a single length byte recording +// the magnitude's byte length, capping magnitudes at 255 bytes (~2040 bits) — +// far beyond any realistic id or timestamp. Larger bigints throw +// ERR_KVSTORE_BIGINT_TOO_LARGE rather than silently truncating or corrupting +// order. +const MAX_BIGINT_MAGNITUDE_BYTES = 255; + +// Fixed internal table name — the `namespace` option was cut from the +// public surface entirely, so there's only ever one table. +const TABLE_NAME = "kvstore_v1"; + +// Chunk size for getMany's IN (...) query. Kept comfortably below every +// known SQLITE_MAX_VARIABLE_NUMBER default (999 pre-3.32.0, 32766 from +// 3.32.0 on) so getMany works for arbitrarily large key lists without +// depending on — or crashing past — the runtime's compiled-in limit. +const MAX_IN_CLAUSE_VARIABLES = 500; + +module.exports = { + MAX_BIGINT_MAGNITUDE_BYTES, + MAX_IN_CLAUSE_VARIABLES, + PREFIX_LOW_MARKER, + STRING_ESCAPE, + STRING_TERMINATOR, + TABLE_NAME, + TAG_BIGINT, + TAG_BOOLEAN, + TAG_NUMBER, + TAG_STRING, +}; diff --git a/lib/internal/kvstore/keys.js b/lib/internal/kvstore/keys.js new file mode 100644 index 00000000000000..d4dc76f8b32398 --- /dev/null +++ b/lib/internal/kvstore/keys.js @@ -0,0 +1,423 @@ +// Auto-generated by scripts/bundle.mjs — do not edit; edit src/keys.mjs in NoNoSQLite. +'use strict'; + +const { + ArrayIsArray, + ArrayPrototypeJoin, + ArrayPrototypeMap, + ArrayPrototypeSlice, + ArrayPrototypeSome, + BigIntPrototypeToString, + NumberIsNaN, + ObjectIs, + StringFromCharCode, + StringPrototypeCharCodeAt, + StringPrototypeSlice, + StringPrototypeStartsWith, + TypedArrayPrototypeMap, +} = primordials; + +const { + MAX_BIGINT_MAGNITUDE_BYTES, + PREFIX_LOW_MARKER, + STRING_ESCAPE, + STRING_TERMINATOR, + TAG_BIGINT, + TAG_BOOLEAN, + TAG_NUMBER, + TAG_STRING, +} = require('internal/kvstore/constants'); +const { + exceedsKeyByteLimit, + getType, + isString, + UNKNOWN_TYPE, +} = require('internal/kvstore/utils'); +const { + ERR_INVALID_ARG_TYPE, + ERR_INVALID_ARG_VALUE, + ERR_INVALID_STATE, + ERR_KVSTORE_BIGINT_TOO_LARGE, + ERR_OUT_OF_RANGE, +} = require('internal/errors').codes; + + +// context: 'key' | 'prefix' — determines which error code is thrown. +// Two separate branches (one per context) so every throw is a direct +// `throw new ERR_KVSTORE_*` that can carry a `// @core:` annotation for +// the Node-core CJS bundle target. +function validateSegments(keys, context) { + if (ArrayPrototypeSome(keys, (key) => isString(key) && key === "")) { + if (context === "key") { + throw new ERR_INVALID_ARG_VALUE('key', keys, 'segments cannot be an empty string'); + } + throw new ERR_INVALID_ARG_VALUE('selector.prefix', keys, 'segments cannot be an empty string'); + } + if (ArrayPrototypeSome(keys, (key) => getType(key) === UNKNOWN_TYPE)) { + if (context === "key") { + throw new ERR_INVALID_ARG_VALUE('key', keys, 'segments must be a string, number, bigint, or boolean'); + } + throw new ERR_INVALID_ARG_VALUE('selector.prefix', keys, 'segments must be a string, number, bigint, or boolean'); + } + if (ArrayPrototypeSome(keys, (key) => typeof key === "number" && NumberIsNaN(key))) { + if (context === "key") { + throw new ERR_INVALID_ARG_VALUE('key', keys, 'NaN is not a valid key segment'); + } + throw new ERR_INVALID_ARG_VALUE('selector.prefix', keys, 'NaN is not a valid key segment'); + } +} + +function validateKeys(keys) { + if (!ArrayIsArray(keys)) { + throw new ERR_INVALID_ARG_TYPE('key', 'Array or string', keys); + } + if (keys.length < 1) { + throw new ERR_INVALID_ARG_VALUE('key', keys, 'must have at least one segment'); + } + validateSegments(keys, "key"); +} + +/** + * Looser than `validateKeys`: an empty array is a valid prefix meaning + * "no constraint, match every key" — `{ prefix: [] }` is the documented + * full-scan recipe. + */ +function validatePrefixSegments(keys) { + if (!ArrayIsArray(keys)) { + throw new ERR_INVALID_ARG_TYPE('selector.prefix', 'Array or string', keys); + } + validateSegments(keys, "prefix"); +} + +/** + * Copies an array input so the normalized key returned to callers (and + * stashed in `get`/`getMany`/`delete`/`watch` results) never aliases the + * caller's own array — mirrors the clone-on-read guarantee already made for + * values (see `set()`'s doc comment in kv.mjs). The string-shorthand path + * already allocates a fresh array, so only the array path needs a copy. + */ +function toOwnedArray(rawKeys) { + if (isString(rawKeys)) return [rawKeys]; + return ArrayIsArray(rawKeys) ? ArrayPrototypeSlice(rawKeys) : rawKeys; +} + +function normalizeKeys(rawKeys) { + const keys = toOwnedArray(rawKeys); + validateKeys(keys); + return keys; +} + +function normalizePrefixKeys(rawKeys) { + const keys = toOwnedArray(rawKeys); + validatePrefixSegments(keys); + return keys; +} + +function toBinaryString(buf) { + return buf.toString("latin1"); +} + +function toBuffer(binaryString) { + return Buffer.from(binaryString, "latin1"); +} + +// --- string segment: escape+terminate, preserves UTF-8 byte order --------- + +function encodeStringSegment(value) { + const bytes = Buffer.from(value, "utf8"); + let out = StringFromCharCode(TAG_STRING); + for (const byte of bytes) { + const ch = StringFromCharCode(byte); + out += + ch === STRING_TERMINATOR || ch === STRING_ESCAPE + ? STRING_ESCAPE + ch + : ch; + } + return out + STRING_TERMINATOR; +} + +function decodeStringSegment(str, offset) { + let raw = ""; + let i = offset; + for (;;) { + const ch = str[i]; + if (ch === undefined) { + throw new ERR_INVALID_STATE('corrupt key encoding: truncated string segment'); + } + if (ch === STRING_ESCAPE) { + raw += str[i + 1]; + i += 2; + continue; + } + if (ch === STRING_TERMINATOR) { + i += 1; + break; + } + raw += ch; + i += 1; + } + return { value: toBuffer(raw).toString("utf8"), nextOffset: i }; +} + +// --- number segment: sign-flipped float64, order-preserving --------------- + +const SIGN_MASK = 0x8000000000000000n; +const ALL_ONES_64 = 0xffffffffffffffffn; + +function encodeNumberSegment(value) { + if (NumberIsNaN(value)) { + throw new ERR_INVALID_ARG_VALUE('key', value, 'NaN is not a valid key segment'); + } + const normalized = ObjectIs(value, -0) ? 0 : value; + const raw = Buffer.alloc(8); + raw.writeDoubleBE(normalized, 0); + const bits = raw.readBigUInt64BE(0); + const flipped = bits & SIGN_MASK ? ~bits & ALL_ONES_64 : bits | SIGN_MASK; + const out = Buffer.alloc(8); + out.writeBigUInt64BE(flipped, 0); + return StringFromCharCode(TAG_NUMBER) + toBinaryString(out); +} + +function decodeNumberSegment(str, offset) { + const buf = toBuffer(str.slice(offset, offset + 8)); + const bits = buf.readBigUInt64BE(0); + const unflipped = + bits & SIGN_MASK ? bits & ~SIGN_MASK & ALL_ONES_64 : ~bits & ALL_ONES_64; + const out = Buffer.alloc(8); + out.writeBigUInt64BE(unflipped, 0); + return { value: out.readDoubleBE(0), nextOffset: offset + 8 }; +} + +// --- bigint segment: sign + order-corrected length + magnitude ------------ + +function bigintMagnitudeBytes(abs) { + if (abs === 0n) return Buffer.alloc(0); + let hex = BigIntPrototypeToString(abs, 16); + if (hex.length % 2 !== 0) hex = `0${hex}`; + return Buffer.from(hex, "hex"); +} + +function encodeBigintSegment(value) { + const negative = value < 0n; + const magnitude = bigintMagnitudeBytes(negative ? -value : value); + if (magnitude.length > MAX_BIGINT_MAGNITUDE_BYTES) { + throw new ERR_KVSTORE_BIGINT_TOO_LARGE(MAX_BIGINT_MAGNITUDE_BYTES); + } + // For negative values, both the length byte and every magnitude byte are + // bit-complemented so that a larger magnitude (a more negative number) + // sorts *before* a smaller one. + const signByte = negative ? 0 : 1; + const lengthByte = negative ? 0xff ^ magnitude.length : magnitude.length; + const magBytes = negative + ? Buffer.from(TypedArrayPrototypeMap(magnitude, (b) => 0xff ^ b)) + : magnitude; + return ( + StringFromCharCode(TAG_BIGINT, signByte, lengthByte) + + toBinaryString(magBytes) + ); +} + +function decodeBigintSegment(str, offset) { + const signByte = StringPrototypeCharCodeAt(str, offset); + const negative = signByte === 0; + const rawLengthByte = StringPrototypeCharCodeAt(str, offset + 1); + const length = negative ? 0xff ^ rawLengthByte : rawLengthByte; + const start = offset + 2; + let magBytes = toBuffer(StringPrototypeSlice(str, start, start + length)); + if (negative) magBytes = Buffer.from(TypedArrayPrototypeMap(magBytes, (b) => 0xff ^ b)); + const abs = + magBytes.length === 0 ? 0n : BigInt(`0x${magBytes.toString("hex")}`); + return { value: negative ? -abs : abs, nextOffset: start + length }; +} + +// --- boolean segment: single byte ------------------------------------------ + +function encodeBooleanSegment(value) { + return StringFromCharCode(TAG_BOOLEAN, value ? 1 : 0); +} + +function decodeBooleanSegment(str, offset) { + return { value: StringPrototypeCharCodeAt(str, offset) === 1, nextOffset: offset + 1 }; +} + +// --- segment dispatch ------------------------------------------------------- + +function encodeSegment(segment) { + const type = getType(segment); + switch (type) { + case "string": + return encodeStringSegment(segment); + case "number": + return encodeNumberSegment(segment); + case "bigint": + return encodeBigintSegment(segment); + case "boolean": + return encodeBooleanSegment(segment); + default: + throw new ERR_INVALID_ARG_VALUE('key', segment, 'segments must be a string, number, bigint, or boolean'); + } +} + +function decodeSegment(str, offset) { + const tag = StringPrototypeCharCodeAt(str, offset); + switch (tag) { + case TAG_STRING: + return decodeStringSegment(str, offset + 1); + case TAG_NUMBER: + return decodeNumberSegment(str, offset + 1); + case TAG_BIGINT: + return decodeBigintSegment(str, offset + 1); + case TAG_BOOLEAN: + return decodeBooleanSegment(str, offset + 1); + default: + throw new ERR_INVALID_STATE('corrupt key encoding: unknown segment tag'); + } +} + +function serializeKeys(keys) { + return ArrayPrototypeJoin(ArrayPrototypeMap(keys, encodeSegment), ""); +} + +function deserializeKeys(serialized) { + const segments = []; + let offset = 0; + while (offset < serialized.length) { + const { value, nextOffset } = decodeSegment(serialized, offset); + segments.push(value); + offset = nextOffset; + } + return segments; +} + +/** + * Normalize and serialize a raw key in one pass — the entry point for a + * *concrete* key (get/set/delete, range bounds, watch key/keys). Requires + * at least one segment. + * @param {string|Array} rawKey + * @returns {{normalized: Array, encoded: string}} + */ +function encodeKey(rawKey) { + const normalized = normalizeKeys(rawKey); + const encoded = serializeKeys(normalized); + if (exceedsKeyByteLimit(encoded)) { + throw new ERR_OUT_OF_RANGE('key', `<= ${MAX_KEY_BYTE_LENGTH} bytes`, `${encoded.length} bytes`); + } + return { normalized, encoded }; +} + +/** + * Like `encodeKey`, but for a selector's `prefix` field: an empty array is + * valid and means "match every key" (see `validatePrefixSegments`). + * @param {string|Array} rawPrefix + * @returns {{normalized: Array, encoded: string}} + */ +function encodePrefix(rawPrefix) { + const normalized = normalizePrefixKeys(rawPrefix); + const encoded = serializeKeys(normalized); + if (exceedsKeyByteLimit(encoded)) { + throw new ERR_OUT_OF_RANGE('selector.prefix', `<= ${MAX_KEY_BYTE_LENGTH} bytes`, `${encoded.length} bytes`); + } + return { normalized, encoded }; +} + +/** + * Compute the [start, end) bounds of an encoded prefix for sargable range + * queries under BLOB memcmp ordering. `end` is `null` when the prefix has + * no finite successor (every byte is 0xFF) — callers must treat that as + * "no upper bound" rather than binding a literal null-as-value. + * @param {string} encodedPrefix + * @returns {{start: string, end: string|null}} + */ +function prefixBounds(encodedPrefix) { + return { + start: encodedPrefix + PREFIX_LOW_MARKER, + end: successor(encodedPrefix), + }; +} + +function successor(str) { + for (let i = str.length - 1; i >= 0; i--) { + const code = StringPrototypeCharCodeAt(str, i); + if (code < 0xff) { + return StringPrototypeSlice(str, 0, i) + StringFromCharCode(code + 1); + } + } + return null; +} + +function isGreater(startKey, endKey) { + return encodeKey(startKey).encoded > encodeKey(endKey).encoded; +} + +function isInPrefix(key, encodedPrefix) { + const encodedKey = encodeKey(key).encoded; + return ( + StringPrototypeStartsWith(encodedKey, encodedPrefix) && + encodedKey.length > encodedPrefix.length + ); +} + +// Presence checks below use `!== undefined`, not truthiness — `""` is a +// falsy-but-valid KeyValue (shorthand for a single empty-string segment, +// which is itself invalid and must fail with a clear "empty string segment" +// error from encodePrefix/encodeKey, not be silently treated as "absent"). +function validateSelectorKeys(selector) { + if (!selector || typeof selector !== "object") { + throw new ERR_INVALID_ARG_TYPE('selector', 'Object', selector); + } + if (selector.prefix !== undefined) { + const { encoded: encodedPrefix } = encodePrefix(selector.prefix); + if ( + selector.start !== undefined && + !isInPrefix(selector.start, encodedPrefix) + ) { + throw new ERR_INVALID_ARG_VALUE('selector.start', selector.start, 'is not in the key space defined by prefix'); + } + if ( + selector.end !== undefined && + !isInPrefix(selector.end, encodedPrefix) + ) { + throw new ERR_INVALID_ARG_VALUE('selector.end', selector.end, 'is not in the key space defined by prefix'); + } + if ( + selector.start !== undefined && + selector.end !== undefined && + isGreater(selector.start, selector.end) + ) { + throw new ERR_INVALID_ARG_VALUE('selector.start', selector.start, 'is greater than end key'); + } + return; + } + if (selector.start === undefined || selector.end === undefined) { + throw new ERR_INVALID_ARG_VALUE('selector', selector, 'range selector requires both start and end keys'); + } + if (isGreater(selector.start, selector.end)) { + throw new ERR_INVALID_ARG_VALUE('selector.start', selector.start, 'is greater than end key'); + } +} + +function findBySelector(selector, db) { + validateSelectorKeys(selector); + if (selector.prefix === undefined) { + return db.range( + encodeKey(selector.start).encoded, + encodeKey(selector.end).encoded, + ); + } + const { encoded: encodedPrefix } = encodePrefix(selector.prefix); + return db.prefix(encodedPrefix, { + start: + selector.start != null ? encodeKey(selector.start).encoded : undefined, + end: selector.end != null ? encodeKey(selector.end).encoded : undefined, + }); +} + +module.exports = { + deserializeKeys, + encodeKey, + encodePrefix, + findBySelector, + normalizeKeys, + prefixBounds, + serializeKeys, +}; diff --git a/lib/internal/kvstore/statements.js b/lib/internal/kvstore/statements.js new file mode 100644 index 00000000000000..71ee90130b7013 --- /dev/null +++ b/lib/internal/kvstore/statements.js @@ -0,0 +1,192 @@ +// Auto-generated by scripts/bundle.mjs — do not edit; edit src/statements.mjs in NoNoSQLite. +'use strict'; + +const { + ArrayPrototypeFill, + ArrayPrototypeJoin, + ArrayPrototypeMap, + ArrayPrototypeSlice, + SafeMap, +} = primordials; + +const { MAX_IN_CLAUSE_VARIABLES, TABLE_NAME } = require('internal/kvstore/constants'); +const { deserializeKeys, prefixBounds } = require('internal/kvstore/keys'); +const { deserializeValue, serializeValue } = require('internal/kvstore/utils'); + +/** + * @typedef {Object} DbInstance + */ + +function toBlob(binaryString) { + return Buffer.from(binaryString, "latin1"); +} + +function fromBlob(value) { + return Buffer.from(value).toString("latin1"); +} + +/** + * Pool of prepared statements for one fixed SQL text. `range()`/`prefix()` + * need a statement each has exclusive use of for the lifetime of its + * `.iterate()` cursor (a single sqlite3_stmt can only have one active + * execution at a time — see the pool usage below), so concurrent/interleaved + * iterators must never share one. Sequential calls, the common case, reuse a + * released statement instead of recompiling identical SQL every time. + */ +function makeStatementPool(database, sql) { + const idle = []; + return { + acquire: () => idle.pop() ?? database.prepare(sql), + release: (stmt) => idle.push(stmt), + }; +} + +/** + * Prepare a custom DB interface backed by a fixed table name. Keys are + * stored as BLOB (memcmp comparison, no text-encoding re-interpretation) so + * the order-preserving key encoding in keys.mjs sorts exactly as designed. + * @param {DatabaseSync} database + * @param {{isMemory: boolean}} options + * @returns {DbInstance} + */ +function prepareDb(database, { isMemory } = {}) { + // WAL + NORMAL synchronous is always-on for file-backed stores (never for + // :memory:, where journaling is irrelevant). This creates `-wal`/`-shm` + // sidecar files next to the database file while it's open. + if (!isMemory) { + database.exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;"); + } + + database.exec(` + CREATE TABLE IF NOT EXISTS ${TABLE_NAME} ( + key BLOB PRIMARY KEY, + value BLOB + )`); + + const upsertStm = database.prepare( + `INSERT INTO ${TABLE_NAME} (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, + ); + + const selectStm = database.prepare( + `SELECT value FROM ${TABLE_NAME} WHERE key = :key`, + ); + + const deleteStm = database.prepare( + `DELETE FROM ${TABLE_NAME} WHERE key = :key`, + ); + + const clearStm = database.prepare(`DELETE FROM ${TABLE_NAME}`); + + // range/prefix use `.iterate()`, which holds a live cursor on the + // underlying prepared statement — see `makeStatementPool` above for why + // each iteration pass needs its own statement. + const rangeSql = `SELECT key, value FROM ${TABLE_NAME} WHERE key >= :start AND key <= :end`; + const prefixSql = `SELECT key, value FROM ${TABLE_NAME} + WHERE key >= :prefixStart AND (:prefixEnd IS NULL OR key < :prefixEnd) + AND (:start IS NULL OR key >= :start) + AND (:end IS NULL OR key <= :end)`; + const rangePool = makeStatementPool(database, rangeSql); + const prefixPool = makeStatementPool(database, prefixSql); + + // getMany's chunk statements have no live cursor (each is a single .all() + // call, not .iterate()), so — unlike range/prefix — they can be cached + // outright, keyed by chunk length, with no concurrency caveat. + const inClauseStatements = new SafeMap(); + function getInClauseStatement(count) { + let stmt = inClauseStatements.get(count); + if (!stmt) { + const placeholders = ArrayPrototypeJoin(ArrayPrototypeFill(Array(count), "?"), ", "); + stmt = database.prepare( + `SELECT key, value FROM ${TABLE_NAME} WHERE key IN (${placeholders})`, + ); + inClauseStatements.set(count, stmt); + } + return stmt; + } + + function* mapEntries(rows) { + for (const entry of rows) { + yield { key: deserializeKeys(fromBlob(entry.key)) }; + } + } + + return { + upsert: (serializedKey, value) => { + upsertStm.run(toBlob(serializedKey), serializeValue(value)); + }, + get: (serializedKey) => { + const entry = selectStm.get({ key: toBlob(serializedKey) }); + return entry ? { value: deserializeValue(entry.value) } : { value: null }; + }, + getMany: (serializedKeys) => { + const byKey = new SafeMap(); + if (serializedKeys.length === 0) return byKey; + // BLOB keys can't round-trip through json_each (JSON has no binary + // type, and SQLite never treats a TEXT value as equal to a BLOB one + // regardless of column affinity) — build parameterized IN(...) lists + // instead, chunked to stay under SQLITE_MAX_VARIABLE_NUMBER. + for ( + let offset = 0; + offset < serializedKeys.length; + offset += MAX_IN_CLAUSE_VARIABLES + ) { + const chunk = ArrayPrototypeSlice(serializedKeys, offset, offset + MAX_IN_CLAUSE_VARIABLES); + const stmt = getInClauseStatement(chunk.length); + const rows = stmt.all(...ArrayPrototypeMap(chunk, toBlob)); + for (const row of rows) { + byKey.set(fromBlob(row.key), deserializeValue(row.value)); + } + } + return byKey; + }, + delete: (serializedKey) => { + return deleteStm.run({ key: toBlob(serializedKey) }); + }, + clear: () => { + clearStm.run(); + }, + range: (serializedStart, serializedEnd) => { + const params = { + start: toBlob(serializedStart), + end: toBlob(serializedEnd), + }; + // A statement is acquired only when iteration actually starts (not + // here, at range() call time) and released once this pass is fully + // consumed, so the returned object stays a true re-iterable and + // concurrent/interleaved iterators still never share one statement. + return { + *[Symbol.iterator]() { + const stmt = rangePool.acquire(); + try { + yield* mapEntries(stmt.iterate(params)); + } finally { + rangePool.release(stmt); + } + }, + }; + }, + prefix: (serializedPrefix, options) => { + const bounds = prefixBounds(serializedPrefix); + const params = { + prefixStart: toBlob(bounds.start), + prefixEnd: bounds.end != null ? toBlob(bounds.end) : null, + start: options.start != null ? toBlob(options.start) : null, + end: options.end != null ? toBlob(options.end) : null, + }; + return { + *[Symbol.iterator]() { + const stmt = prefixPool.acquire(); + try { + yield* mapEntries(stmt.iterate(params)); + } finally { + prefixPool.release(stmt); + } + }, + }; + }, + }; +} + +module.exports = { + prepareDb, +}; diff --git a/lib/internal/kvstore/utils.js b/lib/internal/kvstore/utils.js new file mode 100644 index 00000000000000..e41652bdbf26a7 --- /dev/null +++ b/lib/internal/kvstore/utils.js @@ -0,0 +1,59 @@ +// Auto-generated by scripts/bundle.mjs — do not edit; edit src/utils.mjs in NoNoSQLite. +'use strict'; + +const { deserialize, serialize } = require('v8'); + +const UNKNOWN_TYPE = "unknown"; + +function isString(value) { + return typeof value === "string"; +} + +function getType(value) { + const t = typeof value; + if (t === "string" || t === "number" || t === "boolean" || t === "bigint") { + return t; + } + return UNKNOWN_TYPE; +} + +const MAX_KEY_BYTE_LENGTH = 1024; + +/** + * `encoded` is a "binary string": one JS UTF-16 code unit (range 0x00-0xFF) + * per raw byte (see keys.mjs). Under that mapping `.length` already equals + * the byte count directly — no `Buffer.byteLength` re-encoding needed. + */ +function exceedsKeyByteLimit(encoded) { + return encoded.length > MAX_KEY_BYTE_LENGTH; +} + +/** + * Serialize a value for storage using the Node `v8` structured-clone codec. + * + * Covers everything `postMessage` can transfer: primitives (incl. `undefined`, + * `BigInt`), `Date`, `RegExp`, `Map`, `Set`, typed arrays, `ArrayBuffer`, + * sparse arrays, and circular references. Functions and class identities + * still cannot be preserved — class instances round-trip as plain objects. + * + * The byte format is V8-internal: stable across Node versions, but not + * portable to other runtimes (Bun/Deno/browser). Switch to a wire format + * like CBOR if cross-runtime portability becomes a requirement. + */ +function serializeValue(value) { + return serialize(value); +} + +function deserializeValue(buffer) { + return deserialize(buffer); +} + +module.exports = { + MAX_KEY_BYTE_LENGTH, + UNKNOWN_TYPE, + deserializeValue, + exceedsKeyByteLimit, + getType, + isString, + serializeValue, +}; diff --git a/lib/internal/process/pre_execution.js b/lib/internal/process/pre_execution.js index 38ea6675928ad8..d3a9f7539c1512 100644 --- a/lib/internal/process/pre_execution.js +++ b/lib/internal/process/pre_execution.js @@ -116,6 +116,7 @@ function prepareExecution(options) { setupWarningHandler(); setupFFI(); setupSQLite(); + setupKVStore(); setupStreamIter(); setupDTLS(); setupVfs(); @@ -398,6 +399,15 @@ function setupSQLite() { BuiltinModule.allowRequireByUsers('sqlite'); } +function setupKVStore() { + if (getOptionValue('--no-experimental-kvstore')) { + return; + } + + const { BuiltinModule } = require('internal/bootstrap/realm'); + BuiltinModule.allowRequireByUsers('kvstore'); +} + function initializeConfigFileSupport() { if (getOptionValue('--experimental-config-file')) { emitExperimentalWarning('--experimental-config-file'); diff --git a/lib/kvstore.js b/lib/kvstore.js new file mode 100644 index 00000000000000..61bb2c266424cf --- /dev/null +++ b/lib/kvstore.js @@ -0,0 +1,410 @@ +// Auto-generated by scripts/bundle.mjs — do not edit; edit src/kv.mjs in NoNoSQLite. +'use strict'; + +const { + ArrayIsArray, + ArrayPrototypeFilter, + ArrayPrototypeFind, + ArrayPrototypeMap, + JSONStringify, + SafeSet, + StructuredClone, +} = primordials; + +const EventEmitter = require('events'); +const { DatabaseSync } = require('node:sqlite'); +const { Readable } = require('stream'); +const { + encodeKey, + encodePrefix, + findBySelector, + prefixBounds, +} = require('internal/kvstore/keys'); +const { prepareDb } = require('internal/kvstore/statements'); +const { + ERR_INVALID_ARG_TYPE, + ERR_INVALID_ARG_VALUE, + ERR_KVSTORE_CLOSED, + ERR_OUT_OF_RANGE, +} = require('internal/errors').codes; + + +const IN_MEMORY_DB = ":memory:"; + +/** + * A key made of either a single string, or a list of a string, number, bigint or boolean values. + * @typedef {string|(string|number|bigint|boolean)[]} KeyValue + */ + +/** + * @typedef {Object} EntryValue + * @property {*} value The value for the entry. It is set to `null` if not present. + * @property {KeyValue} key The key for the entry. + */ + +/** + * @typedef {{start: KeyValue, end: KeyValue}} RangeSelector + */ + +/** + * @typedef {Object} PrefixSelector + * @property {KeyValue} prefix The prefix for the entry. An empty array (`[]`) + * matches every key in the store. + * @property {KeyValue} [start] Optional start bound within the prefix. + * @property {KeyValue} [end] Optional end bound within the prefix. + */ + +/** + * A key selector used to select a range of data. + * @typedef {PrefixSelector|RangeSelector} KeySelector + */ + +const VALID_KEY_EVENTS = new SafeSet(["set", "delete", "clear"]); +const KEY_CHANGED = "keyChanged"; +const DEFAULT_SUB_HIGHWATERMARK = 1024; +const SELECTOR_SHAPE_MESSAGE = + "watch selector must be exactly one of { key }, { keys }, { prefix }, or { topic }"; + +class KVStore { + #closed = false; + #database; + #statements; + #keyEmitter = new EventEmitter(); + #topicEmitter = new EventEmitter(); + #subscriptions = new SafeSet(); + + /** + * Create a new instance of a Key Value database. + * @param {{path?: string}} [options] + * - `path` (default: in-memory): on-disk database file. When omitted, + * the store is ephemeral (`:memory:`) — `openKv()` with no arguments + * has no disk side effect. + */ + constructor(options = {}) { + const { path } = options; + const isMemory = path === undefined; + + this.#database = new DatabaseSync(isMemory ? IN_MEMORY_DB : path); + this.#statements = prepareDb(this.#database, { isMemory }); + // Every watch() call adds one listener to these shared emitters — that's + // the normal, expected shape of the public API (many independent + // watchers on the same store), not a leak, so the default cap of 10 + // would otherwise print a spurious MaxListenersExceededWarning. + this.#keyEmitter.setMaxListeners(0); + this.#topicEmitter.setMaxListeners(0); + } + + #assertIsNotClosed() { + if (this.#closed) { + throw new ERR_KVSTORE_CLOSED(); + } + } + + /** + * Close the connection to the database. Idempotent. + * @returns {void} + */ + close() { + if (!this.#closed) { + this.#database.close(); + this.#closed = true; + this.#keyEmitter.removeAllListeners(); + this.#topicEmitter.removeAllListeners(); + for (const stream of this.#subscriptions) stream.push(null); + this.#subscriptions.clear(); + } + } + + /** + * Delete the value associated with the given key. + * @param {KeyValue} key + * @returns {boolean} `true` if a row was removed, `false` if no row matched. + */ + delete(key) { + this.#assertIsNotClosed(); + const { normalized, encoded } = encodeKey(key); + const removed = this.#statements.delete(encoded).changes > 0; + if (removed) { + this.#keyEmitter.emit(KEY_CHANGED, { + type: "delete", + encodedKey: encoded, + normalizedKey: normalized, + }); + } + return removed; + } + + /** + * Delete all the keys in the database. + * @returns {void} + */ + clear() { + this.#assertIsNotClosed(); + this.#statements.clear(); + this.#keyEmitter.emit(KEY_CHANGED, { type: "clear" }); + } + + /** + * Retrieve the value associated with the given key. + * + * Note: a `null` value is returned both when the key is absent and when + * the stored value genuinely is `null` — there is no way to distinguish + * the two in 1.0. A future `versionstamp` return (tracked for 1.x) would + * resolve this. + * @param {KeyValue} key + * @returns {EntryValue} + */ + get(key) { + this.#assertIsNotClosed(); + const { normalized, encoded } = encodeKey(key); + const row = this.#statements.get(encoded); + return { key: normalized, value: row.value }; + } + + /** + * Retrieve the values associated with the given keys. + * The returned array preserves the order of the input keys. + * @param {KeyValue[]} manyKeys + * @returns {EntryValue[]} + */ + getMany(manyKeys) { + this.#assertIsNotClosed(); + if (!ArrayIsArray(manyKeys)) { + throw new ERR_INVALID_ARG_TYPE('manyKeys', 'Array', manyKeys); + } + if (manyKeys.length < 1) { + throw new ERR_INVALID_ARG_VALUE('manyKeys', manyKeys, 'must have at least one key'); + } + const encoded = new Array(manyKeys.length); + const normalized = new Array(manyKeys.length); + for (let i = 0; i < manyKeys.length; i++) { + const k = encodeKey(manyKeys[i]); + encoded[i] = k.encoded; + normalized[i] = k.normalized; + } + const byKey = this.#statements.getMany(encoded); + const result = new Array(manyKeys.length); + for (let i = 0; i < manyKeys.length; i++) { + result[i] = { + key: normalized[i], + value: byKey.has(encoded[i]) ? byKey.get(encoded[i]) : null, + }; + } + return result; + } + + /** + * Retrieve the keys associated with the given selector. + * @param {KeySelector} selector + * @returns {Iterable<{key: KeyValue}>} + */ + keys(selector) { + this.#assertIsNotClosed(); + return findBySelector(selector, this.#statements); + } + + /** + * Set the value associated with the given key. Throws on error; does not + * return a value — a `{ ok }` envelope would be misleading under SQLite + * UPSERT semantics, where a no-op write of an identical value also + * reports zero changed rows. + * + * Watchers observe a fresh clone of `value` (via the platform + * `structuredClone`), not the reference passed in — consistent with + * `get()` and immune to the caller mutating `value` after this returns. + * Cloned directly from the input rather than round-tripped through the + * v8-serialized storage bytes, since the two codecs accept the same set + * of types (see utils.mjs `serializeValue`) and skipping the round-trip + * avoids doing it at all when nothing is watching (guarded below). + * @param {KeyValue} key + * @param {*} value - any value supported by the v8 structured-clone codec + * (see utils.mjs `serializeValue`), including `BigInt` and `undefined`. + * @returns {void} + */ + set(key, value) { + this.#assertIsNotClosed(); + const { normalized, encoded } = encodeKey(key); + this.#statements.upsert(encoded, value); + if (this.#keyEmitter.listenerCount(KEY_CHANGED) > 0) { + this.#keyEmitter.emit(KEY_CHANGED, { + type: "set", + encodedKey: encoded, + normalizedKey: normalized, + value: StructuredClone(value), + }); + } + } + + /** + * Watch for changes on the store or for messages on a custom topic. + * + * Selector forms (mutually exclusive): + * { key } — fires on set/delete of that exact key (and on clear) + * { keys } — fires on set/delete of any of the listed exact keys (and on clear) + * { prefix } — fires on set/delete of any key under prefix (and on clear) + * { topic } — fires on kv.publish(topic, payload) + * + * Optional `events` filter (key/keys/prefix only): array of 'set' | 'delete' | 'clear'. + * + * Backpressure: if a watcher's buffer reaches `highWaterMark`, intervening + * events are dropped and a single `{type:'lag', dropped}` event is delivered + * once the buffer drains again. + * + * Key/keys/prefix streams emit: + * { type:'set', key, value } + * { type:'delete', key } + * { type:'clear' } + * { type:'lag', dropped } + * + * Topic streams emit the published payload directly (no envelope). + * + * @param {{key?: any, keys?: any[], prefix?: any, topic?: string, events?: string[]}} selector + * @param {{highWaterMark?: number}} [options] + * @returns {Readable} object-mode stream emitting one event per push. + */ + watch(selector, options = {}) { + this.#assertIsNotClosed(); + const { highWaterMark = DEFAULT_SUB_HIGHWATERMARK } = options; + if (!(highWaterMark >= 1)) { + throw new ERR_OUT_OF_RANGE('options.highWaterMark', '>= 1', highWaterMark); + } + const { predicate, kind } = this.#compileSelector(selector); + const eventFilter = this.#compileEventFilter(selector.events); + + const stream = new Readable({ objectMode: true, highWaterMark, read() {} }); + this.#subscriptions.add(stream); + + let dropped = 0; + const tryPush = (event) => { + if (stream.readableLength >= highWaterMark) { + dropped++; + return; + } + if (dropped > 0) { + stream.push({ type: "lag", dropped }); + dropped = 0; + } + stream.push(event); + }; + + let detach; + if (kind === "topic") { + const listener = (payload) => tryPush(payload); + this.#topicEmitter.on(selector.topic, listener); + detach = () => this.#topicEmitter.off(selector.topic, listener); + } else { + const listener = (event) => { + if (!predicate(event)) return; + if (!eventFilter(event.type)) return; + if (event.type === "clear") { + tryPush({ type: "clear" }); + } else if (event.type === "set") { + tryPush({ + type: "set", + key: event.normalizedKey, + value: event.value, + }); + } else { + tryPush({ type: "delete", key: event.normalizedKey }); + } + }; + this.#keyEmitter.on(KEY_CHANGED, listener); + detach = () => this.#keyEmitter.off(KEY_CHANGED, listener); + } + + stream.on("close", () => { + detach(); + this.#subscriptions.delete(stream); + }); + return stream; + } + + /** + * Publish a payload on a custom topic. Topics live in their own namespace + * (independent of the key store) and have no event-type taxonomy. + * @param {string} topic + * @param {*} payload + */ + publish(topic, payload) { + this.#assertIsNotClosed(); + if (typeof topic !== "string" || topic.length === 0) { + throw new ERR_INVALID_ARG_VALUE('topic', topic, 'must be a non-empty string'); + } + this.#topicEmitter.emit(topic, payload); + } + + #compileSelector(selector) { + if (!selector || typeof selector !== "object") { + throw new ERR_INVALID_ARG_TYPE('selector', 'Object', selector); + } + const { key, keys, prefix, topic } = selector; + const provided = ArrayPrototypeFilter([key, keys, prefix, topic], (v) => v !== undefined).length; + if (provided !== 1) { + throw new ERR_INVALID_ARG_VALUE('selector', selector, SELECTOR_SHAPE_MESSAGE); + } + if (topic !== undefined) { + if (typeof topic !== "string" || topic.length === 0) { + throw new ERR_INVALID_ARG_VALUE('selector.topic', topic, 'must be a non-empty string'); + } + return { kind: "topic", predicate: null }; + } + if (key !== undefined) { + const { encoded } = encodeKey(key); + return { + kind: "key", + predicate: (e) => + e.type === "clear" ? true : e.encodedKey === encoded, + }; + } + if (keys !== undefined) { + if (!ArrayIsArray(keys) || keys.length === 0) { + throw new ERR_INVALID_ARG_VALUE('selector.keys', keys, 'must be a non-empty array of keys'); + } + const encodedSet = new SafeSet(ArrayPrototypeMap(keys, (k) => encodeKey(k).encoded)); + return { + kind: "keys", + predicate: (e) => + e.type === "clear" ? true : encodedSet.has(e.encodedKey), + }; + } + // prefix + const { encoded } = encodePrefix(prefix); + const bounds = prefixBounds(encoded); + return { + kind: "prefix", + predicate: (e) => { + if (e.type === "clear") return true; + if (e.encodedKey < bounds.start) return false; + return bounds.end === null || e.encodedKey < bounds.end; + }, + }; + } + + #compileEventFilter(events) { + if (events === undefined) return () => true; + if (!ArrayIsArray(events) || events.length === 0) { + throw new ERR_INVALID_ARG_VALUE('options.events', events, 'must be a non-empty array'); + } + const invalid = ArrayPrototypeFind(events, (e) => !VALID_KEY_EVENTS.has(e)); + if (invalid !== undefined) { + throw new ERR_INVALID_ARG_VALUE('options.events', invalid, `contains an invalid event type: ${JSONStringify(invalid)} (expected "set", "delete", or "clear")`); + } + const allowed = new SafeSet(events); + return (type) => allowed.has(type); + } +} + +/** + * Open a KV store. Documented entry point of the module — `KVStore` itself + * is exported too, for `instanceof` checks and typing. + * @param {{path?: string}} [options] + * @returns {KVStore} + */ +function openKv(options) { + return new KVStore(options); +} + +module.exports = { + openKv, + KVStore, +}; diff --git a/src/node_options.cc b/src/node_options.cc index ca12586e479b99..412b78a29e34a9 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -606,6 +606,11 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() { &EnvironmentOptions::experimental_sqlite, kAllowedInEnvvar, HAVE_SQLITE); + AddOption("--experimental-kvstore", + "experimental node:kvstore module", + &EnvironmentOptions::experimental_kvstore, + kAllowedInEnvvar, + HAVE_SQLITE); AddOption("--experimental-stream-iter", "experimental iterable streams API (node:stream/iter)", &EnvironmentOptions::experimental_stream_iter, diff --git a/src/node_options.h b/src/node_options.h index eb5aadc2347e25..e59dc61d68294c 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -131,6 +131,7 @@ class EnvironmentOptions : public Options { bool experimental_ffi = EXPERIMENTALS_DEFAULT_VALUE; bool experimental_websocket = true; bool experimental_sqlite = HAVE_SQLITE; + bool experimental_kvstore = HAVE_SQLITE; bool experimental_stream_iter = EXPERIMENTALS_DEFAULT_VALUE; bool experimental_vfs = EXPERIMENTALS_DEFAULT_VALUE; bool webstorage = HAVE_SQLITE; diff --git a/test/common/index.js b/test/common/index.js index ac0c400581c439..b99e7fa1e608aa 100755 --- a/test/common/index.js +++ b/test/common/index.js @@ -70,6 +70,7 @@ const hasCrypto = Boolean(process.versions.openssl) && const hasInspector = Boolean(process.features.inspector); const hasSQLite = Boolean(process.versions.sqlite); +const hasKVStore = hasSQLite; const hasFFI = Boolean(process.config.variables.node_use_ffi); const hasDtls = hasCrypto && !!process.features.dtls; @@ -763,6 +764,12 @@ function skipIfSQLiteMissing() { } } +function skipIfKVStoreMissing() { + if (!hasKVStore) { + skip('missing KVStore (requires SQLite)'); + } +} + function skipIfFFIMissing() { if (!hasFFI) { skip('missing FFI'); @@ -1011,6 +1018,7 @@ const common = { hasQuic, hasInspector, hasSQLite, + hasKVStore, hasFFI, hasLocalStorage, invalidArgTypeHelper, @@ -1047,6 +1055,7 @@ const common = { skipIfInspectorDisabled, skipIfFFIMissing, skipIfSQLiteMissing, + skipIfKVStoreMissing, spawnPromisified, sleepSync, usesSharedLibrary, diff --git a/test/parallel/test-kvstore-key-codec.js b/test/parallel/test-kvstore-key-codec.js new file mode 100644 index 00000000000000..95a59e2ff55e59 --- /dev/null +++ b/test/parallel/test-kvstore-key-codec.js @@ -0,0 +1,319 @@ +// Auto-generated by scripts/bundle.mjs from src/keys.test.mjs — do not edit directly. +// Regenerate with: npm run bundle +'use strict'; + +const { skipIfKVStoreMissing } = require('../common'); +skipIfKVStoreMissing(); + +const assert = require('node:assert/strict'); +const { describe, it } = require('node:test'); +const { deserializeKeys, encodeKey, encodePrefix, normalizeKeys, serializeKeys } = require('internal/kvstore/keys'); + +// Constants from internal/kvstore/constants (inlined for test portability) +const TAG_STRING = 0x01; + +function sortByEncoded(values) { + return [...values] + .map((v) => ({ v, encoded: serializeKeys(normalizeKeys([v])) })) + .sort((a, b) => + a.encoded < b.encoded ? -1 : a.encoded > b.encoded ? 1 : 0, + ) + .map((e) => e.v); +} + +describe("normalizeKeys", () => { + it("should not throw for valid keys", () => { + const validKeys = [ + // string based keys + "key", + ["key"], + ["key", "key2"], + // number based keys + [8], + [8, 8], + [Infinity, -Infinity], + // bigInt based keys + [8n], + [8n, 8n], + // boolean based keys + [true], + [false], + [true, false], + [true, true], + [false, false], + // whitespace-only string segments are allowed — only "" is rejected + [" "], + ["\t\n"], + ]; + for (const validKey of validKeys) { + assert.doesNotThrow(() => normalizeKeys(validKey)); + } + }); + + it("should not throw for stringified edge case keys", () => { + const validKeys = [ + // string based keys + "null", + ["null", "undefined", "Infinity", "NaN", "true", "false", "🤷", "康"], + ]; + for (const validKey of validKeys) { + assert.doesNotThrow(() => normalizeKeys(validKey)); + } + }); + + it("should throw ERR_KVSTORE_INVALID_KEY for an empty array", () => { + assert.throws(() => normalizeKeys([]), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + }); + + it("should throw ERR_KVSTORE_INVALID_KEY for an empty string segment", () => { + assert.throws(() => normalizeKeys([""]), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + }); + + it("should throw for NaN — it cannot compare equal to itself as a key", () => { + assert.throws(() => normalizeKeys([NaN]), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + }); + + it("should throw for invalid segment types", () => { + const invalidKeys = [[undefined], [null], [Symbol()], [{}], [() => {}]]; + for (const invalidKey of invalidKeys) { + assert.throws(() => normalizeKeys(invalidKey), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + } + }); + + it("should throw ERR_KVSTORE_INVALID_KEY when the raw key itself isn't a string or array (regression: coverage gap)", () => { + for (const raw of [null, undefined, 42, true, {}, Symbol()]) { + assert.throws( + () => normalizeKeys(raw), + (err) => { + assert.ok(err instanceof TypeError); + assert.equal(err.code, "ERR_INVALID_ARG_TYPE"); + return true; + }, + ); + } + }); + + it("encodeKey should throw when the encoded form exceeds 1024 bytes", () => { + assert.throws(() => encodeKey(Array(210).fill("hello")), { + name: "RangeError", + code: "ERR_OUT_OF_RANGE", + }); + }); + + it("encodeKey should throw ERR_KVSTORE_BIGINT_TOO_LARGE for an oversized bigint", () => { + const huge = 2n ** 2100n; // > 255 bytes of magnitude + assert.throws(() => encodeKey([huge]), { + name: "RangeError", + code: "ERR_KVSTORE_BIGINT_TOO_LARGE", + }); + }); +}); + +describe("encodePrefix", () => { + it("accepts an empty array — matches every key", () => { + assert.doesNotThrow(() => encodePrefix([])); + assert.equal(encodePrefix([]).encoded, ""); + }); + + it("still rejects an empty string segment", () => { + assert.throws(() => encodePrefix([""]), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + }); + + it("still rejects NaN", () => { + assert.throws(() => encodePrefix([NaN]), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + }); + + it("should throw ERR_KVSTORE_INVALID_SELECTOR when the raw prefix itself isn't a string or array (regression: coverage gap)", () => { + for (const raw of [null, undefined, 42, true, {}]) { + assert.throws( + () => encodePrefix(raw), + (err) => { + assert.ok(err instanceof TypeError); + assert.equal(err.code, "ERR_INVALID_ARG_TYPE"); + return true; + }, + ); + } + }); + + it("should throw when the encoded prefix form exceeds 1024 bytes (regression: coverage gap)", () => { + assert.throws(() => encodePrefix(Array(210).fill("hello")), { + name: "RangeError", + code: "ERR_OUT_OF_RANGE", + }); + }); +}); + +describe("malformed input at the encode/decode layer (bypassing normalizeKeys' validation — regression: coverage gap)", () => { + it("serializeKeys rejects NaN even when called directly, without normalizeKeys", () => { + assert.throws( + () => serializeKeys([NaN]), + (err) => { + assert.ok(err instanceof TypeError); + assert.equal(err.code, "ERR_INVALID_ARG_VALUE"); + return true; + }, + ); + }); + + it("serializeKeys rejects an unsupported segment type even when called directly", () => { + for (const bad of [undefined, null, {}, Symbol(), () => {}]) { + assert.throws( + () => serializeKeys([bad]), + (err) => { + assert.ok(err instanceof TypeError); + assert.equal(err.code, "ERR_INVALID_ARG_VALUE"); + err.message, + /key segments must be a string, number, bigint, or boolean/, + ); + return true; + }, + ); + } + }); + + it("deserializeKeys throws on a truncated string segment", () => { + const truncated = `${String.fromCharCode(TAG_STRING)}abc`; // no terminator + assert.throws( + () => deserializeKeys(truncated), + (err) => { + assert.ok(err instanceof Error); + assert.equal(err.code, "ERR_INVALID_STATE"); + assert.match(err.message, /truncated string segment/); + return true; + }, + ); + }); + + it("deserializeKeys throws on an unrecognized segment tag", () => { + const corrupt = String.fromCharCode(0x09); // not any real TAG_* value + assert.throws( + () => deserializeKeys(corrupt), + (err) => { + assert.ok(err instanceof Error); + assert.equal(err.code, "ERR_INVALID_STATE"); + assert.match(err.message, /unknown segment tag/); + return true; + }, + ); + }); +}); + +describe("serializeKeys / deserializeKeys round-trip", () => { + const fixtures = [ + ["plain string"], + ["multi", "segment"], + [42], + [42n], + [true, false], + // numeric strings must not collide with numbers + ["5"], + ["5n"], + // strings containing the new terminator byte (U+0000) — must be escaped + ["foo\x00bar"], + // strings containing the new escape byte (U+0001) — must round-trip too + ["foo\x01bar"], + // adjacent escape + terminator + ["\x01\x00"], + // the bytes used by the *old* text-based encoding scheme, to prove + // they are now ordinary content with no special meaning + ["foo\x1Fbar", "foo\x1Ebar"], + // unicode + ["🤷", "康"], + // mixed types in one key + [false, "a", 1, 2n], + // edge numbers, including negatives and -0 + [Number.MAX_SAFE_INTEGER, Infinity, -Infinity, -0, 0, -1.5, 1.5], + // bigints spanning sign and magnitude-length boundaries + [0n, -1n, 1n, -255n, 255n, 256n, -256n], + ]; + + for (const fixture of fixtures) { + it(`round-trips ${JSON.stringify(fixture, (_, v) => (typeof v === "bigint" ? `${v}n` : v))}`, () => { + const normalized = normalizeKeys(fixture); + const round = deserializeKeys(serializeKeys(normalized)); + // -0 normalizes to 0 by design (see the float64 encoding in keys.mjs). + const expected = normalized.map((v) => (Object.is(v, -0) ? 0 : v)); + assert.deepEqual(round, expected); + }); + } + + it("distinguishes [5] from ['5']", () => { + assert.notStrictEqual( + serializeKeys(normalizeKeys([5])), + serializeKeys(normalizeKeys(["5"])), + ); + }); + + it("distinguishes [true] from ['true']", () => { + assert.notStrictEqual( + serializeKeys(normalizeKeys([true])), + serializeKeys(normalizeKeys(["true"])), + ); + }); + + it("distinguishes 1 (number) from 1n (bigint) — never coerced together", () => { + assert.notStrictEqual( + serializeKeys(normalizeKeys([1])), + serializeKeys(normalizeKeys([1n])), + ); + }); + + it("-0 encodes identically to 0", () => { + assert.equal( + serializeKeys(normalizeKeys([-0])), + serializeKeys(normalizeKeys([0])), + ); + }); +}); + +describe("order-preserving encoding", () => { + it("sorts numbers numerically, not lexicographically", () => { + const values = [1, 2, 3, 9, 10, 11, 20, 100, -1, -300, -3, 0, 1.5, -1.5]; + const expected = [...values].sort((a, b) => a - b); + assert.deepEqual(sortByEncoded(values), expected); + }); + + it("sorts bigints numerically across sign and magnitude-length boundaries", () => { + const values = [0n, -1n, 1n, -255n, 255n, 256n, -256n, 5n, -5n]; + const expected = [...values].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + assert.deepEqual(sortByEncoded(values), expected); + }); + + it("sorts strings by UTF-8 byte order", () => { + const values = ["b", "a", "ab", "abc"]; + assert.deepEqual(sortByEncoded(values), ["a", "ab", "abc", "b"]); + }); + + it("sorts booleans false < true", () => { + assert.deepEqual(sortByEncoded([true, false]), [false, true]); + }); + + it("cross-type order is string < number < bigint < boolean", () => { + assert.deepEqual(sortByEncoded([true, 1n, 1, "a"]), ["a", 1, 1n, true]); + }); + + it("Infinity/-Infinity sort at the extremes of the number bucket", () => { + const values = [Infinity, 0, -Infinity, 100, -100]; + const expected = [-Infinity, -100, 0, 100, Infinity]; + assert.deepEqual(sortByEncoded(values), expected); + }); +}); diff --git a/test/parallel/test-kvstore-module-flags.js b/test/parallel/test-kvstore-module-flags.js new file mode 100644 index 00000000000000..2b7597c4d0e759 --- /dev/null +++ b/test/parallel/test-kvstore-module-flags.js @@ -0,0 +1,38 @@ +// Tests for node:kvstore experimental module access-control: +// 1. Schemeless require('kvstore') must throw MODULE_NOT_FOUND. +// 2. --no-experimental-kvstore disables require('node:kvstore'). +'use strict'; + +const { spawnPromisified, skipIfKVStoreMissing } = require('../common'); +skipIfKVStoreMissing(); + +const { suite, test } = require('node:test'); + +suite('accessing the node:kvstore module', () => { + test('cannot be accessed without the node: scheme', (t) => { + t.assert.throws(() => { + require('kvstore'); + }, { + code: 'MODULE_NOT_FOUND', + message: /Cannot find module 'kvstore'/, + }); + }); + + test('can be disabled with --no-experimental-kvstore flag', async (t) => { + const { + stdout, + stderr, + code, + signal, + } = await spawnPromisified(process.execPath, [ + '--no-experimental-kvstore', + '-e', + 'require("node:kvstore")', + ]); + + t.assert.strictEqual(stdout, ''); + t.assert.match(stderr, /No such built-in module: node:kvstore/); + t.assert.notStrictEqual(code, 0); + t.assert.strictEqual(signal, null); + }); +}); diff --git a/test/parallel/test-kvstore.js b/test/parallel/test-kvstore.js new file mode 100644 index 00000000000000..6f0c115ca7e80c --- /dev/null +++ b/test/parallel/test-kvstore.js @@ -0,0 +1,955 @@ +// Auto-generated by scripts/bundle.mjs from src/kv.test.mjs — do not edit directly. +// Regenerate with: npm run bundle +'use strict'; + +const { skipIfKVStoreMissing } = require('../common'); +skipIfKVStoreMissing(); + +const assert = require('node:assert/strict'); +const { existsSync, mkdtempSync, rmSync } = require('node:fs'); +const { tmpdir } = require('node:os'); +const { join } = require('node:path'); +const { afterEach, beforeEach, describe, it } = require('node:test'); +const { KVStore, openKv } = require('node:kvstore'); + +async function collectN(stream, n) { + const out = []; + for await (const event of stream) { + out.push(event); + if (out.length === n) break; + } + return out; +} + +describe("KV instance", () => { + let kv; + + beforeEach(() => { + kv = openKv(); + }); + + afterEach(() => { + kv.close(); + }); + + it("openKv() returns a KVStore instance", () => { + assert.ok(kv instanceof KVStore); + }); + + it("should set and get a value", () => { + const testKey = [1, "test", false, 2n]; + const testValue = { foo: "bar" }; + + kv.set(testKey, testValue); + + const result = kv.get(testKey); + assert.deepEqual(result, { key: testKey, value: testValue }); + }); + + it("set() returns undefined", () => { + assert.strictEqual(kv.set(["a"], 1), undefined); + }); + + it("get() returns a key independent from the caller's input array (regression: key-aliasing)", () => { + const key = ["a", 1]; + kv.set(key, "v"); + const result = kv.get(key); + key.push("mutated-after-get"); + assert.deepEqual(result.key, ["a", 1]); + assert.notStrictEqual(result.key, key); + }); + + it("getMany() returns keys independent from the caller's input arrays (regression: key-aliasing)", () => { + const key = ["a", 1]; + kv.set(key, "v"); + const [result] = kv.getMany([key]); + key.push("mutated-after-getMany"); + assert.deepEqual(result.key, ["a", 1]); + assert.notStrictEqual(result.key, key); + }); + + it("delete()'s watch event carries a key independent from the caller's input array (regression: key-aliasing)", async () => { + const key = ["a", 1]; + kv.set(key, "v"); + const sub = kv.watch({ key }); + kv.delete(key); + key.push("mutated-after-delete"); + const [event] = await collectN(sub, 1); + assert.deepEqual(event, { type: "delete", key: ["a", 1] }); + sub.destroy(); + }); + + it("should get multiple entry at once", () => { + const testKeys = [ + [1, "test", false, 2n], + [2, "test2", true, 3n], + ]; + const testValues = [{ foo: "bar" }, { foo: "baz" }]; + + kv.set(testKeys[0], testValues[0]); + kv.set(testKeys[1], testValues[1]); + + const results = kv.getMany(testKeys); + assert.deepEqual(results, [ + { key: testKeys[0], value: testValues[0] }, + { key: testKeys[1], value: testValues[1] }, + ]); + }); + + it("should round-trip rich value types (Date, BigInt, Map, Set, RegExp, undefined)", () => { + const value = { + when: new Date("2024-01-15T10:30:00Z"), + count: 9007199254740993n, + tags: new Set(["a", "b"]), + lookup: new Map([["x", 1]]), + pattern: /foo/gi, + missing: undefined, + nested: { typed: new Uint8Array([1, 2, 3]) }, + }; + kv.set(["rich"], value); + const { value: round } = kv.get(["rich"]); + assert.deepStrictEqual(round, value); + }); + + it("should round-trip circular references", () => { + const a = { name: "a" }; + a.self = a; + kv.set(["cyc"], a); + const { value } = kv.get(["cyc"]); + assert.strictEqual(value.self, value); + assert.strictEqual(value.name, "a"); + }); + + it("getMany should reject non-array input", () => { + for (const bad of ["not-an-array", null, undefined, 123]) { + assert.throws(() => kv.getMany(bad), { + name: "TypeError", + code: "ERR_INVALID_ARG_TYPE", + }); + } + }); + + it("getMany should reject an empty array", () => { + assert.throws(() => kv.getMany([]), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + }); + + it("getMany should preserve input order", () => { + kv.set(["a"], 1); + kv.set(["b"], 2); + kv.set(["c"], 3); + const result = kv.getMany([["c"], ["a"], ["b"]]); + assert.deepEqual(result, [ + { key: ["c"], value: 3 }, + { key: ["a"], value: 1 }, + { key: ["b"], value: 2 }, + ]); + }); + + it("getMany should yield duplicates when the input contains them", () => { + kv.set(["a"], 1); + const result = kv.getMany([["a"], ["a"]]); + assert.deepEqual(result, [ + { key: ["a"], value: 1 }, + { key: ["a"], value: 1 }, + ]); + }); + + it("getMany should return null for missing keys", () => { + kv.set(["existing"], "exists"); + const result = kv.getMany([["existing"], ["non-existing"]]); + assert.deepEqual(result, [ + { key: ["existing"], value: "exists" }, + { key: ["non-existing"], value: null }, + ]); + }); + + it("getMany should correctly match keys of mixed segment types (regression: BLOB vs json_each)", () => { + kv.set(["a"], "string-key"); + kv.set([2], "number-key"); + kv.set([3n], "bigint-key"); + kv.set([true], "bool-key"); + const result = kv.getMany([["a"], [2], [3n], [true], ["missing"]]); + assert.deepEqual(result, [ + { key: ["a"], value: "string-key" }, + { key: [2], value: "number-key" }, + { key: [3n], value: "bigint-key" }, + { key: [true], value: "bool-key" }, + { key: ["missing"], value: null }, + ]); + }); + + it("getMany works across IN(...) chunk boundaries (regression: H3 chunking)", () => { + const n = 1200; // > MAX_IN_CLAUSE_VARIABLES (500), spans 3 chunks + for (let i = 0; i < n; i++) kv.set([i], `v${i}`); + const keys = Array.from({ length: n }, (_, i) => [i]); + const result = kv.getMany(keys); + assert.equal(result.length, n); + for (let i = 0; i < n; i++) { + assert.deepEqual(result[i], { key: [i], value: `v${i}` }); + } + }); + + it("getMany does not throw past the historical SQLite variable limit (regression: H3)", () => { + const n = 40000; // > the old, unchunked SQLITE_MAX_VARIABLE_NUMBER ceiling + for (let i = 0; i < n; i++) kv.set([i], i); + const keys = Array.from({ length: n }, (_, i) => [i]); + const result = kv.getMany(keys); + assert.equal(result.length, n); + assert.deepEqual(result[0], { key: [0], value: 0 }); + assert.deepEqual(result[n - 1], { key: [n - 1], value: n - 1 }); + }); + + it("should return false when deleting a missing key", () => { + assert.strictEqual(kv.delete(["never-set"]), false); + }); + + it("should delete a entry by the provided key", () => { + const testKey = [1, "test", false, 2n]; + const testValue = { foo: "bar" }; + + kv.set(testKey, testValue); + + assert.deepEqual(kv.get(testKey), { key: testKey, value: testValue }); + + kv.delete(testKey); + + assert.deepEqual(kv.get(testKey), { key: testKey, value: null }); + }); + + function testKeyIterator(iterator, nValues, criteria) { + let counter = 0; + for (const { key } of iterator) { + assert.ok(criteria(key)); + counter++; + } + assert.strictEqual(counter, nValues); + } + + it("should retrieve keys by range query - numeric values in true numeric order (regression: N1)", () => { + const values = [1, 2, 3, 9, 10, 11, 20, 100]; + for (const n of values) kv.set([n], `value${n}`); + + const got = [...kv.keys({ start: [1], end: [100] })].map((e) => e.key[0]); + assert.deepEqual(got, values); + }); + + it("should retrieve keys by range query - negative and fractional numbers in true numeric order", () => { + const values = [-300, -5, -3, -1.5, 0, 1.5, 5]; + for (const n of values) kv.set([n], `value${n}`); + + const got = [...kv.keys({ start: [-300], end: [5] })].map((e) => e.key[0]); + assert.deepEqual(got, values); + }); + + it("should retrieve keys by range query - bigint values in true numeric order", () => { + const values = [-256n, -5n, -3n, 0n, 5n, 256n]; + for (const n of values) kv.set([n], `value${n}`); + + const got = [...kv.keys({ start: [-256n], end: [256n] })].map( + (e) => e.key[0], + ); + assert.deepEqual(got, values); + }); + + it("should retrieve keys by range query - string values", () => { + const entries = [ + { key: ["a"], value: "value1" }, + { key: ["b"], value: "value2" }, + { key: ["c"], value: "value3" }, + ]; + for (const { key, value } of entries) { + kv.set(key, value); + } + + const iterator = kv.keys({ start: ["a"], end: ["c"] }); + testKeyIterator(iterator, 3, (key) => ["a", "b", "c"].includes(key[0])); + }); + + it("should retrieve keys by range query avoiding partial matching - nested values", () => { + const entries = [ + { key: ["users", "alice"], value: "value1" }, + { key: ["users", "bob"], value: "value2" }, + ]; + for (const { key, value } of entries) { + kv.set(key, value); + } + + const iterator = kv.keys({ start: ["users", "a"], end: ["users", "b"] }); + testKeyIterator(iterator, 1, (key) => key[1] === "alice"); + }); + + it("should retrieve keys by range query - nested values", () => { + const entries = [ + { key: ["users", "alice"], value: "value1" }, + { key: ["users", "bob"], value: "value2" }, + ]; + for (const { key, value } of entries) { + kv.set(key, value); + } + + const iterator = kv.keys({ start: ["users", "a"], end: ["users", "c"] }); + testKeyIterator(iterator, 2, (key) => ["alice", "bob"].includes(key[1])); + }); + + it("should throw if start key is greater than end key", () => { + assert.throws( + () => kv.keys({ start: ["users", "b"], end: ["users", "a"] }), + { name: "TypeError", code: "ERR_INVALID_ARG_VALUE" }, + ); + }); + + it("should throw if range query is missing either start or end", () => { + assert.throws(() => kv.keys({ start: ["users", "b"] }), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + assert.throws(() => kv.keys({ end: ["users", "b"] }), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + }); + + it("should throw for a selector that isn't an object (regression: coverage gap)", () => { + for (const bad of [null, undefined, 42, "foo", true]) { + assert.throws( + () => kv.keys(bad), + (err) => { + assert.ok(err instanceof TypeError); + assert.equal(err.code, "ERR_INVALID_ARG_TYPE"); + return true; + }, + ); + } + }); + + it("should throw if start key is greater than end key within a prefix selector (regression: coverage gap)", () => { + assert.throws( + () => + kv.keys({ + prefix: ["users"], + start: ["users", "c"], + end: ["users", "b"], + }), + (err) => { + assert.ok(err instanceof TypeError); + assert.equal(err.code, "ERR_INVALID_ARG_VALUE"); + return true; + }, + ); + }); + + it("{ prefix: '' } reports the empty-string-segment error, not a range-selector error (regression: falsy-prefix)", () => { + assert.throws( + () => kv.keys({ prefix: "" }), + (err) => { + assert.ok(err instanceof TypeError); + assert.equal(err.code, "ERR_INVALID_ARG_VALUE"); + assert.match(err.message, /empty string/); + return true; + }, + ); + }); + + it("should retrieve keys by prefix query", () => { + const entries = [ + { key: ["users", "alice"], value: "value1" }, + { key: ["users", "bob"], value: "value2" }, + { key: ["admins", "carol"], value: "value3" }, + ]; + for (const { key, value } of entries) { + kv.set(key, value); + } + + const iterator = kv.keys({ prefix: ["users"] }); + testKeyIterator(iterator, 2, (key) => key[0] === "users"); + }); + + it("should retrieve keys by prefix query without partial matching", () => { + const entries = [ + { key: ["users", "alice"], value: "value1" }, + { key: ["users", "bob"], value: "value2" }, + { key: ["admins", "carol"], value: "value3" }, + ]; + for (const { key, value } of entries) { + kv.set(key, value); + } + + const iterator = kv.keys({ prefix: ["users", "a"] }); + testKeyIterator(iterator, 0, () => true); + }); + + it("prefix query does not match the prefix key itself", () => { + kv.set(["user"], "exact"); + kv.set(["user", 1], "child"); + const got = [...kv.keys({ prefix: ["user"] })].map((e) => e.key); + assert.deepEqual(got, [["user", 1]]); + }); + + it("should retrieve keys by prefix query with start key provided", () => { + const entries = [ + { key: ["users", "alice"], value: "value1" }, + { key: ["users", "bob"], value: "value2" }, + { key: ["admins", "carol"], value: "value3" }, + ]; + for (const { key, value } of entries) { + kv.set(key, value); + } + + const iterator = kv.keys({ prefix: ["users"], start: ["users", "b"] }); + testKeyIterator( + iterator, + 1, + (key) => key[0] === "users" && key[1] === "bob", + ); + }); + + it("should throw if start key is not within prefix", () => { + assert.throws( + () => kv.keys({ prefix: ["users", "a"], start: ["users", "a"] }), + { name: "TypeError", code: "ERR_INVALID_ARG_VALUE" }, + ); + assert.throws( + () => kv.keys({ prefix: ["users", "a"], start: ["users", "b"] }), + { name: "TypeError", code: "ERR_INVALID_ARG_VALUE" }, + ); + // an empty array isn't a valid concrete key at all, regardless of prefix + assert.throws(() => kv.keys({ prefix: ["users", "a"], start: [] }), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + }); + + it("should retrieve keys by prefix query with end key provided", () => { + const entries = [ + { key: ["users", "alice"], value: "value1" }, + { key: ["users", "bob"], value: "value2" }, + { key: ["admins", "carol"], value: "value3" }, + ]; + for (const { key, value } of entries) { + kv.set(key, value); + } + + const iterator = kv.keys({ prefix: ["users"], end: ["users", "b"] }); + testKeyIterator( + iterator, + 1, + (key) => key[0] === "users" && key[1] === "alice", + ); + }); + + it("should retrieve keys by prefix query with both start and end", () => { + const entries = [ + { key: ["users", "alice"], value: "value1" }, + { key: ["users", "bob"], value: "value2" }, + { key: ["users", "carol"], value: "value3" }, + ]; + for (const { key, value } of entries) { + kv.set(key, value); + } + + const iterator = kv.keys({ + prefix: ["users"], + start: ["users", "b"], + end: ["users", "c"], + }); + testKeyIterator(iterator, 1, (key) => key[1] === "bob"); + }); + + it("should throw if end key is not within prefix", () => { + assert.throws( + () => kv.keys({ prefix: ["users", "a"], end: ["users", "a"] }), + { name: "TypeError", code: "ERR_INVALID_ARG_VALUE" }, + ); + assert.throws( + () => kv.keys({ prefix: ["users", "a"], end: ["users", "b"] }), + { name: "TypeError", code: "ERR_INVALID_ARG_VALUE" }, + ); + // an empty array isn't a valid concrete key at all, regardless of prefix + assert.throws(() => kv.keys({ prefix: ["users", "a"], end: [] }), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + }); + + it("{ prefix: [] } scans every key in the store (regression: N2)", () => { + kv.set(["users", "alice"], 1); + kv.set(["admins", "carol"], 2); + kv.set([42], 3); + const got = [...kv.keys({ prefix: [] })].map((e) => e.key); + assert.equal(got.length, 3); + assert.ok(got.some((k) => k[0] === "users")); + assert.ok(got.some((k) => k[0] === "admins")); + assert.ok(got.some((k) => k[0] === 42)); + }); + + it("two interleaved keys() iterators each yield their own full, correct result set (regression: N3)", () => { + for (let i = 0; i < 10; i++) kv.set([i], i); + + const it1 = kv.keys({ start: [0], end: [4] })[Symbol.iterator](); + const it2 = kv.keys({ start: [5], end: [9] })[Symbol.iterator](); + const r1 = []; + const r2 = []; + let a = it1.next(); + let b = it2.next(); + while (!a.done || !b.done) { + if (!a.done) { + r1.push(a.value.key[0]); + a = it1.next(); + } + if (!b.done) { + r2.push(b.value.key[0]); + b = it2.next(); + } + } + assert.deepEqual(r1, [0, 1, 2, 3, 4]); + assert.deepEqual(r2, [5, 6, 7, 8, 9]); + }); + + it("keys() iterable can be safely re-iterated (regression: H4)", () => { + kv.set(["a"], 1); + kv.set(["b"], 2); + const iterable = kv.keys({ prefix: [] }); + const first = [...iterable].map((e) => e.key); + const second = [...iterable].map((e) => e.key); + assert.deepEqual(first, [["a"], ["b"]]); + assert.deepEqual(second, [["a"], ["b"]]); + }); + + it("NaN is not a valid key segment", () => { + assert.throws(() => kv.set([NaN], "x"), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + }); + + it("a bigint exceeding the 255-byte magnitude limit throws ERR_KVSTORE_BIGINT_TOO_LARGE", () => { + const huge = 2n ** 2100n; + assert.throws( + () => kv.set([huge], "x"), + (err) => { + assert.ok(err instanceof RangeError); + assert.equal(err.code, "ERR_KVSTORE_BIGINT_TOO_LARGE"); + return true; + }, + ); + }); + + it("whitespace-only string segments are valid keys", () => { + kv.set([" "], "space"); + assert.deepEqual(kv.get([" "]), { key: [" "], value: "space" }); + }); + + it("empty string segments are still rejected", () => { + assert.throws(() => kv.set([""], "x"), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + }); + + it("should throw on operations after close()", () => { + kv.close(); + const closedMatcher = { name: "Error", code: "ERR_KVSTORE_CLOSED" }; + assert.throws(() => kv.get(["x"]), closedMatcher); + assert.throws(() => kv.set(["x"], "v"), closedMatcher); + assert.throws(() => kv.delete(["x"]), closedMatcher); + assert.throws(() => kv.clear(), closedMatcher); + // re-open for the afterEach close + kv = openKv(); + }); + + it("close() should be idempotent", () => { + kv.close(); + assert.doesNotThrow(() => kv.close()); + kv = openKv(); + }); +}); + +describe("KV constructor", () => { + it("defaults to an in-memory store with no disk side effect", () => { + const a = openKv(); + a.set(["x"], "in-memory"); + assert.deepEqual(a.get(["x"]), { key: ["x"], value: "in-memory" }); + a.close(); + }); + + it("two default (in-memory) instances should not share state", () => { + const a = openKv(); + const b = openKv(); + a.set(["x"], "from-a"); + assert.deepEqual(b.get(["x"]), { key: ["x"], value: null }); + assert.deepEqual(a.get(["x"]), { key: ["x"], value: "from-a" }); + a.close(); + b.close(); + }); + + it("closing one instance should not affect another", () => { + const a = openKv(); + const b = openKv(); + a.close(); + assert.doesNotThrow(() => b.set(["y"], "still-works")); + b.close(); + }); +}); + +describe("watch / publish", () => { + let kv; + + beforeEach(() => { + kv = openKv(); + }); + + afterEach(() => { + kv.close(); + }); + + it("exact-key watcher sees set and delete on its key only", async () => { + const sub = kv.watch({ key: ["a"] }); + kv.set(["a"], 1); + kv.set(["b"], 2); // ignored + kv.delete(["a"]); + const events = await collectN(sub, 2); + assert.deepEqual(events, [ + { type: "set", key: ["a"], value: 1 }, + { type: "delete", key: ["a"] }, + ]); + sub.destroy(); + }); + + it("watch() delivers an isolated clone, not the caller's live object (regression: H2)", async () => { + const sub = kv.watch({ key: ["a"] }); + const obj = { foo: "bar" }; + kv.set(["a"], obj); + obj.foo = "mutated-after-set"; + + const [event] = await collectN(sub, 1); + assert.deepEqual(event, { + type: "set", + key: ["a"], + value: { foo: "bar" }, + }); + assert.notStrictEqual(event.value, obj); + + const stored = kv.get(["a"]); + assert.deepEqual(stored.value, { foo: "bar" }); + sub.destroy(); + }); + + it("watch() delivers a key independent from the caller's live array (regression: key-aliasing)", async () => { + const sub = kv.watch({ prefix: [] }); + const key = ["a", 1]; + kv.set(key, "v"); + key.push("mutated-after-set"); + + const [event] = await collectN(sub, 1); + assert.deepEqual(event, { type: "set", key: ["a", 1], value: "v" }); + assert.notStrictEqual(event.key, key); + sub.destroy(); + }); + + it("prefix watcher matches nested keys, ignores siblings", async () => { + const sub = kv.watch({ prefix: ["user"] }); + kv.set(["user", 1], "alice"); + kv.set(["admin", 1], "carol"); // ignored + kv.set(["user", 2, "profile"], { v: 1 }); + kv.delete(["user", 1]); + const events = await collectN(sub, 3); + assert.deepEqual(events, [ + { type: "set", key: ["user", 1], value: "alice" }, + { type: "set", key: ["user", 2, "profile"], value: { v: 1 } }, + { type: "delete", key: ["user", 1] }, + ]); + sub.destroy(); + }); + + it("prefix watcher does NOT match the prefix key itself", async () => { + const sub = kv.watch({ prefix: ["user"] }); + kv.set(["user"], "exact"); // ignored — at prefix, not under it + kv.set(["user", 1], "child"); + const [event] = await collectN(sub, 1); + assert.deepEqual(event, { type: "set", key: ["user", 1], value: "child" }); + sub.destroy(); + }); + + it("keys watcher fires for any of the listed keys, ignores others", async () => { + const sub = kv.watch({ keys: [["a"], ["b"]] }); + kv.set(["a"], 1); + kv.set(["c"], 99); // ignored + kv.set(["b"], 2); + kv.delete(["a"]); + const events = await collectN(sub, 3); + assert.deepEqual(events, [ + { type: "set", key: ["a"], value: 1 }, + { type: "set", key: ["b"], value: 2 }, + { type: "delete", key: ["a"] }, + ]); + sub.destroy(); + }); + + it("keys watcher receives clear like other key watchers", async () => { + const sub = kv.watch({ keys: [["a"], ["b"]] }); + kv.set(["a"], 1); + kv.clear(); + const events = await collectN(sub, 2); + assert.deepEqual(events, [ + { type: "set", key: ["a"], value: 1 }, + { type: "clear" }, + ]); + sub.destroy(); + }); + + it("keys watcher rejects empty array", () => { + assert.throws(() => kv.watch({ keys: [] }), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + }); + + it("events filter excludes unwanted types", async () => { + const sub = kv.watch({ prefix: ["x"], events: ["set"] }); + kv.set(["x", 1], "v"); + kv.delete(["x", 1]); // ignored + kv.set(["x", 2], "w"); + const events = await collectN(sub, 2); + assert.deepEqual(events, [ + { type: "set", key: ["x", 1], value: "v" }, + { type: "set", key: ["x", 2], value: "w" }, + ]); + sub.destroy(); + }); + + it("clear fires on every key/prefix watcher", async () => { + const a = kv.watch({ key: ["a"] }); + const b = kv.watch({ prefix: ["b"] }); + kv.set(["a"], 1); + kv.set(["b", 1], 2); + kv.clear(); + const aEvents = await collectN(a, 2); + const bEvents = await collectN(b, 2); + assert.deepEqual(aEvents[1], { type: "clear" }); + assert.deepEqual(bEvents[1], { type: "clear" }); + a.destroy(); + b.destroy(); + }); + + it("topic watcher receives publish payloads as-is", async () => { + const sub = kv.watch({ topic: "login" }); + kv.publish("login", { user: "alice" }); + kv.publish("login", { user: "bob" }); + const events = await collectN(sub, 2); + assert.deepEqual(events, [{ user: "alice" }, { user: "bob" }]); + sub.destroy(); + }); + + it("topics and key channels do not cross", async () => { + const keySub = kv.watch({ key: ["x"] }); + const topicSub = kv.watch({ topic: "x" }); + + let topicFired = false; + topicSub.on("data", () => { + topicFired = true; + }); + + kv.set(["x"], 1); + await new Promise((r) => setImmediate(r)); + assert.strictEqual(topicFired, false); + + const [event] = await collectN(keySub, 1); + assert.deepEqual(event, { type: "set", key: ["x"], value: 1 }); + keySub.destroy(); + topicSub.destroy(); + }); + + it("close() ends outstanding subscriptions", async () => { + const sub = kv.watch({ prefix: ["a"] }); + kv.close(); + const events = []; + for await (const e of sub) events.push(e); + assert.deepEqual(events, []); + // re-open for the afterEach close + kv = openKv(); + }); + + it("watch throws after close", () => { + kv.close(); + assert.throws(() => kv.watch({ key: ["a"] }), { + name: "Error", + code: "ERR_KVSTORE_CLOSED", + }); + kv = openKv(); + }); + + it("publish throws after close", () => { + kv.close(); + assert.throws(() => kv.publish("t", 1), { + name: "Error", + code: "ERR_KVSTORE_CLOSED", + }); + kv = openKv(); + }); + + it("rejects malformed selectors", () => { + const bad = [ + undefined, + null, + "foo", + {}, + { key: "a", topic: "b" }, + { key: "a", keys: [["b"]] }, + { keys: "not-an-array" }, + { keys: [] }, + { topic: "" }, + { topic: 123 }, + { key: "a", events: [] }, + { key: "a", events: ["bogus"] }, + ]; + for (const sel of bad) { + assert.throws(() => kv.watch(sel), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + } + }); + + + it("rejects malformed publish topics", () => { + for (const topic of [undefined, null, "", 42, {}]) { + assert.throws(() => kv.publish(topic, 1), { + name: "TypeError", + code: "ERR_INVALID_ARG_VALUE", + }); + } + }); + + + it("rejects highWaterMark < 1", () => { + const matcher = { name: "TypeError", code: "ERR_OUT_OF_RANGE, name: RangeError" }; + assert.throws( + () => kv.watch({ key: ["a"] }, { highWaterMark: 0 }), + matcher, + ); + assert.throws( + () => kv.watch({ key: ["a"] }, { highWaterMark: -1 }), + matcher, + ); + }); + + it("emits a lag event when watcher falls behind", async () => { + const sub = kv.watch({ prefix: ["x"] }, { highWaterMark: 4 }); + + // Collect everything via a single 'data' listener. + const received = []; + sub.on("data", (event) => received.push(event)); + + // Burst: first 4 buffered (then immediately drained because the listener + // puts the stream in flowing mode), the remaining 46 also push without + // dropping because each push drains synchronously. + // To force drops, we need to pause the stream before the burst. + sub.pause(); + for (let i = 0; i < 50; i++) kv.set(["x", i], i); + + // Resume to drain the 4 buffered events. + sub.resume(); + await new Promise((r) => setImmediate(r)); + assert.strictEqual(received.length, 4); + + // Buffer is empty; emit one more — it should be preceded by a lag event. + kv.set(["x", 999], 999); + await new Promise((r) => setImmediate(r)); + + assert.deepEqual(received[4], { type: "lag", dropped: 46 }); + assert.deepEqual(received[5], { + type: "set", + key: ["x", 999], + value: 999, + }); + + sub.destroy(); + }); + + it("does not deliver events emitted after destroy()", () => { + const sub = kv.watch({ key: ["a"] }); + const received = []; + sub.on("data", (e) => received.push(e)); + sub.destroy(); + kv.set(["a"], "after-destroy"); + assert.deepEqual(received, []); + }); + + it("more than 10 concurrent watchers does not trigger MaxListenersExceededWarning (regression: max-listeners)", async () => { + const warnings = []; + const onWarning = (w) => warnings.push(w); + process.on("warning", onWarning); + try { + const subs = Array.from({ length: 20 }, (_, i) => + kv.watch({ key: [`k${i}`] }), + ); + kv.set(["k0"], 1); + await new Promise((r) => setImmediate(r)); + for (const sub of subs) sub.destroy(); + // give any (unwanted) async warning a chance to surface + await new Promise((r) => setImmediate(r)); + } finally { + process.off("warning", onWarning); + } + assert.deepEqual(warnings, []); + }); +}); + +describe("file-backed store", () => { + let dir; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "nonosqlite-test-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("persists across close/reopen with correct ordering", () => { + const path = join(dir, "store.sqlite"); + const a = openKv({ path }); + for (const n of [10, 1, 100, 2]) a.set([n], `v${n}`); + a.close(); + + const b = openKv({ path }); + const got = [...b.keys({ prefix: [] })].map((e) => e.key[0]); + assert.deepEqual(got, [1, 2, 10, 100]); + b.close(); + }); + + it("creates WAL sidecar files while open, cleaned up on close", () => { + const path = join(dir, "wal.sqlite"); + const kv = openKv({ path }); + kv.set(["x"], 1); + assert.ok(existsSync(`${path}-wal`)); + kv.close(); + assert.ok(!existsSync(`${path}-wal`)); + }); + + it("watch() does not observe writes made through a different KVStore instance on the same file (regression: cross-instance scope)", async () => { + const path = join(dir, "shared.sqlite"); + const writer = openKv({ path }); + const reader = openKv({ path }); + + const sub = reader.watch({ prefix: [] }); + const seen = []; + sub.on("data", (event) => seen.push(event)); + + writer.set(["from-writer"], 1); + writer.delete(["from-writer"]); + writer.clear(); + // Give any (wrongly) cross-instance event a chance to arrive before + // asserting its absence. + await new Promise((resolve) => setImmediate(resolve)); + + assert.deepEqual(seen, []); + + sub.destroy(); + writer.close(); + reader.close(); + }); +}); diff --git a/test/parallel/test-process-get-builtin.mjs b/test/parallel/test-process-get-builtin.mjs index fa92dc52f96916..06f102a127a48c 100644 --- a/test/parallel/test-process-get-builtin.mjs +++ b/test/parallel/test-process-get-builtin.mjs @@ -1,4 +1,4 @@ -import { hasCrypto, hasIntl, hasInspector, hasSQLite } from '../common/index.mjs'; +import { hasCrypto, hasIntl, hasInspector, hasSQLite, hasKVStore } from '../common/index.mjs'; import assert from 'node:assert'; import { builtinModules } from 'node:module'; import { isMainThread } from 'node:worker_threads'; @@ -50,6 +50,9 @@ if (!hasInspector) { if (!hasSQLite) { publicBuiltins.delete('node:sqlite'); } +if (!hasKVStore) { + publicBuiltins.delete('node:kvstore'); +} // TODO: Remove this once node:ffi graduates from unflagged. publicBuiltins.delete('node:ffi'); From afa13093e7294ac7f02a13922072f69258af2023 Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 15 Jul 2026 17:22:17 +0200 Subject: [PATCH 2/4] :pencil2: Add examples to doc --- doc/api/kvstore.md | 408 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 407 insertions(+), 1 deletion(-) diff --git a/doc/api/kvstore.md b/doc/api/kvstore.md index 93103ae3488666..941d1e8aa51181 100644 --- a/doc/api/kvstore.md +++ b/doc/api/kvstore.md @@ -156,6 +156,28 @@ added: v27.0.0 Deletes every key in the database. +```mjs +import { openKv } from 'node:kvstore'; +const kv = openKv(); + +kv.set('a', 1); +kv.set('b', 2); +kv.clear(); +console.log(kv.get('a').value); // null +kv.close(); +``` + +```cjs +const { openKv } = require('node:kvstore'); +const kv = openKv(); + +kv.set('a', 1); +kv.set('b', 2); +kv.clear(); +console.log(kv.get('a').value); // null +kv.close(); +``` + ### `kv.close()`