From e9e8cca5fccf6473a7f8f45961529cd1ea87f911 Mon Sep 17 00:00:00 2001 From: Vincent Gao Date: Tue, 23 Jun 2026 22:10:59 +0200 Subject: [PATCH 1/2] Resolve string concatenation expressions in field values ResolveStringReferencesMiddleware only resolved a field whose entire value was a single string reference. A concatenation expression such as `10 # "~" # jan` was looked up as one key, failed, and was left raw (issue #396); an all-quoted expression like `"a" # "b"` was skipped by the enclosed-value guard and later mangled by RemoveEnclosingMiddleware. Detect top-level `#` concatenation (respecting quotes and braces), resolve each token (number, quoted/braced literal, or string reference) and join the results. The resolved value is kept enclosed in braces so the downstream enclosing middlewares treat it as a plain string and it round-trips to valid BibTeX. If any reference is unknown, the original expression is left untouched. --- bibtexparser/middlewares/interpolate.py | 78 ++++++++++++++++++++++ tests/middleware_tests/test_interpolate.py | 63 +++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/bibtexparser/middlewares/interpolate.py b/bibtexparser/middlewares/interpolate.py index 2e5d832..0842f9d 100644 --- a/bibtexparser/middlewares/interpolate.py +++ b/bibtexparser/middlewares/interpolate.py @@ -21,6 +21,72 @@ def _value_is_nonstring_or_enclosed(value: Any) -> bool: return False +def _split_concatenation(value: str) -> "list[str] | None": + """Split a value on top-level ``#`` concatenation operators. + + Returns the list of stripped tokens if the value contains at least one + ``#`` outside of any ``"..."`` or ``{...}`` group, otherwise ``None`` + (i.e., the value is not a concatenation expression). + """ + tokens = [] + current = [] + depth = 0 + in_quotes = False + found_separator = False + for char in value: + if char == '"' and depth == 0: + in_quotes = not in_quotes + current.append(char) + elif char == "{" and not in_quotes: + depth += 1 + current.append(char) + elif char == "}" and not in_quotes: + depth = max(0, depth - 1) + current.append(char) + elif char == "#" and depth == 0 and not in_quotes: + found_separator = True + tokens.append("".join(current).strip()) + current = [] + else: + current.append(char) + + if not found_separator: + return None + + tokens.append("".join(current).strip()) + return tokens + + +def _resolve_concatenation(tokens: "list[str]", string_values: "dict[str, str]") -> "str | None": + """Resolve concatenation tokens to their joined string content. + + Each token is a number (kept verbatim), a quoted or braced string (its + inner content is used), or a string reference (resolved via + ``string_values``). Returns ``None`` if any reference is unknown, so the + caller can leave the original expression untouched. + """ + resolved = [] + for token in tokens: + if not token: + return None + if token.startswith('"') and token.endswith('"'): + resolved.append(token[1:-1]) + elif token.startswith("{") and token.endswith("}"): + resolved.append(token[1:-1]) + elif token.isdigit(): + resolved.append(token) + else: + try: + referenced = string_values[token.lower()] + except KeyError: + return None + if referenced.startswith(('"', "{")) and referenced.endswith(('"', "}")): + referenced = referenced[1:-1] + resolved.append(referenced) + + return "".join(resolved) + + class ResolveStringReferencesMiddleware(LibraryMiddleware): """Replace strings references with their values.""" @@ -59,6 +125,18 @@ def transform(self, library: Library) -> Library: field: Field for field in entry.fields: + if isinstance(field.value, str): + tokens = _split_concatenation(field.value) + if tokens is not None: + joined = _resolve_concatenation(tokens, string_values) + if joined is not None: + # Keep the result enclosed (in braces) so the + # downstream enclosing middlewares treat it as a + # plain string rather than an unenclosed reference. + field.value = "{" + joined + "}" + resolved_fields.append(field.key) + continue + if _value_is_nonstring_or_enclosed(field.value): continue try: diff --git a/tests/middleware_tests/test_interpolate.py b/tests/middleware_tests/test_interpolate.py index 264373c..a223598 100644 --- a/tests/middleware_tests/test_interpolate.py +++ b/tests/middleware_tests/test_interpolate.py @@ -1,5 +1,6 @@ import pytest +import bibtexparser from bibtexparser.middlewares.enclosing import RemoveEnclosingMiddleware from bibtexparser.middlewares.interpolate import ResolveStringReferencesMiddleware from bibtexparser.splitter import Splitter @@ -66,3 +67,65 @@ def test_warning_is_raised_if_enclosings_are_removed(): assert len(record) == 1 assert "RemoveEnclosing" in record[0].message.args[0] + + +def test_string_interpolation_resolves_concatenation(): + bibtex = """ + @string{jan = "Jan."} + + @inbook{test_inbook, + month = 10 # "~" # jan, + title = "hello" # " " # "world", + mixed = jan # { } # "X", + numbers = 1 # 2 # 3 + } + """ + library = Splitter(bibtex).split() + + m = ResolveStringReferencesMiddleware() + library = m.transform(library) + + fields = library.entries_dict["test_inbook"].fields_dict + # Concatenations are resolved and kept enclosed in braces. + assert fields["month"].value == "{10~Jan.}" + assert fields["title"].value == "{hello world}" + assert fields["mixed"].value == "{Jan. X}" + assert fields["numbers"].value == "{123}" + + +def test_string_interpolation_leaves_unresolvable_concatenation_untouched(): + bibtex = """ + @inbook{test_inbook, + note = unknown_macro # " suffix" + } + """ + library = Splitter(bibtex).split() + + m = ResolveStringReferencesMiddleware() + library = m.transform(library) + + # An unknown reference means the expression is left exactly as-is. + assert ( + library.entries_dict["test_inbook"].fields_dict["note"].value == 'unknown_macro # " suffix"' + ) + + +def test_parse_string_resolves_concatenation_end_to_end(): + # Exact reproduction of issue #396. + bibtex_str = """ + @STRING{ jan = "Jan." } + + @INBOOK{inbook-full, + month = 10 # "~" # jan, + } + """ + library = bibtexparser.parse_string(bibtex_str) + month = library.entries[0].fields_dict["month"].value + assert month == "10~Jan." + + # The resolved value writes as a valid, brace-enclosed string and + # round-trips back to the same value. + written = bibtexparser.write_string(library) + assert "month = {10~Jan.}" in written + reparsed = bibtexparser.parse_string(written) + assert reparsed.entries[0].fields_dict["month"].value == "10~Jan." From 4c47d13822bfc1d453c2c27b8a300caf034d23a4 Mon Sep 17 00:00:00 2001 From: Vincent Gao Date: Mon, 27 Jul 2026 01:41:41 +0200 Subject: [PATCH 2/2] Resolve concatenation in string definitions and keep unresolved ones intact The remaining places where a value that is a `#` expression is treated as a single token: - A string definition was substituted verbatim, so a definition that is itself an expression or another reference was spliced into the field unevaluated (`asiacrypt91name = asiacryptname # "'91"` from #396). Definitions are now resolved up front, iteratively, so chains resolve and a reference cycle simply never resolves instead of producing a partial value. - `RemoveEnclosingMiddleware._strip_enclosing` took the first and last character of an expression for one enclosing pair, so an expression bounded by delimiters was corrupted, both in a field (`"a" # nope # "b"`, when nothing resolves it first) and in a `@string` block, where a valid file was rewritten as `@string{s = {Asia" # "crypt}}`. Both middlewares now share one expression scanner, which follows the escaping the splitter applies (a delimiter after a backslash is content). Cases and the comparison branch that prompted them: @claell on #565. Co-authored-by: Claudius Ellsel --- bibtexparser/middlewares/enclosing.py | 71 ++++++++++++ bibtexparser/middlewares/interpolate.py | 129 ++++++++++----------- tests/middleware_tests/test_enclosing.py | 42 +++++++ tests/middleware_tests/test_interpolate.py | 94 +++++++++++++++ 4 files changed, 271 insertions(+), 65 deletions(-) diff --git a/bibtexparser/middlewares/enclosing.py b/bibtexparser/middlewares/enclosing.py index a3c09ca..8b2ac9d 100644 --- a/bibtexparser/middlewares/enclosing.py +++ b/bibtexparser/middlewares/enclosing.py @@ -20,6 +20,73 @@ ] +def _is_escaped(value: str, index: int) -> bool: + """Whether the character at `index` is escaped, as the splitter defines it.""" + return index > 0 and value[index - 1] == "\\" + + +def _split_concatenation(value: str) -> list[str] | None: + """Split a top-level bibtex `#` concatenation into its tokens. + + Returns None if `value` is not a concatenation expression, i.e., if it holds + no `#` outside of a `"..."` or `{...}` group, or if its delimiters are + unbalanced (in which case its structure is unknown and must not be guessed). + """ + tokens = [] + token_start = 0 + depth = 0 + in_quotes = False + for index, char in enumerate(value): + if _is_escaped(value, index): + continue + if char == '"' and depth == 0: + in_quotes = not in_quotes + elif in_quotes: + continue + elif char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth < 0: + return None + elif char == "#" and depth == 0: + tokens.append(value[token_start:index].strip()) + token_start = index + 1 + + if not tokens or in_quotes or depth != 0: + return None + + tokens.append(value[token_start:].strip()) + return tokens + + +def _literal_content(token: str) -> str | None: + """The content of a token that is one fully quoted or brace-enclosed literal.""" + if len(token) < 2: + return None + + if token.startswith('"'): + for index in range(1, len(token)): + if token[index] == '"' and not _is_escaped(token, index): + return token[1:index] if index == len(token) - 1 else None + return None + + if token.startswith("{"): + depth = 0 + for index, char in enumerate(token): + if _is_escaped(token, index): + continue + if char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + return token[1:index] if index == len(token) - 1 else None + if depth < 0: + return None + return None + + class RemoveEnclosingMiddleware(BlockMiddleware): """Remove enclosing characters from values such as field and strings. @@ -51,6 +118,10 @@ def metadata_key(cls) -> str: @staticmethod def _strip_enclosing(value: str) -> tuple[str, str | None]: value = value.strip() + if _split_concatenation(value) is not None: + # The first and last character of a `#` expression do not enclose it, + # so stripping them would corrupt the expression. + return value, "no-enclosing" if value.startswith("{") and value.endswith("}"): return value[1:-1], "{" if value.startswith('"') and value.endswith('"'): diff --git a/bibtexparser/middlewares/interpolate.py b/bibtexparser/middlewares/interpolate.py index 0842f9d..a31ae42 100644 --- a/bibtexparser/middlewares/interpolate.py +++ b/bibtexparser/middlewares/interpolate.py @@ -7,6 +7,8 @@ from bibtexparser.model import Field from .enclosing import REMOVED_ENCLOSING_KEY +from .enclosing import _literal_content +from .enclosing import _split_concatenation from .middleware import LibraryMiddleware @@ -21,72 +23,54 @@ def _value_is_nonstring_or_enclosed(value: Any) -> bool: return False -def _split_concatenation(value: str) -> "list[str] | None": - """Split a value on top-level ``#`` concatenation operators. +def _resolve_tokens(tokens: list[str], strings: dict[str, str]) -> str | None: + """Join the resolved content of the tokens of a concatenation. - Returns the list of stripped tokens if the value contains at least one - ``#`` outside of any ``"..."`` or ``{...}`` group, otherwise ``None`` - (i.e., the value is not a concatenation expression). - """ - tokens = [] - current = [] - depth = 0 - in_quotes = False - found_separator = False - for char in value: - if char == '"' and depth == 0: - in_quotes = not in_quotes - current.append(char) - elif char == "{" and not in_quotes: - depth += 1 - current.append(char) - elif char == "}" and not in_quotes: - depth = max(0, depth - 1) - current.append(char) - elif char == "#" and depth == 0 and not in_quotes: - found_separator = True - tokens.append("".join(current).strip()) - current = [] - else: - current.append(char) - - if not found_separator: - return None - - tokens.append("".join(current).strip()) - return tokens - - -def _resolve_concatenation(tokens: "list[str]", string_values: "dict[str, str]") -> "str | None": - """Resolve concatenation tokens to their joined string content. - - Each token is a number (kept verbatim), a quoted or braced string (its - inner content is used), or a string reference (resolved via - ``string_values``). Returns ``None`` if any reference is unknown, so the - caller can leave the original expression untouched. + Each token is a quoted or braced literal, a number, or a string reference. + Returns None if any of them is unresolvable, so that the caller can keep the + original expression instead of resolving it partially. """ resolved = [] for token in tokens: - if not token: + content = _literal_content(token) + if content is None: + content = token if token.isdigit() else strings.get(token.lower()) + if content is None: return None - if token.startswith('"') and token.endswith('"'): - resolved.append(token[1:-1]) - elif token.startswith("{") and token.endswith("}"): - resolved.append(token[1:-1]) - elif token.isdigit(): - resolved.append(token) - else: - try: - referenced = string_values[token.lower()] - except KeyError: - return None - if referenced.startswith(('"', "{")) and referenced.endswith(('"', "}")): - referenced = referenced[1:-1] - resolved.append(referenced) - + resolved.append(content) return "".join(resolved) +def _resolve_value(value: str, strings: dict[str, str]) -> str | None: + """Resolve a literal, a number, a string reference or a `#` expression.""" + tokens = _split_concatenation(value) + return _resolve_tokens([value.strip()] if tokens is None else tokens, strings) + + +def _resolve_string_definitions(definitions: dict[str, str]) -> dict[str, str]: + """Resolve the content of every string definition that can be resolved. + + Definitions may reference each other, so this iterates until no further + definition resolves. Iterating rather than recursing means that an + arbitrarily long chain of definitions cannot exhaust the interpreter stack, + and that definitions on a reference cycle simply never resolve. + """ + resolved: dict[str, str] = {} + pending = dict(definitions) + while pending: + progressed = False + for key, value in list(pending.items()): + content = _resolve_value(value, resolved) + if content is None: + continue + resolved[key] = content + del pending[key] + progressed = True + if not progressed: + break + return resolved + + class ResolveStringReferencesMiddleware(LibraryMiddleware): """Replace strings references with their values.""" @@ -106,6 +90,7 @@ def transform(self, library: Library) -> Library: # BibTeX string keys are case-insensitive; later definitions win. string_values = {key.lower(): s.value for key, s in library.strings_dict.items()} + resolved_strings = _resolve_string_definitions(string_values) entry: Entry raised_enclosing_warning = False @@ -128,21 +113,35 @@ def transform(self, library: Library) -> Library: if isinstance(field.value, str): tokens = _split_concatenation(field.value) if tokens is not None: - joined = _resolve_concatenation(tokens, string_values) - if joined is not None: - # Keep the result enclosed (in braces) so the - # downstream enclosing middlewares treat it as a - # plain string rather than an unenclosed reference. - field.value = "{" + joined + "}" + content = _resolve_tokens(tokens, resolved_strings) + if content is not None: + # Braces keep the result a plain value: unenclosed, it + # would be read back as a reference when written. + field.value = "{" + content + "}" resolved_fields.append(field.key) + # An unresolvable expression is left exactly as it is, + # which the enclosing middlewares preserve. continue if _value_is_nonstring_or_enclosed(field.value): continue + key = field.value.lower() try: - field.value = string_values[field.value.lower()] + referenced = string_values[key] except KeyError: continue + + if ( + _split_concatenation(referenced) is not None + or referenced.lower() in string_values + ): + # The definition is itself an expression or another reference, + # so it must be resolved rather than substituted verbatim. + content = resolved_strings.get(key) + if content is None: + continue + referenced = "{" + content + "}" + field.value = referenced resolved_fields.append(field.key) if resolved_fields: diff --git a/tests/middleware_tests/test_enclosing.py b/tests/middleware_tests/test_enclosing.py index e5e021f..998ba3c 100644 --- a/tests/middleware_tests/test_enclosing.py +++ b/tests/middleware_tests/test_enclosing.py @@ -421,4 +421,46 @@ def test_string_reference_roundtrip(): assert "year = {2019}" in written +@pytest.mark.parametrize( + "expression", + [ + '"first" # "second"', + "{first} # {second}", + '"prefix" # unknown_macro # "suffix"', + # The splitter treats a backslash-escaped delimiter as content, so this + # is a two-token expression rather than one quoted value. + '"sch\\"on" # "!"', + ], +) +def test_removal_keeps_multi_token_expressions_intact(expression: str): + """The first and last character of a `#` expression are not one enclosing pair. + + Cases adapted from @claell's comparison branch on issue #396. + """ + library = Library( + [ + Entry( + entry_type="article", key="someKey", fields=[Field(key="title", value=expression)] + ), + String(key="someString", value=expression), + ] + ) + + transformed = RemoveEnclosingMiddleware().transform(library) + + assert transformed.entries[0].fields[0].value == expression + assert transformed.entries[0].fields[0].enclosing == "no-enclosing" + assert transformed.strings[0].value == expression + assert transformed.strings[0].enclosing == "no-enclosing" + + +def test_string_definition_expression_roundtrip(): + """A definition whose expression starts and ends with a delimiter must not be stripped.""" + bibtex = '@string{quoted = "Asia" # "crypt"}\n@string{braced = {Asia} # {crypt}}' + written = bibtexparser.write_string(bibtexparser.parse_string(bibtex)) + + assert 'quoted = "Asia" # "crypt"' in written + assert "braced = {Asia} # {crypt}" in written + + # TODO round-trip tests (removal -> addition -> removal) diff --git a/tests/middleware_tests/test_interpolate.py b/tests/middleware_tests/test_interpolate.py index a223598..b720b52 100644 --- a/tests/middleware_tests/test_interpolate.py +++ b/tests/middleware_tests/test_interpolate.py @@ -129,3 +129,97 @@ def test_parse_string_resolves_concatenation_end_to_end(): assert "month = {10~Jan.}" in written reparsed = bibtexparser.parse_string(written) assert reparsed.entries[0].fields_dict["month"].value == "10~Jan." + + +def test_string_interpolation_resolves_string_definitions_recursively(): + """A definition may itself be an expression or another reference (cases via @claell).""" + bibtex = """ + @string{asiacryptname = "Asiacrypt"} + @string{asiacrypt91name = asiacryptname # "'91"} + @string{alias = asiacrypt91name} + @string{quoted = "Asia" # "crypt"} + @string{vol = 1 # 0} + + @inbook{test_inbook, + booktitle = asiacrypt91name, + series = alias, + note = asiacrypt91name # " proceedings", + title = quoted, + volume = vol + } + """ + library = bibtexparser.parse_string(bibtex) + + fields = library.entries_dict["test_inbook"].fields_dict + assert fields["booktitle"].value == "Asiacrypt'91" + assert fields["series"].value == "Asiacrypt'91" + assert fields["note"].value == "Asiacrypt'91 proceedings" + assert fields["title"].value == "Asiacrypt" + assert fields["volume"].value == "10" + + # The definitions keep their expression form, so the file still writes validly. + written = bibtexparser.write_string(library) + assert 'asiacrypt91name = asiacryptname # "\'91"' in written + assert 'quoted = "Asia" # "crypt"' in written + + +def test_string_interpolation_leaves_cyclic_definitions_unresolved(): + """Cyclic definitions resolve to nothing rather than to a partial value (case via @claell).""" + bibtex = """ + @string{first = second # " one"} + @string{second = first # " two"} + @string{itself = itself} + + @inbook{test_inbook, + title = first # "!", + note = second, + other = itself + } + """ + library = bibtexparser.parse_string(bibtex) + + fields = library.entries_dict["test_inbook"].fields_dict + assert fields["title"].value == 'first # "!"' + assert fields["note"].value == "second" + assert fields["other"].value == "itself" + + # Writing and re-parsing is a fixpoint, i.e. it does not substitute one more level. + written = bibtexparser.write_string(library) + reparsed = bibtexparser.parse_string(written).entries_dict["test_inbook"].fields_dict + assert reparsed["title"].value == 'first # "!"' + assert reparsed["note"].value == "second" + assert reparsed["other"].value == "itself" + + +def test_string_interpolation_resolves_long_definition_chains(): + """A chain of definitions is resolved iteratively, so it cannot exhaust the stack.""" + length = 500 + bibtex = "".join(f'@string{{s{i} = s{i + 1} # "x"}}\n' for i in range(length)) + bibtex += f'@string{{s{length} = "end"}}\n@inbook{{test_inbook, title = s0}}' + + library = bibtexparser.parse_string(bibtex) + + title = library.entries_dict["test_inbook"].fields_dict["title"] + assert title.value == "end" + "x" * length + + +def test_unresolvable_concatenation_survives_a_round_trip(): + """An expression with an unknown reference is written back exactly as it was read.""" + bibtex = ( + "@inbook{test_inbook,\n" + ' note = "prefix" # unknown_macro # "suffix",\n' + " title = {first} # unknown_macro # {second}\n" + "}" + ) + library = bibtexparser.parse_string(bibtex) + + fields = library.entries_dict["test_inbook"].fields_dict + assert fields["note"].value == '"prefix" # unknown_macro # "suffix"' + assert fields["title"].value == "{first} # unknown_macro # {second}" + + written = bibtexparser.write_string(library) + assert 'note = "prefix" # unknown_macro # "suffix"' in written + assert "title = {first} # unknown_macro # {second}" in written + reparsed = bibtexparser.parse_string(written).entries_dict["test_inbook"].fields_dict + assert reparsed["note"].value == fields["note"].value + assert reparsed["title"].value == fields["title"].value