diff --git a/bibtexparser/middlewares/enclosing.py b/bibtexparser/middlewares/enclosing.py index a3c09ca..8b2ac9d 100644 --- a/bibtexparser/middlewares/enclosing.py +++ b/bibtexparser/middlewares/enclosing.py @@ -20,6 +20,73 @@ ] +def _is_escaped(value: str, index: int) -> bool: + """Whether the character at `index` is escaped, as the splitter defines it.""" + return index > 0 and value[index - 1] == "\\" + + +def _split_concatenation(value: str) -> list[str] | None: + """Split a top-level bibtex `#` concatenation into its tokens. + + Returns None if `value` is not a concatenation expression, i.e., if it holds + no `#` outside of a `"..."` or `{...}` group, or if its delimiters are + unbalanced (in which case its structure is unknown and must not be guessed). + """ + tokens = [] + token_start = 0 + depth = 0 + in_quotes = False + for index, char in enumerate(value): + if _is_escaped(value, index): + continue + if char == '"' and depth == 0: + in_quotes = not in_quotes + elif in_quotes: + continue + elif char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth < 0: + return None + elif char == "#" and depth == 0: + tokens.append(value[token_start:index].strip()) + token_start = index + 1 + + if not tokens or in_quotes or depth != 0: + return None + + tokens.append(value[token_start:].strip()) + return tokens + + +def _literal_content(token: str) -> str | None: + """The content of a token that is one fully quoted or brace-enclosed literal.""" + if len(token) < 2: + return None + + if token.startswith('"'): + for index in range(1, len(token)): + if token[index] == '"' and not _is_escaped(token, index): + return token[1:index] if index == len(token) - 1 else None + return None + + if token.startswith("{"): + depth = 0 + for index, char in enumerate(token): + if _is_escaped(token, index): + continue + if char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + return token[1:index] if index == len(token) - 1 else None + if depth < 0: + return None + return None + + class RemoveEnclosingMiddleware(BlockMiddleware): """Remove enclosing characters from values such as field and strings. @@ -51,6 +118,10 @@ def metadata_key(cls) -> str: @staticmethod def _strip_enclosing(value: str) -> tuple[str, str | None]: value = value.strip() + if _split_concatenation(value) is not None: + # The first and last character of a `#` expression do not enclose it, + # so stripping them would corrupt the expression. + return value, "no-enclosing" if value.startswith("{") and value.endswith("}"): return value[1:-1], "{" if value.startswith('"') and value.endswith('"'): diff --git a/bibtexparser/middlewares/interpolate.py b/bibtexparser/middlewares/interpolate.py index 2e5d832..a31ae42 100644 --- a/bibtexparser/middlewares/interpolate.py +++ b/bibtexparser/middlewares/interpolate.py @@ -7,6 +7,8 @@ from bibtexparser.model import Field from .enclosing import REMOVED_ENCLOSING_KEY +from .enclosing import _literal_content +from .enclosing import _split_concatenation from .middleware import LibraryMiddleware @@ -21,6 +23,54 @@ def _value_is_nonstring_or_enclosed(value: Any) -> bool: return False +def _resolve_tokens(tokens: list[str], strings: dict[str, str]) -> str | None: + """Join the resolved content of the tokens of a concatenation. + + Each token is a quoted or braced literal, a number, or a string reference. + Returns None if any of them is unresolvable, so that the caller can keep the + original expression instead of resolving it partially. + """ + resolved = [] + for token in tokens: + content = _literal_content(token) + if content is None: + content = token if token.isdigit() else strings.get(token.lower()) + if content is None: + return None + resolved.append(content) + return "".join(resolved) + + +def _resolve_value(value: str, strings: dict[str, str]) -> str | None: + """Resolve a literal, a number, a string reference or a `#` expression.""" + tokens = _split_concatenation(value) + return _resolve_tokens([value.strip()] if tokens is None else tokens, strings) + + +def _resolve_string_definitions(definitions: dict[str, str]) -> dict[str, str]: + """Resolve the content of every string definition that can be resolved. + + Definitions may reference each other, so this iterates until no further + definition resolves. Iterating rather than recursing means that an + arbitrarily long chain of definitions cannot exhaust the interpreter stack, + and that definitions on a reference cycle simply never resolve. + """ + resolved: dict[str, str] = {} + pending = dict(definitions) + while pending: + progressed = False + for key, value in list(pending.items()): + content = _resolve_value(value, resolved) + if content is None: + continue + resolved[key] = content + del pending[key] + progressed = True + if not progressed: + break + return resolved + + class ResolveStringReferencesMiddleware(LibraryMiddleware): """Replace strings references with their values.""" @@ -40,6 +90,7 @@ def transform(self, library: Library) -> Library: # BibTeX string keys are case-insensitive; later definitions win. string_values = {key.lower(): s.value for key, s in library.strings_dict.items()} + resolved_strings = _resolve_string_definitions(string_values) entry: Entry raised_enclosing_warning = False @@ -59,12 +110,38 @@ def transform(self, library: Library) -> Library: field: Field for field in entry.fields: + if isinstance(field.value, str): + tokens = _split_concatenation(field.value) + if tokens is not None: + content = _resolve_tokens(tokens, resolved_strings) + if content is not None: + # Braces keep the result a plain value: unenclosed, it + # would be read back as a reference when written. + field.value = "{" + content + "}" + resolved_fields.append(field.key) + # An unresolvable expression is left exactly as it is, + # which the enclosing middlewares preserve. + continue + if _value_is_nonstring_or_enclosed(field.value): continue + key = field.value.lower() try: - field.value = string_values[field.value.lower()] + referenced = string_values[key] except KeyError: continue + + if ( + _split_concatenation(referenced) is not None + or referenced.lower() in string_values + ): + # The definition is itself an expression or another reference, + # so it must be resolved rather than substituted verbatim. + content = resolved_strings.get(key) + if content is None: + continue + referenced = "{" + content + "}" + field.value = referenced resolved_fields.append(field.key) if resolved_fields: diff --git a/tests/middleware_tests/test_enclosing.py b/tests/middleware_tests/test_enclosing.py index e5e021f..998ba3c 100644 --- a/tests/middleware_tests/test_enclosing.py +++ b/tests/middleware_tests/test_enclosing.py @@ -421,4 +421,46 @@ def test_string_reference_roundtrip(): assert "year = {2019}" in written +@pytest.mark.parametrize( + "expression", + [ + '"first" # "second"', + "{first} # {second}", + '"prefix" # unknown_macro # "suffix"', + # The splitter treats a backslash-escaped delimiter as content, so this + # is a two-token expression rather than one quoted value. + '"sch\\"on" # "!"', + ], +) +def test_removal_keeps_multi_token_expressions_intact(expression: str): + """The first and last character of a `#` expression are not one enclosing pair. + + Cases adapted from @claell's comparison branch on issue #396. + """ + library = Library( + [ + Entry( + entry_type="article", key="someKey", fields=[Field(key="title", value=expression)] + ), + String(key="someString", value=expression), + ] + ) + + transformed = RemoveEnclosingMiddleware().transform(library) + + assert transformed.entries[0].fields[0].value == expression + assert transformed.entries[0].fields[0].enclosing == "no-enclosing" + assert transformed.strings[0].value == expression + assert transformed.strings[0].enclosing == "no-enclosing" + + +def test_string_definition_expression_roundtrip(): + """A definition whose expression starts and ends with a delimiter must not be stripped.""" + bibtex = '@string{quoted = "Asia" # "crypt"}\n@string{braced = {Asia} # {crypt}}' + written = bibtexparser.write_string(bibtexparser.parse_string(bibtex)) + + assert 'quoted = "Asia" # "crypt"' in written + assert "braced = {Asia} # {crypt}" in written + + # TODO round-trip tests (removal -> addition -> removal) diff --git a/tests/middleware_tests/test_interpolate.py b/tests/middleware_tests/test_interpolate.py index 264373c..b720b52 100644 --- a/tests/middleware_tests/test_interpolate.py +++ b/tests/middleware_tests/test_interpolate.py @@ -1,5 +1,6 @@ import pytest +import bibtexparser from bibtexparser.middlewares.enclosing import RemoveEnclosingMiddleware from bibtexparser.middlewares.interpolate import ResolveStringReferencesMiddleware from bibtexparser.splitter import Splitter @@ -66,3 +67,159 @@ def test_warning_is_raised_if_enclosings_are_removed(): assert len(record) == 1 assert "RemoveEnclosing" in record[0].message.args[0] + + +def test_string_interpolation_resolves_concatenation(): + bibtex = """ + @string{jan = "Jan."} + + @inbook{test_inbook, + month = 10 # "~" # jan, + title = "hello" # " " # "world", + mixed = jan # { } # "X", + numbers = 1 # 2 # 3 + } + """ + library = Splitter(bibtex).split() + + m = ResolveStringReferencesMiddleware() + library = m.transform(library) + + fields = library.entries_dict["test_inbook"].fields_dict + # Concatenations are resolved and kept enclosed in braces. + assert fields["month"].value == "{10~Jan.}" + assert fields["title"].value == "{hello world}" + assert fields["mixed"].value == "{Jan. X}" + assert fields["numbers"].value == "{123}" + + +def test_string_interpolation_leaves_unresolvable_concatenation_untouched(): + bibtex = """ + @inbook{test_inbook, + note = unknown_macro # " suffix" + } + """ + library = Splitter(bibtex).split() + + m = ResolveStringReferencesMiddleware() + library = m.transform(library) + + # An unknown reference means the expression is left exactly as-is. + assert ( + library.entries_dict["test_inbook"].fields_dict["note"].value == 'unknown_macro # " suffix"' + ) + + +def test_parse_string_resolves_concatenation_end_to_end(): + # Exact reproduction of issue #396. + bibtex_str = """ + @STRING{ jan = "Jan." } + + @INBOOK{inbook-full, + month = 10 # "~" # jan, + } + """ + library = bibtexparser.parse_string(bibtex_str) + month = library.entries[0].fields_dict["month"].value + assert month == "10~Jan." + + # The resolved value writes as a valid, brace-enclosed string and + # round-trips back to the same value. + written = bibtexparser.write_string(library) + assert "month = {10~Jan.}" in written + reparsed = bibtexparser.parse_string(written) + assert reparsed.entries[0].fields_dict["month"].value == "10~Jan." + + +def test_string_interpolation_resolves_string_definitions_recursively(): + """A definition may itself be an expression or another reference (cases via @claell).""" + bibtex = """ + @string{asiacryptname = "Asiacrypt"} + @string{asiacrypt91name = asiacryptname # "'91"} + @string{alias = asiacrypt91name} + @string{quoted = "Asia" # "crypt"} + @string{vol = 1 # 0} + + @inbook{test_inbook, + booktitle = asiacrypt91name, + series = alias, + note = asiacrypt91name # " proceedings", + title = quoted, + volume = vol + } + """ + library = bibtexparser.parse_string(bibtex) + + fields = library.entries_dict["test_inbook"].fields_dict + assert fields["booktitle"].value == "Asiacrypt'91" + assert fields["series"].value == "Asiacrypt'91" + assert fields["note"].value == "Asiacrypt'91 proceedings" + assert fields["title"].value == "Asiacrypt" + assert fields["volume"].value == "10" + + # The definitions keep their expression form, so the file still writes validly. + written = bibtexparser.write_string(library) + assert 'asiacrypt91name = asiacryptname # "\'91"' in written + assert 'quoted = "Asia" # "crypt"' in written + + +def test_string_interpolation_leaves_cyclic_definitions_unresolved(): + """Cyclic definitions resolve to nothing rather than to a partial value (case via @claell).""" + bibtex = """ + @string{first = second # " one"} + @string{second = first # " two"} + @string{itself = itself} + + @inbook{test_inbook, + title = first # "!", + note = second, + other = itself + } + """ + library = bibtexparser.parse_string(bibtex) + + fields = library.entries_dict["test_inbook"].fields_dict + assert fields["title"].value == 'first # "!"' + assert fields["note"].value == "second" + assert fields["other"].value == "itself" + + # Writing and re-parsing is a fixpoint, i.e. it does not substitute one more level. + written = bibtexparser.write_string(library) + reparsed = bibtexparser.parse_string(written).entries_dict["test_inbook"].fields_dict + assert reparsed["title"].value == 'first # "!"' + assert reparsed["note"].value == "second" + assert reparsed["other"].value == "itself" + + +def test_string_interpolation_resolves_long_definition_chains(): + """A chain of definitions is resolved iteratively, so it cannot exhaust the stack.""" + length = 500 + bibtex = "".join(f'@string{{s{i} = s{i + 1} # "x"}}\n' for i in range(length)) + bibtex += f'@string{{s{length} = "end"}}\n@inbook{{test_inbook, title = s0}}' + + library = bibtexparser.parse_string(bibtex) + + title = library.entries_dict["test_inbook"].fields_dict["title"] + assert title.value == "end" + "x" * length + + +def test_unresolvable_concatenation_survives_a_round_trip(): + """An expression with an unknown reference is written back exactly as it was read.""" + bibtex = ( + "@inbook{test_inbook,\n" + ' note = "prefix" # unknown_macro # "suffix",\n' + " title = {first} # unknown_macro # {second}\n" + "}" + ) + library = bibtexparser.parse_string(bibtex) + + fields = library.entries_dict["test_inbook"].fields_dict + assert fields["note"].value == '"prefix" # unknown_macro # "suffix"' + assert fields["title"].value == "{first} # unknown_macro # {second}" + + written = bibtexparser.write_string(library) + assert 'note = "prefix" # unknown_macro # "suffix"' in written + assert "title = {first} # unknown_macro # {second}" in written + reparsed = bibtexparser.parse_string(written).entries_dict["test_inbook"].fields_dict + assert reparsed["note"].value == fields["note"].value + assert reparsed["title"].value == fields["title"].value