Resolve string concatenation expressions in field values (#396)

bibtexparser/middlewares/enclosing.py

@@ -20,6 +20,73 @@
 ]
 
 
+def _is_escaped(value: str, index: int) -> bool:
+    """Whether the character at `index` is escaped, as the splitter defines it."""
+    return index > 0 and value[index - 1] == "\\"
+
+
+def _split_concatenation(value: str) -> list[str] | None:
+    """Split a top-level bibtex `#` concatenation into its tokens.
+
+    Returns None if `value` is not a concatenation expression, i.e., if it holds
+    no `#` outside of a `"..."` or `{...}` group, or if its delimiters are
+    unbalanced (in which case its structure is unknown and must not be guessed).
+    """
+    tokens = []
+    token_start = 0
+    depth = 0
+    in_quotes = False
+    for index, char in enumerate(value):
+        if _is_escaped(value, index):
+            continue
+        if char == '"' and depth == 0:
+            in_quotes = not in_quotes
+        elif in_quotes:
+            continue
+        elif char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth < 0:
+                return None
+        elif char == "#" and depth == 0:
+            tokens.append(value[token_start:index].strip())
+            token_start = index + 1
+
+    if not tokens or in_quotes or depth != 0:
+        return None
+
+    tokens.append(value[token_start:].strip())
+    return tokens
+
+
+def _literal_content(token: str) -> str | None:
+    """The content of a token that is one fully quoted or brace-enclosed literal."""
+    if len(token) < 2:
+        return None
+
+    if token.startswith('"'):
+        for index in range(1, len(token)):
+            if token[index] == '"' and not _is_escaped(token, index):
+                return token[1:index] if index == len(token) - 1 else None
+        return None
+
+    if token.startswith("{"):
+        depth = 0
+        for index, char in enumerate(token):
+            if _is_escaped(token, index):
+                continue
+            if char == "{":
+                depth += 1
+            elif char == "}":
+                depth -= 1
+                if depth == 0:
+                    return token[1:index] if index == len(token) - 1 else None
+                if depth < 0:
+                    return None
+    return None
+
+
 class RemoveEnclosingMiddleware(BlockMiddleware):
     """Remove enclosing characters from values such as field and strings.
 
@@ -51,6 +118,10 @@ def metadata_key(cls) -> str:
     @staticmethod
     def _strip_enclosing(value: str) -> tuple[str, str | None]:
         value = value.strip()
+        if _split_concatenation(value) is not None:
+            # The first and last character of a `#` expression do not enclose it,
+            # so stripping them would corrupt the expression.
+            return value, "no-enclosing"
         if value.startswith("{") and value.endswith("}"):
             return value[1:-1], "{"
         if value.startswith('"') and value.endswith('"'):

bibtexparser/middlewares/interpolate.py

@@ -7,6 +7,8 @@
 from bibtexparser.model import Field
 
 from .enclosing import REMOVED_ENCLOSING_KEY
+from .enclosing import _literal_content
+from .enclosing import _split_concatenation
 from .middleware import LibraryMiddleware
 
 
@@ -21,6 +23,54 @@ def _value_is_nonstring_or_enclosed(value: Any) -> bool:
     return False
 
 
+def _resolve_tokens(tokens: list[str], strings: dict[str, str]) -> str | None:
+    """Join the resolved content of the tokens of a concatenation.
+
+    Each token is a quoted or braced literal, a number, or a string reference.
+    Returns None if any of them is unresolvable, so that the caller can keep the
+    original expression instead of resolving it partially.
+    """
+    resolved = []
+    for token in tokens:
+        content = _literal_content(token)
+        if content is None:
+            content = token if token.isdigit() else strings.get(token.lower())
+        if content is None:
+            return None
+        resolved.append(content)
+    return "".join(resolved)
+
+
+def _resolve_value(value: str, strings: dict[str, str]) -> str | None:
+    """Resolve a literal, a number, a string reference or a `#` expression."""
+    tokens = _split_concatenation(value)
+    return _resolve_tokens([value.strip()] if tokens is None else tokens, strings)
+
+
+def _resolve_string_definitions(definitions: dict[str, str]) -> dict[str, str]:
+    """Resolve the content of every string definition that can be resolved.
+
+    Definitions may reference each other, so this iterates until no further
+    definition resolves. Iterating rather than recursing means that an
+    arbitrarily long chain of definitions cannot exhaust the interpreter stack,
+    and that definitions on a reference cycle simply never resolve.
+    """
+    resolved: dict[str, str] = {}
+    pending = dict(definitions)
+    while pending:
+        progressed = False
+        for key, value in list(pending.items()):
+            content = _resolve_value(value, resolved)
+            if content is None:
+                continue
+            resolved[key] = content
+            del pending[key]
+            progressed = True
+        if not progressed:
+            break
+    return resolved
+
+
 class ResolveStringReferencesMiddleware(LibraryMiddleware):
     """Replace strings references with their values."""
 
@@ -40,6 +90,7 @@ def transform(self, library: Library) -> Library:
 
         # BibTeX string keys are case-insensitive; later definitions win.
         string_values = {key.lower(): s.value for key, s in library.strings_dict.items()}
+        resolved_strings = _resolve_string_definitions(string_values)
 
         entry: Entry
         raised_enclosing_warning = False
@@ -59,12 +110,38 @@ def transform(self, library: Library) -> Library:
 
             field: Field
             for field in entry.fields:
+                if isinstance(field.value, str):
+                    tokens = _split_concatenation(field.value)
+                    if tokens is not None:
+                        content = _resolve_tokens(tokens, resolved_strings)
+                        if content is not None:
+                            # Braces keep the result a plain value: unenclosed, it
+                            # would be read back as a reference when written.
+                            field.value = "{" + content + "}"
+                            resolved_fields.append(field.key)
+                        # An unresolvable expression is left exactly as it is,
+                        # which the enclosing middlewares preserve.
+                        continue
+
                 if _value_is_nonstring_or_enclosed(field.value):
                     continue
+                key = field.value.lower()
                 try:
-                    field.value = string_values[field.value.lower()]
+                    referenced = string_values[key]
                 except KeyError:
                     continue
+
+                if (
+                    _split_concatenation(referenced) is not None
+                    or referenced.lower() in string_values
+                ):
+                    # The definition is itself an expression or another reference,
+                    # so it must be resolved rather than substituted verbatim.
+                    content = resolved_strings.get(key)
+                    if content is None:
+                        continue
+                    referenced = "{" + content + "}"
+                field.value = referenced
                 resolved_fields.append(field.key)
 
             if resolved_fields:

tests/middleware_tests/test_enclosing.py

@@ -421,4 +421,46 @@ def test_string_reference_roundtrip():
     assert "year = {2019}" in written
 
 
+@pytest.mark.parametrize(
+    "expression",
+    [
+        '"first" # "second"',
+        "{first} # {second}",
+        '"prefix" # unknown_macro # "suffix"',
+        # The splitter treats a backslash-escaped delimiter as content, so this
+        # is a two-token expression rather than one quoted value.
+        '"sch\\"on" # "!"',
+    ],
+)
+def test_removal_keeps_multi_token_expressions_intact(expression: str):
+    """The first and last character of a `#` expression are not one enclosing pair.
+
+    Cases adapted from @claell's comparison branch on issue #396.
+    """
+    library = Library(
+        [
+            Entry(
+                entry_type="article", key="someKey", fields=[Field(key="title", value=expression)]
+            ),
+            String(key="someString", value=expression),
+        ]
+    )
+
+    transformed = RemoveEnclosingMiddleware().transform(library)
+
+    assert transformed.entries[0].fields[0].value == expression
+    assert transformed.entries[0].fields[0].enclosing == "no-enclosing"
+    assert transformed.strings[0].value == expression
+    assert transformed.strings[0].enclosing == "no-enclosing"
+
+
+def test_string_definition_expression_roundtrip():
+    """A definition whose expression starts and ends with a delimiter must not be stripped."""
+    bibtex = '@string{quoted = "Asia" # "crypt"}\n@string{braced = {Asia} # {crypt}}'
+    written = bibtexparser.write_string(bibtexparser.parse_string(bibtex))
+
+    assert 'quoted = "Asia" # "crypt"' in written
+    assert "braced = {Asia} # {crypt}" in written
+
+
 # TODO round-trip tests (removal -> addition -> removal)