gh-153569: store formatted-string text as source spans

Formatted-string expressions and comments can outlive the buffer window where
scanning began. Pointer boundaries require buffer relocation to repair tokenizer
state.

Record logical source ranges instead. Retain the active input window while a
formatted string is open, give each mode ownership of its comment spans, and
materialize text through shared span views.

Author: pablogsal

Lib/test/test_fstring.py

@@ -1679,6 +1679,20 @@ def __repr__(self):
 
         self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'')
         self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'')
+        self.assertEqual(f'{"""a" # inside"""=}',
+                         '"""a" # inside"""=\'a" # inside\'')
+        self.assertEqual(f"{'''a' # inside'''=}",
+                         "'''a' # inside'''=\"a' # inside\"")
+        self.assertEqual(f'{"""a""""#" # outside
+=}', '"""a""""#" \n=\'a#\'')
+
+        x, y = 1, 2
+        self.assertEqual(f'{x != y # outside
+=}', 'x != y \n=True')
+
+        d = {'a#b': 42}
+        self.assertEqual(f'''{f"{d["a#b"]}"=}''',
+                         'f"{d["a#b"]}"=\'42\'')
 
         self.assertEqual(f'{ # some comment goes here
   """hello"""=}',  ' \n  """hello"""=\'hello\'')

Lib/test/test_tokenize.py

@@ -2573,6 +2573,25 @@ def test_degraded_fstring_format_spec(self):
             ("f-string: single '}' is not allowed", (1, 11)),
         )
 
+    def test_carriage_return_after_debug_comment(self):
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                tokens = self._get_tokens(f"{prefix}'''{{x=# comment\r}}'''")
+                self.assertEqual(tokens[4].string, "# comment\r}")
+
+    def test_incomplete_formatted_string_comment_after_carriage_return(self):
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                for extra_tokens in (False, True):
+                    with self.assertRaises(tokenize.TokenError) as caught:
+                        self._get_tokens(
+                            f"{prefix}'{{#\r!", extra_tokens=extra_tokens
+                        )
+                    self.assertEqual(
+                        caught.exception.args,
+                        ("unexpected EOF in multi-line statement", (1, 7)),
+                    )
+
     def test_escaped_fstring_brace_has_a_position_gap(self):
         tokens = self._get_tokens('f"a{{"', extra_tokens=True)
         self.assertEqual(

Lib/test/test_tstring.py

@@ -287,5 +287,22 @@ def test_triple_quoted(self):
         )
         self.assertEqual(fstring(t), "\n        Hello,\n        Python\n        ")
 
+        t = t'{"""a" # inside"""}'
+        self.assertEqual(t.interpolations[0].expression,
+                         '"""a" # inside"""')
+
+        t = t'{"""a""""#" # outside
+}'
+        self.assertEqual(t.interpolations[0].expression, '"""a""""#"')
+
+        x, y = 1, 2
+        t = t'{x != y # outside
+}'
+        self.assertEqual(t.interpolations[0].expression, 'x != y')
+
+        d = {'a#b': 42}
+        t = t'''{f"{d["a#b"]}"}'''
+        self.assertEqual(t.interpolations[0].expression, 'f"{d["a#b"]}"')
+
 if __name__ == '__main__':
     unittest.main()

Parser/lexer/buffer.c

@@ -15,12 +15,6 @@ _PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base,
         ? -1 : tok->line_start - tok->buf;
     pointers->multi_line_start_from_buf = tok->multi_line_start == NULL
         ? -1 : tok->multi_line_start - tok->buf;
-    for (int index = tok->tok_mode_stack_index; index > 0; --index) {
-        tokenizer_mode *mode = &tok->tok_mode_stack[index];
-        mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
-        mode->multi_line_start_offset = mode->multi_line_start == NULL
-            ? -1 : mode->multi_line_start - tok->buf;
-    }
 }
 
 void
@@ -36,11 +30,4 @@ _PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base,
         ? NULL : tok->buf + pointers->line_start_from_buf;
     tok->multi_line_start = pointers->multi_line_start_from_buf < 0
         ? NULL : tok->buf + pointers->multi_line_start_from_buf;
-    for (int index = tok->tok_mode_stack_index; index > 0; --index) {
-        tokenizer_mode *mode = &tok->tok_mode_stack[index];
-        mode->start = mode->start_offset < 0
-            ? NULL : tok->buf + mode->start_offset;
-        mode->multi_line_start = mode->multi_line_start_offset < 0
-            ? NULL : tok->buf + mode->multi_line_start_offset;
-    }
 }

Parser/lexer/lexer.c

@@ -317,6 +317,18 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             c = tok_nextc(tok);
         }
 
+        if (INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) {
+            const char *comment_end = tok->cur;
+            if (c == '\n' || c == '\r') {
+                comment_end--;
+            }
+            if (_PyLexer_record_ftstring_comment(
+                    tok, tok->start, comment_end) < 0) {
+                tok->done = E_NOMEM;
+                return MAKE_TOKEN(ERRORTOKEN);
+            }
+        }
+
         if (tok->tok_extra_tokens) {
             p = tok->start;
         }
@@ -544,10 +556,18 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
          int cursor_in_format_with_debug =
              cursor == 1 && (current_tok->in_debug || in_format_spec);
          int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
-        if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
-            return MAKE_TOKEN(ENDMARKER);
+        if (cursor_valid && c == '!') {
+            int c2 = tok_nextc(tok);
+            if (c2 == '=') {
+                cursor_valid = 0;
+            }
+            tok_backup(tok, c2);
+        }
+        if (cursor_valid) {
+            _PyLexer_update_ftstring_expr(tok, c);
         }
-        if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) {
+        if (cursor_valid && c != '{' &&
+                _PyLexer_set_ftstring_expr_metadata(tok, token)) {
             return MAKE_TOKEN(ERRORTOKEN);
         }
 

Parser/lexer/lexer.h

@@ -3,8 +3,6 @@
 
 #include "state.h"
 
-int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur);
-
 int _PyTokenizer_Get(struct tok_state *, struct token *);
 
 /* The view points into the current input window. The next
@@ -19,13 +17,7 @@ _PyToken_TextView(const struct tok_state *tok, const struct token *token,
         *length = 0;
         return "";
     }
-    assert(_PyTok_SpanIsValid(token->span));
-    assert(tok->buf != NULL);
-    assert(tok->inp >= tok->buf);
-    assert(token->span.start >= tok->buf_offset);
-    assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf);
-    *length = token->span.end - token->span.start;
-    return tok->buf + (token->span.start - tok->buf_offset);
+    return _PyLexer_BufferSpanView(tok, token->span, length);
 }
 
 #endif

Parser/lexer/lexer_internal.h

@@ -46,7 +46,10 @@ TOK_NEXT_MODE(struct tok_state *tok)
 
 int _PyLexer_nextc(struct tok_state *);
 void _PyLexer_backup(struct tok_state *, int);
-int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char);
+void _PyLexer_update_ftstring_expr(struct tok_state *, char);
+int _PyLexer_record_ftstring_comment(
+    struct tok_state *, const char *, const char *);
+int _PyLexer_set_ftstring_expr_metadata(struct tok_state *, struct token *);
 int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
 int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
 int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int);

Parser/lexer/state.c

@@ -60,24 +60,6 @@ _PyTokenizer_tok_new(void)
     return tok;
 }
 
-static void
-free_fstring_expressions(struct tok_state *tok)
-{
-    int index;
-    tokenizer_mode *mode;
-
-    for (index = tok->tok_mode_stack_index; index >= 0; --index) {
-        mode = &(tok->tok_mode_stack[index]);
-        if (mode->last_expr_buffer != NULL) {
-            PyMem_Free(mode->last_expr_buffer);
-            mode->last_expr_buffer = NULL;
-            mode->last_expr_size = 0;
-            mode->last_expr_end = -1;
-            mode->in_format_spec = 0;
-        }
-    }
-}
-
 /* Free a tok_state structure */
 void
 _PyTokenizer_Free(struct tok_state *tok)
@@ -89,7 +71,9 @@ _PyTokenizer_Free(struct tok_state *tok)
     Py_XDECREF(tok->module);
     _PyTok_ReaderFree(tok);
     _PyTok_SourceClear(&tok->source);
-    free_fstring_expressions(tok);
+    for (int i = 0; i <= tok->tok_mode_stack_index; i++) {
+        PyMem_Free(tok->tok_mode_stack[i].comments);
+    }
     PyMem_Free(tok);
 }
 
@@ -108,31 +92,11 @@ _PyToken_Init(struct token *token) {
     token->metadata = NULL;
 }
 
-static inline _PyTok_Span
-buffer_span(const struct tok_state *tok, const char *start, const char *end)
-{
-    if (start == NULL) {
-        assert(end == NULL);
-        return (_PyTok_Span){-1, -1};
-    }
-    assert(end != NULL);
-    const char *base = tok->buf;
-    assert(base != NULL);
-    assert(tok->inp >= base);
-    Py_ssize_t start_offset = start - base;
-    Py_ssize_t end_offset = end - base;
-    assert(start_offset >= 0 && start_offset <= end_offset);
-    assert(end_offset <= tok->inp - base);
-    assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset);
-    return _PyTok_SpanFromBounds(
-        tok->buf_offset + start_offset, tok->buf_offset + end_offset);
-}
-
 int
 _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
 {
     token->level = tok->level;
-    token->span = buffer_span(tok, start, end);
+    token->span = _PyLexer_BufferSpan(tok, start, end);
     int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;
     token->start_loc = (_PyTok_Loc){lineno, -1};
     token->end_loc = (_PyTok_Loc){tok->lineno, -1};

Parser/lexer/state.h

@@ -41,6 +41,12 @@ enum string_kind_t {
 
 #define MAX_EXPR_NESTING 3
 
+typedef struct _tokenizer_comments {
+    Py_ssize_t count;
+    Py_ssize_t capacity;
+    _PyTok_Span spans[];
+} tokenizer_comments;
+
 typedef struct _tokenizer_mode {
     enum tokenizer_mode_kind_t kind;
 
@@ -50,20 +56,16 @@ typedef struct _tokenizer_mode {
     char quote;
     int quote_size;
     int raw;
-    const char* start;
-    const char* multi_line_start;
+    _PyTok_Off start;
+    _PyTok_Off multi_line_start;
     int first_line;
 
-    Py_ssize_t start_offset;
-    Py_ssize_t multi_line_start_offset;
-
-    Py_ssize_t last_expr_size;
-    Py_ssize_t last_expr_end;
-    char* last_expr_buffer;
+    _PyTok_Span expr_span;
     int in_debug;
     int in_format_spec;
 
     enum string_kind_t string_kind;
+    tokenizer_comments *comments;
 } tokenizer_mode;
 
 /* Tokenizer state */
@@ -129,6 +131,53 @@ struct tok_state {
 #endif
 };
 
+static inline _PyTok_Off
+_PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
+{
+    assert(tok->buf != NULL);
+    assert(tok->inp >= tok->buf);
+    assert(position >= tok->buf && position <= tok->inp);
+    Py_ssize_t offset = position - tok->buf;
+    assert(tok->buf_offset <= PY_SSIZE_T_MAX - offset);
+    return tok->buf_offset + offset;
+}
+
+static inline char *
+_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
+{
+    assert(tok->buf != NULL);
+    assert(tok->inp >= tok->buf);
+    assert(offset >= tok->buf_offset);
+    assert(offset - tok->buf_offset <= tok->inp - tok->buf);
+    return tok->buf + (offset - tok->buf_offset);
+}
+
+static inline const char *
+_PyLexer_BufferSpanView(const struct tok_state *tok, _PyTok_Span span,
+                        Py_ssize_t *length)
+{
+    assert(length != NULL);
+    assert(_PyTok_SpanIsValid(span));
+    *length = span.end - span.start;
+    (void)_PyLexer_BufferPointer(tok, span.end);
+    return _PyLexer_BufferPointer(tok, span.start);
+}
+
+static inline _PyTok_Span
+_PyLexer_BufferSpan(const struct tok_state *tok, const char *start,
+                    const char *end)
+{
+    if (start == NULL) {
+        assert(end == NULL);
+        return (_PyTok_Span){-1, -1};
+    }
+    assert(end != NULL);
+    assert(start <= end);
+    return _PyTok_SpanFromBounds(
+        _PyLexer_BufferOffset(tok, start),
+        _PyLexer_BufferOffset(tok, end));
+}
+
 int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
 
 struct tok_state *_PyTokenizer_tok_new(void);