diff --git a/Lib/test/test_capi/test_tokenizer.py b/Lib/test/test_capi/test_tokenizer.py
index 2fe1fef241e90ae..eb04f6c0136022d 100644
--- a/Lib/test/test_capi/test_tokenizer.py
+++ b/Lib/test/test_capi/test_tokenizer.py
@@ -9,6 +9,9 @@ class TokenizerTests(unittest.TestCase):
     def test_source(self):
         _testinternalcapi.test_tokenizer_source()
 
+    def test_source_discard(self):
+        _testinternalcapi.test_tokenizer_source_discard()
+
     def test_cursor(self):
         _testinternalcapi.test_tokenizer_cursor()
 
diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py
index ea0f72e0e8d4568..372c110783bce7a 100644
--- a/Lib/test/test_repl.py
+++ b/Lib/test/test_repl.py
@@ -198,6 +198,22 @@ def test_lexer_buffer_realloc_with_null_start(self):
         self.assertEqual(p.returncode, 0)
         self.assertIn(long_value, output)
 
+    @cpython_only
+    def test_multiline_fstring_source_reallocation(self):
+        long_line = " " * 9000 + "+ 2"
+        user_input = (
+            'value = f"""{(\n'
+            '1\n'
+            f'{long_line}\n'
+            ')}"""\n'
+            'print(value)\n'
+        )
+        p = spawn_repl()
+        p.stdin.write(user_input)
+        output = kill_python(p)
+        self.assertEqual(p.returncode, 0)
+        self.assertIn(">>> 3\n>>> ", output)
+
     def test_close_stdin(self):
         user_input = dedent('''
             import os
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index c471f857660ec90..948b341a5dd72ec 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -2427,6 +2427,30 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self):
             (token.ENDMARKER, "", (1, 0), (1, 0), ""),
         )
 
+    def test_fstring_offsets_survive_buffer_reallocation(self):
+        for prefix in ("f", "t"):
+            for extra_tokens in (False, True):
+                with self.subTest(prefix=prefix, extra_tokens=extra_tokens):
+                    physical_lines = [
+                        prefix + '"""\n',
+                        "{(\n",
+                        " " * 9000 + "1\n",
+                        ")=:>{2}}\n",
+                        '"""\n',
+                    ]
+                    source = "".join(physical_lines)
+                    chunks = iter([
+                        "".join(physical_lines[:2]),
+                        "".join(physical_lines[2:4]),
+                        physical_lines[4],
+                        "",
+                    ])
+                    expected = self._get_tokens(
+                        source, extra_tokens=extra_tokens)
+                    tokens = list(tokenize._generate_tokens_from_c_tokenizer(
+                        chunks.__next__, extra_tokens=extra_tokens))
+                    self.assertEqual(tokens, expected)
+
     def test_extra_tokens_relaxes_lexer_errors(self):
         cases = [
             (
diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c
index 0b292410d3eb4ef..df481cb832a4363 100644
--- a/Modules/_testinternalcapi/tokenizer.c
+++ b/Modules/_testinternalcapi/tokenizer.c
@@ -195,6 +195,28 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
         goto error;
     }
 
+    _PyTok_SourceDiscard(&source);
+    if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4,
+              "wrong retained source offset") < 0 ||
+            _PyTok_SourceLine(&source, 1, &line) < 0 ||
+            check(line.start == 4 && line.end == 6,
+                  "wrong retained source line") < 0 ||
+            _PyTok_SourceLocation(
+                &source, 4, _PYTOK_AFFINITY_LEFT, &loc) < 0 ||
+            check(loc.lineno == 1 && loc.byte_col == 0,
+                  "wrong retained source location") < 0) {
+        goto error;
+    }
+    view = _PyTok_SourceSpanView(
+        &source, _PyTok_SpanFromBounds(4, 5), &view_len);
+    if (check(view != NULL && view_len == 1 && view[0] == 'a',
+              "wrong retained source span") < 0 ||
+            check_system_error(_PyTok_SourceSpanView(
+                &source, _PyTok_SpanFromBounds(0, 1), &view_len) == NULL,
+                "accepted discarded source span") < 0) {
+        goto error;
+    }
+
     _PyTok_SourceClear(&source);
     Py_RETURN_NONE;
 
@@ -296,6 +318,88 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module),
     }
 #endif
 
+    _PyTok_Off base = source.len;
+    _PyTok_SourceDiscard(&source);
+    if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 ||
+            _PyTok_SourceAppendLine(&source, "cd", 2, 0) < 0) {
+        goto error;
+    }
+    _PyTok_CursorInit(&cursor, &source);
+    if (_PyTok_CursorSetLine(&cursor, 1) < 0 ||
+            check(cursor.pos == base && _PyTok_CursorPeek(&cursor, 1) == 'b',
+                  "wrong retained cursor line") < 0 ||
+            _PyTok_CursorSetLine(&cursor, 2) < 0 ||
+            check(_PyTok_CursorAdvance(&cursor) == 'c',
+                  "wrong retained cursor byte") < 0 ||
+            _PyTok_CursorSetOffset(&cursor, base + 5) < 0 ||
+            check(cursor.lineno == 2 && _PyTok_CursorAdvance(&cursor) == EOF,
+                  "wrong retained cursor EOF") < 0) {
+        goto error;
+    }
+
+    _PyTok_SourceClear(&source);
+    Py_RETURN_NONE;
+
+error:
+    _PyTok_SourceClear(&source);
+    return NULL;
+}
+
+static PyObject *
+test_tokenizer_source_discard(PyObject *Py_UNUSED(module),
+                             PyObject *Py_UNUSED(args))
+{
+    _PyTok_SourceText source;
+    _PyTok_SourceInit(&source);
+    for (int i = 0; i < 260; i++) {
+        if (_PyTok_SourceAppendLine(&source, "x\n", 2, 1) < 0) {
+            goto error;
+        }
+    }
+    char *bytes = source.bytes;
+    _PyTok_Off capacity = source.cap;
+    _PyTok_SourceDiscard(&source);
+    if (check(source.base_offset == 520 && source.len == 0 &&
+                  source.nlines == 0 && source.bytes == bytes &&
+                  source.cap == capacity && source.bytes[0] == '\0',
+              "discard did not preserve source allocation") < 0) {
+        goto error;
+    }
+    for (int i = 0; i < 260; i++) {
+        if (check(_PyTok_SourceAppendLine(&source, "y\n", 2, 0) == 520 + 2 * i,
+                  "wrong source offset after discard") < 0 ||
+                check(!_PyTok_SourceLineIsImplicit(&source, i + 1),
+                      "discard preserved implicit newline flag") < 0) {
+            goto error;
+        }
+    }
+    if (check(source.bytes == bytes && source.cap == capacity,
+              "discarded allocation was not reused") < 0) {
+        goto error;
+    }
+    _PyTok_SourceDiscard(&source);
+    if (check(_PyTok_SourceAppendLine(&source, "tail", 4, 0) == 1040,
+              "wrong source offset after repeated discard") < 0) {
+        goto error;
+    }
+    _PyTok_SourceDiscard(&source);
+    if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) == 1044,
+              "cannot append after discarding unterminated line") < 0) {
+        goto error;
+    }
+    _PyTok_SourceDiscard(&source);
+    source.base_offset = PY_SSIZE_T_MAX - 1;
+    if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) < 0 &&
+                  PyErr_ExceptionMatches(PyExc_MemoryError),
+              "accepted overflowing logical source offset") < 0) {
+        goto error;
+    }
+    PyErr_Clear();
+    if (check(source.len == 0 && source.nlines == 0 &&
+                  source.base_offset == PY_SSIZE_T_MAX - 1,
+              "overflow changed retained source") < 0) {
+        goto error;
+    }
     _PyTok_SourceClear(&source);
     Py_RETURN_NONE;
 
@@ -307,6 +411,7 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module),
 static PyMethodDef test_methods[] = {
     {"test_tokenizer_source", test_tokenizer_source, METH_NOARGS},
     {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS},
+    {"test_tokenizer_source_discard", test_tokenizer_source_discard, METH_NOARGS},
     {NULL},
 };
 
diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c
index cd6885a7d01040a..9c39544ca7c4790 100644
--- a/Parser/lexer/buffer.c
+++ b/Parser/lexer/buffer.c
@@ -1,62 +1,46 @@
 #include "Python.h"
-#include "errcode.h"
-
+#include "buffer.h"
 #include "state.h"
 
-/* Traverse and remember all f-string buffers, in order to be able to restore
-   them after reallocating tok->buf */
 void
-_PyLexer_remember_fstring_buffers(struct tok_state *tok)
+_PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base,
+                            _PyLexer_BufferPointers *pointers)
 {
-    int index;
-    tokenizer_mode *mode;
-
-    for (index = tok->tok_mode_stack_index; index >= 0; --index) {
-        mode = &(tok->tok_mode_stack[index]);
+    pointers->buf_from_base = tok->buf - base;
+    pointers->cur_from_buf = tok->cur - tok->buf;
+    pointers->inp_from_buf = tok->inp - tok->buf;
+    pointers->start_from_buf = tok->start == NULL
+        ? -1 : tok->start - tok->buf;
+    pointers->line_start_from_buf = tok->line_start == NULL
+        ? -1 : tok->line_start - tok->buf;
+    pointers->multi_line_start_from_buf = tok->multi_line_start == NULL
+        ? -1 : tok->multi_line_start - tok->buf;
+    for (int index = tok->tok_mode_stack_index; index > 0; --index) {
+        tokenizer_mode *mode = &tok->tok_mode_stack[index];
         mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
-        mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf;
+        mode->multi_line_start_offset = mode->multi_line_start == NULL
+            ? -1 : mode->multi_line_start - tok->buf;
     }
 }
 
-/* Traverse and restore all f-string buffers after reallocating tok->buf */
 void
-_PyLexer_restore_fstring_buffers(struct tok_state *tok)
-{
-    int index;
-    tokenizer_mode *mode;
-
-    for (index = tok->tok_mode_stack_index; index >= 0; --index) {
-        mode = &(tok->tok_mode_stack[index]);
-        mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset;
-        mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset;
-    }
-}
-
-int
-_PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size)
+_PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base,
+                               const _PyLexer_BufferPointers *pointers)
 {
-    Py_ssize_t cur = tok->cur - tok->buf;
-    Py_ssize_t oldsize = tok->inp - tok->buf;
-    Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1);
-    if (newsize > tok->end - tok->buf) {
-        char *newbuf = tok->buf;
-        Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf;
-        Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf;
-        Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf;
-        _PyLexer_remember_fstring_buffers(tok);
-        newbuf = (char *)PyMem_Realloc(newbuf, newsize);
-        if (newbuf == NULL) {
-            tok->done = E_NOMEM;
-            return 0;
-        }
-        tok->buf = newbuf;
-        tok->cur = tok->buf + cur;
-        tok->inp = tok->buf + oldsize;
-        tok->end = tok->buf + newsize;
-        tok->start = start < 0 ? NULL : tok->buf + start;
-        tok->line_start = line_start < 0 ? NULL : tok->buf + line_start;
-        tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start;
-        _PyLexer_restore_fstring_buffers(tok);
+    tok->buf = base + pointers->buf_from_base;
+    tok->cur = tok->buf + pointers->cur_from_buf;
+    tok->inp = tok->buf + pointers->inp_from_buf;
+    tok->start = pointers->start_from_buf < 0
+        ? NULL : tok->buf + pointers->start_from_buf;
+    tok->line_start = pointers->line_start_from_buf < 0
+        ? NULL : tok->buf + pointers->line_start_from_buf;
+    tok->multi_line_start = pointers->multi_line_start_from_buf < 0
+        ? NULL : tok->buf + pointers->multi_line_start_from_buf;
+    for (int index = tok->tok_mode_stack_index; index > 0; --index) {
+        tokenizer_mode *mode = &tok->tok_mode_stack[index];
+        mode->start = mode->start_offset < 0
+            ? NULL : tok->buf + mode->start_offset;
+        mode->multi_line_start = mode->multi_line_start_offset < 0
+            ? NULL : tok->buf + mode->multi_line_start_offset;
     }
-    return 1;
 }
diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h
index bb218162ff48453..285da124226d50e 100644
--- a/Parser/lexer/buffer.h
+++ b/Parser/lexer/buffer.h
@@ -3,8 +3,20 @@
 
 #include "pyport.h"
 
-void _PyLexer_remember_fstring_buffers(struct tok_state *tok);
-void _PyLexer_restore_fstring_buffers(struct tok_state *tok);
-int _PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size);
+struct tok_state;
+
+typedef struct {
+    Py_ssize_t buf_from_base;
+    Py_ssize_t cur_from_buf;
+    Py_ssize_t inp_from_buf;
+    Py_ssize_t start_from_buf;
+    Py_ssize_t line_start_from_buf;
+    Py_ssize_t multi_line_start_from_buf;
+} _PyLexer_BufferPointers;
+
+void _PyLexer_SaveBufferPointers(
+    struct tok_state *, const char *, _PyLexer_BufferPointers *);
+void _PyLexer_RestoreBufferPointers(
+    struct tok_state *, char *, const _PyLexer_BufferPointers *);
 
 #endif
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index a96362c8961023a..f96b31b9d2f38a1 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -12,8 +12,6 @@
 
 
 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
-#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\
-                _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end))
 
 /* Spaces in this constant are treated as "zero or more spaces or tabs" when
    tokenizing. */
@@ -360,21 +358,25 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
                     && !(tok->cur > ignore_end
                          && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));
 
+                int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT;
+                int start_col_offset = is_type_ignore
+                    ? ignore_end_col_offset : current_starting_col_offset;
+                p_end = tok->cur;
                 if (is_type_ignore) {
                     p_start = ignore_end;
-                    p_end = tok->cur;
 
                     /* If this type ignore is the only thing on the line, consume the newline also. */
                     if (blankline) {
                         tok_nextc(tok);
                         tok->atbol = 1;
                     }
-                    return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset);
                 } else {
                     p_start = type_start;
-                    p_end = tok->cur;
-                    return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset);
                 }
+                _PyLexer_token_setup(tok, token, type, p_start, p_end);
+                token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
+                token->end_loc = (_PyTok_Loc){tok->lineno, tok->col_offset};
+                return type;
             }
         }
         if (tok->tok_extra_tokens) {
diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h
index 1d97ac57b745b09..040935a7e689138 100644
--- a/Parser/lexer/lexer.h
+++ b/Parser/lexer/lexer.h
@@ -7,4 +7,25 @@ int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur);
 
 int _PyTokenizer_Get(struct tok_state *, struct token *);
 
+/* The view points into the current input window. The next
+   _PyTokenizer_Get() call may discard it. */
+static inline const char *
+_PyToken_TextView(const struct tok_state *tok, const struct token *token,
+                  Py_ssize_t *length)
+{
+    assert(length != NULL);
+    if (token->span.start < 0) {
+        assert(token->span.start == -1 && token->span.end == -1);
+        *length = 0;
+        return "";
+    }
+    assert(_PyTok_SpanIsValid(token->span));
+    assert(tok->buf != NULL);
+    assert(tok->inp >= tok->buf);
+    assert(token->span.start >= tok->buf_offset);
+    assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf);
+    *length = token->span.end - token->span.start;
+    return tok->buf + (token->span.start - tok->buf_offset);
+}
+
 #endif
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index 2a6408bef927a36..d82a7d0f296bac0 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -26,7 +26,6 @@ _PyTokenizer_tok_new(void)
     tok->interactive_src_start = NULL;
     tok->interactive_src_end = NULL;
     tok->start = NULL;
-    tok->end = NULL;
     tok->done = E_OK;
     tok->fp = NULL;
     tok->tabsize = TABSIZE;
@@ -101,41 +100,46 @@ _PyToken_Free(struct token *token) {
 
 void
 _PyToken_Init(struct token *token) {
+#ifdef Py_DEBUG
+    token->span = (_PyTok_Span){-1, -1};
+    token->start_loc = (_PyTok_Loc){-1, -1};
+    token->end_loc = (_PyTok_Loc){-1, -1};
+#endif
     token->metadata = NULL;
 }
 
-int
-_PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
-                         int end_col_offset, const char *start, const char *end)
+static inline _PyTok_Span
+buffer_span(const struct tok_state *tok, const char *start, const char *end)
 {
-    token->level = tok->level;
-    token->lineno = token->end_lineno = tok->lineno;
-    token->col_offset = col_offset;
-    token->end_col_offset = end_col_offset;
-    token->start = start;
-    token->end = end;
-    return type;
+    if (start == NULL) {
+        assert(end == NULL);
+        return (_PyTok_Span){-1, -1};
+    }
+    assert(end != NULL);
+    const char *base = tok->buf;
+    assert(base != NULL);
+    assert(tok->inp >= base);
+    Py_ssize_t start_offset = start - base;
+    Py_ssize_t end_offset = end - base;
+    assert(start_offset >= 0 && start_offset <= end_offset);
+    assert(end_offset <= tok->inp - base);
+    assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset);
+    return _PyTok_SpanFromBounds(
+        tok->buf_offset + start_offset, tok->buf_offset + end_offset);
 }
 
 int
 _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
 {
-    assert((start == NULL && end == NULL) || (start != NULL && end != NULL));
     token->level = tok->level;
-    if (ISSTRINGLIT(type)) {
-        token->lineno = tok->first_lineno;
-    }
-    else {
-        token->lineno = tok->lineno;
-    }
-    token->end_lineno = tok->lineno;
-    token->col_offset = token->end_col_offset = -1;
-    token->start = start;
-    token->end = end;
+    token->span = buffer_span(tok, start, end);
+    int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;
+    token->start_loc = (_PyTok_Loc){lineno, -1};
+    token->end_loc = (_PyTok_Loc){tok->lineno, -1};
 
     if (start != NULL && end != NULL) {
-        token->col_offset = tok->starting_col_offset;
-        token->end_col_offset = tok->col_offset;
+        token->start_loc.byte_col = tok->starting_col_offset;
+        token->end_loc.byte_col = tok->col_offset;
     }
     return type;
 }
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 0824785195491ee..6d19e685bd7f81b 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -23,8 +23,9 @@ enum interactive_underflow_t {
 
 struct token {
     int level;
-    int lineno, col_offset, end_lineno, end_col_offset;
-    const char *start, *end;
+    _PyTok_Span span;
+    _PyTok_Loc start_loc;
+    _PyTok_Loc end_loc;
     PyObject *metadata;
 };
 
@@ -67,15 +68,15 @@ typedef struct _tokenizer_mode {
 
 /* Tokenizer state */
 struct tok_state {
-    /* Input state; buf <= cur <= inp <= end */
+    /* Input state; buf <= cur <= inp */
     /* NB an entire line is held in the buffer */
-    char *buf;          /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */
+    char *buf;
     char *cur;          /* Next character in buffer */
     char *inp;          /* End of data in buffer */
+    _PyTok_Off buf_offset; /* Logical offset of buf[0]. */
     int fp_interactive; /* If the file descriptor is interactive */
     char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
     char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
-    const char *end;    /* End of input buffer if buf != NULL */
     const char *start;  /* Start of current token if not NULL */
     int done;           /* E_OK normally, E_EOF at EOF, otherwise error code */
     /* NB If done != E_OK, cur must be == inp!!! */
@@ -128,8 +129,6 @@ struct tok_state {
 #endif
 };
 
-int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
-                         int end_col_offset, const char *start, const char *end);
 int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
 
 struct tok_state *_PyTokenizer_tok_new(void);
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index d67c48f7f678eda..fc0299c5c7c592f 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -125,7 +125,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
 {
     assert(tok->cur != NULL);
 
-    Py_ssize_t size = strlen(tok->cur);
+    Py_ssize_t size = cur == 0
+        ? tok->inp - tok->cur : (Py_ssize_t)strlen(tok->cur);
     tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
 
     switch (cur) {
@@ -142,7 +143,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
                 goto error;
             }
             tok_mode->last_expr_buffer = new_buffer;
-            strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size);
+            memcpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size,
+                   tok->cur, size);
             tok_mode->last_expr_size += size;
             break;
         case '{':
@@ -155,7 +157,7 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
             }
             tok_mode->last_expr_size = size;
             tok_mode->last_expr_end = -1;
-            strncpy(tok_mode->last_expr_buffer, tok->cur, size);
+            memcpy(tok_mode->last_expr_buffer, tok->cur, size);
             break;
         case '}':
         case '!':
diff --git a/Parser/pegen.c b/Parser/pegen.c
index fcec810037e98d4..d86dd22444e6a7b 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -171,18 +171,17 @@ growable_comment_array_deallocate(growable_comment_array *arr) {
 }
 
 static int
-_get_keyword_or_name_type(Parser *p, struct token *new_token)
+_get_keyword_or_name_type(Parser *p, const char *text, Py_ssize_t length)
 {
-    Py_ssize_t name_len = new_token->end_col_offset - new_token->col_offset;
-    assert(name_len > 0);
+    assert(length > 0);
 
-    if (name_len >= p->n_keyword_lists ||
-        p->keywords[name_len] == NULL ||
-        p->keywords[name_len]->type == -1) {
+    if (length >= p->n_keyword_lists ||
+        p->keywords[length] == NULL ||
+        p->keywords[length]->type == -1) {
         return NAME;
     }
-    for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) {
-        if (strncmp(k->str, new_token->start, (size_t)name_len) == 0) {
+    for (KeywordToken *k = p->keywords[length]; k != NULL && k->type != -1; k++) {
+        if (memcmp(k->str, text, (size_t)length) == 0) {
             return k->type;
         }
     }
@@ -193,8 +192,11 @@ static int
 initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) {
     assert(parser_token != NULL);
 
-    parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type;
-    parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start);
+    Py_ssize_t length;
+    const char *text = _PyToken_TextView(p->tok, new_token, &length);
+    parser_token->type = token_type == NAME
+        ? _get_keyword_or_name_type(p, text, length) : token_type;
+    parser_token->bytes = PyBytes_FromStringAndSize(text, length);
     if (parser_token->bytes == NULL) {
         return -1;
     }
@@ -214,12 +216,14 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
     }
 
     parser_token->level = new_token->level;
-    parser_token->lineno = new_token->lineno;
-    parser_token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->col_offset
-                                                                    : new_token->col_offset;
-    parser_token->end_lineno = new_token->end_lineno;
-    parser_token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->end_col_offset
-                                                                 : new_token->end_col_offset;
+    parser_token->lineno = new_token->start_loc.lineno;
+    parser_token->col_offset = p->tok->lineno == p->starting_lineno
+        ? p->starting_col_offset + new_token->start_loc.byte_col
+        : new_token->start_loc.byte_col;
+    parser_token->end_lineno = new_token->end_loc.lineno;
+    parser_token->end_col_offset = p->tok->lineno == p->starting_lineno
+        ? p->starting_col_offset + new_token->end_loc.byte_col
+        : new_token->end_loc.byte_col;
 
     p->fill += 1;
 
@@ -261,13 +265,14 @@ _PyPegen_fill_token(Parser *p)
 
     // Record and skip '# type: ignore' comments
     while (type == TYPE_IGNORE) {
-        Py_ssize_t len = new_token.end_col_offset - new_token.col_offset;
+        Py_ssize_t len;
+        const char *text = _PyToken_TextView(p->tok, &new_token, &len);
         char *tag = PyMem_Malloc((size_t)len + 1);
         if (tag == NULL) {
             PyErr_NoMemory();
             goto error;
         }
-        strncpy(tag, new_token.start, (size_t)len);
+        memcpy(tag, text, (size_t)len);
         tag[len] = '\0';
         // Ownership of tag passes to the growable array
         if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
diff --git a/Parser/tokenizer/cursor.c b/Parser/tokenizer/cursor.c
index 698a26a740fd249..523b99dedc6160a 100644
--- a/Parser/tokenizer/cursor.c
+++ b/Parser/tokenizer/cursor.c
@@ -23,7 +23,7 @@ _PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno)
     if (lineno > 0 && cursor->lineno == lineno - 1 &&
             lineno <= source->nlines) {
         _PyTok_Off start = cursor->line_end;
-        _PyTok_Off end = source->len;
+        _PyTok_Off end = source->base_offset + source->len;
         if (lineno < source->nlines) {
             end = _PyTok_SourceFindLineEnd(source, start);
             if (end < 0) {
@@ -53,8 +53,9 @@ _PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset)
     int stays_on_line = cursor->lineno > 0 &&
         offset >= cursor->line_start && offset < cursor->line_end;
     if (!stays_on_line && cursor->lineno > 0 &&
-            offset == cursor->line_end && offset == source->len &&
-            (offset == 0 || source->bytes[offset - 1] != '\n')) {
+            offset == cursor->line_end &&
+            offset - source->base_offset == source->len &&
+            (source->len == 0 || source->bytes[source->len - 1] != '\n')) {
         stays_on_line = 1;
     }
     if (stays_on_line) {
@@ -68,7 +69,7 @@ _PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset)
         return -1;
     }
     _PyTok_Off start = offset - loc.byte_col;
-    _PyTok_Off end = source->len;
+    _PyTok_Off end = source->base_offset + source->len;
     if (loc.lineno < source->nlines) {
         end = _PyTok_SourceFindLineEnd(source, start);
         if (end < 0) {
diff --git a/Parser/tokenizer/cursor.h b/Parser/tokenizer/cursor.h
index d0fd9cf80b77b71..18e404251316f0c 100644
--- a/Parser/tokenizer/cursor.h
+++ b/Parser/tokenizer/cursor.h
@@ -21,8 +21,12 @@ PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off);
 static inline void
 _PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source)
 {
+    _PyTok_Off base = source != NULL ? source->base_offset : 0;
     *cursor = (_PyTok_Cursor){
         .source = source,
+        .pos = base,
+        .line_start = base,
+        .line_end = base,
     };
 }
 
@@ -35,14 +39,16 @@ _PyTok_CursorAdvance(_PyTok_Cursor *cursor)
     assert(cursor->source != NULL);
     assert(cursor->pos >= cursor->line_start);
     assert(cursor->pos <= cursor->line_end);
-    assert(cursor->line_end <= cursor->source->len);
+    assert(cursor->line_start >= cursor->source->base_offset);
+    assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len);
     if (cursor->pos >= cursor->line_end) {
         return EOF;
     }
     if (cursor->pos - cursor->line_start >= INT_MAX) {
         return EOF;
     }
-    return Py_CHARMASK(cursor->source->bytes[cursor->pos++]);
+    return Py_CHARMASK(cursor->source->bytes[
+        cursor->pos++ - cursor->source->base_offset]);
 }
 
 /* Return the byte at a nonnegative distance within the current line, or EOF
@@ -53,13 +59,15 @@ _PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance)
     assert(cursor->source != NULL);
     assert(cursor->pos >= cursor->line_start);
     assert(cursor->pos <= cursor->line_end);
-    assert(cursor->line_end <= cursor->source->len);
+    assert(cursor->line_start >= cursor->source->base_offset);
+    assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len);
     assert(distance >= 0);
     if (distance < 0 ||
             distance >= cursor->line_end - cursor->pos) {
         return EOF;
     }
-    return Py_CHARMASK(cursor->source->bytes[cursor->pos + distance]);
+    return Py_CHARMASK(cursor->source->bytes[
+        cursor->pos - cursor->source->base_offset + distance]);
 }
 
 #endif
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index 82b824f56374fcf..b9b4a4610874419 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -13,6 +13,12 @@
 #  include <unistd.h>
 #endif
 
+static inline int
+reader_is_streaming(_PyTok_ReaderKind kind)
+{
+    return kind == _PYTOK_READER_FILE || kind == _PYTOK_READER_READLINE;
+}
+
 void
 _PyTok_ReaderFree(struct tok_state *tok)
 {
@@ -28,10 +34,7 @@ _PyTok_ReaderFree(struct tok_state *tok)
     }
     PyMem_Free(reader->file_buffer);
     PyMem_Free(reader->decoded);
-    if (reader->kind != _PYTOK_READER_PREPARED) {
-        PyMem_Free(tok->buf);
-        tok->buf = NULL;
-    }
+    tok->buf = NULL;
     PyMem_Free(reader);
     tok->reader = NULL;
 }
@@ -529,19 +532,31 @@ reader_next(struct tok_state *tok, _PyTok_Chunk *chunk)
     Py_UNREACHABLE();
 }
 
+static void
+reset_streaming_buffer(struct tok_state *tok)
+{
+    assert(tok->buf != NULL);
+    assert(tok->cur >= tok->buf && tok->cur <= tok->inp);
+    _PyTok_SourceDiscard(&tok->source);
+    tok->buf_offset = tok->source.base_offset;
+    tok->buf = tok->cur = tok->inp = (char *)_PyTok_SourceData(&tok->source);
+    tok->line_start = tok->buf;
+}
+
 int
 _PyTok_ReaderUnderflow(struct tok_state *tok)
 {
-    int prepared = tok->reader->kind == _PYTOK_READER_PREPARED;
+    _PyTok_ReaderKind kind = tok->reader->kind;
+    int prepared = kind == _PYTOK_READER_PREPARED;
+    int streaming = reader_is_streaming(kind);
     int reset_buffer = !prepared && tok->start == NULL && !INSIDE_FSTRING(tok);
 
-    if (reset_buffer && tok->reader->kind != _PYTOK_READER_INTERACTIVE) {
-        tok->cur = tok->inp = tok->buf;
-    }
-
     _PyTok_Chunk chunk;
     _PyTok_ReadResult result = reader_next(tok, &chunk);
     if (result != _PYTOK_READ_LINE) {
+        if (reset_buffer && streaming) {
+            reset_streaming_buffer(tok);
+        }
         if (result == _PYTOK_READ_EOF) {
             tok->done = E_EOF;
         }
@@ -558,34 +573,51 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
                     ? E_NOMEM : E_ERROR;
             }
         }
-        if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
+        if (kind == _PYTOK_READER_INTERACTIVE &&
                 result != _PYTOK_READ_STOPPED) {
             PySys_WriteStderr("\n");
         }
         return 0;
     }
 
-    Py_ssize_t copy_len = chunk.len;
-    if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
+    Py_ssize_t scan_len = chunk.len;
+    if (kind == _PYTOK_READER_INTERACTIVE &&
             chunk.implicit_newline) {
-        copy_len--;
-    }
-    if (reset_buffer && tok->reader->kind == _PYTOK_READER_INTERACTIVE) {
-        tok->cur = tok->inp = tok->buf;
-    }
-    if (!prepared && !_PyLexer_tok_reserve_buf(tok, copy_len + 1)) {
-        _PyTok_ChunkClear(&chunk);
-        tok->input_error = 1;
-        return 0;
-    }
-    if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
-            _PyTok_SourceAppendLine(&tok->source, chunk.data, chunk.len,
-                                    chunk.implicit_newline) < 0) {
-        _PyTok_ChunkClear(&chunk);
-        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
-            ? E_NOMEM : E_ERROR;
-        tok->input_error = 1;
-        return 0;
+        scan_len--;
+    }
+    if (!prepared) {
+        if (streaming && reset_buffer) {
+            reset_streaming_buffer(tok);
+        }
+        _PyLexer_BufferPointers pointers;
+        if (!reset_buffer) {
+            _PyLexer_SaveBufferPointers(
+                tok, tok->source.bytes, &pointers);
+        }
+        _PyTok_Off source_start = _PyTok_SourceAppendLine(
+            &tok->source, chunk.data, chunk.len,
+            chunk.implicit_newline);
+        if (source_start < 0) {
+            _PyTok_ChunkClear(&chunk);
+            tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
+                ? E_NOMEM : E_ERROR;
+            tok->input_error = 1;
+            return 0;
+        }
+        if (reset_buffer) {
+            tok->buf = tok->cur =
+                tok->source.bytes + (source_start - tok->source.base_offset);
+            tok->buf_offset = source_start;
+            tok->line_start = tok->buf;
+            tok->start = NULL;
+            tok->multi_line_start = NULL;
+        }
+        else {
+            _PyLexer_RestoreBufferPointers(
+                tok, tok->source.bytes, &pointers);
+        }
+        tok->inp = tok->source.bytes +
+            (source_start - tok->source.base_offset) + scan_len;
     }
     if (tok->fp_interactive) {
         tok->interactive_src_start = tok->source.bytes;
@@ -594,14 +626,11 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
     if (prepared) {
         if (tok->start == NULL) {
             tok->buf = tok->cur;
+            tok->buf_offset = tok->source.base_offset +
+                (chunk.data - tok->source.bytes);
         }
         tok->inp = chunk.data + chunk.len;
     }
-    else {
-        memcpy(tok->inp, chunk.data, (size_t)copy_len);
-        tok->inp += copy_len;
-        *tok->inp = '\0';
-    }
     tok->implicit_newline = chunk.implicit_newline;
 
     if (!prepared && tok->tok_mode_stack_index &&
@@ -611,7 +640,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         return 0;
     }
     ADVANCE_LINENO();
-    if (tok->reader->kind == _PYTOK_READER_FILE &&
+    if (kind == _PYTOK_READER_FILE &&
             (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
             !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) {
         _PyTok_ChunkClear(&chunk);
@@ -639,14 +668,10 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind)
     if (kind == _PYTOK_READER_PREPARED) {
         return tok;
     }
-    tok->buf = PyMem_Malloc(BUFSIZ);
-    if (tok->buf == NULL) {
-        PyErr_NoMemory();
-        _PyTokenizer_Free(tok);
-        return NULL;
+    if (reader_is_streaming(kind)) {
+        tok->buf = tok->cur = tok->inp =
+            (char *)_PyTok_SourceData(&tok->source);
     }
-    tok->cur = tok->inp = tok->buf;
-    tok->end = tok->buf + BUFSIZ;
     return tok;
 }
 
@@ -664,7 +689,6 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input,
         return NULL;
     }
     tok->buf = tok->cur = tok->inp = tok->str;
-    tok->end = tok->buf;
     return tok;
 }
 
diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c
index c0f7925e33f8b97..2f2aaf2589246d9 100644
--- a/Parser/tokenizer/source.c
+++ b/Parser/tokenizer/source.c
@@ -19,6 +19,22 @@ _PyTok_SourceClear(_PyTok_SourceText *source)
     _PyTok_SourceInit(source);
 }
 
+void
+_PyTok_SourceDiscard(_PyTok_SourceText *source)
+{
+    assert(source->base_offset <= PY_SSIZE_T_MAX - source->len);
+    source->base_offset += source->len;
+    source->len = 0;
+    if (source->bytes != NULL) {
+        source->bytes[0] = '\0';
+    }
+    if (source->implicit_lines != NULL) {
+        Py_ssize_t used = source->nlines / 8 + (source->nlines % 8 != 0);
+        memset(source->implicit_lines, 0, Py_MIN(used, source->implicit_cap));
+    }
+    source->nlines = 0;
+}
+
 static int
 reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed)
 {
@@ -89,7 +105,7 @@ reserve_checkpoints(_PyTok_SourceText *source, int needed)
 static int
 reserve_implicit_lines(_PyTok_SourceText *source, int nlines)
 {
-    Py_ssize_t needed = ((Py_ssize_t)nlines + 7) / 8;
+    Py_ssize_t needed = nlines / 8 + (nlines % 8 != 0);
     if (needed <= source->implicit_cap) {
         return 0;
     }
@@ -143,7 +159,8 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
     if (validate_line(source, bytes, len, implicit_newline) < 0) {
         return -1;
     }
-    if (source->len > PY_SSIZE_T_MAX - len - 1) {
+    if (source->len > PY_SSIZE_T_MAX - len - 1 ||
+            source->base_offset > PY_SSIZE_T_MAX - source->len - len) {
         PyErr_NoMemory();
         return -1;
     }
@@ -162,26 +179,28 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
     source->len += len;
     source->bytes[source->len] = '\0';
     if (checkpoint) {
-        source->line_checkpoints[checkpoint_count - 1] = start;
+        source->line_checkpoints[checkpoint_count - 1] =
+            source->base_offset + start;
     }
     if (implicit_newline) {
         source->implicit_lines[(nlines - 1) / 8] |=
             (unsigned char)(1U << ((nlines - 1) & 7));
     }
     source->nlines = nlines;
-    return start;
+    return source->base_offset + start;
 }
 
 const char *
 _PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
                       Py_ssize_t *len)
 {
-    if (!_PyTok_SpanIsValid(span) || span.end > source->len || len == NULL) {
+    if (!_PyTok_SpanIsValid(span) || span.start < source->base_offset ||
+            span.end - source->base_offset > source->len || len == NULL) {
         PyErr_SetString(PyExc_SystemError, "invalid tokenizer source span");
         return NULL;
     }
     *len = span.end - span.start;
-    return source->bytes == NULL ? "" : source->bytes + span.start;
+    return _PyTok_SourceData(source) + (span.start - source->base_offset);
 }
 
 int
@@ -220,8 +239,8 @@ _PyTok_SourceLine(const _PyTok_SourceText *source, int lineno,
     }
     if (lineno > source->nlines) {
         *line = (_PyTok_Line){
-            .start = source->len,
-            .end = source->len,
+            .start = source->base_offset + source->len,
+            .end = source->base_offset + source->len,
         };
         return 0;
     }
@@ -236,7 +255,7 @@ _PyTok_SourceLine(const _PyTok_SourceText *source, int lineno,
         }
         current++;
     }
-    _PyTok_Off end = source->len;
+    _PyTok_Off end = source->base_offset + source->len;
     if (lineno < source->nlines) {
         end = _PyTok_SourceFindLineEnd(source, start);
         if (end < 0) {
@@ -248,7 +267,8 @@ _PyTok_SourceLine(const _PyTok_SourceText *source, int lineno,
         .end = end,
         .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno),
         .contains_nul = memchr(
-            source->bytes + start, 0, end - start) != NULL,
+            source->bytes + (start - source->base_offset),
+            0, end - start) != NULL,
     };
     return 0;
 }
@@ -257,21 +277,23 @@ int
 _PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset,
                       _PyTok_Affinity affinity, _PyTok_Loc *loc)
 {
-    if (offset < 0 || offset > source->len || loc == NULL ||
+    if (offset < source->base_offset ||
+            offset - source->base_offset > source->len || loc == NULL ||
             (affinity != _PYTOK_AFFINITY_LEFT &&
              affinity != _PYTOK_AFFINITY_RIGHT)) {
         PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset");
         return -1;
     }
     if (source->nlines == 0 ||
-            (offset == source->len && source_ends_in_newline(source) &&
+            (offset - source->base_offset == source->len &&
+             source_ends_in_newline(source) &&
              affinity == _PYTOK_AFFINITY_RIGHT)) {
         *loc = (_PyTok_Loc){eof_lineno(source), 0};
         return 0;
     }
 
     _PyTok_Off key = offset;
-    if (affinity == _PYTOK_AFFINITY_LEFT && key > 0) {
+    if (affinity == _PYTOK_AFFINITY_LEFT && key > source->base_offset) {
         key--;
     }
     int low = 0;
diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h
index b42ecda1b31aa50..7a2f46f73aff470 100644
--- a/Parser/tokenizer/source.h
+++ b/Parser/tokenizer/source.h
@@ -5,7 +5,8 @@
 
 typedef Py_ssize_t _PyTok_Off;
 
-/* Half-open byte offsets into a _PyTok_SourceText. */
+/* Spans use half-open logical byte offsets into decoded input. Their backing
+   storage may retain only the current input window. */
 typedef struct {
     _PyTok_Off start;
     _PyTok_Off end;
@@ -32,6 +33,7 @@ typedef struct {
 
 typedef struct {
     char *bytes;
+    _PyTok_Off base_offset;
     _PyTok_Off len;
     _PyTok_Off cap;
     _PyTok_Off *line_checkpoints;
@@ -41,9 +43,18 @@ typedef struct {
     Py_ssize_t implicit_cap;
 } _PyTok_SourceText;
 
+static inline const char *
+_PyTok_SourceData(const _PyTok_SourceText *source)
+{
+    return source->bytes != NULL ? source->bytes : "";
+}
+
 PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *);
 /* Clear invalidates all cursors, spans, and views for the source. */
 PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);
+/* Discard the retained window and invalidate its cursors, spans, and views.
+   Keep its allocation and advance the logical base to the end of the window. */
+PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *);
 /* Append one nonempty logical line and return its start offset. The input may
    contain one newline, as its final byte. An unterminated line must be the
    final line. implicit_newline means that the final newline was synthesized.
@@ -54,8 +65,8 @@ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
 /* The returned view is invalidated by SourceAppendLine and SourceClear. */
 PyAPI_FUNC(const char *) _PyTok_SourceSpanView(
     const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *);
-/* Look up a 1-based line. Empty and newline-terminated sources have an empty
-   virtual line at EOF. */
+/* Look up a 1-based line in the retained window. Empty and newline-terminated
+   sources have an empty virtual line at EOF. */
 PyAPI_FUNC(int) _PyTok_SourceLine(
     const _PyTok_SourceText *, int, _PyTok_Line *);
 /* Return false for invalid line numbers and the virtual EOF line. */
@@ -81,19 +92,21 @@ _PyTok_SpanIsValid(_PyTok_Span span)
 static inline _PyTok_Off
 _PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start)
 {
-    if (source->bytes == NULL || start < 0 || start >= source->len) {
+    if (source->bytes == NULL || start < source->base_offset ||
+            start - source->base_offset >= source->len) {
         PyErr_SetString(PyExc_SystemError,
                         "corrupt tokenizer source line index");
         return -1;
     }
+    _PyTok_Off relative_start = start - source->base_offset;
     const char *newline = memchr(
-        source->bytes + start, '\n', source->len - start);
+        source->bytes + relative_start, '\n', source->len - relative_start);
     if (newline == NULL) {
         PyErr_SetString(PyExc_SystemError,
                         "corrupt tokenizer source line index");
         return -1;
     }
-    return newline - source->bytes + 1;
+    return source->base_offset + (newline - source->bytes) + 1;
 }
 
 #endif
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index 762b7b3e4c8d71d..71f236b08d93c8f 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -203,14 +203,19 @@ _get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t si
 }
 
 static void
-_get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_start,
-                 PyObject *line, int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno,
+_get_col_offsets(tokenizeriterobject *it, const struct token *token,
+                 const char *token_start, const char *line_start,
+                 PyObject *line, int line_changed,
                  Py_ssize_t *col_offset, Py_ssize_t *end_col_offset)
 {
     _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
+    const char *token_end = token_start == NULL
+        ? NULL : token_start + token->span.end - token->span.start;
+    Py_ssize_t lineno = token->start_loc.lineno;
+    Py_ssize_t end_lineno = token->end_loc.lineno;
     Py_ssize_t byte_offset = -1;
-    if (token.start != NULL && token.start >= line_start) {
-        byte_offset = token.start - line_start;
+    if (token_start != NULL && token_start >= line_start) {
+        byte_offset = token_start - line_start;
         if (line_changed) {
             *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);
             it->byte_col_offset_diff = byte_offset - *col_offset;
@@ -220,15 +225,13 @@ _get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_s
         }
     }
 
-    if (token.end != NULL && token.end >= it->tok->line_start) {
-        Py_ssize_t end_byte_offset = token.end - it->tok->line_start;
+    if (token_end != NULL && token_end >= it->tok->line_start) {
+        Py_ssize_t end_byte_offset = token_end - it->tok->line_start;
         if (lineno == end_lineno) {
-            // If the whole token is at the same line, we can just use the token.start
-            // buffer for figuring out the new column offset, since using line is not
-            // performant for very long lines.
+            // Avoid rescanning the prefix of a very long line.
             Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset);
             *end_col_offset = *col_offset + token_col_offset;
-            it->byte_col_offset_diff += token.end - token.start - token_col_offset;
+            it->byte_col_offset_diff += token_end - token_start - token_col_offset;
         }
         else {
             *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset);
@@ -263,12 +266,17 @@ tokenizeriter_next(PyObject *op)
         it->done = 1;
         goto exit;
     }
-    PyObject *str = NULL;
-    if (token.start == NULL || token.end == NULL) {
+    const char *token_start = NULL;
+    PyObject *str;
+    if (token.span.start < 0) {
+        assert(token.span.start == -1 && token.span.end == -1);
         str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
     }
     else {
-        str = PyUnicode_FromStringAndSize(token.start, token.end - token.start);
+        Py_ssize_t token_length;
+        token_start = _PyToken_TextView(
+            it->tok, &token, &token_length);
+        str = PyUnicode_FromStringAndSize(token_start, token_length);
     }
     if (str == NULL) {
         goto exit;
@@ -297,12 +305,12 @@ tokenizeriter_next(PyObject *op)
         goto exit;
     }
 
-    Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
-    Py_ssize_t end_lineno = it->tok->lineno;
+    Py_ssize_t lineno = token.start_loc.lineno;
+    Py_ssize_t end_lineno = token.end_loc.lineno;
     Py_ssize_t col_offset = -1;
     Py_ssize_t end_col_offset = -1;
-    _get_col_offsets(it, token, line_start, line, line_changed,
-                     lineno, end_lineno, &col_offset, &end_col_offset);
+    _get_col_offsets(it, &token, token_start, line_start, line, line_changed,
+                     &col_offset, &end_col_offset);
 
     if (it->tok->tok_extra_tokens) {
         if (is_trailing_token) {
@@ -317,7 +325,8 @@ tokenizeriter_next(PyObject *op)
         else if (type == NEWLINE) {
             Py_DECREF(str);
             if (!it->tok->implicit_newline) {
-                if (it->tok->start[0] == '\r') {
+                assert(token_start != NULL);
+                if (token_start[0] == '\r') {
                     str = PyUnicode_FromString("\r\n");
                 } else {
                     str = PyUnicode_FromString("\n");