From 8d3359b1c6aaac3871c6d7a5ac2f4ab430931f64 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 6 Sep 2026 22:25:45 +0100 Subject: [PATCH 1/5] gh-153569: move tokenizer input state and relocation into the reader --- Lib/test/test_repl.py | 5 +- Makefile.pre.in | 2 - PCbuild/_freeze_module.vcxproj | 1 - PCbuild/_freeze_module.vcxproj.filters | 3 - PCbuild/pythoncore.vcxproj | 4 +- PCbuild/pythoncore.vcxproj.filters | 12 +- Parser/lexer/buffer.c | 46 ------- Parser/lexer/buffer.h | 22 ---- Parser/lexer/lexer.c | 5 +- Parser/lexer/state.c | 55 --------- Parser/lexer/state.h | 18 --- Parser/pegen.c | 24 ++-- Parser/pegen_errors.c | 44 +++---- Parser/tokenizer/decoder.c | 7 +- Parser/tokenizer/helpers.c | 8 -- Parser/tokenizer/reader.c | 162 +++++++++++++++++++------ Parser/tokenizer/reader.h | 2 + Parser/tokenizer/reader_internal.h | 19 ++- Parser/tokenizer/tokenizer.h | 6 + Tools/peg_generator/pegen/build.py | 1 - 20 files changed, 174 insertions(+), 272 deletions(-) delete mode 100644 Parser/lexer/buffer.c delete mode 100644 Parser/lexer/buffer.h diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 372c110783bce7a..e6eb7df16d23214 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -184,9 +184,8 @@ def read_until(marker, start=0): @cpython_only def test_lexer_buffer_realloc_with_null_start(self): - # gh-144759: NULL pointer arithmetic in the lexer when start and - # multi_line_start are NULL (uninitialized in tok_mode_stack[0]) - # and the lexer buffer is reallocated while parsing long input. + # gh-144759: NULL pointer arithmetic when the lexer buffer grows + # while parsing long input. long_value = "a" * 2000 user_input = dedent(f"""\ x = f'{{{long_value!r}}}' diff --git a/Makefile.pre.in b/Makefile.pre.in index 78a486623181fa8..7a6cd270af2a195 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -394,7 +394,6 @@ PEGEN_OBJS= \ Parser/peg_api.o TOKENIZER_OBJS= \ - Parser/lexer/buffer.o \ Parser/lexer/lexer.o \ Parser/lexer/number.o \ Parser/lexer/state.o \ @@ -411,7 +410,6 @@ PEGEN_HEADERS= \ $(srcdir)/Parser/string_parser.h TOKENIZER_HEADERS= \ - Parser/lexer/buffer.h \ Parser/lexer/lexer.h \ Parser/lexer/lexer_internal.h \ Parser/lexer/state.h \ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 469fd77cc8be9dc..a6a37d7be9608f3 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -181,7 +181,6 @@ - diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 976c99b7d24bdfd..27d47ba14e6c2a0 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -469,9 +469,6 @@ Source Files - - Source Files - Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 79dfc9ccf39ec26..1495282d54d63a8 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -423,10 +423,9 @@ - - + @@ -593,7 +592,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 765b4d46b12dd00..6260c965625cc57 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -330,18 +330,15 @@ Parser - - Parser - - - Parser - Parser Parser + + Parser + Parser @@ -1361,9 +1358,6 @@ Parser - - Parser - Parser diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c deleted file mode 100644 index 9c39544ca7c4790..000000000000000 --- a/Parser/lexer/buffer.c +++ /dev/null @@ -1,46 +0,0 @@ -#include "Python.h" -#include "buffer.h" -#include "state.h" - -void -_PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base, - _PyLexer_BufferPointers *pointers) -{ - pointers->buf_from_base = tok->buf - base; - pointers->cur_from_buf = tok->cur - tok->buf; - pointers->inp_from_buf = tok->inp - tok->buf; - pointers->start_from_buf = tok->start == NULL - ? -1 : tok->start - tok->buf; - pointers->line_start_from_buf = tok->line_start == NULL - ? -1 : tok->line_start - tok->buf; - pointers->multi_line_start_from_buf = tok->multi_line_start == NULL - ? -1 : tok->multi_line_start - tok->buf; - for (int index = tok->tok_mode_stack_index; index > 0; --index) { - tokenizer_mode *mode = &tok->tok_mode_stack[index]; - mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf; - mode->multi_line_start_offset = mode->multi_line_start == NULL - ? -1 : mode->multi_line_start - tok->buf; - } -} - -void -_PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base, - const _PyLexer_BufferPointers *pointers) -{ - tok->buf = base + pointers->buf_from_base; - tok->cur = tok->buf + pointers->cur_from_buf; - tok->inp = tok->buf + pointers->inp_from_buf; - tok->start = pointers->start_from_buf < 0 - ? NULL : tok->buf + pointers->start_from_buf; - tok->line_start = pointers->line_start_from_buf < 0 - ? NULL : tok->buf + pointers->line_start_from_buf; - tok->multi_line_start = pointers->multi_line_start_from_buf < 0 - ? NULL : tok->buf + pointers->multi_line_start_from_buf; - for (int index = tok->tok_mode_stack_index; index > 0; --index) { - tokenizer_mode *mode = &tok->tok_mode_stack[index]; - mode->start = mode->start_offset < 0 - ? NULL : tok->buf + mode->start_offset; - mode->multi_line_start = mode->multi_line_start_offset < 0 - ? NULL : tok->buf + mode->multi_line_start_offset; - } -} diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h deleted file mode 100644 index 285da124226d50e..000000000000000 --- a/Parser/lexer/buffer.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef _LEXER_BUFFER_H_ -#define _LEXER_BUFFER_H_ - -#include "pyport.h" - -struct tok_state; - -typedef struct { - Py_ssize_t buf_from_base; - Py_ssize_t cur_from_buf; - Py_ssize_t inp_from_buf; - Py_ssize_t start_from_buf; - Py_ssize_t line_start_from_buf; - Py_ssize_t multi_line_start_from_buf; -} _PyLexer_BufferPointers; - -void _PyLexer_SaveBufferPointers( - struct tok_state *, const char *, _PyLexer_BufferPointers *); -void _PyLexer_RestoreBufferPointers( - struct tok_state *, char *, const _PyLexer_BufferPointers *); - -#endif diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index f96b31b9d2f38a1..1b148d7c72c4e05 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -206,15 +206,16 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str } tok_backup(tok, c); if (c == '#' || c == '\n' || c == '\r') { + int interactive = _PyTok_ReaderIsInteractive(tok); /* Lines with only whitespace and/or comments shouldn't affect the indentation and are not passed to the parser as NEWLINE tokens, except *totally* empty lines in interactive mode, which signal the end of a command group. */ - if (col == 0 && c == '\n' && tok->prompt != NULL) { + if (col == 0 && c == '\n' && interactive) { blankline = 0; /* Let it through */ } - else if (tok->prompt != NULL && tok->lineno == 1) { + else if (interactive && tok->lineno == 1) { /* In interactive mode, if the first line contains only spaces and/or a comment, let it through. */ blankline = 0; diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index d82a7d0f296bac0..b8f308e2e812115 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -1,65 +1,10 @@ #include "Python.h" -#include "pycore_pystate.h" #include "pycore_token.h" #include "errcode.h" #include "state.h" #include "../tokenizer/reader.h" -/* Never change this */ -#define TABSIZE 8 - -/* Create and initialize a new tok_state structure */ -struct tok_state * -_PyTokenizer_tok_new(void) -{ - struct tok_state *tok = (struct tok_state *)PyMem_Calloc( - 1, - sizeof(struct tok_state)); - if (tok == NULL) { - PyErr_NoMemory(); - return NULL; - } - - tok->buf = tok->cur = tok->inp = NULL; - tok->fp_interactive = 0; - tok->interactive_src_start = NULL; - tok->interactive_src_end = NULL; - tok->start = NULL; - tok->done = E_OK; - tok->fp = NULL; - tok->tabsize = TABSIZE; - tok->indent = 0; - tok->indstack[0] = 0; - tok->atbol = 1; - tok->pendin = 0; - tok->prompt = NULL; - tok->lineno = 0; - tok->starting_col_offset = -1; - tok->col_offset = -1; - tok->level = 0; - tok->altindstack[0] = 0; - tok->input_error = 0; - tok->encoding = NULL; - tok->filename = NULL; - tok->module = NULL; - tok->type_comments = 0; - tok->interactive_underflow = IUNDERFLOW_NORMAL; - tok->str = NULL; - tok->report_warnings = 1; - tok->tok_extra_tokens = 0; - tok->comment_newline = 0; - tok->implicit_newline = 0; - _PyTok_SourceInit(&tok->source); - tok->reader = NULL; - tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .quote='\0', .quote_size = 0, .in_debug=0}; - tok->tok_mode_stack_index = 0; -#ifdef Py_DEBUG - tok->debug = _Py_GetConfig()->parser_debug; -#endif - return tok; -} - static void free_fstring_expressions(struct tok_state *tok) { diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 6d19e685bd7f81b..9bfaad6b5abbd3a 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -13,14 +13,6 @@ #define INSIDE_FSTRING_EXPR_AT_TOP(tok) \ (tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1) -enum interactive_underflow_t { - /* Normal mode of operation: return a new token when asked in interactive mode */ - IUNDERFLOW_NORMAL, - /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This - * can be used to prevent the tokenizer to prompt the user for new tokens */ - IUNDERFLOW_STOP, -}; - struct token { int level; _PyTok_Span span; @@ -74,9 +66,6 @@ struct tok_state { char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ _PyTok_Off buf_offset; /* Logical offset of buf[0]. */ - int fp_interactive; /* If the file descriptor is interactive */ - char *interactive_src_start; /* The start of the source parsed so far in interactive mode */ - char *interactive_src_end; /* The end of the source parsed so far in interactive mode */ const char *start; /* Start of current token if not NULL */ int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ /* NB If done != E_OK, cur must be == inp!!! */ @@ -86,7 +75,6 @@ struct tok_state { int indstack[MAXINDENT]; /* Stack of indents */ int atbol; /* Nonzero if at begin of new line */ int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ - const char *prompt; /* For interactive prompting */ int lineno; /* Current line number */ int first_lineno; /* First line of a single line or multi line string expression (cf. issue 16806) */ @@ -108,17 +96,12 @@ struct tok_state { const char* multi_line_start; /* pointer to start of first line of a single line or multi line string expression (cf. issue 16806) */ - char* str; /* Source string being tokenized (if tokenizing from a string)*/ _PyTok_SourceText source; struct _PyTok_Reader *reader; int type_comments; /* Whether to look for type comments */ - /* How to proceed when asked for a new token in interactive mode */ - enum interactive_underflow_t interactive_underflow; - int report_warnings; - // TODO: Factor this into its own thing tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL]; int tok_mode_stack_index; int tok_extra_tokens; @@ -131,7 +114,6 @@ struct tok_state { int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); -struct tok_state *_PyTokenizer_tok_new(void); void _PyTokenizer_Free(struct tok_state *); void _PyToken_Free(struct token *); void _PyToken_Init(struct token *); diff --git a/Parser/pegen.c b/Parser/pegen.c index d86dd22444e6a7b..72275b4eaad2ee7 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -8,8 +8,9 @@ #include #include "lexer/lexer.h" -#include "tokenizer/tokenizer.h" #include "tokenizer/helpers.h" +#include "tokenizer/reader.h" +#include "tokenizer/tokenizer.h" #include "pegen.h" #define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two. @@ -943,9 +944,7 @@ reset_parser_state_for_error_pass(Parser *p) } p->mark = 0; p->call_invalid_rules = 1; - // Don't try to get extra tokens in interactive mode when trying to - // raise specialized errors in the second pass. - p->tok->interactive_underflow = IUNDERFLOW_STOP; + _PyTok_ReaderStopInteractive(p->tok); } static inline int @@ -961,13 +960,7 @@ _PyPegen_set_syntax_error_metadata(Parser *p) { PyErr_SetRaisedException(exc); return; } - const char *source = NULL; - if (p->tok->str != NULL) { - source = p->tok->str; - } - if (!source && p->tok->fp_interactive && p->tok->interactive_src_start) { - source = p->tok->interactive_src_start; - } + const char *source = _PyTokenizer_RetainedSource(p->tok); PyObject* the_source = NULL; if (source) { if (p->tok->encoding == NULL) { @@ -1070,10 +1063,6 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena } return NULL; } - if (!tok->fp || ps1 != NULL || ps2 != NULL || - PyUnicode_CompareWithASCIIString(filename_ob, "") == 0) { - tok->fp_interactive = 1; - } // This transfers the ownership to the tokenizer tok->filename = Py_NewRef(filename_ob); @@ -1095,8 +1084,9 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena result = _PyPegen_run_parser(p); _PyPegen_Parser_Free(p); - if (tok->fp_interactive && tok->interactive_src_start && result && interactive_src != NULL) { - *interactive_src = PyUnicode_FromString(tok->interactive_src_start); + const char *source = _PyTokenizer_RetainedSource(tok); + if (source != NULL && result && interactive_src != NULL) { + *interactive_src = PyUnicode_FromString(source); if (!*interactive_src || _PyArena_AddPyObject(arena, *interactive_src) < 0) { Py_XDECREF(*interactive_src); result = NULL; diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index b13e1c079220a92..a1340d9661c53f2 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -7,6 +7,8 @@ #include "lexer/state.h" #include "lexer/lexer.h" #include "pegen.h" +#include "tokenizer/reader.h" +#include "tokenizer/tokenizer.h" // TOKENIZER ERRORS @@ -122,7 +124,7 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { // before the one that we had for the generic error. // We don't want to tokenize to the end for interactive input - if (p->tok->prompt != NULL) { + if (_PyTok_ReaderIsInteractive(p->tok)) { return 0; } @@ -225,45 +227,26 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err } static PyObject * -get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno) +get_error_line_from_source(Parser *p, Py_ssize_t lineno) { - /* If the file descriptor is interactive, the source lines of the current - * (multi-line) statement are stored in p->tok->interactive_src_start. - * If not, we're parsing from a string, which means that the whole source - * is stored in p->tok->str. */ - assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp != NULL); - - char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str; + const char *cur_line = _PyTokenizer_RetainedSource(p->tok); if (cur_line == NULL) { - assert(p->tok->fp_interactive); - // We can reach this point if the tokenizer buffers for interactive source have not been - // initialized because we failed to decode the original source with the given locale. return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno; - const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end : p->tok->inp; - - if (buf_end < cur_line) { - buf_end = cur_line + strlen(cur_line); - } + const char *buf_end = cur_line + p->tok->source.len; for (int i = 0; i < relative_lineno - 1; i++) { - char *new_line = strchr(cur_line, '\n'); - // The assert is here for debug builds but the conditional that - // follows is there so in release builds we do not crash at the cost - // to report a potentially wrong line. - assert(new_line != NULL && new_line + 1 < buf_end); - if (new_line == NULL || new_line + 1 > buf_end) { + const char *new_line = memchr(cur_line, '\n', buf_end - cur_line); + if (new_line == NULL) { break; } cur_line = new_line + 1; } - char *next_newline; - if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line - next_newline = cur_line + strlen(cur_line); - } + const char *next_newline = memchr(cur_line, '\n', buf_end - cur_line); + next_newline = next_newline != NULL ? next_newline : buf_end; return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace"); } @@ -295,8 +278,9 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, goto error; } - if (p->tok->fp_interactive && p->tok->interactive_src_start != NULL) { - error_line = get_error_line_from_tokenizer_buffers(p, lineno); + if (_PyTok_ReaderIsInteractive(p->tok) && + _PyTokenizer_RetainedSource(p->tok) != NULL) { + error_line = get_error_line_from_source(p, lineno); } else if (p->start_rule == Py_file_input) { error_line = _PyErr_ProgramDecodedTextObject(p->tok->filename, @@ -318,7 +302,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size, "replace"); } else if (p->tok->fp == NULL || p->tok->fp == stdin) { - error_line = get_error_line_from_tokenizer_buffers(p, lineno); + error_line = get_error_line_from_source(p, lineno); } else { error_line = Py_GetConstant(Py_CONSTANT_EMPTY_STR); diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index af17b8b63235f51..023455d29d6d073 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -103,7 +103,9 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf, } result[write] = '\0'; *out_len = write; - *implicit_newline = implicit; + if (implicit_newline != NULL) { + *implicit_newline = implicit; + } return result; } @@ -399,10 +401,9 @@ _PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only, if (stored < 0) { return -1; } - tok->str = tok->source.bytes != NULL ? tok->source.bytes : (char *)""; if (!utf8_only && (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) && - !_PyTokenizer_ensure_utf8(tok->str, tok, 1)) { + !_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) { return -1; } return 0; diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index e99be70ea5e9801..646f8e5932e9546 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -118,10 +118,6 @@ _PyTokenizer_indenterror(struct tok_state *tok) int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char) { - if (!tok->report_warnings) { - return 0; - } - PyObject *msg = PyUnicode_FromFormat( "\"\\%c\" is an invalid escape sequence. " "Such sequences will not work in the future. " @@ -207,10 +203,6 @@ _PyTokenizer_raise_init_error(PyObject *filename) int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) { - if (!tok->report_warnings) { - return 0; - } - PyObject *errmsg; va_list vargs; va_start(vargs, format); diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index b9b4a4610874419..769ee4083bc4d44 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -1,13 +1,13 @@ #include "Python.h" #include "pycore_fileutils.h" +#include "pycore_pystate.h" #include "errcode.h" #include "helpers.h" #include "reader.h" #include "reader_internal.h" -#include "../lexer/buffer.h" -#include "../lexer/lexer.h" #include "../lexer/state.h" +#include "../lexer/lexer.h" #ifdef HAVE_UNISTD_H # include @@ -19,6 +19,56 @@ reader_is_streaming(_PyTok_ReaderKind kind) return kind == _PYTOK_READER_FILE || kind == _PYTOK_READER_READLINE; } +typedef struct { + Py_ssize_t buf; + Py_ssize_t cur; + Py_ssize_t inp; + Py_ssize_t start; + Py_ssize_t line_start; + Py_ssize_t multi_line_start; +} BufferOffsets; + +static BufferOffsets +save_buffer_offsets(struct tok_state *tok, const char *base) +{ + for (int index = tok->tok_mode_stack_index; index > 0; --index) { + tokenizer_mode *mode = &tok->tok_mode_stack[index]; + mode->start_offset = mode->start == NULL ? -1 : mode->start - base; + mode->multi_line_start_offset = mode->multi_line_start == NULL + ? -1 : mode->multi_line_start - base; + } + return (BufferOffsets) { + .buf = tok->buf - base, + .cur = tok->cur - base, + .inp = tok->inp - base, + .start = tok->start == NULL ? -1 : tok->start - base, + .line_start = tok->line_start == NULL + ? -1 : tok->line_start - base, + .multi_line_start = tok->multi_line_start == NULL + ? -1 : tok->multi_line_start - base, + }; +} + +static void +restore_buffer_offsets(struct tok_state *tok, char *base, + const BufferOffsets *offsets) +{ + tok->buf = base + offsets->buf; + tok->cur = base + offsets->cur; + tok->inp = base + offsets->inp; + tok->start = offsets->start < 0 ? NULL : base + offsets->start; + tok->line_start = offsets->line_start < 0 + ? NULL : base + offsets->line_start; + tok->multi_line_start = offsets->multi_line_start < 0 + ? NULL : base + offsets->multi_line_start; + for (int index = tok->tok_mode_stack_index; index > 0; --index) { + tokenizer_mode *mode = &tok->tok_mode_stack[index]; + mode->start = mode->start_offset < 0 ? NULL : base + mode->start_offset; + mode->multi_line_start = mode->multi_line_start_offset < 0 + ? NULL : base + mode->multi_line_start_offset; + } +} + void _PyTok_ReaderFree(struct tok_state *tok) { @@ -137,10 +187,10 @@ chunk_is_line(const _PyTok_Chunk *chunk) static _PyTok_ReadResult next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) { - int lineno = tok->lineno + 1; - if (lineno > tok->source.nlines) { + if (tok->lineno >= tok->source.nlines) { return _PYTOK_READ_EOF; } + int lineno = tok->lineno + 1; const char *start = tok->inp; const char *newline = memchr( start, '\n', tok->source.bytes + tok->source.len - start); @@ -206,7 +256,6 @@ initialize_file(struct tok_state *tok) if (result != _PYTOK_READ_LINE) { return -1; } - reader->prefetched_count = 1; Py_ssize_t bom_len; _PyTok_EncodingResult detection = _PyTok_DetectEncoding( tok, &reader->prefetched_lines[0], NULL, 0, &bom_len); @@ -224,16 +273,13 @@ initialize_file(struct tok_state *tok) reader->prefetched_lines[0].data = first; reader->prefetched_lines[0].ownership = _PYTOK_CHUNK_PYMEM; result = read_file_line(tok, &reader->prefetched_lines[1]); - if (result == _PYTOK_READ_LINE) { - reader->prefetched_count = 2; - } - else if (result == _PYTOK_READ_EOF) { + if (result == _PYTOK_READ_EOF) { reader->file_eof = 1; } - else { + else if (result != _PYTOK_READ_LINE) { return -1; } - _PyTok_Chunk *second = reader->prefetched_count == 2 + _PyTok_Chunk *second = reader->prefetched_lines[1].data != NULL ? &reader->prefetched_lines[1] : NULL; detection = _PyTok_DetectEncoding( tok, &reader->prefetched_lines[0], second, 1, &bom_len); @@ -303,10 +349,13 @@ next_file(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } _PyTok_Chunk input = {0}; - if (reader->prefetched_index < reader->prefetched_count) { - input = reader->prefetched_lines[reader->prefetched_index]; - reader->prefetched_lines[reader->prefetched_index++] = - (_PyTok_Chunk){0}; + if (reader->prefetched_lines[0].data != NULL) { + input = reader->prefetched_lines[0]; + reader->prefetched_lines[0] = (_PyTok_Chunk){0}; + } + else if (reader->prefetched_lines[1].data != NULL) { + input = reader->prefetched_lines[1]; + reader->prefetched_lines[1] = (_PyTok_Chunk){0}; } else if (!reader->file_eof) { _PyTok_ReadResult result = read_file_line(tok, &input); @@ -475,13 +524,13 @@ static _PyTok_ReadResult next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) { _PyTok_Reader *reader = tok->reader; - if (tok->interactive_underflow == IUNDERFLOW_STOP) { + if (reader->stop_interactive) { return _PYTOK_READ_STOPPED; } char *input = PyOS_Readline( - tok->fp != NULL ? tok->fp : stdin, stdout, tok->prompt); + tok->fp != NULL ? tok->fp : stdin, stdout, reader->prompt); if (reader->nextprompt != NULL) { - tok->prompt = reader->nextprompt; + reader->prompt = reader->nextprompt; } if (input == NULL) { return _PYTOK_READ_INTERRUPT; @@ -504,7 +553,7 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) } chunk->data = _PyTok_NormalizeNewlines( decoded.data, decoded.len, 0, 0, - &chunk->len, &chunk->implicit_newline); + &chunk->len, NULL); _PyTok_ChunkClear(&decoded); if (chunk->data == NULL) { PyErr_NoMemory(); @@ -515,6 +564,32 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } +int +_PyTok_ReaderIsInteractive(const struct tok_state *tok) +{ + return tok->reader->kind == _PYTOK_READER_INTERACTIVE; +} + +const char * +_PyTokenizer_RetainedSource(const struct tok_state *tok) +{ + if (tok->reader->kind == _PYTOK_READER_PREPARED) { + return _PyTok_SourceData(&tok->source); + } + if (tok->reader->kind == _PYTOK_READER_INTERACTIVE) { + return tok->source.bytes; + } + return NULL; +} + +void +_PyTok_ReaderStopInteractive(struct tok_state *tok) +{ + if (_PyTok_ReaderIsInteractive(tok)) { + tok->reader->stop_interactive = 1; + } +} + static _PyTok_ReadResult reader_next(struct tok_state *tok, _PyTok_Chunk *chunk) { @@ -579,6 +654,12 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) } return 0; } + if (tok->lineno == INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "too many tokenizer source lines"); + tok->done = E_ERROR; + _PyTok_ChunkClear(&chunk); + return 0; + } Py_ssize_t scan_len = chunk.len; if (kind == _PYTOK_READER_INTERACTIVE && @@ -589,10 +670,9 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) if (streaming && reset_buffer) { reset_streaming_buffer(tok); } - _PyLexer_BufferPointers pointers; + BufferOffsets offsets; if (!reset_buffer) { - _PyLexer_SaveBufferPointers( - tok, tok->source.bytes, &pointers); + offsets = save_buffer_offsets(tok, tok->source.bytes); } _PyTok_Off source_start = _PyTok_SourceAppendLine( &tok->source, chunk.data, chunk.len, @@ -613,18 +693,13 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) tok->multi_line_start = NULL; } else { - _PyLexer_RestoreBufferPointers( - tok, tok->source.bytes, &pointers); + restore_buffer_offsets(tok, tok->source.bytes, &offsets); } tok->inp = tok->source.bytes + (source_start - tok->source.base_offset) + scan_len; } - if (tok->fp_interactive) { - tok->interactive_src_start = tok->source.bytes; - tok->interactive_src_end = tok->source.bytes + tok->source.len; - } if (prepared) { - if (tok->start == NULL) { + if (tok->start == NULL && !INSIDE_FSTRING(tok)) { tok->buf = tok->cur; tok->buf_offset = tok->source.base_offset + (chunk.data - tok->source.bytes); @@ -654,10 +729,20 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) static struct tok_state * tokenizer_new_with_reader(_PyTok_ReaderKind kind) { - struct tok_state *tok = _PyTokenizer_tok_new(); + struct tok_state *tok = PyMem_Calloc(1, sizeof(*tok)); if (tok == NULL) { + PyErr_NoMemory(); return NULL; } + tok->done = E_OK; + tok->atbol = 1; + tok->starting_col_offset = -1; + tok->col_offset = -1; + tok->tabsize = 8; + tok->tok_mode_stack[0].kind = TOK_REGULAR_MODE; +#ifdef Py_DEBUG + tok->debug = _Py_GetConfig()->parser_debug; +#endif tok->reader = PyMem_Calloc(1, sizeof(*tok->reader)); if (tok->reader == NULL) { PyErr_NoMemory(); @@ -688,7 +773,9 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input, _PyTokenizer_Free(tok); return NULL; } - tok->buf = tok->cur = tok->inp = tok->str; + char *source = (char *)_PyTok_SourceData(&tok->source); + tok->buf = tok->cur = tok->inp = source; + tok->line_start = source; return tok; } @@ -734,7 +821,7 @@ _PyTokenizer_FromFile(FILE *fp, const char *encoding, return NULL; } tok->fp = fp; - tok->prompt = ps1; + tok->reader->prompt = ps1; tok->reader->nextprompt = ps2; return tok; } @@ -788,13 +875,10 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) _PyTokenizer_Free(tok); return NULL; } - /* Reporting a warning here could recursively ask for the encoding. */ - tok->report_warnings = 0; - while (tok->lineno < 2 && tok->done == E_OK) { - struct token token; - _PyToken_Init(&token); - _PyTokenizer_Get(tok, &token); - _PyToken_Free(&token); + if (initialize_file(tok) < 0) { + fclose(fp); + _PyTokenizer_Free(tok); + return NULL; } fclose(fp); char *encoding = tok->encoding == NULL diff --git a/Parser/tokenizer/reader.h b/Parser/tokenizer/reader.h index c27bc2aa3fb8197..2913e52b9d563b4 100644 --- a/Parser/tokenizer/reader.h +++ b/Parser/tokenizer/reader.h @@ -5,5 +5,7 @@ struct tok_state; void _PyTok_ReaderFree(struct tok_state *); int _PyTok_ReaderUnderflow(struct tok_state *); +int _PyTok_ReaderIsInteractive(const struct tok_state *); +void _PyTok_ReaderStopInteractive(struct tok_state *); #endif diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 121d0f96f6698a2..b4f6807c207a4aa 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -32,33 +32,32 @@ typedef enum { typedef struct { char *data; - Py_ssize_t len; - int implicit_newline; PyObject *owner; + Py_ssize_t len; _PyTok_ChunkOwnership ownership; + unsigned char implicit_newline; } _PyTok_Chunk; typedef struct _PyTok_Reader { - _PyTok_ReaderKind kind; PyObject *readline; PyObject *decoder; + const char *prompt; const char *nextprompt; char *file_buffer; Py_ssize_t file_buffer_cap; _PyTok_Chunk prefetched_lines[2]; - int prefetched_index; - int prefetched_count; char *decoded; Py_ssize_t decoded_pos; Py_ssize_t decoded_len; Py_ssize_t decoded_cap; - int decoded_tail_is_implicit; - - int file_initialized; - int file_eof; - int decoder_finalized; + _PyTok_ReaderKind kind; + unsigned char decoded_tail_is_implicit; + unsigned char file_initialized; + unsigned char file_eof; + unsigned char decoder_finalized; + unsigned char stop_interactive; } _PyTok_Reader; struct tok_state; diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h index d8c889115cfd73a..6bff1311046f20d 100644 --- a/Parser/tokenizer/tokenizer.h +++ b/Parser/tokenizer/tokenizer.h @@ -3,6 +3,12 @@ #include "Python.h" +struct tok_state; + +/* Return NUL-terminated retained input, or NULL without setting an exception + for streaming input or interactive input before its first line. */ +const char *_PyTokenizer_RetainedSource(const struct tok_state *); + struct tok_state *_PyTokenizer_FromString(const char *, int, int); struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int); struct tok_state *_PyTokenizer_FromReadline(PyObject *, const char *); diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index bfd8e43c6912e86..af8027db27234ad 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -128,7 +128,6 @@ def compile_c_extension( str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"), - str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "buffer.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "decoder.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "reader.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "helpers.c"), From 4f39a2fcfbe3c31c44a699376bc5590b182f5f7d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 6 Sep 2026 22:28:33 +0100 Subject: [PATCH 2/5] gh-153569: borrow diagnostic lines through the source API --- Modules/_testinternalcapi/tokenizer.c | 38 ++++++++++++++++++++++++--- Parser/pegen_errors.c | 20 +++----------- Parser/tokenizer/source.c | 20 ++++++++++++++ Parser/tokenizer/source.h | 6 +++++ 4 files changed, 64 insertions(+), 20 deletions(-) diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index df481cb832a4363..602dda8ee94a8bf 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -23,6 +23,17 @@ check_system_error(int failed, const char *message) return 0; } +static int +check_line_view(const _PyTok_SourceText *source, Py_ssize_t lineno, + const char *expected) +{ + Py_ssize_t len; + const char *line = _PyTok_SourceLineView(source, lineno, &len); + return check(len == (Py_ssize_t)strlen(expected) && + memcmp(line, expected, len) == 0, + "wrong source line view"); +} + static int same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right) { @@ -40,6 +51,10 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), _PyTok_SourceText source; _PyTok_SourceInit(&source); + if (check_line_view(&source, 1, "") < 0) { + goto error; + } + _PyTok_Loc loc; _PyTok_Line line; if (check(_PyTok_SourceLocation( @@ -67,10 +82,20 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( &source, "\xce\xb2\n", 3, 1) == 6, - "wrong second source offset") < 0 || - check(_PyTok_SourceAppendLine( - &source, "nul\0x\n", 6, 0) == 9, - "wrong third source offset") < 0) { + "wrong second source offset") < 0) { + goto error; + } + + if (check_line_view(&source, PY_SSIZE_T_MIN, "alpha") < 0 || + check_line_view(&source, 1, "alpha") < 0 || + check_line_view(&source, 2, "\xce\xb2") < 0 || + check_line_view(&source, 3, "") < 0 || + check_line_view(&source, PY_SSIZE_T_MAX, "") < 0) { + goto error; + } + + if (check(_PyTok_SourceAppendLine(&source, "nul\0x\n", 6, 0) == 9, + "wrong third source offset") < 0) { goto error; } @@ -195,6 +220,11 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } + if (check_line_view(&source, 1, "tail") < 0 || + check_line_view(&source, PY_SSIZE_T_MAX, "tail") < 0) { + goto error; + } + _PyTok_SourceDiscard(&source); if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4, "wrong retained source offset") < 0 || diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index a1340d9661c53f2..839beb1761ad59e 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -229,25 +229,13 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err static PyObject * get_error_line_from_source(Parser *p, Py_ssize_t lineno) { - const char *cur_line = _PyTokenizer_RetainedSource(p->tok); - if (cur_line == NULL) { + if (_PyTokenizer_RetainedSource(p->tok) == NULL) { return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } - Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno; - const char *buf_end = cur_line + p->tok->source.len; - - for (int i = 0; i < relative_lineno - 1; i++) { - const char *new_line = memchr(cur_line, '\n', buf_end - cur_line); - if (new_line == NULL) { - break; - } - cur_line = new_line + 1; - } - - const char *next_newline = memchr(cur_line, '\n', buf_end - cur_line); - next_newline = next_newline != NULL ? next_newline : buf_end; - return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace"); + Py_ssize_t len; + const char *line = _PyTok_SourceLineView(&p->tok->source, relative_lineno, &len); + return PyUnicode_DecodeUTF8(line, len, "replace"); } void * diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index 2f2aaf2589246d9..989944afd9fea96 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -190,6 +190,26 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, return source->base_offset + start; } +const char * +_PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno, + Py_ssize_t *len) +{ + assert(len != NULL); + const char *line = _PyTok_SourceData(source); + const char *end = line + source->len; + while (lineno > 1) { + const char *newline = memchr(line, '\n', end - line); + if (newline == NULL) { + break; + } + line = newline + 1; + lineno--; + } + const char *newline = memchr(line, '\n', end - line); + *len = (newline != NULL ? newline : end) - line; + return line; +} + const char * _PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span, Py_ssize_t *len) diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 7a2f46f73aff470..7ae1e6e21abbc54 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -62,6 +62,12 @@ PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( _PyTok_SourceText *source, const char *bytes, Py_ssize_t len, int implicit_newline); +/* Return borrowed bytes excluding '\n', writing the byte length to *len. + Line numbers are 1-based and clamp to the first or final line; a trailing + '\n' adds an empty final line. The view need not be NUL-terminated. + This does not set an exception. Append, discard, and clear invalidate the view. */ +PyAPI_FUNC(const char *) _PyTok_SourceLineView( + const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len); /* The returned view is invalidated by SourceAppendLine and SourceClear. */ PyAPI_FUNC(const char *) _PyTok_SourceSpanView( const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *); From 728c4d90fd967b34dfda59c27487e812342472b4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 7 Sep 2026 00:06:12 +0100 Subject: [PATCH 3/5] gh-153569: group indentation and logical-line state --- Makefile.pre.in | 1 + PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Parser/lexer/layout.c | 195 +++++++++++++++++++++++++ Parser/lexer/lexer.c | 179 ++--------------------- Parser/lexer/lexer_internal.h | 6 + Parser/lexer/state.h | 24 ++- Parser/pegen.c | 5 +- Parser/tokenizer/reader.c | 3 +- Tools/peg_generator/pegen/build.py | 1 + 12 files changed, 242 insertions(+), 180 deletions(-) create mode 100644 Parser/lexer/layout.c diff --git a/Makefile.pre.in b/Makefile.pre.in index 7a6cd270af2a195..ac0db1a1c5923fa 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -395,6 +395,7 @@ PEGEN_OBJS= \ TOKENIZER_OBJS= \ Parser/lexer/lexer.o \ + Parser/lexer/layout.o \ Parser/lexer/number.o \ Parser/lexer/state.o \ Parser/lexer/string.o \ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index a6a37d7be9608f3..9374a9a6687e491 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -183,6 +183,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 27d47ba14e6c2a0..2264b5312f526e7 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -463,6 +463,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 1495282d54d63a8..ceb6b792cc0c621 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -590,6 +590,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 6260c965625cc57..a2c4eb431f373ef 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1349,6 +1349,9 @@ Parser + + Parser + Parser diff --git a/Parser/lexer/layout.c b/Parser/lexer/layout.c new file mode 100644 index 000000000000000..e460574b6e97c44 --- /dev/null +++ b/Parser/lexer/layout.c @@ -0,0 +1,195 @@ +#include "Python.h" +#include "errcode.h" +#include "pycore_token.h" + +#include "lexer_internal.h" +#include "../tokenizer/helpers.h" +#include "../tokenizer/reader.h" + +#define TABSIZE 8 +#define ALTTABSIZE 1 + +int +_PyLexer_ContinueLine(struct tok_state *tok) +{ + int c = tok_nextc(tok); + if (c == '\r') { + c = tok_nextc(tok); + } + if (c != '\n') { + tok->done = E_LINECONT; + return -1; + } + c = tok_nextc(tok); + if (c == EOF) { + tok->done = E_EOF; + tok->cur = tok->inp; + return -1; + } else { + tok_backup(tok, c); + } + return c; +} + + +static int +update_indentation(struct tok_state *tok, int col, int altcol) +{ + lexer_layout_state *layout = &tok->layout; + if (col == layout->stack[layout->depth].column) { + if (altcol != layout->stack[layout->depth].alternate_column) { + _PyTokenizer_indenterror(tok); + return -1; + } + } + else if (col > layout->stack[layout->depth].column) { + if (layout->depth + 1 >= MAXINDENT) { + tok->done = E_TOODEEP; + tok->cur = tok->inp; + return -1; + } + if (altcol <= layout->stack[layout->depth].alternate_column) { + _PyTokenizer_indenterror(tok); + return -1; + } + layout->pending++; + layout->stack[++layout->depth] = (indentation_level){col, altcol}; + } + else { + while (layout->depth > 0 && + col < layout->stack[layout->depth].column) { + layout->pending--; + layout->depth--; + } + if (col != layout->stack[layout->depth].column) { + tok->done = E_DEDENT; + tok->cur = tok->inp; + return -1; + } + if (altcol != layout->stack[layout->depth].alternate_column) { + _PyTokenizer_indenterror(tok); + return -1; + } + } + return 0; +} + +int +_PyLexer_BeginLine(struct tok_state *tok) +{ + assert(tok->layout.at_bol); + int c; + int blankline = 0; + int col = 0; + int altcol = 0; + tok->layout.at_bol = 0; + int cont_line_col = 0; + for (;;) { + c = tok_nextc(tok); + if (c == ' ') { + col++, altcol++; + } + else if (c == '\t') { + col = (col / TABSIZE + 1) * TABSIZE; + altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE; + } + else if (c == '\014') {/* Control-L (formfeed) */ + col = altcol = 0; /* For Emacs users */ + } + else if (c == '\\') { + // Indentation cannot be split over multiple physical lines + // using backslashes. This means that if we found a backslash + // preceded by whitespace, **the first one we find** determines + // the level of indentation of whatever comes next. + cont_line_col = cont_line_col ? cont_line_col : col; + if ((c = _PyLexer_ContinueLine(tok)) == -1) { + return -1; + } + } + else if (c == EOF && PyErr_Occurred()) { + return -1; + } + else { + break; + } + } + tok_backup(tok, c); + if (c == '#' || c == '\n' || c == '\r') { + int interactive = _PyTok_ReaderIsInteractive(tok); + /* Lines with only whitespace and/or comments + shouldn't affect the indentation and are + not passed to the parser as NEWLINE tokens, + except *totally* empty lines in interactive + mode, which signal the end of a command group. */ + if (col == 0 && c == '\n' && interactive) { + blankline = 0; /* Let it through */ + } + else if (interactive && tok->lineno == 1) { + /* In interactive mode, if the first line contains + only spaces and/or a comment, let it through. */ + blankline = 0; + col = altcol = 0; + } + else { + blankline = 1; /* Ignore completely */ + } + } + if (!blankline && tok->level == 0) { + col = cont_line_col ? cont_line_col : col; + altcol = cont_line_col ? cont_line_col : altcol; + if (update_indentation(tok, col, altcol) < 0) { + return -1; + } + } + return blankline; +} + +int +_PyLexer_IndentationToken(struct tok_state *tok, struct token *token) +{ + assert(tok->layout.pending != 0); + const char *p_start = NULL; + const char *p_end = NULL; + if (tok->layout.pending < 0) { + if (tok->tok_extra_tokens) { + p_start = tok->cur; + p_end = tok->cur; + } + tok->layout.pending++; + return _PyLexer_token_setup(tok, token, DEDENT, p_start, p_end); + } + else { + if (tok->tok_extra_tokens) { + p_start = tok->buf; + p_end = tok->cur; + } + tok->layout.pending--; + return _PyLexer_token_setup(tok, token, INDENT, p_start, p_end); + } +} + +int +_PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline) +{ + tok->layout.at_bol = 1; + if (blankline || tok->level > 0) { + if (!tok->tok_extra_tokens) { + return 0; + } + } + else if (!tok->layout.comment_newline || !tok->tok_extra_tokens) { + return _PyLexer_token_setup(tok, token, NEWLINE, + tok->start, tok->cur - 1); + } + tok->layout.comment_newline = 0; + return _PyLexer_token_setup(tok, token, NL, tok->start, tok->cur); +} + +void +_PyLexer_ImplyDedents(struct tok_state *tok) +{ + if (tok->layout.depth != 0) { + tok->layout.pending = -tok->layout.depth; + tok->layout.depth = 0; + } +} diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 1b148d7c72c4e05..01d6d221000f32a 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -7,10 +7,6 @@ #include "../tokenizer/helpers.h" #include "../tokenizer/reader.h" -/* Alternate tab spacing */ -#define ALTTABSIZE 1 - - #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) /* Spaces in this constant are treated as "zero or more spaces or tabs" when @@ -130,31 +126,6 @@ verify_identifier(struct tok_state *tok) return 1; } - - -static inline int -tok_continuation_line(struct tok_state *tok) { - int c = tok_nextc(tok); - if (c == '\r') { - c = tok_nextc(tok); - } - if (c != '\n') { - tok->done = E_LINECONT; - return -1; - } - c = tok_nextc(tok); - if (c == EOF) { - tok->done = E_EOF; - tok->cur = tok->inp; - return -1; - } else { - tok_backup(tok, c); - } - return c; -} - - - int _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token) { @@ -169,127 +140,18 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str blankline = 0; - /* Get indentation level */ - if (tok->atbol) { - int col = 0; - int altcol = 0; - tok->atbol = 0; - int cont_line_col = 0; - for (;;) { - c = tok_nextc(tok); - if (c == ' ') { - col++, altcol++; - } - else if (c == '\t') { - col = (col / tok->tabsize + 1) * tok->tabsize; - altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE; - } - else if (c == '\014') {/* Control-L (formfeed) */ - col = altcol = 0; /* For Emacs users */ - } - else if (c == '\\') { - // Indentation cannot be split over multiple physical lines - // using backslashes. This means that if we found a backslash - // preceded by whitespace, **the first one we find** determines - // the level of indentation of whatever comes next. - cont_line_col = cont_line_col ? cont_line_col : col; - if ((c = tok_continuation_line(tok)) == -1) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - else if (c == EOF && PyErr_Occurred()) { - return MAKE_TOKEN(ERRORTOKEN); - } - else { - break; - } - } - tok_backup(tok, c); - if (c == '#' || c == '\n' || c == '\r') { - int interactive = _PyTok_ReaderIsInteractive(tok); - /* Lines with only whitespace and/or comments - shouldn't affect the indentation and are - not passed to the parser as NEWLINE tokens, - except *totally* empty lines in interactive - mode, which signal the end of a command group. */ - if (col == 0 && c == '\n' && interactive) { - blankline = 0; /* Let it through */ - } - else if (interactive && tok->lineno == 1) { - /* In interactive mode, if the first line contains - only spaces and/or a comment, let it through. */ - blankline = 0; - col = altcol = 0; - } - else { - blankline = 1; /* Ignore completely */ - } - /* We can't jump back right here since we still - may need to skip to the end of a comment */ - } - if (!blankline && tok->level == 0) { - col = cont_line_col ? cont_line_col : col; - altcol = cont_line_col ? cont_line_col : altcol; - if (col == tok->indstack[tok->indent]) { - /* No change */ - if (altcol != tok->altindstack[tok->indent]) { - return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); - } - } - else if (col > tok->indstack[tok->indent]) { - /* Indent -- always one */ - if (tok->indent+1 >= MAXINDENT) { - tok->done = E_TOODEEP; - tok->cur = tok->inp; - return MAKE_TOKEN(ERRORTOKEN); - } - if (altcol <= tok->altindstack[tok->indent]) { - return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); - } - tok->pendin++; - tok->indstack[++tok->indent] = col; - tok->altindstack[tok->indent] = altcol; - } - else /* col < tok->indstack[tok->indent] */ { - /* Dedent -- any number, must be consistent */ - while (tok->indent > 0 && - col < tok->indstack[tok->indent]) { - tok->pendin--; - tok->indent--; - } - if (col != tok->indstack[tok->indent]) { - tok->done = E_DEDENT; - tok->cur = tok->inp; - return MAKE_TOKEN(ERRORTOKEN); - } - if (altcol != tok->altindstack[tok->indent]) { - return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); - } - } + if (tok->layout.at_bol) { + blankline = _PyLexer_BeginLine(tok); + if (blankline < 0) { + return MAKE_TOKEN(ERRORTOKEN); } } tok->start = tok->cur; tok->starting_col_offset = tok->col_offset; - /* Return pending indents/dedents */ - if (tok->pendin != 0) { - if (tok->pendin < 0) { - if (tok->tok_extra_tokens) { - p_start = tok->cur; - p_end = tok->cur; - } - tok->pendin++; - return MAKE_TOKEN(DEDENT); - } - else { - if (tok->tok_extra_tokens) { - p_start = tok->buf; - p_end = tok->cur; - } - tok->pendin--; - return MAKE_TOKEN(INDENT); - } + if (tok->layout.pending != 0) { + return _PyLexer_IndentationToken(tok, token); } /* Peek ahead at the next character */ @@ -369,7 +231,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str /* If this type ignore is the only thing on the line, consume the newline also. */ if (blankline) { tok_nextc(tok); - tok->atbol = 1; + tok->layout.at_bol = 1; } } else { p_start = type_start; @@ -384,7 +246,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str tok_backup(tok, c); /* don't eat the newline or EOF */ p_start = p; p_end = tok->cur; - tok->comment_newline = blankline; + tok->layout.comment_newline = blankline; return MAKE_TOKEN(COMMENT); } } @@ -465,29 +327,12 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str c = tok_nextc(tok); } - /* Newline */ if (c == '\n') { - tok->atbol = 1; - if (blankline || tok->level > 0) { - if (tok->tok_extra_tokens) { - if (tok->comment_newline) { - tok->comment_newline = 0; - } - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(NL); - } + int type = _PyLexer_Newline(tok, token, blankline); + if (type == 0) { goto nextline; } - if (tok->comment_newline && tok->tok_extra_tokens) { - tok->comment_newline = 0; - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(NL); - } - p_start = tok->start; - p_end = tok->cur - 1; /* Leave '\n' out of the string */ - return MAKE_TOKEN(NEWLINE); + return type; } /* Period or number starting with period? */ @@ -528,7 +373,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str /* Line continuation */ if (c == '\\') { - if ((c = tok_continuation_line(tok)) == -1) { + if ((c = _PyLexer_ContinueLine(tok)) == -1) { return MAKE_TOKEN(ERRORTOKEN); } goto again; /* Read next line */ diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index c6d3b9045c72921..d0e10d40e43c8bf 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -44,6 +44,12 @@ TOK_NEXT_MODE(struct tok_state *tok) #define tok_nextc _PyLexer_nextc #define tok_backup _PyLexer_backup +/* Return -1 on error, otherwise whether the line is blank. */ +int _PyLexer_BeginLine(struct tok_state *); +int _PyLexer_ContinueLine(struct tok_state *); +int _PyLexer_IndentationToken(struct tok_state *, struct token *); +/* Return zero when the newline is suppressed, otherwise its token type. */ +int _PyLexer_Newline(struct tok_state *, struct token *, int); int _PyLexer_nextc(struct tok_state *); void _PyLexer_backup(struct tok_state *, int); int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char); diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 9bfaad6b5abbd3a..2901cd3d3187808 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -58,6 +58,19 @@ typedef struct _tokenizer_mode { enum string_kind_t string_kind; } tokenizer_mode; +typedef struct { + int column; + int alternate_column; +} indentation_level; + +typedef struct { + int depth; + int pending; + int at_bol; + int comment_newline; + indentation_level stack[MAXINDENT]; +} lexer_layout_state; + /* Tokenizer state */ struct tok_state { /* Input state; buf <= cur <= inp */ @@ -70,11 +83,7 @@ struct tok_state { int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ /* NB If done != E_OK, cur must be == inp!!! */ FILE *fp; /* Rest of input; NULL if tokenizing a string */ - int tabsize; /* Tab spacing */ - int indent; /* Current indentation index */ - int indstack[MAXINDENT]; /* Stack of indents */ - int atbol; /* Nonzero if at begin of new line */ - int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ + lexer_layout_state layout; int lineno; /* Current line number */ int first_lineno; /* First line of a single line or multi line string expression (cf. issue 16806) */ @@ -87,8 +96,6 @@ struct tok_state { int parencolstack[MAXLEVEL]; PyObject *filename; PyObject *module; - /* Stuff for checking on different tab sizes */ - int altindstack[MAXINDENT]; /* Stack of alternate indents */ /* Stuff for PEP 0263 */ int input_error; char *encoding; /* Source encoding. */ @@ -105,7 +112,6 @@ struct tok_state { tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL]; int tok_mode_stack_index; int tok_extra_tokens; - int comment_newline; int implicit_newline; #ifdef Py_DEBUG int debug; @@ -114,6 +120,8 @@ struct tok_state { int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); +void _PyLexer_ImplyDedents(struct tok_state *); + void _PyTokenizer_Free(struct tok_state *); void _PyToken_Free(struct token *); void _PyToken_Init(struct token *); diff --git a/Parser/pegen.c b/Parser/pegen.c index 72275b4eaad2ee7..cb8ee231fd452d8 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -288,9 +288,8 @@ _PyPegen_fill_token(Parser *p) type = NEWLINE; /* Add an extra newline */ p->parsing_started = 0; - if (p->tok->indent && !(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) { - p->tok->pendin = -p->tok->indent; - p->tok->indent = 0; + if (!(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) { + _PyLexer_ImplyDedents(p->tok); } } else { diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 769ee4083bc4d44..9c1865234637f70 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -735,10 +735,9 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind) return NULL; } tok->done = E_OK; - tok->atbol = 1; + tok->layout.at_bol = 1; tok->starting_col_offset = -1; tok->col_offset = -1; - tok->tabsize = 8; tok->tok_mode_stack[0].kind = TOK_REGULAR_MODE; #ifdef Py_DEBUG tok->debug = _Py_GetConfig()->parser_debug; diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index af8027db27234ad..a20b78271cf3a4b 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -125,6 +125,7 @@ def compile_c_extension( str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"), str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c"), + str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "layout.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"), From 2cf51c4d805d0eb66ed06125da047b41b1951210 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 6 Sep 2026 22:19:20 +0100 Subject: [PATCH 4/5] gh-153569: remove unused tokenizer cursor and source lookup APIs --- Lib/test/test_capi/test_tokenizer.py | 3 - Makefile.pre.in | 4 +- Modules/_testinternalcapi/tokenizer.c | 308 ++------------------------ PCbuild/pythoncore.vcxproj | 2 - PCbuild/pythoncore.vcxproj.filters | 6 - Parser/tokenizer/cursor.c | 82 ------- Parser/tokenizer/cursor.h | 73 ------ Parser/tokenizer/source.c | 175 +-------------- Parser/tokenizer/source.h | 50 +---- 9 files changed, 20 insertions(+), 683 deletions(-) delete mode 100644 Parser/tokenizer/cursor.c delete mode 100644 Parser/tokenizer/cursor.h diff --git a/Lib/test/test_capi/test_tokenizer.py b/Lib/test/test_capi/test_tokenizer.py index eb04f6c0136022d..57d0a3c2f2e99e8 100644 --- a/Lib/test/test_capi/test_tokenizer.py +++ b/Lib/test/test_capi/test_tokenizer.py @@ -12,9 +12,6 @@ def test_source(self): def test_source_discard(self): _testinternalcapi.test_tokenizer_source_discard() - def test_cursor(self): - _testinternalcapi.test_tokenizer_cursor() - if __name__ == "__main__": unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index ac0db1a1c5923fa..5eb8d4099f7c4bc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -399,7 +399,6 @@ TOKENIZER_OBJS= \ Parser/lexer/number.o \ Parser/lexer/state.o \ Parser/lexer/string.o \ - Parser/tokenizer/cursor.o \ Parser/tokenizer/decoder.o \ Parser/tokenizer/reader.o \ Parser/tokenizer/source.o \ @@ -414,7 +413,6 @@ TOKENIZER_HEADERS= \ Parser/lexer/lexer.h \ Parser/lexer/lexer_internal.h \ Parser/lexer/state.h \ - Parser/tokenizer/cursor.h \ Parser/tokenizer/reader.h \ Parser/tokenizer/reader_internal.h \ Parser/tokenizer/source.h \ @@ -3462,7 +3460,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo. MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h -MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h +MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index 602dda8ee94a8bf..1f89c12f223c7d9 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -1,6 +1,6 @@ #include "parts.h" -#include "../../Parser/tokenizer/cursor.h" +#include "../../Parser/tokenizer/source.h" static int check(int condition, const char *message) @@ -34,16 +34,6 @@ check_line_view(const _PyTok_SourceText *source, Py_ssize_t lineno, "wrong source line view"); } -static int -same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right) -{ - return left->source == right->source && - left->pos == right->pos && - left->line_start == right->line_start && - left->line_end == right->line_end && - left->lineno == right->lineno; -} - static PyObject * test_tokenizer_source(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) @@ -55,34 +45,24 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } - _PyTok_Loc loc; - _PyTok_Line line; - if (check(_PyTok_SourceLocation( - &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate empty source") < 0 || - check(loc.lineno == 1 && loc.byte_col == 0, - "wrong empty source location") < 0 || - check(_PyTok_SourceLine(&source, 1, &line) == 0, - "cannot find empty source line") < 0 || - check(line.start == 0 && line.end == 0, - "wrong empty source line") < 0 || - check_system_error( - _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, - "accepted empty source line") < 0 || + if (check_system_error( + _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, + "accepted empty source line") < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0, "accepted multiple source lines") < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0, - "accepted missing implicit newline") < 0) { - goto error; - } - - if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0, - "wrong first source offset") < 0 || + "accepted missing implicit newline") < 0 || + check(_PyTok_SourceAppendLine( + &source, "alpha\n", 6, 0) == 0, + "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( &source, "\xce\xb2\n", 3, 1) == 6, - "wrong second source offset") < 0) { + "wrong second source offset") < 0 || + check(!_PyTok_SourceLineIsImplicit(&source, 1) && + _PyTok_SourceLineIsImplicit(&source, 2), + "wrong implicit newline flags") < 0) { goto error; } @@ -94,279 +74,24 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } - if (check(_PyTok_SourceAppendLine(&source, "nul\0x\n", 6, 0) == 9, - "wrong third source offset") < 0) { - goto error; - } - - int marker_line = 257; - int final_line = 300; - _PyTok_Off marker_start = -1; - for (int lineno = 4; lineno <= final_line; lineno++) { - const char *text = lineno == marker_line ? "marker\n" : "x\n"; - Py_ssize_t len = (Py_ssize_t)strlen(text); - _PyTok_Off start = _PyTok_SourceAppendLine( - &source, text, len, lineno == final_line); - if (start < 0) { - goto error; - } - if (lineno == marker_line) { - marker_start = start; - } - } - - if (check(source.nlines == final_line, "wrong source line count") < 0 || - check(_PyTok_SourceLine(&source, marker_line, &line) == 0, - "cannot find late source line") < 0 || - check(line.start == marker_start && - line.end == marker_start + 7, - "wrong late source line") < 0 || - check(!line.implicit_newline && !line.contains_nul, - "wrong late source flags") < 0 || - check(_PyTok_SourceLine(&source, 2, &line) == 0, - "cannot find second source line") < 0 || - check(line.start == 6 && line.end == 9 && - line.implicit_newline && !line.contains_nul, - "wrong second source line") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, 1) && - _PyTok_SourceLineIsImplicit(&source, 2), - "wrong early implicit newline flags") < 0 || - check(_PyTok_SourceLine(&source, 3, &line) == 0, - "cannot find third source line") < 0 || - check(line.contains_nul, "missing null byte flag") < 0 || - check(_PyTok_SourceLine(&source, final_line, &line) == 0, - "cannot find final source line") < 0 || - check(line.implicit_newline && - _PyTok_SourceLineIsImplicit(&source, final_line), - "missing late implicit newline flag") < 0) { - goto error; - } - - Py_ssize_t view_len; - const char *view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(6, 8), &view_len); - if (check(view != NULL && view_len == 2 && - memcmp(view, "\xce\xb2", 2) == 0, - "wrong source span view") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start, - _PYTOK_AFFINITY_LEFT, &loc) == 0, - "cannot locate left line boundary") < 0 || - check(loc.lineno == marker_line - 1 && loc.byte_col == 2, - "wrong left boundary location") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate right line boundary") < 0 || - check(loc.lineno == marker_line && loc.byte_col == 0, - "wrong right boundary location") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start + 1, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate late source byte") < 0 || - check(loc.lineno == marker_line && loc.byte_col == 1, - "wrong late source location") < 0) { - goto error; - } - - if (check(_PyTok_SourceLocation( - &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0, - "cannot locate left EOF") < 0 || - check(loc.lineno == final_line && loc.byte_col == 2, - "wrong left EOF location") < 0 || - check(_PyTok_SourceLocation( - &source, source.len, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate right EOF") < 0 || - check(loc.lineno == final_line + 1 && loc.byte_col == 0, - "wrong right EOF location") < 0 || - check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0, - "cannot find virtual EOF line") < 0 || - check(line.start == source.len && line.end == source.len, - "wrong virtual EOF line") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, 0) && - !_PyTok_SourceLineIsImplicit( - &source, final_line + 1), - "virtual or invalid line is implicit") < 0) { - goto error; - } - - view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len); - if (check_system_error(view == NULL, "accepted invalid source span") < 0 || - check_system_error( - _PyTok_SourceLocation( - &source, source.len + 1, - _PYTOK_AFFINITY_RIGHT, &loc) < 0, - "accepted invalid source offset") < 0 || - check_system_error( - _PyTok_SourceLine(&source, final_line + 2, &line) < 0, - "accepted invalid source line") < 0) { + if (check(source.len == 9 && + memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0, + "wrong source contents") < 0) { goto error; } _PyTok_SourceClear(&source); - _PyTok_SourceInit(&source); if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0, - "appended after unterminated source line") < 0 || - check(_PyTok_SourceLocation( - &source, source.len, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate unterminated EOF") < 0 || - check(loc.lineno == 1 && loc.byte_col == 4, - "wrong unterminated EOF location") < 0) { + "appended after unterminated source line") < 0) { goto error; } - if (check_line_view(&source, 1, "tail") < 0 || check_line_view(&source, PY_SSIZE_T_MAX, "tail") < 0) { goto error; } - _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4, - "wrong retained source offset") < 0 || - _PyTok_SourceLine(&source, 1, &line) < 0 || - check(line.start == 4 && line.end == 6, - "wrong retained source line") < 0 || - _PyTok_SourceLocation( - &source, 4, _PYTOK_AFFINITY_LEFT, &loc) < 0 || - check(loc.lineno == 1 && loc.byte_col == 0, - "wrong retained source location") < 0) { - goto error; - } - view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(4, 5), &view_len); - if (check(view != NULL && view_len == 1 && view[0] == 'a', - "wrong retained source span") < 0 || - check_system_error(_PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(0, 1), &view_len) == NULL, - "accepted discarded source span") < 0) { - goto error; - } - - _PyTok_SourceClear(&source); - Py_RETURN_NONE; - -error: - _PyTok_SourceClear(&source); - return NULL; -} - -static PyObject * -test_tokenizer_cursor(PyObject *Py_UNUSED(module), - PyObject *Py_UNUSED(args)) -{ - _PyTok_SourceText source; - _PyTok_SourceInit(&source); - if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 || - _PyTok_SourceAppendLine(&source, "cd\n", 3, 0) < 0) { - goto error; - } - - _PyTok_Cursor cursor; - _PyTok_CursorInit(&cursor, &source); - if (_PyTok_CursorSetOffset(&cursor, source.len) < 0 || - check(cursor.lineno == 3 && cursor.pos == source.len, - "wrong cursor at virtual EOF") < 0 || - _PyTok_CursorSetLine(&cursor, 1) < 0) { - goto error; - } - - char large[BUFSIZ + 1]; - memset(large, 'z', sizeof(large)); - large[sizeof(large) - 1] = '\n'; - if (_PyTok_SourceAppendLine(&source, large, sizeof(large), 0) < 0) { - goto error; - } - - if (check(_PyTok_CursorPeek(&cursor, 0) == 'a', - "wrong cursor peek after relocation") < 0 || - check(_PyTok_CursorPeek(&cursor, 1) == 'b', - "wrong distant cursor peek") < 0 || - check(_PyTok_CursorAdvance(&cursor) == 'a', - "wrong first cursor byte") < 0 || - check(_PyTok_CursorAdvance(&cursor) == 'b', - "wrong second cursor byte") < 0 || - check(_PyTok_CursorAdvance(&cursor) == '\n', - "wrong final cursor byte") < 0 || - check(_PyTok_CursorAdvance(&cursor) == EOF, - "cursor advanced past line") < 0 || - check(_PyTok_CursorSetOffset(&cursor, 2) == 0, - "cannot seek cursor offset") < 0 || - check(_PyTok_CursorAdvance(&cursor) == '\n', - "wrong cursor byte after seek") < 0 || - check(_PyTok_CursorSetOffset(&cursor, 3) == 0, - "cannot seek line boundary") < 0 || - check(cursor.lineno == 2 && cursor.line_start == 3 && - _PyTok_CursorAdvance(&cursor) == 'c', - "wrong cursor at line boundary") < 0 || - check(_PyTok_CursorSetLine(&cursor, 3) == 0, - "cannot advance cursor to final line") < 0 || - check(cursor.line_start == 6 && - _PyTok_CursorAdvance(&cursor) == 'z', - "wrong cursor byte on final line") < 0) { - goto error; - } - - _PyTok_Cursor saved = cursor; - if (check_system_error( - _PyTok_CursorSetOffset(&cursor, source.len + 1) < 0, - "accepted invalid cursor offset") < 0 || - check(same_cursor(&cursor, &saved), - "invalid offset changed cursor") < 0 || - check_system_error( - _PyTok_CursorSetLine(&cursor, source.nlines + 2) < 0, - "accepted invalid cursor line") < 0 || - check(same_cursor(&cursor, &saved), - "invalid line changed cursor") < 0 || - check(_PyTok_CursorSetOffset(&cursor, source.len) == 0, - "cannot set cursor to EOF") < 0 || - check(cursor.lineno == 4 && cursor.pos == source.len, - "wrong cursor at EOF") < 0) { - goto error; - } - -#if SIZEOF_VOID_P > 4 - char byte = 0; - _PyTok_SourceText huge_source = { - .bytes = &byte, - .len = (_PyTok_Off)INT_MAX + 1, - }; - _PyTok_Cursor huge_cursor = { - .source = &huge_source, - .pos = INT_MAX, - .line_end = (_PyTok_Off)INT_MAX + 1, - .lineno = 1, - }; - if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF && - huge_cursor.pos == INT_MAX, - "cursor advanced past maximum column") < 0) { - goto error; - } -#endif - - _PyTok_Off base = source.len; - _PyTok_SourceDiscard(&source); - if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 || - _PyTok_SourceAppendLine(&source, "cd", 2, 0) < 0) { - goto error; - } - _PyTok_CursorInit(&cursor, &source); - if (_PyTok_CursorSetLine(&cursor, 1) < 0 || - check(cursor.pos == base && _PyTok_CursorPeek(&cursor, 1) == 'b', - "wrong retained cursor line") < 0 || - _PyTok_CursorSetLine(&cursor, 2) < 0 || - check(_PyTok_CursorAdvance(&cursor) == 'c', - "wrong retained cursor byte") < 0 || - _PyTok_CursorSetOffset(&cursor, base + 5) < 0 || - check(cursor.lineno == 2 && _PyTok_CursorAdvance(&cursor) == EOF, - "wrong retained cursor EOF") < 0) { - goto error; - } - _PyTok_SourceClear(&source); Py_RETURN_NONE; @@ -440,7 +165,6 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), static PyMethodDef test_methods[] = { {"test_tokenizer_source", test_tokenizer_source, METH_NOARGS}, - {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS}, {"test_tokenizer_source_discard", test_tokenizer_source_discard, METH_NOARGS}, {NULL}, }; diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index ceb6b792cc0c621..c90eeb95ff60da7 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -425,7 +425,6 @@ - @@ -593,7 +592,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index a2c4eb431f373ef..4a1bd70c499ff26 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -336,9 +336,6 @@ Parser - - Parser - Parser @@ -1361,9 +1358,6 @@ Parser - - Parser - Parser diff --git a/Parser/tokenizer/cursor.c b/Parser/tokenizer/cursor.c deleted file mode 100644 index 523b99dedc6160a..000000000000000 --- a/Parser/tokenizer/cursor.c +++ /dev/null @@ -1,82 +0,0 @@ -#include "Python.h" - -#include "cursor.h" - -static void -set_line(_PyTok_Cursor *cursor, int lineno, _PyTok_Off start, - _PyTok_Off end) -{ - cursor->pos = start; - cursor->line_start = start; - cursor->line_end = end; - cursor->lineno = lineno; -} - -int -_PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno) -{ - if (cursor->source == NULL) { - PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source"); - return -1; - } - const _PyTok_SourceText *source = cursor->source; - if (lineno > 0 && cursor->lineno == lineno - 1 && - lineno <= source->nlines) { - _PyTok_Off start = cursor->line_end; - _PyTok_Off end = source->base_offset + source->len; - if (lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - set_line(cursor, lineno, start, end); - return 0; - } - - _PyTok_Line line; - if (_PyTok_SourceLine(source, lineno, &line) < 0) { - return -1; - } - set_line(cursor, lineno, line.start, line.end); - return 0; -} - -int -_PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset) -{ - if (cursor->source == NULL) { - PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source"); - return -1; - } - const _PyTok_SourceText *source = cursor->source; - int stays_on_line = cursor->lineno > 0 && - offset >= cursor->line_start && offset < cursor->line_end; - if (!stays_on_line && cursor->lineno > 0 && - offset == cursor->line_end && - offset - source->base_offset == source->len && - (source->len == 0 || source->bytes[source->len - 1] != '\n')) { - stays_on_line = 1; - } - if (stays_on_line) { - cursor->pos = offset; - return 0; - } - - _PyTok_Loc loc; - if (_PyTok_SourceLocation( - source, offset, _PYTOK_AFFINITY_RIGHT, &loc) < 0) { - return -1; - } - _PyTok_Off start = offset - loc.byte_col; - _PyTok_Off end = source->base_offset + source->len; - if (loc.lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - set_line(cursor, loc.lineno, start, end); - cursor->pos = offset; - return 0; -} diff --git a/Parser/tokenizer/cursor.h b/Parser/tokenizer/cursor.h deleted file mode 100644 index 18e404251316f0c..000000000000000 --- a/Parser/tokenizer/cursor.h +++ /dev/null @@ -1,73 +0,0 @@ -#ifndef Py_TOKENIZER_CURSOR_H -#define Py_TOKENIZER_CURSOR_H - -#include "source.h" - -typedef struct { - /* The source must remain initialized at this address while in use. */ - const _PyTok_SourceText *source; - _PyTok_Off pos; - _PyTok_Off line_start; - _PyTok_Off line_end; - int lineno; -} _PyTok_Cursor; - -/* Move to the start of a 1-based line. Both setters preserve the cursor on - error. */ -PyAPI_FUNC(int) _PyTok_CursorSetLine(_PyTok_Cursor *, int); -/* Move to an offset. A line boundary selects the following line. */ -PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off); - -static inline void -_PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source) -{ - _PyTok_Off base = source != NULL ? source->base_offset : 0; - *cursor = (_PyTok_Cursor){ - .source = source, - .pos = base, - .line_start = base, - .line_end = base, - }; -} - -/* Read one byte from the current line, including its terminating newline. - EOF marks the line boundary, not necessarily the end of the source. It is - also returned if advancing would make the byte column unrepresentable. */ -static inline int -_PyTok_CursorAdvance(_PyTok_Cursor *cursor) -{ - assert(cursor->source != NULL); - assert(cursor->pos >= cursor->line_start); - assert(cursor->pos <= cursor->line_end); - assert(cursor->line_start >= cursor->source->base_offset); - assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len); - if (cursor->pos >= cursor->line_end) { - return EOF; - } - if (cursor->pos - cursor->line_start >= INT_MAX) { - return EOF; - } - return Py_CHARMASK(cursor->source->bytes[ - cursor->pos++ - cursor->source->base_offset]); -} - -/* Return the byte at a nonnegative distance within the current line, or EOF - if the distance reaches or crosses the line boundary. */ -static inline int -_PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance) -{ - assert(cursor->source != NULL); - assert(cursor->pos >= cursor->line_start); - assert(cursor->pos <= cursor->line_end); - assert(cursor->line_start >= cursor->source->base_offset); - assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len); - assert(distance >= 0); - if (distance < 0 || - distance >= cursor->line_end - cursor->pos) { - return EOF; - } - return Py_CHARMASK(cursor->source->bytes[ - cursor->pos - cursor->source->base_offset + distance]); -} - -#endif diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index 989944afd9fea96..d69eab93923e81c 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -2,8 +2,6 @@ #include "source.h" -#define LINE_CHECKPOINT_INTERVAL 256 - void _PyTok_SourceInit(_PyTok_SourceText *source) { @@ -14,7 +12,6 @@ void _PyTok_SourceClear(_PyTok_SourceText *source) { PyMem_Free(source->bytes); - PyMem_Free(source->line_checkpoints); PyMem_Free(source->implicit_lines); _PyTok_SourceInit(source); } @@ -74,34 +71,6 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) return 0; } -static int -reserve_checkpoints(_PyTok_SourceText *source, int needed) -{ - if (needed <= source->checkpoints_cap) { - return 0; - } - int cap; - if (source->checkpoints_cap == 0) { - cap = 16; - } - else if (source->checkpoints_cap <= INT_MAX / 2) { - cap = source->checkpoints_cap * 2; - } - else { - PyErr_NoMemory(); - return -1; - } - _PyTok_Off *checkpoints = source->line_checkpoints; - PyMem_Resize(checkpoints, _PyTok_Off, cap); - if (checkpoints == NULL) { - PyErr_NoMemory(); - return -1; - } - source->line_checkpoints = checkpoints; - source->checkpoints_cap = cap; - return 0; -} - static int reserve_implicit_lines(_PyTok_SourceText *source, int nlines) { @@ -165,11 +134,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, return -1; } int nlines = source->nlines + 1; - int checkpoint = ((nlines - 1) % LINE_CHECKPOINT_INTERVAL) == 0; - int checkpoint_count = (nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1; - if ((checkpoint && - reserve_checkpoints(source, checkpoint_count) < 0) || - (implicit_newline && reserve_implicit_lines(source, nlines) < 0) || + if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) || reserve_bytes(source, source->len + len + 1) < 0) { return -1; } @@ -178,10 +143,6 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, memcpy(source->bytes + start, bytes, len); source->len += len; source->bytes[source->len] = '\0'; - if (checkpoint) { - source->line_checkpoints[checkpoint_count - 1] = - source->base_offset + start; - } if (implicit_newline) { source->implicit_lines[(nlines - 1) / 8] |= (unsigned char)(1U << ((nlines - 1) & 7)); @@ -210,19 +171,6 @@ _PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno, return line; } -const char * -_PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span, - Py_ssize_t *len) -{ - if (!_PyTok_SpanIsValid(span) || span.start < source->base_offset || - span.end - source->base_offset > source->len || len == NULL) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source span"); - return NULL; - } - *len = span.end - span.start; - return _PyTok_SourceData(source) + (span.start - source->base_offset); -} - int _PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno) { @@ -233,124 +181,3 @@ _PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno) return (source->implicit_lines[(lineno - 1) / 8] >> ((lineno - 1) & 7)) & 1; } - -static int -source_ends_in_newline(const _PyTok_SourceText *source) -{ - return source->len > 0 && source->bytes[source->len - 1] == '\n'; -} - -static int -eof_lineno(const _PyTok_SourceText *source) -{ - if (source->nlines == 0) { - return 1; - } - return source->nlines + source_ends_in_newline(source); -} - -int -_PyTok_SourceLine(const _PyTok_SourceText *source, int lineno, - _PyTok_Line *line) -{ - if (line == NULL || lineno < 1 || lineno > eof_lineno(source)) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line"); - return -1; - } - if (lineno > source->nlines) { - *line = (_PyTok_Line){ - .start = source->base_offset + source->len, - .end = source->base_offset + source->len, - }; - return 0; - } - - int checkpoint = (lineno - 1) / LINE_CHECKPOINT_INTERVAL; - int current = checkpoint * LINE_CHECKPOINT_INTERVAL + 1; - _PyTok_Off start = source->line_checkpoints[checkpoint]; - while (current < lineno) { - start = _PyTok_SourceFindLineEnd(source, start); - if (start < 0) { - return -1; - } - current++; - } - _PyTok_Off end = source->base_offset + source->len; - if (lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - *line = (_PyTok_Line){ - .start = start, - .end = end, - .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno), - .contains_nul = memchr( - source->bytes + (start - source->base_offset), - 0, end - start) != NULL, - }; - return 0; -} - -int -_PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset, - _PyTok_Affinity affinity, _PyTok_Loc *loc) -{ - if (offset < source->base_offset || - offset - source->base_offset > source->len || loc == NULL || - (affinity != _PYTOK_AFFINITY_LEFT && - affinity != _PYTOK_AFFINITY_RIGHT)) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset"); - return -1; - } - if (source->nlines == 0 || - (offset - source->base_offset == source->len && - source_ends_in_newline(source) && - affinity == _PYTOK_AFFINITY_RIGHT)) { - *loc = (_PyTok_Loc){eof_lineno(source), 0}; - return 0; - } - - _PyTok_Off key = offset; - if (affinity == _PYTOK_AFFINITY_LEFT && key > source->base_offset) { - key--; - } - int low = 0; - int high = (source->nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1; - while (low < high) { - int middle = low + (high - low) / 2; - if (source->line_checkpoints[middle] <= key) { - low = middle + 1; - } - else { - high = middle; - } - } - int checkpoint = low - 1; - if (checkpoint < 0) { - PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index"); - return -1; - } - int lineno = checkpoint * LINE_CHECKPOINT_INTERVAL + 1; - _PyTok_Off start = source->line_checkpoints[checkpoint]; - while (lineno < source->nlines) { - _PyTok_Off end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - if (offset < end || - (offset == end && affinity == _PYTOK_AFFINITY_LEFT)) { - break; - } - start = end; - lineno++; - } - _PyTok_Off byte_col = offset - start; - if (byte_col > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "tokenizer column is too large"); - return -1; - } - *loc = (_PyTok_Loc){lineno, (int)byte_col}; - return 0; -} diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 7ae1e6e21abbc54..6972caf8a36aa24 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -18,28 +18,13 @@ typedef struct { int byte_col; } _PyTok_Loc; -typedef enum { - _PYTOK_AFFINITY_LEFT, - _PYTOK_AFFINITY_RIGHT, -} _PyTok_Affinity; - -/* The half-open range includes the terminating newline when present. */ -typedef struct { - _PyTok_Off start; - _PyTok_Off end; - unsigned implicit_newline : 1; - unsigned contains_nul : 1; -} _PyTok_Line; - typedef struct { char *bytes; _PyTok_Off base_offset; _PyTok_Off len; _PyTok_Off cap; - _PyTok_Off *line_checkpoints; unsigned char *implicit_lines; int nlines; - int checkpoints_cap; Py_ssize_t implicit_cap; } _PyTok_SourceText; @@ -50,9 +35,9 @@ _PyTok_SourceData(const _PyTok_SourceText *source) } PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *); -/* Clear invalidates all cursors, spans, and views for the source. */ +/* Clear invalidates all spans and views for the source. */ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); -/* Discard the retained window and invalidate its cursors, spans, and views. +/* Discard the retained window and invalidate its spans and views. Keep its allocation and advance the logical base to the end of the window. */ PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); /* Append one nonempty logical line and return its start offset. The input may @@ -68,20 +53,9 @@ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( This does not set an exception. Append, discard, and clear invalidate the view. */ PyAPI_FUNC(const char *) _PyTok_SourceLineView( const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len); -/* The returned view is invalidated by SourceAppendLine and SourceClear. */ -PyAPI_FUNC(const char *) _PyTok_SourceSpanView( - const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *); -/* Look up a 1-based line in the retained window. Empty and newline-terminated - sources have an empty virtual line at EOF. */ -PyAPI_FUNC(int) _PyTok_SourceLine( - const _PyTok_SourceText *, int, _PyTok_Line *); /* Return false for invalid line numbers and the virtual EOF line. */ PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit( const _PyTok_SourceText *, int); -/* At a line boundary, left affinity selects the preceding line at its end; - right affinity selects the following line at byte column zero. */ -PyAPI_FUNC(int) _PyTok_SourceLocation( - const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *); static inline _PyTok_Span _PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end) @@ -95,24 +69,4 @@ _PyTok_SpanIsValid(_PyTok_Span span) return span.start >= 0 && span.end >= span.start; } -static inline _PyTok_Off -_PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start) -{ - if (source->bytes == NULL || start < source->base_offset || - start - source->base_offset >= source->len) { - PyErr_SetString(PyExc_SystemError, - "corrupt tokenizer source line index"); - return -1; - } - _PyTok_Off relative_start = start - source->base_offset; - const char *newline = memchr( - source->bytes + relative_start, '\n', source->len - relative_start); - if (newline == NULL) { - PyErr_SetString(PyExc_SystemError, - "corrupt tokenizer source line index"); - return -1; - } - return source->base_offset + (newline - source->bytes) + 1; -} - #endif From 1d75e178b229c31d3932e52a9576878614c370aa Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 6 Sep 2026 22:22:51 +0100 Subject: [PATCH 5/5] gh-153569: report tokenizer diagnostics without rewinding the scanner --- Lib/test/test_codeop.py | 11 ++++ Lib/test/test_source_encoding.py | 26 ++++++-- Lib/test/test_tstring.py | 2 + Parser/lexer/lexer.c | 13 +++- Parser/lexer/state.h | 66 +++++++++++++++++++ Parser/lexer/string.c | 108 +++++++++++++++++-------------- Parser/pegen.c | 4 +- Parser/pegen_errors.c | 34 ++++++---- Parser/tokenizer/decoder.c | 14 +--- Parser/tokenizer/helpers.c | 53 +++++++++------ Parser/tokenizer/helpers.h | 7 ++ 11 files changed, 239 insertions(+), 99 deletions(-) diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index d57452602ce5574..c75c0f8627bf084 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -113,6 +113,17 @@ def test_valid(self, compiler): av("def f():\n pass\n#foo\n") av("@a.b.c\ndef f():\n pass\n") + @subTests('symbol', ('single', 'exec')) + @subTests('prefix', ('', 'f', 't')) + def test_incomplete_string_diagnostics(self, symbol, prefix): + opening = f' á = {prefix}"""first\n' + source = 'if True:\n' + opening + 'second' + with self.assertRaises(_IncompleteInputError) as cm: + Compile()(source, '', symbol) + text = opening + 'second' + ('\n' if symbol == 'exec' else '') + self.assertEqual(cm.exception.args, ( + 'incomplete input', ('', 2, 9, text, 2, -1))) + @subTests('compiler', COMPILERS) def test_incomplete(self, compiler): ai = functools.partial(self.assertIncomplete, compiler=compiler) diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index 862a20a058be75a..ec98e609c4e98f9 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -3,8 +3,8 @@ import unittest from test import support from test.support import script_helper -from test.support.os_helper import TESTFN, unlink, rmtree -from test.support.import_helper import unload +from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree +from test.support.import_helper import import_module, unload import importlib import os import sys @@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self): self.assertRaises(SyntaxError, compile, seq, '', 'exec') def test_invalid_utf8_offset_after_non_ascii(self): + for name in ('é', 'éé', '𝒜'): + with self.subTest(name=name): + source = ('x = ' + name).encode() + b'\xff\n' + with self.assertRaises(SyntaxError) as caught: + compile(source, '', 'exec') + error = caught.exception + self.assertEqual( + (error.lineno, error.offset, error.end_lineno, error.end_offset), + (1, 5 + len(name), 1, 5 + len(name)), + ) + + @support.cpython_only + def test_invalid_utf8_file_offset_after_non_ascii(self): + _testcapi = import_module('_testcapi') + self.addCleanup(unlink, TESTFN_ASCII) + with open(TESTFN_ASCII, 'wb') as f: + f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n') with self.assertRaises(SyntaxError) as caught: - compile(b"x = \xc3\xa9\xff\n", "", "exec") + _testcapi.run_file( + os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {}) error = caught.exception self.assertEqual( (error.lineno, error.offset, error.end_lineno, error.end_offset), - (1, 6, 1, 6), + (2, 7, 2, 7), ) def test_long_bom_conflict_message_is_not_truncated(self): diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index 74653c77c55de17..c5813849cac7c8b 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -198,6 +198,8 @@ def test_nested_templates(self): def test_syntax_errors(self): for case, err in ( + ('t"""{(\n1\n)}\ntail', "unterminated triple-quoted t-string literal"), + ('f"""{(\n1\n)}\ntail', "unterminated triple-quoted f-string literal"), ("t'", "unterminated t-string literal"), ("t'''", "unterminated triple-quoted t-string literal"), ("t''''", "unterminated triple-quoted t-string literal"), diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 01d6d221000f32a..59c2d896962c61d 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -101,6 +101,7 @@ verify_identifier(struct tok_state *tok) assert(PyUnicode_GET_LENGTH(s) > 0); if (invalid < PyUnicode_GET_LENGTH(s)) { Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid); + const char *error_cursor = tok->cur; if (invalid + 1 < PyUnicode_GET_LENGTH(s)) { /* Determine the offset in UTF-8 encoded input */ Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1)); @@ -111,14 +112,20 @@ verify_identifier(struct tok_state *tok) tok->done = E_ERROR; return 0; } - tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s); + error_cursor = tok->start + PyBytes_GET_SIZE(s); } Py_DECREF(s); if (Py_UNICODE_ISPRINTABLE(ch)) { - _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch); + _PyTokenizer_syntaxerror_at( + tok, tok->line_start, + error_cursor - tok->line_start, tok->lineno, -1, -1, + "invalid character '%c' (U+%04X)", ch, ch); } else { - _PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch); + _PyTokenizer_syntaxerror_at( + tok, tok->line_start, + error_cursor - tok->line_start, tok->lineno, -1, -1, + "invalid non-printable character U+%04X", ch); } return 0; } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 2901cd3d3187808..99407158cc8bf2a 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -71,6 +71,14 @@ typedef struct { indentation_level stack[MAXINDENT]; } lexer_layout_state; +/* Supplemental source context for a terminal error. location is the reporting + cursor, independent of the scanner cursor; lineno == 0 means absent. + The text span may cover multiple physical lines. */ +typedef struct { + _PyTok_Loc location; + _PyTok_Span text_span; +} _PyTokenizer_Diagnostic; + /* Tokenizer state */ struct tok_state { /* Input state; buf <= cur <= inp */ @@ -89,6 +97,7 @@ struct tok_state { expression (cf. issue 16806) */ int starting_col_offset; /* The column offset at the beginning of a token */ int col_offset; /* Current col offset */ + _PyTokenizer_Diagnostic diagnostic; int level; /* () [] {} Parentheses nesting level */ /* Used to allow free continuations inside them */ char parenstack[MAXLEVEL]; @@ -118,6 +127,63 @@ struct tok_state { #endif }; +static inline _PyTok_Off +_PyLexer_BufferOffset(const struct tok_state *tok, const char *position) +{ + assert(tok->buf != NULL); + assert(tok->inp >= tok->buf); + assert(position >= tok->buf && position <= tok->inp); + Py_ssize_t offset = position - tok->buf; + assert(tok->buf_offset <= PY_SSIZE_T_MAX - offset); + return tok->buf_offset + offset; +} + +static inline char * +_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset) +{ + assert(tok->buf != NULL); + assert(tok->inp >= tok->buf); + assert(offset >= tok->buf_offset); + assert(offset - tok->buf_offset <= tok->inp - tok->buf); + return tok->buf + (offset - tok->buf_offset); +} + +static inline const char * +_PyLexer_BufferSpanView(const struct tok_state *tok, _PyTok_Span span, + Py_ssize_t *length) +{ + assert(length != NULL); + assert(_PyTok_SpanIsValid(span)); + *length = span.end - span.start; + (void)_PyLexer_BufferPointer(tok, span.end); + return _PyLexer_BufferPointer(tok, span.start); +} + +static inline int +_PyLexer_ByteColumn(const struct tok_state *tok) +{ + assert(tok->line_start != NULL); + assert(tok->cur >= tok->line_start); + Py_ssize_t column = tok->cur - tok->line_start; + assert(column <= INT_MAX); + return (int)column; +} + +static inline _PyTok_Span +_PyLexer_BufferSpan(const struct tok_state *tok, const char *start, + const char *end) +{ + if (start == NULL) { + assert(end == NULL); + return (_PyTok_Span){-1, -1}; + } + assert(end != NULL); + assert(start <= end); + return _PyTok_SpanFromBounds( + _PyLexer_BufferOffset(tok, start), + _PyLexer_BufferOffset(tok, end)); +} + int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); void _PyLexer_ImplyDedents(struct tok_state *); diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index fc0299c5c7c592f..397c90c2e5b9e33 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -7,6 +7,20 @@ #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) +static int +string_error_token(struct tok_state *tok, struct token *token, + const char *start, _PyTok_Loc location) +{ + tok->diagnostic = (_PyTokenizer_Diagnostic){ + .location = {location.lineno, location.byte_col + 1}, + .text_span = _PyLexer_BufferSpan(tok, start - location.byte_col, tok->inp), + }; + int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, NULL, NULL); + token->start_loc = location; + token->end_loc = (_PyTok_Loc){location.lineno, -1}; + return type; +} + int _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { assert(token != NULL); @@ -347,55 +361,52 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) break; } if (c == EOF || (quote_size == 1 && c == '\n')) { - assert(tok->multi_line_start != NULL); - // shift the tok_state's location into - // the start of string, and report the error - // from the initial quote character - tok->cur = (char *)tok->start; - tok->cur++; - tok->line_start = tok->multi_line_start; - int start = tok->lineno; - tok->lineno = tok->first_lineno; - - if (INSIDE_FSTRING(tok)) { - /* When we are in an f-string, before raising the - * unterminated string literal error, check whether - * does the initial quote matches with f-strings quotes - * and if it is, then this must be a missing '}' token - * so raise the proper error */ - tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); - if (the_current_tok->quote == quote && - the_current_tok->quote_size == quote_size) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "%c-string: expecting '}'", TOK_GET_STRING_PREFIX(tok))); + int end_lineno = tok->lineno; + _PyTok_Loc location = {tok->first_lineno, tok->starting_col_offset}; + const char *line = tok->start - location.byte_col; + Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1; + + const tokenizer_mode *state = INSIDE_FSTRING(tok) ? TOK_GET_MODE(tok) : NULL; + if (state != NULL) { + /* A matching quote belongs to the surrounding formatted + * string, so the expression is missing its closing brace. */ + if (state->quote == quote && state->quote_size == quote_size) { + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "%c-string: expecting '}'", + TOK_GET_STRING_PREFIX(tok)); + return string_error_token(tok, token, tok->start, location); } } if (quote_size == 3) { - _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal" - " (detected at line %d)", start); + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated triple-quoted string literal" + " (detected at line %d)", end_lineno); if (c != '\n') { tok->done = E_EOFS; } - return MAKE_TOKEN(ERRORTOKEN); + return string_error_token(tok, token, tok->start, location); } else { if (has_escaped_quote) { - _PyTokenizer_syntaxerror( - tok, + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, "unterminated string literal (detected at line %d); " "perhaps you escaped the end quote?", - start + end_lineno ); } else { - _PyTokenizer_syntaxerror( - tok, "unterminated string literal (detected at line %d)", start + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated string literal (detected at line %d)", end_lineno ); } if (c != '\n') { tok->done = E_EOLS; } - return MAKE_TOKEN(ERRORTOKEN); + return string_error_token(tok, token, tok->start, location); } } if (c == quote) { @@ -516,32 +527,31 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); } - assert(tok->multi_line_start != NULL); - // shift the tok_state's location into - // the start of string, and report the error - // from the initial quote character - tok->cur = (char *)current_tok->start; - tok->cur++; - tok->line_start = current_tok->multi_line_start; - int start = tok->lineno; - - tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); - tok->lineno = the_current_tok->first_line; + int end_lineno = tok->lineno; + _PyTok_Loc location = {current_tok->first_line, + (int)(current_tok->start - current_tok->multi_line_start)}; + const char *line = current_tok->multi_line_start; + Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1; if (current_tok->quote_size == 3) { - _PyTokenizer_syntaxerror(tok, - "unterminated triple-quoted %c-string literal" - " (detected at line %d)", - TOK_GET_STRING_PREFIX(tok), start); + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated triple-quoted %c-string literal" + " (detected at line %d)", + TOK_GET_STRING_PREFIX(tok), end_lineno); if (c != '\n') { tok->done = E_EOFS; } - return MAKE_TOKEN(ERRORTOKEN); + return string_error_token(tok, token, + current_tok->start, location); } else { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "unterminated %c-string literal (detected at" - " line %d)", TOK_GET_STRING_PREFIX(tok), start)); + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated %c-string literal (detected at line %d)", + TOK_GET_STRING_PREFIX(tok), end_lineno); + return string_error_token(tok, token, + current_tok->start, location); } } diff --git a/Parser/pegen.c b/Parser/pegen.c index cb8ee231fd452d8..300cb32424c2d81 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -218,11 +218,11 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to parser_token->level = new_token->level; parser_token->lineno = new_token->start_loc.lineno; - parser_token->col_offset = p->tok->lineno == p->starting_lineno + parser_token->col_offset = new_token->end_loc.lineno == p->starting_lineno ? p->starting_col_offset + new_token->start_loc.byte_col : new_token->start_loc.byte_col; parser_token->end_lineno = new_token->end_loc.lineno; - parser_token->end_col_offset = p->tok->lineno == p->starting_lineno + parser_token->end_col_offset = new_token->end_loc.lineno == p->starting_lineno ? p->starting_col_offset + new_token->end_loc.byte_col : new_token->end_loc.byte_col; diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 839beb1761ad59e..0c787cefa8d73ea 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -204,7 +204,10 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err Py_ssize_t col_offset; Py_ssize_t end_col_offset = -1; if (t->col_offset == -1) { - if (p->tok->cur == p->tok->buf) { + _PyTokenizer_Diagnostic diagnostic = p->tok->diagnostic; + if (diagnostic.location.lineno != 0) { + col_offset = diagnostic.location.byte_col; + } else if (p->tok->cur == p->tok->buf) { col_offset = 0; } else { const char* start = p->tok->buf ? p->tok->line_start : p->tok->buf; @@ -253,12 +256,19 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, PyObject *error_line = NULL; PyObject *tmp = NULL; p->error_indicator = 1; + _PyTokenizer_Diagnostic diagnostic = p->tok->diagnostic; + _PyTok_Loc location = diagnostic.location.lineno != 0 + ? diagnostic.location : (_PyTok_Loc){p->tok->lineno, + p->tok->line_start == NULL ? -1 : _PyLexer_ByteColumn(p->tok)}; + _PyTok_Span text_span = diagnostic.location.lineno != 0 + ? diagnostic.text_span : _PyLexer_BufferSpan( + p->tok, p->tok->line_start, p->tok->inp); if (end_lineno == CURRENT_POS) { - end_lineno = p->tok->lineno; + end_lineno = location.lineno; } if (end_col_offset == CURRENT_POS) { - end_col_offset = p->tok->cur - p->tok->line_start; + end_col_offset = location.byte_col; } errstr = PyUnicode_FromFormatV(errmsg, va); @@ -266,8 +276,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, goto error; } - if (_PyTok_ReaderIsInteractive(p->tok) && - _PyTokenizer_RetainedSource(p->tok) != NULL) { + if (_PyTok_ReaderIsInteractive(p->tok) && _PyTokenizer_RetainedSource(p->tok) != NULL) { error_line = get_error_line_from_source(p, lineno); } else if (p->start_rule == Py_file_input) { @@ -283,13 +292,16 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, we're actually parsing from a file, which has an E_EOF SyntaxError and in that case `PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which does not physically exist */ - assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF); - - if (p->tok->lineno <= lineno && p->tok->inp > p->tok->buf) { - Py_ssize_t size = p->tok->inp - p->tok->line_start; - error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size, "replace"); + assert((p->tok->fp == NULL || p->tok->fp == stdin) || p->tok->done == E_EOF); + + if (location.lineno <= lineno && + p->tok->inp > p->tok->buf) { + Py_ssize_t size; + const char *line = _PyLexer_BufferSpanView( + p->tok, text_span, &size); + error_line = PyUnicode_DecodeUTF8(line, size, "replace"); } - else if (p->tok->fp == NULL || p->tok->fp == stdin) { + else if ((p->tok->fp == NULL || p->tok->fp == stdin)) { error_line = get_error_line_from_source(p, lineno); } else { diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 023455d29d6d073..c36860237fce0b9 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -240,22 +240,14 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, const _PyTok_Chunk *line = cookie_line == 2 ? second : first; const char *line_data = line->data + (cookie_line == 1 ? 3 : 0); Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0); - const char *saved_line_start = tok->line_start; - char *saved_cur = tok->cur; - int saved_lineno = tok->lineno; - tok->line_start = line_data; - tok->cur = (char *)line_data; - tok->lineno = cookie_line; int end_col = (int)Py_MIN(line_len, INT_MAX); if (end_col > 0 && (line_data[end_col - 1] == '\n' || line_data[end_col - 1] == '\r')) { end_col--; } - _PyTokenizer_syntaxerror_known_range( - tok, 0, end_col, "encoding problem: %s with BOM", cookie); - tok->line_start = saved_line_start; - tok->cur = saved_cur; - tok->lineno = saved_lineno; + _PyTokenizer_syntaxerror_at( + tok, line_data, 0, cookie_line, 0, end_col, + "encoding problem: %s with BOM", cookie); PyMem_Free(cookie); return _PYTOK_ENCODING_ERROR; } diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 646f8e5932e9546..66a8b160f687871 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -24,7 +24,8 @@ byte_col_to_char_col(const char *line, int byte_col) } static int -_syntaxerror_range(struct tok_state *tok, const char *format, +_syntaxerror_range(struct tok_state *tok, const char *line_start, + Py_ssize_t cursor_offset, int lineno, const char *format, int col_offset, int end_col_offset, va_list vargs) { @@ -40,7 +41,7 @@ _syntaxerror_range(struct tok_state *tok, const char *format, goto error; } - errtext = PyUnicode_DecodeUTF8(tok->line_start, tok->cur - tok->line_start, + errtext = PyUnicode_DecodeUTF8(line_start, cursor_offset, "replace"); if (!errtext) { goto error; @@ -50,19 +51,19 @@ _syntaxerror_range(struct tok_state *tok, const char *format, col_offset = (int)PyUnicode_GET_LENGTH(errtext); } else if (col_offset > 0) { - col_offset = byte_col_to_char_col(tok->line_start, col_offset); + col_offset = byte_col_to_char_col(line_start, col_offset); } if (end_col_offset == -1) { end_col_offset = col_offset; } else if (end_col_offset > 0) { - end_col_offset = byte_col_to_char_col(tok->line_start, end_col_offset); + end_col_offset = byte_col_to_char_col(line_start, end_col_offset); } - Py_ssize_t line_len = strcspn(tok->line_start, "\n"); - if (line_len != tok->cur - tok->line_start) { + Py_ssize_t line_len = strcspn(line_start, "\n"); + if (line_len != cursor_offset) { Py_DECREF(errtext); - errtext = PyUnicode_DecodeUTF8(tok->line_start, line_len, + errtext = PyUnicode_DecodeUTF8(line_start, line_len, "replace"); } if (!errtext) { @@ -71,8 +72,8 @@ _syntaxerror_range(struct tok_state *tok, const char *format, args = Py_BuildValue("(O(OiiNii))", errmsg, tok->filename ? tok->filename : Py_None, - tok->lineno, col_offset, errtext, - tok->lineno, end_col_offset); + lineno, col_offset, errtext, + lineno, end_col_offset); if (args) { PyErr_SetObject(PyExc_SyntaxError, args); Py_DECREF(args); @@ -90,7 +91,9 @@ _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...) // These errors are cleaned on startup. Todo: Fix it. va_list vargs; va_start(vargs, format); - int ret = _syntaxerror_range(tok, format, -1, -1, vargs); + int ret = _syntaxerror_range( + tok, tok->line_start, + tok->cur - tok->line_start, tok->lineno, format, -1, -1, vargs); va_end(vargs); return ret; } @@ -102,7 +105,24 @@ _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, { va_list vargs; va_start(vargs, format); - int ret = _syntaxerror_range(tok, format, col_offset, end_col_offset, vargs); + int ret = _syntaxerror_range( + tok, tok->line_start, + tok->cur - tok->line_start, tok->lineno, format, + col_offset, end_col_offset, vargs); + va_end(vargs); + return ret; +} + +int +_PyTokenizer_syntaxerror_at( + struct tok_state *tok, const char *line_start, Py_ssize_t cursor_offset, + int lineno, int col_offset, int end_col_offset, const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + int ret = _syntaxerror_range( + tok, line_start, cursor_offset, lineno, format, + col_offset, end_col_offset, vargs); va_end(vargs); return ret; } @@ -302,26 +322,21 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno) const char *badchar = NULL; const char *c; int length; - int col_offset = 0; const char *line_start = line; for (c = line; *c; c += length) { if (!(length = valid_utf8((const unsigned char *)c))) { badchar = c; break; } - col_offset++; if (*c == '\n') { lineno++; - col_offset = 0; line_start = c + 1; } } if (badchar) { - tok->lineno = lineno; - tok->line_start = line_start; - tok->cur = (char *)badchar; - _PyTokenizer_syntaxerror_known_range(tok, - col_offset + 1, col_offset + 1, + _PyTokenizer_syntaxerror_at( + tok, line_start, badchar - line_start + 1, lineno, + -1, -1, "Non-UTF-8 code starting with '\\x%.2x'" "%s%V on line %i, " "but no encoding declared; " diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h index 5edf5a3dfd2e0d1..71f725c0b4c792a 100644 --- a/Parser/tokenizer/helpers.h +++ b/Parser/tokenizer/helpers.h @@ -10,10 +10,17 @@ tok->col_offset = 0; int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...); +/* Positive range columns are 1-based byte columns. A start column of -1 + derives the character column from the reporting cursor; an end column of + -1 uses the start column. */ int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...); +int _PyTokenizer_syntaxerror_at( + struct tok_state *tok, const char *line_start, Py_ssize_t cursor_offset, + int lineno, int col_offset, int end_col_offset, const char *format, ...); int _PyTokenizer_indenterror(struct tok_state *tok); int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char); int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...); + void _PyTokenizer_raise_init_error(PyObject *filename); int _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno);