Skip to content

Commit 0b9a33b

Browse files
pablogsalpablogsal
committed
gh-153569: derive tokenizer locations and failures from scanner state
1 parent baef09e commit 0b9a33b

13 files changed

Lines changed: 64 additions & 84 deletions

File tree

‎Lib/test/test_syntax.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3393,6 +3393,10 @@ def test_invalid_line_continuation_error_position(self):
33933393
self._check_error('\nfgdfgf\n1,\\#\n2\n',
33943394
"unexpected character after line continuation character",
33953395
lineno=3, offset=4)
3396+
for prefix in ("f", "t"):
3397+
self._check_error(f'{prefix}"""{{\n\\ x}}"""',
3398+
"unexpected character after line continuation character",
3399+
lineno=2, offset=2)
33963400

33973401
def test_invalid_line_continuation_left_recursive(self):
33983402
# Check bpo-42218: SyntaxErrors following left-recursive rules

‎Parser/lexer/buffer.c‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@ _PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base,
1313
? -1 : tok->start - tok->buf;
1414
pointers->line_start_from_buf = tok->line_start == NULL
1515
? -1 : tok->line_start - tok->buf;
16-
pointers->multi_line_start_from_buf = tok->multi_line_start == NULL
17-
? -1 : tok->multi_line_start - tok->buf;
1816
}
1917

2018
void
@@ -28,6 +26,4 @@ _PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base,
2826
? NULL : tok->buf + pointers->start_from_buf;
2927
tok->line_start = pointers->line_start_from_buf < 0
3028
? NULL : tok->buf + pointers->line_start_from_buf;
31-
tok->multi_line_start = pointers->multi_line_start_from_buf < 0
32-
? NULL : tok->buf + pointers->multi_line_start_from_buf;
3329
}

‎Parser/lexer/buffer.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ typedef struct {
1111
Py_ssize_t inp_from_buf;
1212
Py_ssize_t start_from_buf;
1313
Py_ssize_t line_start_from_buf;
14-
Py_ssize_t multi_line_start_from_buf;
1514
} _PyLexer_BufferPointers;
1615

1716
void _PyLexer_SaveBufferPointers(

‎Parser/lexer/lexer.c‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
#include "../tokenizer/helpers.h"
88
#include "../tokenizer/reader.h"
99

10-
/* Alternate tab spacing */
10+
#define TABSIZE 8
1111
#define ALTTABSIZE 1
1212

1313

@@ -30,11 +30,10 @@ _PyLexer_nextc(struct tok_state *tok)
3030
int rc;
3131
for (;;) {
3232
if (tok->cur != tok->inp) {
33-
if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) {
33+
if (tok->cur - tok->line_start >= INT_MAX) {
3434
tok->done = E_COLUMNOVERFLOW;
3535
return EOF;
3636
}
37-
tok->col_offset++;
3837
return Py_CHARMASK(*tok->cur++); /* Fast path */
3938
}
4039
if (tok->done != E_OK) {
@@ -74,7 +73,6 @@ _PyLexer_backup(struct tok_state *tok, int c)
7473
if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) {
7574
Py_FatalError("tok_backup: wrong character");
7675
}
77-
tok->col_offset--;
7876
}
7977
}
8078

@@ -88,7 +86,7 @@ verify_identifier(struct tok_state *tok)
8886
return 1;
8987
}
9088
PyObject *s;
91-
if (tok->input_error)
89+
if (tok_failed(tok))
9290
return 0;
9391
s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
9492
if (s == NULL) {
@@ -165,7 +163,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
165163
const char *p_end = NULL;
166164
nextline:
167165
tok->start = NULL;
168-
tok->starting_col_offset = -1;
166+
tok->start_loc = (_PyTok_Loc){tok->lineno, -1};
169167
blankline = 0;
170168

171169

@@ -181,7 +179,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
181179
col++, altcol++;
182180
}
183181
else if (c == '\t') {
184-
col = (col / tok->tabsize + 1) * tok->tabsize;
182+
col = (col / TABSIZE + 1) * TABSIZE;
185183
altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
186184
}
187185
else if (c == '\014') {/* Control-L (formfeed) */
@@ -269,7 +267,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
269267
}
270268

271269
tok->start = tok->cur;
272-
tok->starting_col_offset = tok->col_offset;
270+
tok->start_loc = (_PyTok_Loc){
271+
tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) : -1};
273272

274273
/* Return pending indents/dedents */
275274
if (tok->pendin != 0) {
@@ -304,7 +303,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
304303

305304
/* Set start of current token */
306305
tok->start = tok->cur == NULL ? NULL : tok->cur - 1;
307-
tok->starting_col_offset = tok->col_offset - 1;
306+
tok->start_loc = (_PyTok_Loc){
307+
tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) - 1 : -1};
308308

309309
/* Skip comment, unless it's a type comment */
310310
if (c == '#') {
@@ -335,7 +335,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
335335

336336
if (tok->type_comments) {
337337
p = tok->start;
338-
current_starting_col_offset = tok->starting_col_offset;
338+
current_starting_col_offset = tok->start_loc.byte_col;
339339
prefix = type_comment_prefix;
340340
while (*prefix && p < tok->cur) {
341341
if (*prefix == ' ') {
@@ -387,7 +387,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
387387
}
388388
_PyLexer_token_setup(tok, token, type, p_start, p_end);
389389
token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
390-
token->end_loc = (_PyTok_Loc){tok->lineno, tok->col_offset};
390+
token->end_loc = (_PyTok_Loc){tok->lineno,
391+
_PyLexer_ByteColumn(tok)};
391392
return type;
392393
}
393394
}
@@ -708,7 +709,7 @@ int
708709
_PyTokenizer_Get(struct tok_state *tok, struct token *token)
709710
{
710711
int result = tok_get(tok, token);
711-
if (tok->input_error) {
712+
if (tok_failed(tok)) {
712713
result = ERRORTOKEN;
713714
}
714715
return result;

‎Parser/lexer/lexer_internal.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#ifndef _PY_LEXER_INTERNAL_H_
22
#define _PY_LEXER_INTERNAL_H_
33

4+
#include "errcode.h"
45
#include "lexer.h"
56

67
#define is_potential_identifier_start(c) (\
@@ -44,6 +45,13 @@ TOK_NEXT_MODE(struct tok_state *tok)
4445
#define tok_nextc _PyLexer_nextc
4546
#define tok_backup _PyLexer_backup
4647

48+
static inline int
49+
tok_failed(const struct tok_state *tok)
50+
{
51+
return tok->done != E_OK && tok->done != E_EOF &&
52+
tok->done != E_INTERACT_STOP;
53+
}
54+
4755
int _PyLexer_nextc(struct tok_state *);
4856
void _PyLexer_backup(struct tok_state *, int);
4957
void _PyLexer_update_ftstring_expr(struct tok_state *, char);

‎Parser/lexer/state.c‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,6 @@
66
#include "state.h"
77
#include "../tokenizer/reader.h"
88

9-
/* Never change this */
10-
#define TABSIZE 8
11-
129
/* Create and initialize a new tok_state structure */
1310
struct tok_state *
1411
_PyTokenizer_tok_new(void)
@@ -28,18 +25,15 @@ _PyTokenizer_tok_new(void)
2825
tok->start = NULL;
2926
tok->done = E_OK;
3027
tok->fp = NULL;
31-
tok->tabsize = TABSIZE;
3228
tok->indent = 0;
3329
tok->indstack[0] = 0;
3430
tok->atbol = 1;
3531
tok->pendin = 0;
3632
tok->prompt = NULL;
3733
tok->lineno = 0;
38-
tok->starting_col_offset = -1;
39-
tok->col_offset = -1;
34+
tok->start_loc = (_PyTok_Loc){-1, -1};
4035
tok->level = 0;
4136
tok->altindstack[0] = 0;
42-
tok->input_error = 0;
4337
tok->encoding = NULL;
4438
tok->filename = NULL;
4539
tok->module = NULL;
@@ -97,13 +91,12 @@ _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const
9791
{
9892
token->level = tok->level;
9993
token->span = _PyLexer_BufferSpan(tok, start, end);
100-
int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;
101-
token->start_loc = (_PyTok_Loc){lineno, -1};
102-
token->end_loc = (_PyTok_Loc){tok->lineno, -1};
103-
10494
if (start != NULL && end != NULL) {
105-
token->start_loc.byte_col = tok->starting_col_offset;
106-
token->end_loc.byte_col = tok->col_offset;
95+
token->start_loc = tok->start_loc;
96+
token->end_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};
97+
}
98+
else {
99+
token->start_loc = token->end_loc = (_PyTok_Loc){tok->lineno, -1};
107100
}
108101
return type;
109102
}

‎Parser/lexer/state.h‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -83,17 +83,13 @@ struct tok_state {
8383
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
8484
/* NB If done != E_OK, cur must be == inp!!! */
8585
FILE *fp; /* Rest of input; NULL if tokenizing a string */
86-
int tabsize; /* Tab spacing */
8786
int indent; /* Current indentation index */
8887
int indstack[MAXINDENT]; /* Stack of indents */
8988
int atbol; /* Nonzero if at begin of new line */
9089
int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
9190
const char *prompt; /* For interactive prompting */
9291
int lineno; /* Current line number */
93-
int first_lineno; /* First line of a single line or multi line string
94-
expression (cf. issue 16806) */
95-
int starting_col_offset; /* The column offset at the beginning of a token */
96-
int col_offset; /* Current col offset */
92+
_PyTok_Loc start_loc;
9793
int level; /* () [] {} Parentheses nesting level */
9894
/* Used to allow free continuations inside them */
9995
char parenstack[MAXLEVEL];
@@ -104,12 +100,8 @@ struct tok_state {
104100
/* Stuff for checking on different tab sizes */
105101
int altindstack[MAXINDENT]; /* Stack of alternate indents */
106102
/* Stuff for PEP 0263 */
107-
int input_error;
108103
char *encoding; /* Source encoding. */
109104
const char* line_start; /* pointer to start of current line */
110-
const char* multi_line_start; /* pointer to start of first line of
111-
a single line or multi line string
112-
expression (cf. issue 16806) */
113105
char* str; /* Source string being tokenized (if tokenizing from a string)*/
114106

115107
_PyTok_SourceText source;
@@ -163,6 +155,16 @@ _PyLexer_BufferSpanView(const struct tok_state *tok, _PyTok_Span span,
163155
return _PyLexer_BufferPointer(tok, span.start);
164156
}
165157

158+
static inline int
159+
_PyLexer_ByteColumn(const struct tok_state *tok)
160+
{
161+
assert(tok->line_start != NULL);
162+
assert(tok->cur >= tok->line_start);
163+
Py_ssize_t column = tok->cur - tok->line_start;
164+
assert(column <= INT_MAX);
165+
return (int)column;
166+
}
167+
166168
static inline _PyTok_Span
167169
_PyLexer_BufferSpan(const struct tok_state *tok, const char *start,
168170
const char *end)

‎Parser/lexer/string.c‎

Lines changed: 16 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,15 @@
77

88
#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
99

10+
static void
11+
rewind_to_string_start(struct tok_state *tok, const char *start,
12+
_PyTok_Loc location)
13+
{
14+
tok->cur = (char *)start + 1;
15+
tok->line_start = start - location.byte_col;
16+
tok->lineno = location.lineno;
17+
}
18+
1019
int
1120
_PyLexer_record_ftstring_comment(struct tok_state *tok, const char *start,
1221
const char *end)
@@ -194,13 +203,6 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
194203
int quote = c;
195204
int quote_size = 1; /* 1 or 3 */
196205

197-
/* Nodes of type STRING, especially multi line strings
198-
must be handled differently in order to get both
199-
the starting line number and the column offset right.
200-
(cf. issue 16806) */
201-
tok->first_lineno = tok->lineno;
202-
tok->multi_line_start = tok->line_start;
203-
204206
/* Find the quote size and start of string */
205207
int after_quote = tok_nextc(tok);
206208
if (after_quote == quote) {
@@ -276,13 +278,6 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
276278
int end_quote_size = 0;
277279
int has_escaped_quote = 0;
278280

279-
/* Nodes of type STRING, especially multi line strings
280-
must be handled differently in order to get both
281-
the starting line number and the column offset right.
282-
(cf. issue 16806) */
283-
tok->first_lineno = tok->lineno;
284-
tok->multi_line_start = tok->line_start;
285-
286281
/* Find the quote size and start of string */
287282
c = tok_nextc(tok);
288283
if (c == quote) {
@@ -308,15 +303,8 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
308303
break;
309304
}
310305
if (c == EOF || (quote_size == 1 && c == '\n')) {
311-
assert(tok->multi_line_start != NULL);
312-
// shift the tok_state's location into
313-
// the start of string, and report the error
314-
// from the initial quote character
315-
tok->cur = (char *)tok->start;
316-
tok->cur++;
317-
tok->line_start = tok->multi_line_start;
318-
int start = tok->lineno;
319-
tok->lineno = tok->first_lineno;
306+
int end_lineno = tok->lineno;
307+
rewind_to_string_start(tok, tok->start, tok->start_loc);
320308

321309
if (INSIDE_FSTRING(tok)) {
322310
/* When we are in an f-string, before raising the
@@ -334,7 +322,7 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
334322

335323
if (quote_size == 3) {
336324
_PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal"
337-
" (detected at line %d)", start);
325+
" (detected at line %d)", end_lineno);
338326
if (c != '\n') {
339327
tok->done = E_EOFS;
340328
}
@@ -346,11 +334,11 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
346334
tok,
347335
"unterminated string literal (detected at line %d); "
348336
"perhaps you escaped the end quote?",
349-
start
337+
end_lineno
350338
);
351339
} else {
352340
_PyTokenizer_syntaxerror(
353-
tok, "unterminated string literal (detected at line %d)", start
341+
tok, "unterminated string literal (detected at line %d)", end_lineno
354342
);
355343
}
356344
if (c != '\n') {
@@ -390,8 +378,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
390378
int unicode_escape = 0;
391379

392380
tok->start = tok->cur;
393-
tok->first_lineno = tok->lineno;
394-
tok->starting_col_offset = tok->col_offset;
381+
tok->start_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};
395382

396383
// If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize
397384
// before it.
@@ -432,9 +419,6 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
432419

433420
f_string_middle:
434421

435-
// TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle
436-
// this.
437-
tok->multi_line_start = tok->line_start;
438422
while (end_quote_size != current_tok->quote_size) {
439423
int c = tok_nextc(tok);
440424
if (tok->done == E_ERROR || tok->done == E_DECODE) {
@@ -447,7 +431,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
447431
);
448432

449433
if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) {
450-
if (tok->input_error) {
434+
if (tok_failed(tok)) {
451435
return MAKE_TOKEN(ERRORTOKEN);
452436
}
453437

@@ -472,7 +456,6 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
472456
return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
473457
}
474458

475-
assert(tok->multi_line_start != NULL);
476459
// shift the tok_state's location into
477460
// the start of string, and report the error
478461
// from the initial quote character

‎Parser/pegen.c‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1035,7 +1035,6 @@ _PyPegen_run_parser(Parser *p)
10351035
}
10361036

10371037
if (p->start_rule == Py_single_input && bad_single_statement(p)) {
1038-
p->tok->done = E_BADSINGLE; // This is not necessary for now, but might be in the future
10391038
return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement");
10401039
}
10411040

‎Parser/pegen_errors.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ _Pypegen_tokenizer_error(Parser *p)
6262
msg = "too many levels of indentation";
6363
break;
6464
case E_LINECONT: {
65-
col_offset = p->tok->cur - p->tok->buf - 1;
65+
col_offset = p->tok->cur - p->tok->line_start - 1;
6666
msg = "unexpected character after line continuation character";
6767
break;
6868
}

0 commit comments

Comments
 (0)