Skip to content

Commit aef6c9e

Browse files
pablogsalpablogsal
committed
gh-153569: keep character scanning inline with source offsets
1 parent 1d56ef5 commit aef6c9e

5 files changed

Lines changed: 51 additions & 39 deletions

File tree

‎Lib/test/test_tstring.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,10 @@ def test_syntax_errors(self):
234234
("t'{x=!}'", "t-string: missing conversion character"),
235235
("t'{x!z}'", "t-string: invalid conversion character 'z': "
236236
"expected 's', 'r', or 'a'"),
237+
("f\"{t'{x!z}'}\"", "t-string: invalid conversion character 'z': "
238+
"expected 's', 'r', or 'a'"),
239+
("t'{f\"{x!z}\"}'", "f-string: invalid conversion character 'z': "
240+
"expected 's', 'r', or 'a'"),
237241
("t'{lambda:1}'", "t-string: lambda expressions are not allowed "
238242
"without parentheses"),
239243
("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"),

‎Makefile.pre.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3463,7 +3463,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
34633463
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
34643464
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
34653465
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
3466-
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
3466+
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
34673467
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
34683468
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
34693469

‎Parser/lexer/lexer.c‎

Lines changed: 22 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -23,43 +23,33 @@ contains_null_bytes(const char* str, size_t size)
2323
return memchr(str, 0, size) != NULL;
2424
}
2525

26-
/* Get next char, updating state; error code goes into tok->done */
2726
int
28-
_PyLexer_nextc(struct tok_state *tok)
27+
_PyLexer_refill(struct tok_state *tok)
2928
{
30-
int rc;
31-
for (;;) {
32-
if (tok->cur != tok->inp) {
33-
if (tok->cur - tok->line_start >= INT_MAX) {
34-
tok->done = E_COLUMNOVERFLOW;
35-
return EOF;
36-
}
37-
return Py_CHARMASK(tok->source.bytes[tok->cur++ - tok->source.base_offset]); /* Fast path */
38-
}
39-
if (tok->done != E_OK) {
40-
return EOF;
41-
}
42-
rc = _PyTok_ReaderUnderflow(tok);
29+
if (tok->done != E_OK) {
30+
return 0;
31+
}
32+
int rc = _PyTok_ReaderUnderflow(tok);
4333
#if defined(Py_DEBUG)
44-
if (tok->debug) {
45-
fprintf(stderr, "line[%d] = ", tok->lineno);
46-
_PyTokenizer_print_escape(stderr, _PyLexer_BufferPointer(tok, tok->cur), tok->inp - tok->cur);
47-
fprintf(stderr, " tok->done = %d\n", tok->done);
48-
}
34+
if (tok->debug) {
35+
fprintf(stderr, "line[%d] = ", tok->lineno);
36+
_PyTokenizer_print_escape(stderr, _PyLexer_BufferPointer(tok, tok->cur),
37+
tok->inp - tok->cur);
38+
fprintf(stderr, " tok->done = %d\n", tok->done);
39+
}
4940
#endif
50-
if (!rc) {
51-
tok->cur = tok->inp;
52-
return EOF;
53-
}
54-
tok->line_start = tok->cur;
55-
56-
if (contains_null_bytes(_PyLexer_BufferPointer(tok, tok->line_start), tok->inp - tok->line_start)) {
57-
_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
58-
tok->cur = tok->inp;
59-
return EOF;
60-
}
41+
if (!rc) {
42+
tok->cur = tok->inp;
43+
return 0;
6144
}
62-
Py_UNREACHABLE();
45+
tok->line_start = tok->cur;
46+
if (contains_null_bytes(_PyLexer_BufferPointer(tok, tok->line_start),
47+
tok->inp - tok->line_start)) {
48+
_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
49+
tok->cur = tok->inp;
50+
return 0;
51+
}
52+
return 1;
6353
}
6454

6555
/* Back-up one character */

‎Parser/lexer/lexer_internal.h‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
(_PyLexer_IsTString((state)->kind) ? TSTRING_MIDDLE : FSTRING_MIDDLE)
2222
#define FTSTRING_END(state) \
2323
(_PyLexer_IsTString((state)->kind) ? TSTRING_END : FSTRING_END)
24-
#define tok_nextc _PyLexer_nextc
2524
#define tok_backup _PyLexer_backup
2625

2726
static inline int
@@ -31,7 +30,26 @@ tok_failed(const struct tok_state *tok)
3130
tok->done != E_INTERACT_STOP;
3231
}
3332

34-
int _PyLexer_nextc(struct tok_state *);
33+
int _PyLexer_refill(struct tok_state *);
34+
35+
static inline int
36+
tok_nextc(struct tok_state *tok)
37+
{
38+
while (tok->cur == tok->inp) {
39+
if (!_PyLexer_refill(tok)) {
40+
return EOF;
41+
}
42+
}
43+
assert(tok->cur >= tok->source.base_offset);
44+
assert(tok->cur - tok->source.base_offset < tok->source.len);
45+
if (tok->cur - tok->line_start >= INT_MAX) {
46+
tok->done = E_COLUMNOVERFLOW;
47+
return EOF;
48+
}
49+
return Py_CHARMASK(
50+
tok->source.bytes[tok->cur++ - tok->source.base_offset]);
51+
}
52+
3553
void _PyLexer_backup(struct tok_state *, int);
3654
int _PyLexer_record_ftstring_comment(
3755
struct tok_state *, ftstring_state *, _PyTok_Off, _PyTok_Off);

‎Parser/lexer/state.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,15 @@ _PyLexer_IsRawString(ftstring_kind kind)
6868

6969
/* Tokenizer state */
7070
struct tok_state {
71+
_PyTok_Off buf_offset;
7172
_PyTok_Off cur;
7273
_PyTok_Off inp;
73-
_PyTok_Off buf_offset;
74+
_PyTok_Off start;
75+
_PyTok_Off line_start;
76+
_PyTok_SourceText source;
7477
int fp_interactive; /* If the file descriptor is interactive */
7578
char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
7679
char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
77-
_PyTok_Off start;
7880
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
7981
/* NB If done != E_OK, cur must be == inp!!! */
8082
FILE *fp; /* Rest of input; NULL if tokenizing a string */
@@ -96,10 +98,8 @@ struct tok_state {
9698
int altindstack[MAXINDENT]; /* Stack of alternate indents */
9799
/* Stuff for PEP 0263 */
98100
char *encoding; /* Source encoding. */
99-
_PyTok_Off line_start;
100101
char* str; /* Source string being tokenized (if tokenizing from a string)*/
101102

102-
_PyTok_SourceText source;
103103
struct _PyTok_Reader *reader;
104104

105105
int type_comments; /* Whether to look for type comments */

0 commit comments

Comments
 (0)