Skip to content
This repository was archived by the owner on Feb 13, 2025. It is now read-only.

Commit 6961b07

Browse files
author
Anselm Kruis
committed
merge branch 3.5
2 parents a5d0466 + 61d6e4a commit 6961b07

File tree

8 files changed

+162
-37
lines changed

8 files changed

+162
-37
lines changed

‎Lib/test/test_compile.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,27 @@ def check_limit(prefix, repeated):
542542
check_limit("a", "[0]")
543543
check_limit("a", "*a")
544544

545+
def test_null_terminated(self):
546+
# The source code is null-terminated internally, but bytes-like
547+
# objects are accepted, which could be not terminated.
548+
# Exception changed from TypeError to ValueError in 3.5
549+
with self.assertRaisesRegex(Exception, "cannot contain null"):
550+
compile("123\x00", "<dummy>", "eval")
551+
with self.assertRaisesRegex(Exception, "cannot contain null"):
552+
compile(memoryview(b"123\x00"), "<dummy>", "eval")
553+
code = compile(memoryview(b"123\x00")[1:-1], "<dummy>", "eval")
554+
self.assertEqual(eval(code), 23)
555+
code = compile(memoryview(b"1234")[1:-1], "<dummy>", "eval")
556+
self.assertEqual(eval(code), 23)
557+
code = compile(memoryview(b"$23$")[1:-1], "<dummy>", "eval")
558+
self.assertEqual(eval(code), 23)
559+
560+
# Also test when eval() and exec() do the compilation step
561+
self.assertEqual(eval(memoryview(b"1234")[1:-1]), 23)
562+
namespace = dict()
563+
exec(memoryview(b"ax = 123")[1:-1], namespace)
564+
self.assertEqual(namespace['x'], 12)
565+
545566

546567
class TestStackSize(unittest.TestCase):
547568
# These tests check that the computed stack size for a code object

‎Lib/test/test_float.py‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ def test_float(self):
3131
self.assertEqual(float(3.14), 3.14)
3232
self.assertEqual(float(314), 314.0)
3333
self.assertEqual(float(" 3.14 "), 3.14)
34-
self.assertEqual(float(b" 3.14 "), 3.14)
3534
self.assertRaises(ValueError, float, " 0x3.1 ")
3635
self.assertRaises(ValueError, float, " -0x3.p-1 ")
3736
self.assertRaises(ValueError, float, " +0x3.p-1 ")
@@ -43,7 +42,6 @@ def test_float(self):
4342
self.assertRaises(ValueError, float, "+.inf")
4443
self.assertRaises(ValueError, float, ".")
4544
self.assertRaises(ValueError, float, "-.")
46-
self.assertRaises(ValueError, float, b"-")
4745
self.assertRaises(TypeError, float, {})
4846
self.assertRaisesRegex(TypeError, "not 'dict'", float, {})
4947
# Lone surrogate
@@ -57,6 +55,42 @@ def test_float(self):
5755
float(b'.' + b'1'*1000)
5856
float('.' + '1'*1000)
5957

58+
def test_non_numeric_input_types(self):
59+
# Test possible non-numeric types for the argument x, including
60+
# subclasses of the explicitly documented accepted types.
61+
class CustomStr(str): pass
62+
class CustomBytes(bytes): pass
63+
class CustomByteArray(bytearray): pass
64+
65+
factories = [
66+
bytes,
67+
bytearray,
68+
lambda b: CustomStr(b.decode()),
69+
CustomBytes,
70+
CustomByteArray,
71+
memoryview,
72+
]
73+
try:
74+
from array import array
75+
except ImportError:
76+
pass
77+
else:
78+
factories.append(lambda b: array('B', b))
79+
80+
for f in factories:
81+
x = f(b" 3.14 ")
82+
with self.subTest(type(x)):
83+
self.assertEqual(float(x), 3.14)
84+
with self.assertRaisesRegex(ValueError, "could not convert"):
85+
float(f(b'A' * 0x10))
86+
87+
def test_float_memoryview(self):
88+
self.assertEqual(float(memoryview(b'12.3')[1:4]), 2.3)
89+
self.assertEqual(float(memoryview(b'12.3\x00')[1:4]), 2.3)
90+
self.assertEqual(float(memoryview(b'12.3 ')[1:4]), 2.3)
91+
self.assertEqual(float(memoryview(b'12.3A')[1:4]), 2.3)
92+
self.assertEqual(float(memoryview(b'12.34')[1:4]), 2.3)
93+
6094
def test_error_message(self):
6195
testlist = ('\xbd', '123\xbd', ' 123 456 ')
6296
for s in testlist:

‎Lib/test/test_int.py‎

Lines changed: 34 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -276,16 +276,40 @@ class CustomStr(str): pass
276276
class CustomBytes(bytes): pass
277277
class CustomByteArray(bytearray): pass
278278

279-
values = [b'100',
280-
bytearray(b'100'),
281-
CustomStr('100'),
282-
CustomBytes(b'100'),
283-
CustomByteArray(b'100')]
284-
285-
for x in values:
286-
msg = 'x has type %s' % type(x).__name__
287-
self.assertEqual(int(x), 100, msg=msg)
288-
self.assertEqual(int(x, 2), 4, msg=msg)
279+
factories = [
280+
bytes,
281+
bytearray,
282+
lambda b: CustomStr(b.decode()),
283+
CustomBytes,
284+
CustomByteArray,
285+
memoryview,
286+
]
287+
try:
288+
from array import array
289+
except ImportError:
290+
pass
291+
else:
292+
factories.append(lambda b: array('B', b))
293+
294+
for f in factories:
295+
x = f(b'100')
296+
with self.subTest(type(x)):
297+
self.assertEqual(int(x), 100)
298+
if isinstance(x, (str, bytes, bytearray)):
299+
self.assertEqual(int(x, 2), 4)
300+
else:
301+
msg = "can't convert non-string"
302+
with self.assertRaisesRegex(TypeError, msg):
303+
int(x, 2)
304+
with self.assertRaisesRegex(ValueError, 'invalid literal'):
305+
int(f(b'A' * 0x10))
306+
307+
def test_int_memoryview(self):
308+
self.assertEqual(int(memoryview(b'123')[1:3]), 23)
309+
self.assertEqual(int(memoryview(b'123\x00')[1:3]), 23)
310+
self.assertEqual(int(memoryview(b'123 ')[1:3]), 23)
311+
self.assertEqual(int(memoryview(b'123A')[1:3]), 23)
312+
self.assertEqual(int(memoryview(b'1234')[1:3]), 23)
289313

290314
def test_string_float(self):
291315
self.assertRaises(ValueError, int, '1.2')

‎Misc/NEWS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ Release date: TBA
1111
Core and Builtins
1212
-----------------
1313

14+
- Issue #24802: Avoid buffer overreads when int(), float(), compile(), exec()
15+
and eval() are passed bytes-like objects. These objects are not
16+
necessarily terminated by a null byte, but the functions assumed they were.
17+
1418
- Issue #24726: Fixed a crash and leaking NULL in repr() of OrderedDict that
1519
was mutated by direct calls of dict methods.
1620

‎Objects/abstract.c‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1310,12 +1310,30 @@ PyNumber_Long(PyObject *o)
13101310
/* The below check is done in PyLong_FromUnicode(). */
13111311
return PyLong_FromUnicodeObject(o, 10);
13121312

1313-
if (PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) == 0) {
1313+
if (PyBytes_Check(o))
13141314
/* need to do extra error checking that PyLong_FromString()
13151315
* doesn't do. In particular int('9\x005') must raise an
13161316
* exception, not truncate at the null.
13171317
*/
1318-
PyObject *result = _PyLong_FromBytes(view.buf, view.len, 10);
1318+
return _PyLong_FromBytes(PyBytes_AS_STRING(o),
1319+
PyBytes_GET_SIZE(o), 10);
1320+
1321+
if (PyByteArray_Check(o))
1322+
return _PyLong_FromBytes(PyByteArray_AS_STRING(o),
1323+
PyByteArray_GET_SIZE(o), 10);
1324+
1325+
if (PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) == 0) {
1326+
PyObject *result, *bytes;
1327+
1328+
/* Copy to NUL-terminated buffer. */
1329+
bytes = PyBytes_FromStringAndSize((const char *)view.buf, view.len);
1330+
if (bytes == NULL) {
1331+
PyBuffer_Release(&view);
1332+
return NULL;
1333+
}
1334+
result = _PyLong_FromBytes(PyBytes_AS_STRING(bytes),
1335+
PyBytes_GET_SIZE(bytes), 10);
1336+
Py_DECREF(bytes);
13191337
PyBuffer_Release(&view);
13201338
return result;
13211339
}

‎Objects/complexobject.c‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -767,7 +767,6 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v)
767767
int got_bracket=0;
768768
PyObject *s_buffer = NULL;
769769
Py_ssize_t len;
770-
Py_buffer view = {NULL, NULL};
771770

772771
if (PyUnicode_Check(v)) {
773772
s_buffer = _PyUnicode_TransformDecimalAndSpaceToASCII(v);
@@ -777,10 +776,6 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v)
777776
if (s == NULL)
778777
goto error;
779778
}
780-
else if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) == 0) {
781-
s = (const char *)view.buf;
782-
len = view.len;
783-
}
784779
else {
785780
PyErr_Format(PyExc_TypeError,
786781
"complex() argument must be a string or a number, not '%.200s'",
@@ -895,15 +890,13 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v)
895890
if (s-start != len)
896891
goto parse_error;
897892

898-
PyBuffer_Release(&view);
899893
Py_XDECREF(s_buffer);
900894
return complex_subtype_from_doubles(type, x, y);
901895

902896
parse_error:
903897
PyErr_SetString(PyExc_ValueError,
904898
"complex() arg is a malformed string");
905899
error:
906-
PyBuffer_Release(&view);
907900
Py_XDECREF(s_buffer);
908901
return NULL;
909902
}

‎Objects/floatobject.c‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,9 +144,24 @@ PyFloat_FromString(PyObject *v)
144144
return NULL;
145145
}
146146
}
147+
else if (PyBytes_Check(v)) {
148+
s = PyBytes_AS_STRING(v);
149+
len = PyBytes_GET_SIZE(v);
150+
}
151+
else if (PyByteArray_Check(v)) {
152+
s = PyByteArray_AS_STRING(v);
153+
len = PyByteArray_GET_SIZE(v);
154+
}
147155
else if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) == 0) {
148156
s = (const char *)view.buf;
149157
len = view.len;
158+
/* Copy to NUL-terminated buffer. */
159+
s_buffer = PyBytes_FromStringAndSize(s, len);
160+
if (s_buffer == NULL) {
161+
PyBuffer_Release(&view);
162+
return NULL;
163+
}
164+
s = PyBytes_AS_STRING(s_buffer);
150165
}
151166
else {
152167
PyErr_Format(PyExc_TypeError,

‎Python/bltinmodule.c‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -601,20 +601,37 @@ builtin_chr_impl(PyModuleDef *module, int i)
601601

602602

603603
static const char *
604-
source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, Py_buffer *view)
604+
source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
605605
{
606606
const char *str;
607607
Py_ssize_t size;
608+
Py_buffer view;
608609

610+
*cmd_copy = NULL;
609611
if (PyUnicode_Check(cmd)) {
610612
cf->cf_flags |= PyCF_IGNORE_COOKIE;
611613
str = PyUnicode_AsUTF8AndSize(cmd, &size);
612614
if (str == NULL)
613615
return NULL;
614616
}
615-
else if (PyObject_GetBuffer(cmd, view, PyBUF_SIMPLE) == 0) {
616-
str = (const char *)view->buf;
617-
size = view->len;
617+
else if (PyBytes_Check(cmd)) {
618+
str = PyBytes_AS_STRING(cmd);
619+
size = PyBytes_GET_SIZE(cmd);
620+
}
621+
else if (PyByteArray_Check(cmd)) {
622+
str = PyByteArray_AS_STRING(cmd);
623+
size = PyByteArray_GET_SIZE(cmd);
624+
}
625+
else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {
626+
/* Copy to NUL-terminated buffer. */
627+
*cmd_copy = PyBytes_FromStringAndSize(
628+
(const char *)view.buf, view.len);
629+
PyBuffer_Release(&view);
630+
if (*cmd_copy == NULL) {
631+
return NULL;
632+
}
633+
str = PyBytes_AS_STRING(*cmd_copy);
634+
size = PyBytes_GET_SIZE(*cmd_copy);
618635
}
619636
else {
620637
PyErr_Format(PyExc_TypeError,
@@ -626,7 +643,7 @@ source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompil
626643
if (strlen(str) != (size_t)size) {
627644
PyErr_SetString(PyExc_ValueError,
628645
"source code string cannot contain null bytes");
629-
PyBuffer_Release(view);
646+
Py_CLEAR(*cmd_copy);
630647
return NULL;
631648
}
632649
return str;
@@ -662,7 +679,7 @@ builtin_compile_impl(PyModuleDef *module, PyObject *source,
662679
int dont_inherit, int optimize)
663680
/*[clinic end generated code: output=31881762c1bb90c4 input=9d53e8cfb3c86414]*/
664681
{
665-
Py_buffer view = {NULL, NULL};
682+
PyObject *source_copy;
666683
const char *str;
667684
int compile_mode = -1;
668685
int is_ast;
@@ -734,12 +751,12 @@ builtin_compile_impl(PyModuleDef *module, PyObject *source,
734751
goto finally;
735752
}
736753

737-
str = source_as_string(source, "compile", "string, bytes or AST", &cf, &view);
754+
str = source_as_string(source, "compile", "string, bytes or AST", &cf, &source_copy);
738755
if (str == NULL)
739756
goto error;
740757

741758
result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize);
742-
PyBuffer_Release(&view);
759+
Py_XDECREF(source_copy);
743760
goto finally;
744761

745762
error:
@@ -816,8 +833,7 @@ builtin_eval_impl(PyModuleDef *module, PyObject *source, PyObject *globals,
816833
/*[clinic end generated code: output=7284501fb7b4d666 input=89d323839395e49d]*/
817834
{
818835
STACKLESS_GETARG();
819-
PyObject *result, *tmp = NULL;
820-
Py_buffer view = {NULL, NULL};
836+
PyObject *result, *source_copy;
821837
const char *str;
822838
PyCompilerFlags cf;
823839

@@ -866,7 +882,7 @@ builtin_eval_impl(PyModuleDef *module, PyObject *source, PyObject *globals,
866882
}
867883

868884
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
869-
str = source_as_string(source, "eval", "string, bytes or code", &cf, &view);
885+
str = source_as_string(source, "eval", "string, bytes or code", &cf, &source_copy);
870886
if (str == NULL)
871887
return NULL;
872888

@@ -877,8 +893,7 @@ builtin_eval_impl(PyModuleDef *module, PyObject *source, PyObject *globals,
877893
STACKLESS_PROMOTE_ALL();
878894
result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf);
879895
STACKLESS_ASSERT();
880-
PyBuffer_Release(&view);
881-
Py_XDECREF(tmp);
896+
Py_XDECREF(source_copy);
882897
return result;
883898
}
884899

@@ -953,12 +968,13 @@ builtin_exec_impl(PyModuleDef *module, PyObject *source, PyObject *globals,
953968
STACKLESS_ASSERT();
954969
}
955970
else {
956-
Py_buffer view = {NULL, NULL};
971+
PyObject *source_copy;
957972
const char *str;
958973
PyCompilerFlags cf;
959974
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
960975
str = source_as_string(source, "exec",
961-
"string, bytes or code", &cf, &view);
976+
"string, bytes or code", &cf,
977+
&source_copy);
962978
if (str == NULL)
963979
return NULL;
964980
STACKLESS_PROMOTE_ALL();
@@ -968,7 +984,7 @@ builtin_exec_impl(PyModuleDef *module, PyObject *source, PyObject *globals,
968984
else
969985
v = PyRun_String(str, Py_file_input, globals, locals);
970986
STACKLESS_ASSERT();
971-
PyBuffer_Release(&view);
987+
Py_XDECREF(source_copy);
972988
}
973989
if (v == NULL)
974990
return NULL;

0 commit comments

Comments
 (0)