Skip to content

Commit a506842

Browse files
authored
improve the way statement ranges are caculcated (#58)
* improve the way statement ranges are caculated * fix linting * coverage * coverage * coverage
1 parent 0de8d7a commit a506842

File tree

3 files changed

+184
-71
lines changed

3 files changed

+184
-71
lines changed

devtools/debug.py

Lines changed: 102 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import ast
2+
import dis
23
import inspect
34
import os
45
import pdb
5-
import re
6+
import sys
67
from pathlib import Path
78
from textwrap import dedent, indent
9+
from types import FrameType
810
from typing import Generator, List, Optional, Tuple
911

1012
from .ansi import isatty, sformat
@@ -23,6 +25,10 @@
2325
)
2426

2527

28+
class IntrospectionError(ValueError):
29+
pass
30+
31+
2632
class DebugArgument:
2733
__slots__ = 'value', 'name', 'extra'
2834

@@ -134,38 +140,39 @@ def _env_bool(cls, value, env_name, env_default):
134140
return value
135141

136142
def __call__(self, *args, file_=None, flush_=True, **kwargs) -> None:
137-
d_out = self._process(args, kwargs, r'debug *\(')
143+
d_out = self._process(args, kwargs, 'debug')
138144
highlight = isatty(file_) if self._highlight is None else self._highlight
139145
s = d_out.str(highlight)
140146
print(s, file=file_, flush=flush_)
141147

142148
def format(self, *args, **kwargs) -> DebugOutput:
143-
return self._process(args, kwargs, r'debug.format *\(')
149+
return self._process(args, kwargs, 'format')
144150

145151
def breakpoint(self):
146152
pdb.Pdb(skip=['devtools.*']).set_trace()
147153

148154
def timer(self, name=None, *, verbose=True, file=None, dp=3) -> Timer:
149155
return Timer(name=name, verbose=verbose, file=file, dp=dp)
150156

151-
def _process(self, args, kwargs, func_regex) -> DebugOutput:
152-
curframe = inspect.currentframe()
157+
def _process(self, args, kwargs, func_name: str) -> DebugOutput:
158+
"""
159+
BEWARE: this must be called from a function exactly 2 levels below the top of the stack.
160+
"""
161+
# HELP: any errors other than ValueError from _getframe? If so please submit an issue
153162
try:
154-
frames = inspect.getouterframes(curframe, context=self._frame_context_length)
155-
except IndexError:
156-
# NOTICE: we should really catch all conceivable errors here, if you find one please report.
157-
# IndexError happens in odd situations such as code called from within jinja templates
163+
call_frame: FrameType = sys._getframe(2)
164+
except ValueError:
165+
# "If [ValueError] is deeper than the call stack, ValueError is raised"
158166
return self.output_class(
159167
filename='<unknown>',
160168
lineno=0,
161169
frame='',
162170
arguments=list(self._args_inspection_failed(args, kwargs)),
163-
warning=self._show_warnings and 'error parsing code, IndexError',
171+
warning=self._show_warnings and 'error parsing code, call stack too shallow',
164172
)
165-
# BEWARE: this must be called by a method which in turn is called "directly" for the frame to be correct
166-
call_frame = frames[2]
167173

168-
filename = call_frame.filename
174+
filename = call_frame.f_code.co_filename
175+
function = call_frame.f_code.co_name
169176
if filename.startswith('/'):
170177
# make the path relative
171178
try:
@@ -174,22 +181,29 @@ def _process(self, args, kwargs, func_regex) -> DebugOutput:
174181
# happens if filename path is not within CWD
175182
pass
176183

177-
if call_frame.code_context:
178-
func_ast, code_lines, lineno, warning = self._parse_code(call_frame, func_regex, filename)
179-
if func_ast:
180-
arguments = list(self._process_args(func_ast, code_lines, args, kwargs))
181-
else:
182-
# parsing failed
183-
arguments = list(self._args_inspection_failed(args, kwargs))
184-
else:
185-
lineno = call_frame.lineno
184+
lineno = call_frame.f_lineno
185+
warning = None
186+
187+
try:
188+
file_lines, _ = inspect.findsource(call_frame)
189+
except OSError:
186190
warning = 'no code context for debug call, code inspection impossible'
187191
arguments = list(self._args_inspection_failed(args, kwargs))
192+
else:
193+
try:
194+
first_line, last_line = self._statement_range(call_frame, func_name)
195+
func_ast, code_lines = self._parse_code(filename, file_lines, first_line, last_line)
196+
except IntrospectionError as e:
197+
# parsing failed
198+
warning = e.args[0]
199+
arguments = list(self._args_inspection_failed(args, kwargs))
200+
else:
201+
arguments = list(self._process_args(func_ast, code_lines, args, kwargs))
188202

189203
return self.output_class(
190204
filename=filename,
191205
lineno=lineno,
192-
frame=call_frame.function,
206+
frame=function,
193207
arguments=arguments,
194208
warning=self._show_warnings and warning,
195209
)
@@ -238,34 +252,29 @@ def _process_args(self, func_ast, code_lines, args, kwargs) -> Generator[DebugAr
238252
yield self.output_class.arg_class(value, name=name, variable=kw_arg_names.get(name))
239253

240254
def _parse_code(
241-
self, call_frame, func_regex, filename
242-
) -> Tuple[Optional[ast.AST], Optional[List[str]], int, Optional[str]]:
243-
call_lines = []
244-
for line in range(call_frame.index, -1, -1):
245-
try:
246-
new_line = call_frame.code_context[line]
247-
except IndexError: # pragma: no cover
248-
return None, None, line, 'error parsing code. line not found'
249-
call_lines.append(new_line)
250-
if re.search(func_regex, new_line):
251-
break
252-
call_lines.reverse()
253-
lineno = call_frame.lineno - len(call_lines) + 1
255+
self, filename: str, file_lines: List[str], first_line: int, last_line: int
256+
) -> Tuple[ast.AST, List[str]]:
257+
"""
258+
All we're trying to do here is build an AST of the function call statement. However numerous ugly interfaces,
259+
lack on introspection support and changes between python versions make this extremely hard.
260+
"""
254261

255-
code = dedent(''.join(call_lines))
262+
def get_code(_last_line: int) -> str:
263+
lines = file_lines[first_line - 1 : _last_line]
264+
return dedent(''.join(ln for ln in lines if ln.strip('\n ') and not ln.lstrip(' ').startswith('#')))
265+
266+
code = get_code(last_line)
256267
func_ast = None
257-
tail_index = call_frame.index
258268
try:
259269
func_ast = self._wrap_parse(code, filename)
260270
except (SyntaxError, AttributeError) as e1:
261-
# if the trailing bracket(s) of the function is/are on a new line eg.
271+
# if the trailing bracket(s) of the function is/are on a new line e.g.:
262272
# debug(
263273
# foo, bar,
264274
# )
265275
# inspect ignores it when setting index and we have to add it back
266-
for extra in range(2, 6):
267-
extra_lines = call_frame.code_context[tail_index + 1 : tail_index + extra]
268-
code = dedent(''.join(call_lines + extra_lines))
276+
for extra in range(1, 6):
277+
code = get_code(last_line + extra)
269278
try:
270279
func_ast = self._wrap_parse(code, filename)
271280
except (SyntaxError, AttributeError):
@@ -274,16 +283,64 @@ def _parse_code(
274283
break
275284

276285
if not func_ast:
277-
return None, None, lineno, 'error parsing code, {0.__class__.__name__}: {0}'.format(e1)
286+
raise IntrospectionError('error parsing code, {0.__class__.__name__}: {0}'.format(e1))
278287

279288
if not isinstance(func_ast, ast.Call):
280-
return None, None, lineno, 'error parsing code, found {} not Call'.format(func_ast.__class__)
289+
raise IntrospectionError('error parsing code, found {0.__class__} not Call'.format(func_ast))
281290

282291
code_lines = [line for line in code.split('\n') if line]
283292
# this removes the trailing bracket from the lines of code meaning it doesn't appear in the
284293
# representation of the last argument
285294
code_lines[-1] = code_lines[-1][:-1]
286-
return func_ast, code_lines, lineno, None
295+
return func_ast, code_lines
296+
297+
@staticmethod # noqa: C901
298+
def _statement_range(call_frame: FrameType, func_name: str) -> Tuple[int, int]: # noqa: C901
299+
"""
300+
Try to find the start and end of a frame statement.
301+
"""
302+
# dis.disassemble(call_frame.f_code, call_frame.f_lasti)
303+
# pprint([i for i in dis.get_instructions(call_frame.f_code)])
304+
305+
instructions = iter(dis.get_instructions(call_frame.f_code))
306+
first_line = None
307+
last_line = None
308+
309+
for instr in instructions:
310+
if instr.starts_line:
311+
if instr.opname in {'LOAD_GLOBAL', 'LOAD_NAME'} and instr.argval == func_name:
312+
first_line = instr.starts_line
313+
break
314+
elif instr.opname == 'LOAD_GLOBAL' and instr.argval == 'debug':
315+
if next(instructions).argval == func_name:
316+
first_line = instr.starts_line
317+
break
318+
319+
if first_line is None:
320+
raise IntrospectionError('error parsing code, unable to find "{}" function statement'.format(func_name))
321+
322+
for instr in instructions: # pragma: no branch
323+
if instr.offset == call_frame.f_lasti:
324+
break
325+
326+
for instr in instructions:
327+
if instr.starts_line:
328+
last_line = instr.starts_line - 1
329+
break
330+
331+
if last_line is None:
332+
if sys.version_info >= (3, 8):
333+
# absolutely no reliable way of getting the last line of the statement, complete hack is to
334+
# get the last line of the last statement of the whole code block and go from there
335+
# this assumes (perhaps wrongly?) that the reason we couldn't find last_line is that the statement
336+
# in question was the last of the block
337+
last_line = max(i.starts_line for i in dis.get_instructions(call_frame.f_code) if i.starts_line)
338+
else:
339+
# in older version of python f_lineno is the end of the statement, not the beginning
340+
# so this is a reasonable guess
341+
last_line = call_frame.f_lineno
342+
343+
return first_line, last_line
287344

288345
@staticmethod
289346
def _wrap_parse(code, filename):

tests/test_expr_render.py

Lines changed: 70 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import ast
12
import asyncio
23
import re
34
import sys
@@ -32,7 +33,6 @@ def test_subscription():
3233
) == s
3334

3435

35-
@pytest.mark.xfail(sys.version_info >= (3, 8), reason='TODO fix for python 3.8')
3636
def test_exotic_types():
3737
aa = [1, 2, 3]
3838
v = debug.format(
@@ -139,7 +139,6 @@ def test_kwargs():
139139
) == s
140140

141141

142-
@pytest.mark.xfail(sys.version_info >= (3, 8), reason='TODO fix for python 3.8')
143142
@pytest.mark.skipif(sys.version_info < (3, 6), reason='kwarg order is not guaranteed for 3.5')
144143
def test_kwargs_multiline():
145144
v = debug.format(
@@ -169,41 +168,77 @@ def test_multiple_trailing_lines():
169168
) == s
170169

171170

172-
def test_syntax_warning():
173-
# exceed the 4 extra lines which are normally checked
174-
v = debug.format(
175-
abs(
171+
def test_very_nested_last_statement():
172+
def func():
173+
return debug.format(
176174
abs(
177175
abs(
178176
abs(
179-
-1
177+
abs(
178+
-1
179+
)
180180
)
181181
)
182182
)
183183
)
184+
185+
v = func()
186+
# check only the original code is included in the warning
187+
s = re.sub(r':\d{2,}', ':<line no>', str(v))
188+
assert s == (
189+
'tests/test_expr_render.py:<line no> func\n'
190+
' abs( abs( abs( abs( -1 ) ) ) ): 1 (int)'
184191
)
192+
193+
194+
def test_syntax_warning():
195+
def func():
196+
return debug.format(
197+
abs(
198+
abs(
199+
abs(
200+
abs(
201+
abs(
202+
-1
203+
)
204+
)
205+
)
206+
)
207+
)
208+
)
209+
210+
v = func()
185211
# check only the original code is included in the warning
186212
s = re.sub(r':\d{2,}', ':<line no>', str(v))
187-
assert s.startswith('tests/test_expr_render.py:<line no> test_syntax_warning (error parsing code, '
188-
'SyntaxError: unexpected EOF')
213+
assert s == (
214+
'tests/test_expr_render.py:<line no> func '
215+
'(error parsing code, SyntaxError: unexpected EOF while parsing (test_expr_render.py, line 8))\n'
216+
' 1 (int)'
217+
)
189218

190219

191220
def test_no_syntax_warning():
192221
# exceed the 4 extra lines which are normally checked
193222
debug_ = Debug(warnings=False)
194-
v = debug_.format(
195-
abs(
223+
224+
def func():
225+
return debug_.format(
196226
abs(
197227
abs(
198228
abs(
199-
-1
229+
abs(
230+
abs(
231+
-1
232+
)
233+
)
200234
)
201235
)
202236
)
203237
)
204-
)
238+
239+
v = func()
205240
assert '(error parsing code' not in str(v)
206-
assert 'test_no_syntax_warning' in str(v)
241+
assert 'func' in str(v)
207242

208243

209244
def test_await():
@@ -220,3 +255,24 @@ async def bar():
220255
'tests/test_expr_render.py:<line no> bar\n'
221256
' 1 (int)'
222257
) == s
258+
259+
260+
def test_other_debug_arg():
261+
debug.timer()
262+
v = debug.format([1, 2])
263+
264+
# check only the original code is included in the warning
265+
s = re.sub(r':\d{2,}', ':<line no>', str(v))
266+
assert s == (
267+
'tests/test_expr_render.py:<line no> test_other_debug_arg\n'
268+
' [1, 2] (list) len=2'
269+
)
270+
271+
272+
def test_wrong_ast_type(mocker):
273+
mocked_ast_parse = mocker.patch('ast.parse')
274+
275+
code = 'async def wrapper():\n x = "foobar"'
276+
mocked_ast_parse.return_value = ast.parse(code, filename='testing.py').body[0].body[0].value
277+
v = debug.format('x')
278+
assert "(error parsing code, found <class 'unittest.mock.MagicMock'> not Call)" in v.str()

0 commit comments

Comments
 (0)