Use executing and asttokens (#82)

* Use executing and asttokens

* Test `raise RuntimeError(debug.format(...))`
Shows that #47 is fixed

* Clean up some now unused stuff

* black

* Differentiate source of generator expressions between python versions

* restrict version, other tweaks

* tweaking is_literal

* remove from requirements

Co-authored-by: Samuel Colvin <[email protected]>

Author: alexmojaki

devtools/debug.py

@@ -4,14 +4,13 @@
 from .ansi import sformat
 from .prettier import PrettyFormat
 from .timer import Timer
-from .utils import env_bool, env_true, use_highlight
+from .utils import env_bool, env_true, is_literal, use_highlight
 
 __all__ = 'Debug', 'debug'
 MYPY = False
 if MYPY:
-    import ast
     from types import FrameType
-    from typing import Generator, List, Optional, Tuple
+    from typing import Generator, List, Optional
 
 
 pformat = PrettyFormat(
@@ -22,10 +21,6 @@
 )
 
 
-class IntrospectionError(ValueError):
-    pass
-
-
 class DebugArgument:
     __slots__ = 'value', 'name', 'extra'
 
@@ -43,7 +38,7 @@ def __init__(self, value, *, name=None, **extra):
 
     def str(self, highlight=False) -> str:
         s = ''
-        if self.name:
+        if self.name and not is_literal(self.name):
             s = sformat(self.name, sformat.blue, apply=highlight) + ': '
 
         suffix = sformat(
@@ -108,21 +103,17 @@ def __repr__(self) -> str:
 class Debug:
     output_class = DebugOutput
 
-    def __init__(
-        self, *, warnings: 'Optional[bool]' = None, highlight: 'Optional[bool]' = None, frame_context_length: int = 50
-    ):
+    def __init__(self, *, warnings: 'Optional[bool]' = None, highlight: 'Optional[bool]' = None):
         self._show_warnings = env_bool(warnings, 'PY_DEVTOOLS_WARNINGS', True)
         self._highlight = highlight
-        # 50 lines should be enough to make sure we always get the entire function definition
-        self._frame_context_length = frame_context_length
 
     def __call__(self, *args, file_=None, flush_=True, **kwargs) -> None:
-        d_out = self._process(args, kwargs, 'debug')
+        d_out = self._process(args, kwargs)
         s = d_out.str(use_highlight(self._highlight, file_))
         print(s, file=file_, flush=flush_)
 
     def format(self, *args, **kwargs) -> DebugOutput:
-        return self._process(args, kwargs, 'format')
+        return self._process(args, kwargs)
 
     def breakpoint(self):
         import pdb
@@ -132,7 +123,7 @@ def breakpoint(self):
     def timer(self, name=None, *, verbose=True, file=None, dp=3) -> Timer:
         return Timer(name=name, verbose=verbose, file=file, dp=dp)
 
-    def _process(self, args, kwargs, func_name: str) -> DebugOutput:
+    def _process(self, args, kwargs) -> DebugOutput:
         """
         BEWARE: this must be called from a function exactly 2 levels below the top of the stack.
         """
@@ -165,23 +156,20 @@ def _process(self, args, kwargs, func_name: str) -> DebugOutput:
         lineno = call_frame.f_lineno
         warning = None
 
-        import inspect
+        import executing
 
-        try:
-            file_lines, _ = inspect.findsource(call_frame)
-        except OSError:
+        source = executing.Source.for_frame(call_frame)
+        if not source.text:
             warning = 'no code context for debug call, code inspection impossible'
             arguments = list(self._args_inspection_failed(args, kwargs))
         else:
-            try:
-                first_line, last_line = self._statement_range(call_frame, func_name)
-                func_ast, code_lines = self._parse_code(filename, file_lines, first_line, last_line)
-            except IntrospectionError as e:
-                # parsing failed
-                warning = e.args[0]
+            ex = source.executing(call_frame)
+            # function = ex.code_qualname()
+            if not ex.node:
+                warning = "executing failed to find the calling node"
                 arguments = list(self._args_inspection_failed(args, kwargs))
             else:
-                arguments = list(self._process_args(func_ast, code_lines, args, kwargs))
+                arguments = list(self._process_args(ex, args, kwargs))
 
         return self.output_class(
             filename=filename,
@@ -197,174 +185,25 @@ def _args_inspection_failed(self, args, kwargs):
         for name, value in kwargs.items():
             yield self.output_class.arg_class(value, name=name)
 
-    def _process_args(self, func_ast, code_lines, args, kwargs) -> 'Generator[DebugArgument, None, None]':  # noqa: C901
+    def _process_args(self, ex, args, kwargs) -> 'Generator[DebugArgument, None, None]':
         import ast
 
-        complex_nodes = (
-            ast.Call,
-            ast.Attribute,
-            ast.Subscript,
-            ast.IfExp,
-            ast.BoolOp,
-            ast.BinOp,
-            ast.Compare,
-            ast.DictComp,
-            ast.ListComp,
-            ast.SetComp,
-            ast.GeneratorExp,
-        )
-
-        arg_offsets = list(self._get_offsets(func_ast))
-        for i, arg in enumerate(args):
-            try:
-                ast_node = func_ast.args[i]
-            except IndexError:  # pragma: no cover
-                # happens when code has been commented out and there are fewer func_ast args than real args
-                yield self.output_class.arg_class(arg)
-                continue
-
-            if isinstance(ast_node, ast.Name):
-                yield self.output_class.arg_class(arg, name=ast_node.id)
-            elif isinstance(ast_node, complex_nodes):
-                # TODO replace this hack with astor when it get's round to a new release
-                start_line, start_col = arg_offsets[i]
-
-                if i + 1 < len(arg_offsets):
-                    end_line, end_col = arg_offsets[i + 1]
-                else:
-                    end_line, end_col = len(code_lines) - 1, None
-
-                name_lines = []
-                for l_ in range(start_line, end_line + 1):
-                    start_ = start_col if l_ == start_line else 0
-                    end_ = end_col if l_ == end_line else None
-                    name_lines.append(code_lines[l_][start_:end_].strip(' '))
-                yield self.output_class.arg_class(arg, name=' '.join(name_lines).strip(' ,'))
+        func_ast = ex.node
+        atok = ex.source.asttokens()
+        for arg, ast_arg in zip(args, func_ast.args):
+            if isinstance(ast_arg, ast.Name):
+                yield self.output_class.arg_class(arg, name=ast_arg.id)
             else:
-                yield self.output_class.arg_class(arg)
+                name = ' '.join(map(str.strip, atok.get_text(ast_arg).splitlines()))
+                yield self.output_class.arg_class(arg, name=name)
 
         kw_arg_names = {}
         for kw in func_ast.keywords:
             if isinstance(kw.value, ast.Name):
                 kw_arg_names[kw.arg] = kw.value.id
+
         for name, value in kwargs.items():
             yield self.output_class.arg_class(value, name=name, variable=kw_arg_names.get(name))
 
-    def _parse_code(
-        self, filename: str, file_lines: 'List[str]', first_line: int, last_line: int
-    ) -> 'Tuple[ast.AST, List[str]]':
-        """
-        All we're trying to do here is build an AST of the function call statement. However numerous ugly interfaces,
-        lack on introspection support and changes between python versions make this extremely hard.
-        """
-        import ast
-        from textwrap import dedent
-
-        def get_code(_last_line: int) -> str:
-            lines = file_lines[first_line - 1 : _last_line]
-            return dedent(''.join(ln for ln in lines if ln.strip('\n ') and not ln.lstrip(' ').startswith('#')))
-
-        code = get_code(last_line)
-        func_ast = None
-        try:
-            func_ast = self._wrap_parse(code, filename)
-        except (SyntaxError, AttributeError) as e1:
-            # if the trailing bracket(s) of the function is/are on a new line e.g.:
-            # debug(
-            #     foo, bar,
-            # )
-            # inspect ignores it when setting index and we have to add it back
-            for extra in range(1, 6):
-                code = get_code(last_line + extra)
-                try:
-                    func_ast = self._wrap_parse(code, filename)
-                except (SyntaxError, AttributeError):
-                    pass
-                else:
-                    break
-
-            if not func_ast:
-                raise IntrospectionError('error parsing code, {0.__class__.__name__}: {0}'.format(e1))
-
-        if not isinstance(func_ast, ast.Call):
-            raise IntrospectionError('error parsing code, found {0.__class__} not Call'.format(func_ast))
-
-        code_lines = [line for line in code.split('\n') if line]
-        # this removes the trailing bracket from the lines of code meaning it doesn't appear in the
-        # representation of the last argument
-        code_lines[-1] = code_lines[-1][:-1]
-        return func_ast, code_lines
-
-    @staticmethod  # noqa: C901
-    def _statement_range(call_frame: 'FrameType', func_name: str) -> 'Tuple[int, int]':  # noqa: C901
-        """
-        Try to find the start and end of a frame statement.
-        """
-        import dis
-
-        # dis.disassemble(call_frame.f_code, call_frame.f_lasti)
-        # pprint([i for i in dis.get_instructions(call_frame.f_code)])
-
-        instructions = iter(dis.get_instructions(call_frame.f_code))
-        first_line = None
-        last_line = None
-
-        for instr in instructions:  # pragma: no branch
-            if (
-                instr.starts_line
-                and instr.opname in {'LOAD_GLOBAL', 'LOAD_NAME'}
-                and (instr.argval == func_name or (instr.argval == 'debug' and next(instructions).argval == func_name))
-            ):
-                first_line = instr.starts_line
-            if instr.offset == call_frame.f_lasti:
-                break
-
-        if first_line is None:
-            raise IntrospectionError('error parsing code, unable to find "{}" function statement'.format(func_name))
-
-        for instr in instructions:
-            if instr.starts_line:
-                last_line = instr.starts_line - 1
-                break
-
-        if last_line is None:
-            if sys.version_info >= (3, 8):
-                # absolutely no reliable way of getting the last line of the statement, complete hack is to
-                # get the last line of the last statement of the whole code block and go from there
-                # this assumes (perhaps wrongly?) that the reason we couldn't find last_line is that the statement
-                # in question was the last of the block
-                last_line = max(i.starts_line for i in dis.get_instructions(call_frame.f_code) if i.starts_line)
-            else:
-                # in older version of python f_lineno is the end of the statement, not the beginning
-                # so this is a reasonable guess
-                last_line = call_frame.f_lineno
-
-        return first_line, last_line
-
-    @staticmethod
-    def _wrap_parse(code: str, filename: str) -> 'ast.Call':
-        """
-        async wrapper is required to avoid await calls raising a SyntaxError
-        """
-        import ast
-        from textwrap import indent
-
-        code = 'async def wrapper():\n' + indent(code, ' ')
-        return ast.parse(code, filename=filename).body[0].body[0].value
-
-    @staticmethod
-    def _get_offsets(func_ast):
-        import ast
-
-        for arg in func_ast.args:
-            start_line, start_col = arg.lineno - 2, arg.col_offset - 1
-
-            # horrible hack for http://bugs.python.org/issue31241
-            if isinstance(arg, (ast.ListComp, ast.GeneratorExp)):
-                start_col -= 1
-            yield start_line, start_col
-        for kw in func_ast.keywords:
-            yield kw.value.lineno - 2, kw.value.col_offset - 2 - (len(kw.arg) if kw.arg else 0)
-
 
 debug = Debug()

devtools/utils.py

@@ -95,3 +95,14 @@ def use_highlight(highlight: 'Optional[bool]' = None, file_=None) -> bool:
     if sys.platform == 'win32':  # pragma: no cover
         return isatty(file_) and activate_win_color()
     return isatty(file_)
+
+
+def is_literal(s):
+    import ast
+
+    try:
+        ast.literal_eval(s)
+    except (TypeError, MemoryError, SyntaxError, ValueError):
+        return False
+    else:
+        return True

setup.py

@@ -48,6 +48,10 @@
     license='MIT',
     packages=['devtools'],
     python_requires='>=3.6',
+    install_requires=[
+        'executing>=0.8.0,<1.0.0',
+        'asttokens>=2.0.0,<3.0.0',
+    ],
     extras_require={
         'pygments': ['Pygments>=2.2.0'],
     },