From ef49c55620ee9bbbd4670f802f78b5779776c486 Mon Sep 17 00:00:00 2001 From: Frank Hoffmann <15r10nk-git@polarbit.de> Date: Mon, 7 Feb 2022 23:12:54 +0100 Subject: [PATCH 1/7] feat: support for python 3.11 * This provides a new implementation, which uses co_positions() to lookup the ast node. * It has a simpler implementation and better performance. * Some limitations in the unit tests are removed for 3.11. * support for `and` and `or` * no ambiguities for generators --- executing/executing.py | 41 +++++++++++++++++++++++++++++++++++++++++ tests/test_main.py | 5 ++++- tests/utils.py | 12 ++++++++---- tox.ini | 2 +- 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/executing/executing.py b/executing/executing.py index 1e06155..a941f37 100644 --- a/executing/executing.py +++ b/executing/executing.py @@ -309,6 +309,47 @@ def executing(cls, frame_or_tb): lineno = frame.f_lineno lasti = frame.f_lasti + if sys.version_info >= (3, 11): + # we can use co_positions() since 3.11, which has fewer limitations + + positions = list(frame.f_code.co_positions()) + source = cls.for_frame(frame) + stmts = source.statements_at_line(lineno) + + def find_node(lasti): + position = positions[lasti // 2] + for node in source._nodes_by_line[position[0]]: + if isinstance(node, (ast.expr, ast.stmt)) and position == ( + node.lineno, + node.end_lineno, + node.col_offset, + node.end_col_offset, + ): + return node + + node = find_node(lasti) + assert_(node != None) + + if isinstance(node, (ast.ClassDef, function_node_types)): + # get the decorator by counting all CALL_FUNCTION ops until the next STORE_* + for idx, inst in enumerate( + islice(dis.Bytecode(frame.f_code), lasti // 2, None) + ): + if inst.opname.startswith("STORE_"): + return Executing( + frame, source, node, stmts, node.decorator_list[idx - 1] + ) + + if inst.opname in ("EXTENDED_ARG", "NOP"): + continue + + assert_(inst.opname == "CALL_FUNCTION", inst) + + if isinstance(node, ast.Expr): + node = node.value + + return Executing(frame, source, node, stmts, None) + code = frame.f_code key = (code, id(code), lasti) executing_cache = cls._class_local('__executing_cache', {}) diff --git a/tests/test_main.py b/tests/test_main.py index 91ead33..ad2dbb8 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -122,8 +122,11 @@ def test_comprehensions(self): str([{tester(x) for x in [1]}, list(tester(x) for x in [1])]) # but not if everything is the same # noinspection PyTypeChecker - with self.assertRaises(NotOneValueFound): + if sys.version_info >= (3, 11): str([{tester(x) for x in [1]}, {tester(x) for x in [2]}]) + else: + with self.assertRaises(NotOneValueFound): + str([{tester(x) for x in [1]}, {tester(x) for x in [2]}]) def test_lambda(self): self.assertEqual( diff --git a/tests/utils.py b/tests/utils.py index 90804c7..d3986af 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -89,11 +89,15 @@ def __lt__(self, other): __ne__ = __ge__ = __lt__ def __bool__(self): - try: - self.get_node(None) - except RuntimeError: + if sys.version_info >= (3, 11): + self.get_node(ast.BoolOp) return False - assert 0 + else: + try: + self.get_node(None) + except RuntimeError: + return False + assert 0 __nonzero__ = __bool__ diff --git a/tox.ini b/tox.ini index 2e2765a..d06f963 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,py36,py37,py38,py39,py310,pypy2,pypy35,pypy36 +envlist = py27,py34,py35,py36,py37,py38,py39,py310,py311,pypy2,pypy35,pypy36 [testenv] commands = From 85707fe8b3df56c4bd42cc89c9a4732660b9511b Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Tue, 8 Feb 2022 11:21:45 +0200 Subject: [PATCH 2/7] GHA: test 3.11 and PRs --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e8a979a..4769b1a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,13 +1,13 @@ name: Tests -on: [push] +on: [push, pull_request] jobs: test: runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10-dev, pypy2, pypy-3.6] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', 3.11-dev, pypy2, pypy-3.6] steps: - uses: actions/checkout@v2 From e0524e17e05166d7eab762446fdd88f7eea13301 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Tue, 8 Feb 2022 11:22:23 +0200 Subject: [PATCH 3/7] Use `only` instead of `find_node` function --- executing/executing.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/executing/executing.py b/executing/executing.py index a941f37..5bf9f44 100644 --- a/executing/executing.py +++ b/executing/executing.py @@ -316,19 +316,19 @@ def executing(cls, frame_or_tb): source = cls.for_frame(frame) stmts = source.statements_at_line(lineno) - def find_node(lasti): - position = positions[lasti // 2] - for node in source._nodes_by_line[position[0]]: - if isinstance(node, (ast.expr, ast.stmt)) and position == ( - node.lineno, - node.end_lineno, - node.col_offset, - node.end_col_offset, - ): - return node - - node = find_node(lasti) - assert_(node != None) + position = positions[lasti // 2] + node = only( + node + for node in source._nodes_by_line[position[0]] + if isinstance(node, (ast.expr, ast.stmt)) + if not isinstance(node, ast.Expr) + if position == ( + node.lineno, + node.end_lineno, + node.col_offset, + node.end_col_offset, + ) + ) if isinstance(node, (ast.ClassDef, function_node_types)): # get the decorator by counting all CALL_FUNCTION ops until the next STORE_* @@ -345,9 +345,6 @@ def find_node(lasti): assert_(inst.opname == "CALL_FUNCTION", inst) - if isinstance(node, ast.Expr): - node = node.value - return Executing(frame, source, node, stmts, None) code = frame.f_code From 7831ba9c251b63aadfa5f09f6e26622482dfee5c Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Tue, 8 Feb 2022 11:23:03 +0200 Subject: [PATCH 4/7] Ignore EXTENDED_ARG/NOP when counting CALL_FUNCTION ops --- executing/executing.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/executing/executing.py b/executing/executing.py index 5bf9f44..9ef3d7d 100644 --- a/executing/executing.py +++ b/executing/executing.py @@ -333,16 +333,15 @@ def executing(cls, frame_or_tb): if isinstance(node, (ast.ClassDef, function_node_types)): # get the decorator by counting all CALL_FUNCTION ops until the next STORE_* for idx, inst in enumerate( - islice(dis.Bytecode(frame.f_code), lasti // 2, None) + inst + for inst in islice(dis.Bytecode(frame.f_code), lasti // 2, None) + if inst.opname not in ("EXTENDED_ARG", "NOP") ): if inst.opname.startswith("STORE_"): return Executing( frame, source, node, stmts, node.decorator_list[idx - 1] ) - if inst.opname in ("EXTENDED_ARG", "NOP"): - continue - assert_(inst.opname == "CALL_FUNCTION", inst) return Executing(frame, source, node, stmts, None) From c57a81d907f41e41cb287708e8ca43954a83fbaf Mon Sep 17 00:00:00 2001 From: Frank Hoffmann <15r10nk-git@polarbit.de> Date: Tue, 8 Feb 2022 23:33:24 +0100 Subject: [PATCH 5/7] fix: fixed decorator detection for 3.11.0a5 --- executing/executing.py | 78 ++++++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 25 deletions(-) diff --git a/executing/executing.py b/executing/executing.py index 9ef3d7d..5a466b5 100644 --- a/executing/executing.py +++ b/executing/executing.py @@ -316,33 +316,61 @@ def executing(cls, frame_or_tb): source = cls.for_frame(frame) stmts = source.statements_at_line(lineno) - position = positions[lasti // 2] - node = only( - node - for node in source._nodes_by_line[position[0]] - if isinstance(node, (ast.expr, ast.stmt)) - if not isinstance(node, ast.Expr) - if position == ( - node.lineno, - node.end_lineno, - node.col_offset, - node.end_col_offset, + def find_node(index): + position = positions[index // 2] + return only( + node + for node in source._nodes_by_line[position[0]] + if isinstance(node, (ast.expr, ast.stmt)) + if not isinstance(node, ast.Expr) + if position + == ( + node.lineno, + node.end_lineno, + node.col_offset, + node.end_col_offset, + ) ) - ) - if isinstance(node, (ast.ClassDef, function_node_types)): - # get the decorator by counting all CALL_FUNCTION ops until the next STORE_* - for idx, inst in enumerate( - inst - for inst in islice(dis.Bytecode(frame.f_code), lasti // 2, None) - if inst.opname not in ("EXTENDED_ARG", "NOP") - ): - if inst.opname.startswith("STORE_"): - return Executing( - frame, source, node, stmts, node.decorator_list[idx - 1] - ) - - assert_(inst.opname == "CALL_FUNCTION", inst) + node = find_node(lasti) + + if ( + isinstance(node.parent, (ast.ClassDef, function_node_types)) + and node in node.parent.decorator_list + ): + node_func = node.parent + index = lasti + bc_list = list(dis.Bytecode(frame.f_code)) + + def opname(i): + return bc_list[i // 2].opname + + while True: + # idx-2: PRECALL_FUNCTION + # idx: CALL + # idx+2: STORE_* / PRECALL_FUNCTION + + if not ( + opname(index - 2) == "PRECALL_FUNCTION" + and opname(index) == "CALL" + ): + break + + if ( + bc_list[index // 2 - 1].positions + != bc_list[index // 2].positions + ): + break + + if find_node(index) not in node_func.decorator_list: + break + + if find_node(index + 2) == node_func and opname( + index + 2 + ).startswith("STORE_"): + return Executing(frame, source, node_func, {node_func}, node) + + index += 4 return Executing(frame, source, node, stmts, None) From 6691eec238d0e8574bdc2b5c8dbeb9303764bf78 Mon Sep 17 00:00:00 2001 From: Frank Hoffmann <15r10nk-git@polarbit.de> Date: Thu, 31 Mar 2022 15:30:50 +0200 Subject: [PATCH 6/7] fix: detect decorators in 3.11.a6 correctly and ignore qualname checks for 3.11 --- executing/executing.py | 25 ++++++++++++++----------- tests/test_main.py | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/executing/executing.py b/executing/executing.py index 5a466b5..9ca2bb2 100644 --- a/executing/executing.py +++ b/executing/executing.py @@ -340,24 +340,22 @@ def find_node(index): ): node_func = node.parent index = lasti - bc_list = list(dis.Bytecode(frame.f_code)) + bc_list = list(dis.Bytecode(frame.f_code, show_caches=True)) def opname(i): return bc_list[i // 2].opname while True: - # idx-2: PRECALL_FUNCTION + # idx-4: PRECALL + # idx-2: CACHE # idx: CALL - # idx+2: STORE_* / PRECALL_FUNCTION + # idx+2: STORE_* / CACHE - if not ( - opname(index - 2) == "PRECALL_FUNCTION" - and opname(index) == "CALL" - ): + if not (opname(index - 4) == "PRECALL" and opname(index) == "CALL"): break if ( - bc_list[index // 2 - 1].positions + bc_list[index // 2 - 2].positions != bc_list[index // 2].positions ): break @@ -365,9 +363,14 @@ def opname(i): if find_node(index) not in node_func.decorator_list: break - if find_node(index + 2) == node_func and opname( - index + 2 - ).startswith("STORE_"): + index += 2 + + while opname(index) == "CACHE": + index += 2 + + if find_node(index) == node_func and opname(index).startswith( + "STORE_" + ): return Executing(frame, source, node_func, {node_func}, node) index += 4 diff --git a/tests/test_main.py b/tests/test_main.py index ad2dbb8..cd8ee83 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -470,7 +470,7 @@ def check_filename(self, filename, check_names): print(filename) source = Source.for_filename(filename) - if PY3: + if PY3 and sys.version_info < (3, 11): code = compile(source.text, filename, "exec", dont_inherit=True) for subcode, qualname in find_qualnames(code): if not qualname.endswith(">"): From 1b2b96a23a164a58f512fd35a16690a4d3915e3c Mon Sep 17 00:00:00 2001 From: Frank Hoffmann <15r10nk-git@polarbit.de> Date: Fri, 1 Apr 2022 01:32:02 +0200 Subject: [PATCH 7/7] wip: fix linenumber problem --- executing/executing.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/executing/executing.py b/executing/executing.py index 9ca2bb2..bfc1c2c 100644 --- a/executing/executing.py +++ b/executing/executing.py @@ -78,13 +78,21 @@ def wrapper(*args): # noinspection PyUnresolvedReferences text_type = unicode -try: - # noinspection PyUnresolvedReferences - _get_instructions = dis.get_instructions -except AttributeError: - class Instruction(namedtuple('Instruction', 'offset argval opname starts_line')): - lineno = None +if hasattr(dis, "get_instructions"): + + if sys.version_info >= (3, 11): + + def _get_instructions(co): + for inst in dis.get_instructions(co, show_caches=True): + if inst.opname != "CACHE": + yield inst + + else: + # noinspection PyUnresolvedReferences + _get_instructions = dis.get_instructions + +else: from dis import HAVE_ARGUMENT, EXTENDED_ARG, hasconst, opname, findlinestarts, hasname