有没有办法声明 function 应该使用调用者的 scope ？

Question

is there a feautre similar to C macros which lets you reuse code in an inline manner, without creating a seperate scope for that piece of code?

for example:

a=3
def foo():
    a=4
foo()
print a

will print 3, however i want it to print 4.

i am aware of solutions involving objects like classes or a global dict, however i'm looking for a more primitive solution (like a function decorator for example) that would simply let me make changes inside the scope of the caller instead.

thank you very much

edit:any solution that requires declaring which variables i'm going to use OR declaring a "namespace" like mutabale objects beforehand is not a solution i'm looking for.

i had made an attempt on my own:

def pgame():
a=3
c=5
print locals()
game(a)
print locals()


class inline_func(object):
    def __init__(self, f):
        self.f = f

    def __call__(self, *args, **kwargs):
        return self.f(*args, **kwargs)


#to be @inline_func
def game(b, a=4):
    exec("inspect.stack()[3][0].f_locals.update(inspect.stack()[1] [0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))\ninspect.stack()[1][0].f_locals.update(inspect.stack()[3][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[1][0]),ctypes.c_int(0))")
try:
    print "your code here"
finally:
    exec("inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))")

@inline_func
def strip_game(b, a=4):
    print "your code here"

but i have ran into a serious problem with how to inject code into strip_game without ruining the debugability of the program, because i had only thought of creating a new code object or using exec, both suffering from some severe problems.

MAJOR EDIT:

ok, so i have something close to a working solution, however i encounter a very wierd problem:

import inspect
import ctypes
import struct
import dis
import types



def cgame():
    a=3
    c=5
    print locals()
    strip_game(a)
    print locals()


def pgame():
    a=3
    c=5
    print locals()
    game(a)
    print locals()


class empty_deco(object):
    def __init__(self, f):
        self.f = f

    def __call__(self, *args, **kwargs):
        return self.f(*args, **kwargs)

debug_func = None
class inline_func(object):
    def __init__(self, f):
        self.f = f

    def __call__(self, *args, **kwargs):

        init_exec_string = "inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\n" + \
                           "ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))\n" + \
                           "inspect.stack()[1][0].f_locals.update(inspect.stack()[3][0].f_locals)\n" + \
                           "ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[1][0]),ctypes.c_int(0))"
        fini_exec_string = "inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\n" + \
                           "ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))" 

        co_stacksize = max(6, self.f.func_code.co_stacksize) # make sure we have enough space on the stack for everything
        co_consts = self.f.func_code.co_consts +(init_exec_string, fini_exec_string)
        init = "d"  + struct.pack("H", len(strip_game.f.func_code.co_consts)) #LOAD_CONST init_exec_string
        init += "d\x00\x00\x04U" # LOAD_CONST None, DUP_TOP, EXEC_STMT
        init += "z" + struct.pack("H", len(self.f.func_code.co_code) + 4) #SETUP_FINALLY

        fini = "Wd\x00\x00" # POP_BLOCK, LOAD_CONST None
        fini += "d" + struct.pack("H", len(strip_game.f.func_code.co_consts) + 1) #LOAD_CONST fini_exec_string
        fini += "d\x00\x00\x04UXd\x00\x00S" # LOAD_CONST None, DUP_TOP, EXEC_STMT, END_FINALLY, LOAD_CONST None, RETURN
        co_code = init + self.f.func_code.co_code + fini
        co_lnotab = "\x00\x00\x0b" + self.f.func_code.co_lnotab[1:] # every error in init will be attributed to @inline_func, errors in the function will be treated as expected, errors in fini will be attributed to the last line probably.
        new_code = types.CodeType(
        self.f.func_code.co_argcount,
        self.f.func_code.co_nlocals,
        co_stacksize,
        self.f.func_code.co_flags & ~(1), # optimized functions are problematic for us
        co_code,
        co_consts,
        self.f.func_code.co_names,
        self.f.func_code.co_varnames,
        self.f.func_code.co_filename,
        self.f.func_code.co_name,
        self.f.func_code.co_firstlineno,
        co_lnotab,
        self.f.func_code.co_freevars,
        self.f.func_code.co_cellvars,)

        self.inline_f =  types.FunctionType(new_code, self.f.func_globals, self.f.func_name, self.f.func_defaults, self.f.func_closure)
        #dis.dis(self.inline_f)
        global debug_func
        debug_func = self.inline_f
        return self.inline_f(*args, **kwargs)


@empty_deco
def game(b, a=4):
    exec("inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))\ninspect.stack()[1][0].f_locals.update(inspect.stack()[3][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[1][0]),ctypes.c_int(0))")
    try:
        print "inner locals:"
        print locals()
        print c
        return None
    finally:
        exec("inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))")

@inline_func
def strip_game(b, a=4):
    print "inner locals:"
    print locals()
    print c
    return None



def stupid():
    exec("print 'hello'")
    try:
        a=1
        b=2
        c=3
        d=4
    finally:
        exec("print 'goodbye'")

now this seems to work however, i get the following:

>>>cgame()
{'a': 3, 'c': 5}
{'a': 4, 'c': 5, 'b': 3}
your code here

Traceback (most recent call last):
  File "<pyshell#43>", line 1, in <module>
    cgame()
  File "C:\Python27\somefile.py", line 14, in cgame
    strip_game(a)
  File "C:\Python27\somefile.py", line 78, in __call__
    return self.inline_f(*args, **kwargs)
  File "C:\Python27\somefile.py", line 94, in strip_game
    z = c
NameError: global name 'c' is not defined

now when i disassemble the functions, i get the following very wierd compilation difference between game and strip_game :

in game:

86          16 LOAD_NAME                0 (locals)
             19 CALL_FUNCTION            0
             22 PRINT_ITEM          
             23 PRINT_NEWLINE       

 87          24 **LOAD_NAME**                1 (c)
             27 PRINT_ITEM          
             28 PRINT_NEWLINE       

in strip game:

95          16 LOAD_GLOBAL              0 (locals)
             19 CALL_FUNCTION            0
             22 PRINT_ITEM          
             23 PRINT_NEWLINE       

 96          24 LOAD_GLOBAL              1 (c)
             27 PRINT_ITEM          
             28 PRINT_NEWLINE       

why is does this difference occur?

Answer 1

In this case, just use the global keyword:

a=3
def foo():
    global a
    a=4
foo()
print (a)

That modifies the outer scope, if it is global.

If the outer scope is a function, that is done with the nonlocal keyword instead - which was introduced with Python 3.0.

dynamic scoping

Changing the scope of the caller function however, is not a premise of Python, and is a language characteristic.

It can be done. But just by calling private C api's (to bake 'locals' values back into the fast local variables) and is definettely not a good practice.

DOing it through a magic decorator would also be possible, but the decorator would have to rewrite the bytecode in the inner function - by replacing each access to a 'nonlocal' variable by retrieving and updating the value on the caler locals , and, at the end of the function - https://programtalk.com/python-examples/ctypes.pythonapi.PyFrame_LocalsToFast/

Example

So, that said, here is a proof of concept. It is, of course, thread, and async unsafe as hell - but if the attributes in the proxy class are promoted to threadlocals or context-local (pep 555), it should work. it should be easy to adapt this to search for the local-variables to change up on the call stack (so that changes made in a sub-sub-call could change the grandparents locals, just as in dynamic scoped languages)

As stated in the question, there is no need to declare the variables on the caller as anything - they just must be normal local variables. However, this requires the declaration, on the decorated function, the variables I want to change on the caller scope as 'global', so that changing then will go through an object I can customize. If you can't have even this, you will indeed have to resort to rewrite the bytecode on the decorated function, or use the hooks put in place for writing debuggers (setting "trace on" on the code).

nb the exact behavior of changes locals() was specified to the language recently - prior to 3.8, IIRC, - and "locals_to_fast" seems to be an stable enough API - but it might change in the future.

# Tested in Python 3.8.0

import ctypes
from functools import wraps
from sys import _getframe as getframe
from types import FunctionType


class GlobalProxy(dict):
    __slots__ = ("parent", "frame", "mode")
    def __init__(self, parent):
        self.parent = parent
        self.frame = None
        self.mode = None

    def __getitem__(self, name):
        if self.mode == "target":
            if name in self.frame.f_locals:
                return self.frame.f_locals[name]
            if name in self.parent:
                return self.parent[name]
            return getattr(self.parent["__builtins__"], name)
        return super().__getitem__(name)

    """
    # This is not run - Python's VM STORE_GLOBAL bypasses the custom __setitem__ (although __getitem__ above runs)
    def __setitem__(self, name, value):
        if name in self.frame.f_locals:
            self.frame.f_locals[name] = value
            bake_locals(self.frame)
        self.parent[name] = value
    """

    def bake_locals(self):
        ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(self.frame), ctypes.c_int(1))

    def save_changes(self):
        self.mode = "inner"
        target = self.frame.f_locals
        target_names = set(target.keys())
        for key in self:
            if key in target_names:
                target[key] = self[key]
            else:
                self.parent[key] = self[key]
        self.bake_locals()


def caller_changer(func):
    """Makes all global variable changes on the decorated function affect _local_ variables on the callee function instead.
    """

    code = func.__code__
    # NB: for Python 2, these dunder-attributes for functions have other names.
    # this is for Python 3
    proxy = GlobalProxy(func.__globals__)
    new_function = FunctionType(code, proxy, func.__name__, func.__defaults__, func.__closure__)
    @wraps(func)
    def wrapper(*args, **kw):
        proxy.frame = getframe().f_back
        proxy.mode = "target"
        result = new_function(*args, **kw)
        proxy.save_changes()
        return result

    wrapper.proxy = proxy

    return wrapper


### Example and testing code:


@caller_changer
def blah():
    global iwillchange
    iwillchange = "new value"


def bleh():
    iwillchange = "original value"
    print(iwillchange)
    blah()
    print(iwillchange)

And, pasting all that on an IPython shell:

In [121]: bleh()                                                                                                                     
original value
new value

(I might add that it felt weird testing that, since the functions that have the local variables changed do not need any decorator, or any special declaration to the variables at all)

Answer 2

ok, so after several hours of sitting on this thing i've managed to write a solution, there are some major pitfalls when approaching this and i'll note them below

import inspect
import ctypes
import struct
import dis
import types

def dump(obj):
  for attr in dir(obj):
    print("obj.%s = %r" % (attr, getattr(obj, attr)))

def cgame():
    a=3
    c=5
    print locals()
    strip_game(a)
    print locals()


def pgame():
    a=3
    c=5
    print locals()
    game(a)
    print locals()


class empty_deco(object):
    def __init__(self, f):
        self.f = f

    def __call__(self, *args, **kwargs):
        return self.f(*args, **kwargs)



debug_func = None
class inline_func(object):
    def __init__(self, f):
        self.f = f
    # this is the price we pay for using 2.7
    # also, there is a huge glraing issue here, which is what happens if the user TRIES to access a global variable?
    @staticmethod
    def replace_globals_with_name_lookups(co):
        res = ""
        code = list(co)
        n = len(code)
        i = 0
        while i < n:
            c = code[i]
            op = ord(c)
            if dis.opname[op] == "STORE_GLOBAL":
                code[i] = chr(dis.opmap['STORE_NAME'])
            elif dis.opname[op] == "DELETE_GLOBAL":
                code[i] = chr(dis.opmap['DELETE_NAME'])
            elif dis.opname[op] == "LOAD_GLOBAL":
                code[i] = chr(dis.opmap['LOAD_NAME'])
            i = i+1
            if op >= dis.HAVE_ARGUMENT:
                i = i+2
        return "".join(code)

    def __call__(self, *args, **kwargs):

        init_exec_string = "inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\n" + \
                           "ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))\n" + \
                           "inspect.stack()[1][0].f_locals.update(inspect.stack()[3][0].f_locals)\n" + \
                           "ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[1][0]),ctypes.c_int(0))"
        fini_exec_string = "inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\n" + \
                           "ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))" 

        co_stacksize = max(6, self.f.func_code.co_stacksize) # make sure we have enough space on the stack for everything
        co_consts = self.f.func_code.co_consts +(init_exec_string, fini_exec_string)
        init = "d"  + struct.pack("H", len(strip_game.f.func_code.co_consts)) #LOAD_CONST init_exec_string
        init += "d\x00\x00\x04U" # LOAD_CONST None, DUP_TOP, EXEC_STMT
        init += "z" + struct.pack("H", len(self.f.func_code.co_code) + 4) #SETUP_FINALLY

        fini = "Wd\x00\x00" # POP_BLOCK, LOAD_CONST None
        fini += "d" + struct.pack("H", len(strip_game.f.func_code.co_consts) + 1) #LOAD_CONST fini_exec_string
        fini += "d\x00\x00\x04UXd\x00\x00S" # LOAD_CONST None, DUP_TOP, EXEC_STMT, END_FINALLY, LOAD_CONST None, RETURN
        co_code = init + self.replace_globals_with_name_lookups(self.f.func_code.co_code) + fini
        co_lnotab = "\x00\x00\x0b" + self.f.func_code.co_lnotab[1:] # every error in init will be attributed to @inline_func, errors in the function will be treated as expected, errors in fini will be attributed to the last line probably.
        new_code = types.CodeType(
        self.f.func_code.co_argcount,
        self.f.func_code.co_nlocals,
        co_stacksize,
        self.f.func_code.co_flags & ~(1), # optimized functions are problematic for us
        co_code,
        co_consts,
        self.f.func_code.co_names,
        self.f.func_code.co_varnames,
        self.f.func_code.co_filename,
        self.f.func_code.co_name,
        self.f.func_code.co_firstlineno,
        co_lnotab,
        self.f.func_code.co_freevars,
        self.f.func_code.co_cellvars,)

        self.inline_f =  types.FunctionType(new_code, self.f.func_globals, self.f.func_name, self.f.func_defaults, self.f.func_closure)
        #dis.dis(self.inline_f)
        global debug_func
        debug_func = self.inline_f
        return self.inline_f(*args, **kwargs)


@empty_deco
def game(b, a=4):
    exec("inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))\ninspect.stack()[1][0].f_locals.update(inspect.stack()[3][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[1][0]),ctypes.c_int(0))")
    try:
        print "inner locals:"
        print locals()
        print c
        return None
    finally:
        exec("inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))")

@inline_func
def strip_game(b, a=4):
    print "inner locals:"
    print locals()
    print c
    return None

where the acutal code needed lies in the class inline_func and some of the imports (maybe you can make them internal to the class? i'm really not sure)

so what does this whole thing do? well, it makes it so the code for strip_game and game are (nearly) identical, namely:

1. it inserts a function prologue which updates the locals of the caller, then adds to locals of the caller to the callee.
2. insert a try finally block around the function
3. changes every symbol lookup from a global lookup to a normal (name) lookup, after some thought i had realized that this doens't really have any effects
4. upon entering the finally block, updates the caller locals.

there are some major pitfalls making things like these, i'll list a few problems i've encountered:

1. cpython compiler_nameop function optimizes namespace lookup based on the simplicity of the given function, that means that it will optimize name lookups to global lookups if it can
2. changing the bytecode means affecting the debug-ability of the program, i had addressed this in the co_lnotab variable
3. for large functions this solution won't work as some of the opcodes would have to use extended_args: namely, the loads of the variables and the try-finally block (this point is solvable by using extended_args anyways...)

thank @jsbueno for putting in the time and pointing me to PyFrame_LocalsToFast.

PS this solution works for python 2.7.6, python has some issues when it comes to stability of the API, so for newer versions this might need to be fixed.