遍历复杂的字典时获取对Python对象的引用

Question

I would appreciate it if someone could help me with this little problem. I want to iterate over a complex Python data structure (dict, list, tuple, strings, bytes, ...) and replace all bytes (byte strings) with a base64 encoded version. This is necessary to convert the original complex data structure to JSON (eg json.dumps(complex_data_structure) ) since JSON does not support binary data. My code already does the right thing but there is one Python-specific problem. Here is my code:

import sys
import json
import base64


def iter_object(obj):
    if type(obj) is tuple:
        iter_tuple(obj)
    elif type(obj) is dict:
        iter_dict(obj)
    elif type(obj) is list:
        iter_list(obj)
    else: # non iterable types except of string and bytes etc.
        if type(obj) is bytes:
           # THE PROBLEM IS THE COPY OF OBJ!
           obj = base64.b64encode(obj).decode("ascii")
        else:
            pass # we don't care about other data types


def iter_tuple(obj_tuple):
    for t in obj_tuple:
        iter_object(t)


def iter_list(obj_list):
    for l in obj_list:
        iter_object(l)


def iter_dict(obj_dict):
    for k, v in obj_dict.items():
        iter_object(v)


def main():

    test_dict = {
        "foo": [1, 3, 4, 5, 6, 7],
        "bar": 1,
        "baz": (1, 2),
        "blub": {
            "bla": b"\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41",
            "ble": {
                "blu": [
                    1, 3, b"\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42",
                    (1, [b"\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43"])
                ]
            }
        }
    }

    iter_object(test_dict)

    print(json.dumps(test_dict))

    return 0


if __name__ == "__main__":
    sys.exit(main())

The problem is the line obj = base64.b64encode(obj).decode("ascii") because it works on a copy not a reference (to say it in C++). Here is my question: Is there a workaround to make the above code work?

Thank you very much!

Answer 1

Works on a copy?? No. What is happening is that the function is returning a value instead of changing it in place. This is because byte strings are immutable. There is no concept of pass by value or reference in python. The variables are not boxes which hold objects, they are rather names of some object. And object can be

1. Mutable - list , set , dict
2. Immutable - tuple , str , bytes

So if a function acts on Immutable object it has to return another object. The memory usage is optimized. And this is the defacto way in languages like Haskell.

Answer 2

I found a solution to my problem:

import sys
import json
import base64

class BinaryToBase64Encoder(json.JSONEncoder):
    def default(self, o):
        if isinstance(o, bytes):
            return base64.b64encode(o).decode("ascii")
        return super(BinaryToBase64Encoder, self).default(o)


def main():

    test_dict = {
        "foo": [1, 3, 4, 5, 6, 7],
        "bar": 1,
        "baz": (1, 2),
        "blub": {
            "bla": b"\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41\x41",
            "ble": {
                "blu": [
                    1, 3, b"\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42",
                    (1, [b"\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43\x43"])
                ]
            }
        }
    }

    print(json.dumps(test_dict, cls=BinaryToBase64Encoder))

    return 0


if __name__ == "__main__":
    sys.exit(main())

The JSON output is:

{
    "foo": [1, 3, 4, 5, 6, 7],
    "baz": [1, 2],
    "bar": 1,
    "blub": {
        "ble": {
            "blu": [
                1,
                3,
                "QkJCQkJCQkJCQkJCQkJCQkJCQkI=",
                [
                    1,
                    ["Q0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0M="]
                ]
            ]
        },
        "bla": "QUFBQUFBQUFBQUFBQUFBQUFBQUE="
    }
}