Transition from imp to importlib

Question

A project that I would like to use has parts of code

_VALID_MODULE_TYPES = set((imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION))

[...]

for suffix, mode, type in imp.get_suffixes():
        if type in self._VALID_MODULE_TYPES:
           path = prefix + suffix
           if self.file_system.isfile(path):
                   return path, type

[...]

if type == imp.PY_SOURCE:
        code = self.file_system.readbytes(path).replace(b("\r\n"), b("\n"))
        return compile(code, path, "exec")
elif type == imp.PY_COMPILED:
        code = self.file_system.readbytes(path)
        if code[:4] != imp.get_magic():
                return None
        return marshal.loads(code[8:])
elif type == imp.C_EXTENSION:
        code = self.file_system.readbytes(path)

which use the module imp . It is said that imp is deprecated and that importlib should be used instead, but I do not know how to transition. How do I have to rewrite the code?

Answer 1

1) Inspection

1. We realize that imp.get_magic() can be replaced by importlib.util.MAGIC_NUMBER .

2. Then we inspect imp.get_suffixes

def get_suffixes():
    """**DEPRECATED**"""
    extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES]
    source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES]
    bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES]

    return extensions + source + bytecode

and we notice that

• get_suffixes uses some "magic numbers" imp.PY_SOURCE , imp.PY_COMPILED , imp.C_EXTENSION
• get_suffixes uses machinery from importlib

2) Refactoring to get independent of imp

We rewrite the code from before. We improve further on it by

• using Enum
• using is instead of == in the last chunk

import importlib.machinery

class ModuleType(Enum):
    SEARCH_ERROR = 0
    PY_SOURCE = 1
    PY_COMPILED = 2
    C_EXTENSION = 3
    PY_RESOURCE = 4
    PKG_DIRECTORY = 5
    C_BUILTIN = 6
    PY_FROZEN = 7
    PY_CODERESOURCE = 8
    IMP_HOOK = 9

# _VALID_MODULE_TYPES = {ModuleType.PY_SOURCE, ModuleType.PY_COMPILED}
_VALID_MODULE_TYPES = {ModuleType.PY_SOURCE, ModuleType.PY_COMPILED, ModuleType.C_EXTENSION}


def get_suffixes():
    extensions = [(s, 'rb', C_EXTENSION) for s in importlib.machinery.EXTENSION_SUFFIXES]
    source = [(s, 'r', PY_SOURCE) for s in importlib.machinery.SOURCE_SUFFIXES]
    bytecode = [(s, 'rb', PY_COMPILED) for s in importlib.machinery.BYTECODE_SUFFIXES]

    return extensions + source + bytecode

[...]

for suffix, mode, type in get_suffixes():
        if type in _VALID_MODULE_TYPES:
           path = prefix + suffix
           if self.file_system.isfile(path):
                   return path, type

[...]

if type is PY_SOURCE:
        code = self.file_system.readbytes(path).replace(b("\r\n"), b("\n"))
        return compile(code, path, "exec")
elif type is PY_COMPILED:
        code = self.file_system.readbytes(path)
        if code[:4] != importlib.util.MAGIC_NUMBER:
                return None
        return marshal.loads(code[8:])
elif type is C_EXTENSION:
        code = self.file_system.readbytes(path)

3) Further refactoring

We can also further refactor by writing

_VALID_MODULE_TYPES = [ModuleType.PY_SOURCE, ModuleType.PY_COMPILED, ModuleType.C_EXTENSION]

def get_suffixes(return_as_dict=False):
    if return_as_dict:
        suffixes = {
            ModuleType.C_EXTENSION: (importlib.machinery.EXTENSION_SUFFIXES, 'rb'),
            ModuleType.PY_SOURCE: (importlib.machinery.SOURCE_SUFFIXES, 'r'),
            ModuleType.PY_COMPILED: (importlib.machinery.BYTECODE_SUFFIXES, 'rb')
        }
    else:
        extensions = [(s, 'rb', ModuleType.C_EXTENSION) for s in importlib.machinery.EXTENSION_SUFFIXES]
        source = [(s, 'r', ModuleType.PY_SOURCE) for s in importlib.machinery.SOURCE_SUFFIXES]
        bytecode = [(s, 'rb', ModuleType.PY_COMPILED) for s in importlib.machinery.BYTECODE_SUFFIXES]
        suffixes = extensions + source + bytecode
    return suffixes

so that we can set in which order we want to go through the _VALID_MODULE_TYPES and we also improve by iterating less when

suffixes = get_suffixes(return_as_dict=True)
for module_type in _VALID_MODULE_TYPES:
    for suffix in suffixes[module_type][0]:
        path = prefix + suffix
        if self.file_system.isfile(path):
            return path, module_type
return None, None