如何使用熊猫Series / DataFrame从类字典类的对象中提取数据

Question

It's a homework from school I was doing...

So basically I was asked to scan a given directory and find all the .py files in it, and count given attributes, which are classes and functions(including methods in classes) defined in the file, and total lines and characters for each file. And print all of the data in a table on the terminal.

To print the table, it was suggested by my lecturer to use a package called prettytable , although for me it's not pretty at all.

I want to use pandas .
The reason is simple: for each file count its 4 attributes --> a nested-dict is so naturally recalled here. And pandas.DataFrame is 100% perfect for record a nested-dict.

Scanning and summarizing are the easy part, what actually got me stuck is how to make the data container flexible and scalable.

Built-in dict can't initialize with 4 existed key-value pairs in it, so I build a class CountAttr(MutableMapping) and use another class FileCounter to create and count every attribute for every file.

However, pandas.DataFrame only recognize the first layer of this dict-like object. And I have read the source files of DataFrame and Series, still unable to figure out how to solve this.

So my question is,
how to make pandas.DataFrame/Series extract the data from a dictionary whose values are dict-like objects?

PS I'm open for every advice for the following code, coding style, implementing way, everything. Much appreciate!

from collections.abc import MutableMapping
from collections import defaultdict
import pandas as pd
import os

class CounterAttr(MutableMapping):
""" Initialize a dictionary with 4 keys whose values are all 0,

    keys:value
    - 'class': 0
    - 'function': 0
    - 'line': 0
    - 'char': 0

    interfaces to get and set these attributes """

    def __init__(self):
        """ Initially there are 4 attributes in the storage"""
        # key: counted attributes | value: counting number
        self.__dict__ = {'class': 0, 'function': 0, 'line': 0, 'char': 0}

    def __getitem__(self, key):
        if key in self.__dict__:
            return self.__dict__[key]
        else:
            raise KeyError

    def get(self, key, defaut = None):
        if key in self.__dict__:
            return self.__dict__[key]
        else:
            return defaut

    def __setitem__(self, key, value):
        self.__dict__[key] = value

    def __delitem__(self, key):
        del self.__dict__[key]

    def __len__(self):
        return len(self.__dict__)

    def __iter__(self):
        return iter(self.__dict__)

    def get_all(self):
        """ return a copy of the self._storagem, in case the internal data got polluted"""
        copy = self.__dict__.copy()
        return copy

    def to_dict(self):
        return self.__dict__

    def __repr__(self):
        return '{0.__class__.__name__}()'.format(self)

class FileCounter(MutableMapping):
""" Discribe the object the store all the counters for all .py files

    Attributes:
    - 

"""
    def __init__(self):
        self._storage = dict()

    def __setitem__(self, key, value = CounterAttr()):
        if key not in self._storage.keys():
            self._storage[key] = value
        else:
            print("Attribute exist!")

    def __getitem__(self, key):
        if key in self._storage.keys():
            return self._storage[key]
        else:
            self._storage[key] = CounterAttr()

    def __delitem__(self, key):
        del self._storage[key]

    def __len__(self):
        return len(self._storage)

    def __iter__(self):
        return iter(self._storage)






def scan_summerize_pyfile(directory, give_me_dict = False):
""" Scan the passing directory, find all .py file, count the classes, funcs, lines, chars in each file
    and print out with a table
"""
    file_counter = FileCounter()


    if os.path.isdir(directory):                                            # if the given directory is a valid one

        os.chdir(directory)                                                 # change the CWD
        print("\nThe current working directory is {}\n".format(os.getcwd()))

        file_lst = os.listdir(directory)                                    # get all files in the CWD

        for a_file in file_lst:                                             # traverse the list and find all pyfiles
            if a_file.endswith(".py"):

                file_counter[a_file] 

                try:
                    open_file = open(a_file, 'r')
                except FileNotFoundError:
                    print("File {0} can't be opened!".format(a_file))

                else:

                    with open_file:
                        for line in open_file:

                            if line.lstrip().startswith("class"):           # count the classes
                                file_counter[a_file]['class'] += 1

                            if line.lstrip().startswith("def"):             # count the functions
                                file_counter[a_file]['function'] += 1

                            file_counter[a_file]['line'] += 1               # count the lines

                            file_counter[a_file]['char'] += len(line)       # count the chars, no whitespace

    else:
        print("The directory", directory, "is not existed.\nI'm sorry, program ends.")


    return file_counter

# Haven't had the pandas codes part yet

Answer 1

I don't know why you would need something like what you wrote.. it all seems over engineered to me.

Assume read_file() returns the 4 attribute you want class, function, line, chars and you have a list of python file in a list_of_files , you can just do this:

result = []
for file in list_of_files:
    c, f, l, num_c = read_file(file)
    curr_dict = {'class':c, 'function':f, 'line':l, 'chars':num_c}
    result.append(curr_dict)
your_table = pd.DataFrame(result)

That's all you need.

You should generate the list of file and the function to read them separately, each different thing should live in it's own function - It definitely helps to separate the logic.

Answer 2

So this is my solution for the question. Instead of struggling for what pandas does, I try to figure out how to adjust my solution and make it easy for pandas to read my data. Thanks for the advice from @RockyLi

class FileCounter(object):
""" A class that contains the .py files counted 
    - .py files that are found in the given directory
    - attributes counted for each .py file
    - methods that scan and sumerized .py file
"""
def __init__(self, directory):
    self._directory = directory
    self._data = dict()        # key: file name | value: dict of counted attributes
    self._update_data()

def _read_file(self, filename):
    """ return a dictionary of attributes statistical data

        return type: dictionary
            - key: attributes' name
            - value: counting number of attributes

        it's not available to add a counting attributes interactively
    """

    class_, function_, line_, char_ = 0, 0, 0, 0
    try:
        open_file = open(filename, 'r')
    except FileNotFoundError:
        print("File {0} can't be opened!".format(filename))
    else:

        with open_file:
            for line in open_file:

                if line.lstrip().startswith("class "):           # count the classes
                    class_ += 1

                if line.lstrip().startswith("def "):             # count the functions
                    function_ += 1

                line_ += 1                                       # count the lines

                char_ += len(line)                               # count the chars, no whitespace
    return {'class': class_, 'function': function_, 'line': line_, 'char': char_}

def _scan_dir(self):
    """ return all of the file in the directory
        if the directory is not valid, raise and OSError
    """
    if os.path.isdir(self._directory):
        os.chdir(self._directory)
        return os.listdir(self._directory)

    else:
        raise OSError("The directory doesn't exist!")

def _find_py(self, lst_of_file):
    """ find all of the .py files in the directory"""
    lst_of_pyfile = list()

    for filename in lst_of_file:
        if filename.endswith('.py'):
            lst_of_pyfile.append(filename)

    return lst_of_pyfile

def _update_data(self):
    """ manipulate the _data\n
        this is the ONLY method that manipulate _data
    """
    lst_of_pyfile = self._find_py(self._scan_dir())

    for filename in lst_of_pyfile:
        self._data[filename] = self._read_file(filename)        # only place manipulate _data

def pretty_print(self):
    """ Print the data!"""

    df_prettyprint = pd.DataFrame.from_dict(self._data, orient = 'index')

    if not df_prettyprint.empty:
        print(df_prettyprint)
    else:
        print("Oops, seems like you don't get any .py file.\n You must be Java people :p")

def get_data(self):
    return self._data.copy()                                    # never give them the original data!

This class builds two interface to A. print table B. get the data for further use, also protect the data to be reached and modified directly.