Get files not in hidden folders

Question

The test is failing because it's getting the files from hidden folders too. How can I modify the code so that it skips the hidden folders?

It looks like the post is mostly code but I don't know what else to say, the question is very simple and all I'm going to do by writing more is explain the code in words instead why would I bother writing the code then. It would be easier to just copy a link here.

def get_files_not_in_hidden_folder(parent_folder: str, extension: str) -> List[str]:
    """
    Get all files recursively from parent folder,
    except for the ones that are in hidden folders
    """
    files = []
    for root, _, filenames in os.walk(parent_folder):
        for filename in filenames:
            if filename.endswith(extension) and not root.startswith('.'):
                files.append(os.path.join(root, filename))
    logger.debug(f"get_files_not_in_hidden_folder: {parent_folder}, {extension} -> {files}")
    return files

def test_get_files_not_in_hidden_folder():
    Path('tmp').mkdir(parents=True, exist_ok=True)
    Path('tmp/test.json').touch()
    Path('tmp/tmp/.tmp').mkdir(parents=True, exist_ok=True)
    Path('tmp/tmp/.tmp/test.json').touch()
    Path('tmp/.tmp/tmp').mkdir(parents=True, exist_ok=True)
    Path('tmp/.tmp/tmp/test.json').touch()

    assert get_files_not_in_hidden_folder('tmp', '.json') == ['tmp/test.json']

    shutil.rmtree(Path('tmp'))

Answer 1

What you call root is the full path, including parent names.

If you want to convert to just the directory name, you can use os.path.basename , like:

    for root, _, filenames in os.walk(parent_folder):
        for filename in filenames:
            if filename.endswith(extension) and "/." not in root:
                files.append(os.path.join(root, filename))

Answer 2

I would implement this something like as follows...

def my_walk(root_dir):
    files,dirs = [],[]
    try:
        for fname in os.listdir(root_dir):
            if not fname.startswith("."):
               fpath = os.path.join(root_dir,fname)
               if os.path.isdir(fpath):
                  dirs.append(fpath)
               else:
                  files.append(fpath)
    except:
        print("SKIP:",root_dir)
    yield root_dir,dirs,files
    for d in dirs:
        yield from my_walk(d)

    

I think should work...

for root, _, filenames in my_walk(parent_folder):
    print(f"{root} contains {filenames}")