execute .py file, call python function in for loop

Question

I have a function 'plot_rdm', which creates a plot and saves it as 'rdm.png'. I want several of these plots to be formed, each using a different.json file - so I have the function plot_rdm saved in 'plotrdm.py'.

In the saverdm.py file - I defined the filepath of the.json file I want to create a plot from and then called the plot_rdm function, looping over all of the files I want to create a plot from:

#import libraries
import numpy as np
import matplotlib
import matplotlib.pyplot
import matplotlib.pyplot as plt
import scipy
import scipy.spatial
import scipy.spatial.distance as sd
from scipy.spatial.distance import squareform
import os
import json 

# define fpath 
#i.e. fpath[0] will be the first filepath... 

path = './RDM_researchproject' 
rootdir = path
filepath = []
for subdir, dirs, files in os.walk(rootdir):
    for file in files:
        if file.startswith('Meadows'):
            count=0 # count default
            filepath.append(os.path.join(subdir, file))
            fpath = filepath[count]
            os.system("/home/taran/RDM_researchproject/AVIMA/plotrdm.py")
            count +=1      

The plotrdm.py file with the plot_rdm function is as follows:

def plot_rdm(fpath):
    import numpy as np
    import matplotlib
    import matplotlib.pyplot
    import matplotlib.pyplot as plt
    import scipy
    import scipy.spatial
    import scipy.spatial.distance as sd
    from scipy.spatial.distance import squareform
    import json 
    
    with open(fpath) as fhandle:
        data = json.load(fhandle)
     #inspect rdm stimuli labels 
    stim = data['stimuli']

    #contain all labels for y axis and x axis seperately  
    y_names = []
    for i in stim:
        y_names.append(i['name'])

    x_names = []
    for i in stim:
        x_names.append(i['name'])

    #create rdm array and squareform 
    rdm_array = np.array(data['rdm'])
    srdm = squareform(rdm_array)

    #label x and y axis on rdm 
    fig, ax = plt.subplots()
    rdm = ax.imshow(srdm)

    ax.set_xticks(np.arange(len(x_names)))
    ax.set_yticks(np.arange(len(y_names)))

    ax.set_xticklabels(x_names)
    ax.set_yticklabels(y_names)
    plt.setp(ax.get_xticklabels(), rotation=90, ha="right", rotation_mode="anchor")
    plt.plot(srdm)
    plt.imshow(srdm)
    plt.colorbar(mappable = None, cax = None, ax = None)
    fig.subplots_adjust(bottom=0.23)
import matplotlib.pyplot as plt
plt.savefig('rdm.png')

I am able to create the plots individually (ie when I don't call the plot_rdm function and loop over the files but I specify the filepath each time). But when I use the following code, I get an empty plot forming in the AVIMA folder. I'm not sure what's wrong in the saverdm file making this happen?

https://github.com/Taranks7/RDM_researchproject If I haven't explained what's going on well, this is the project I'm working on.

Thank you

Answer 1

When you want to call a python function from another file, you should not try to run another python process by calling os.system . Just import that function: from plotrdm import plot_rdm

Instead of using os.filewalk and a file.startswith check, we can cleanup the code a lot by using the nice python library glob . I throw in a enumerate for good measure.

Your new rdmsave.py

import glob

from plotrdm import plot_rdm

basedir = "."

if __name__ == "__main__":
    count = 0
    for count, path in enumerate(sorted(glob.glob(f'{basedir}/**/Meadow*.json', recursive=True)), start=1):
        print(f"processing {path}")
        output_image = f'rdm_{count - 1:02}.png'
        print(f"output image will be {output_image}")
        plot_rdm(path, output_image)
    print(f"processed {count} files")

Note that you may need to change basedir back to your local path.

And your plotrdm.py becomes:

import numpy as np
import matplotlib
import matplotlib.pyplot
import matplotlib.pyplot as plt
import scipy
import scipy.spatial
import scipy.spatial.distance as sd
from scipy.spatial.distance import squareform
import json
import matplotlib.pyplot as plt


def plot_rdm(fpath, output_filename):
    with open(fpath) as fhandle:
        data = json.load(fhandle)
        # inspect rdm stimuli labels

    stim = data['stimuli']

    # contain all labels for y axis and x axis seperately
    y_names = []
    for i in stim:
        y_names.append(i['name'])

    x_names = []
    for i in stim:
        x_names.append(i['name'])

    # create rdm array and squareform
    rdm_array = np.array(data['rdm'])
    srdm = squareform(rdm_array)

    # label x and y axis on rdm
    fig, ax = plt.subplots()
    rdm = ax.imshow(srdm)

    ax.set_xticks(np.arange(len(x_names)))
    ax.set_yticks(np.arange(len(y_names)))

    ax.set_xticklabels(x_names)
    ax.set_yticklabels(y_names)
    plt.setp(ax.get_xticklabels(), rotation=90, ha="right", rotation_mode="anchor")
    plt.plot(srdm)
    plt.imshow(srdm)
    plt.colorbar(mappable=None, cax=None, ax=None)
    fig.subplots_adjust(bottom=0.23)

    plt.savefig(output_filename)

I added the second argument output_filename to the plot_rdm function to make it possible to store each image in a new file.

The output on my machine reads

processing ./5/Meadows_avima-image-version1_v_v2_vital-macaw_2_tree.json
output image will be rdm_00.png
processing ./4/Meadows_avima-image-version1_v_v2_quick-louse_2_tree.json
output image will be rdm_01.png
processing ./1/Meadows_avima-image-version1_v_v2_better-hound_2_tree.json
output image will be rdm_02.png
processing ./3/Meadows_avima-image-version1_v_v2_huge-falcon_2_tree.json
output image will be rdm_03.png
processing ./2/Meadows_avima-image-version1_v_v2_guided-koi_2_tree.json
output image will be rdm_04.png
processed 4 files

And 4 png files are created in the current folder.