多次使用子进程和调用脚本来并行运行-如何检查返回码是否有错误

Question

Below is code section which loops through all the namespaces and then calls remove_empty_dirs for each one.

remove_empty_dirs uses subprocess to call a separate python script for each namespace. Using a counter we call up to 25 subprocesses at a time.

I use a ps -ef command to check if the subprocess script is still running.

I need to use some kind of p=subprocess . I think to be able to check on the return code for each execution , but how do I capture up to 25 process objects at once?

And what would I use instead of ps -ef to monitor when subprocess scripts are finished?

import os, sys, datetime, time, glob, re, commands
import subprocess
from sets import Set

def get_all_namespaces_run_remove_empty_dirs(filename):
    """
    Creates set of unique namespaces from applid.namespace.ctl file and calls remove_empty_dirs for each one.
    """

    print "Namespaces to be processed: ", sorted(namespaces)
    for name in sorted(namespaces):
        remove_empty_dirs(name)
        job_ct = 99
        while job_ct > 25:
            cmd = "ps -ef | grep archive_remove_empty_dirs.py | grep -vE 'grep|sh -c' | wc -l"
            status,job_ct = commands.getstatusoutput(cmd)
            print job_ct, " jobs are running"
            job_ct = int(job_ct)
            if job_ct > 25:
                print "waiting for jobs to finish"
                time.sleep(30)

def wait_for_subprocesses_to_end():

    time_ct = 0
    job_ct = 99
    while job_ct > 0:
        cmd = "ps -ef | grep archive_remove_empty_dirs.py | grep -vE 'grep|sh -c' | wc -l"
        status,job_ct = commands.getstatusoutput(cmd)
        job_ct = int(job_ct)
        if job_ct > 0:
            time_ct +=1
            if time_ct > 5:
                logging.info("%d jobs are still running" % job_ct)
                cmd = "ps -ef | grep archive_remove_empty_dirs.py | grep -vE 'grep|sh -c'"
                status,output = commands.getstatusoutput(cmd)
                print output
                time_ct = 0
            time.sleep(10)
    return

def remove_empty_dirs(NS_PREFIX):
    """
    executes archive_remove_empty_dirs.py to remove empty directories for each NS_PREFIX
    """

    global statistics, NS_SITE, app_state

    output_filename = "%s/%s_remove_empty_dirs.out" % (archivelog_dir,NS_PREFIX)
    err_filename = "%s/%s_remove_empty_dirs.err" % (archivelog_dir,NS_PREFIX)
    cmd = "%s/archive_remove_empty_dirs.py" % source_dir

    print "Executing %s %s %s" % (cmd, NS_PREFIX, NS_SITE)
    subprocess.Popen([cmd, NS_PREFIX, NS_SITE],
            stdout=open(output_filename, 'w'),
            stderr=open(err_filename, 'w'),
            preexec_fn=os.setpgrp
            )
    logging.info(NS_PREFIX + " removing empty directories...")

Answer 1

I solved this by

def get_all_namespaces_run_remove_empty_dirs(filename):

global master_ix

namespaces is a list of 50-100 items - call remove_empty_dirs for each one until 26 are called then call wait_for_subprocesses_to_end until less than 26 are running

for name in sorted(namespaces):
    remove_empty_dirs(name)
    while master_ix > 25:
        wait_for_subprocesses_to_end()
        time.sleep(10)

def wait_for_subprocesses_to_end():

this is the solution - the process_list is built by the function remove_empty_dirs and now we loop through this list getting the return code when it completes and removing it from the list.

global subprocess_failures, master_ix
j = len(process_list)
print "begin wait_for_subprocesses_to_end length of process_list = %d" % j
i = 0
while i < j:
    check_process, NS_PREFIX = process_list[i]
    ret_code = check_process.poll()
    #print i, ret_code
    if ret_code == None:
        i += 1
        continue
    else:
        del process_list[i]
        j -= 1
        if ret_code != 0:
            subprocess_failures +=1
            print "%s failure!!! rc=%d" % (NS_PREFIX, ret_code)

master_ix = len(process_list)
print "end wait_for_subprocesses_to_end length of process_list = %d" % master_ix
print "subprocess_failures = %d" % subprocess_failures
return

def remove_empty_dirs(NS_PREFIX):

executes archive_remove_empty_dirs.py to remove empty directories for each NS_PREFIX

this is the solution - save the process object (p) in process_list - each entry in process_list is a tuple of p and unique identifier of process

global statistics, NS_SITE, app_state, master_ix

output_filename = "%s/%s_remove_empty_dirs.out" % (archivelog_dir,NS_PREFIX)
err_filename = "%s/%s_remove_empty_dirs.err" % (archivelog_dir,NS_PREFIX)
cmd = "%s/archive_remove_empty_dirs.py" % source_dir

print "Executing %s %s %s" % (cmd, NS_PREFIX, NS_SITE)
p = subprocess.Popen([cmd, NS_PREFIX, NS_SITE],
        stdout=open(output_filename, 'w'),
        stderr=open(err_filename, 'w'),
        preexec_fn=os.setpgrp
        )
process_list.append((p,NS_PREFIX))
master_ix += 1
logging.info("%d.  %s removing empty directories..." % (master_ix, NS_PREFIX))