How to incorporate multiprocessing into existing python code?

Question

I am having a serious problem wrapping my head around using multiprocessing in my existing python code. Below is my program gauss.py which essentially fits a Gaussian to data. What is the correct way to incorporate multiprocessing and run this script multiple times using different input files? Should I create a separate .py script that calls this script as a function? Or, do I include a main section underneath all of the existing code?

Also, I am currently manually entering in the data input file in the command line when executing the script. I'm guessing that will need to be changed to some sort of queue format?

import json
import sys
import numpy
import pylab
from numpy import * #log, exp, pi
import scipy.stats, scipy
import pymultinest
import os

#-------

infile = sys.argv[1]
file = infile[-5:]
outfile = "out/test/"+init.target+file

wave,flux = numpy.loadtxt(infile, usecols=(0,1), unpack=True)

import init #initialization file

x       = wave[init.start:init.end]
ydata   = abs(flux[init.start:init.end])
maxy    = max(flux[init.start:init.end])
textpos = (.1*(init.plotmax-init.plotmin))+init.plotmin
systemic= (1.+init.red)*init.orig_wave
cont    = flux[init.low1:init.upp1] #select continuum adjacent to emission line
avg     = sum(cont)/len(cont)

stdev   = numpy.std(cont) #stnd dev of continuum flux
noise   = stdev * numpy.sqrt(ydata / avg) #signal dependant noise

dum = 0

###### GAUSSIAN MODEL ######

def make_gauss(mu, sigma, N):

    s = -1.0 / (2 * sigma * sigma)
    def f(x):
        return N * numpy.exp(s * (x - mu)*(x - mu))
    return f

def model1(pos1, width1, height1):
    gaussian1 = make_gauss(pos1, width1, height1)
    return  gaussian1(x) + avg

def prior(cube, ndim, nparams):
    cube[0] = init.minwave + (cube[0]*init.wave_range)      # uniform wavelength prior
    cube[1] = init.minwidth + (cube[1]*(init.maxwidth-init.minwidth))   # uniform width prior
    cube[2] = init.fluxsigma * stdev * 10**(cube[2]*5)  # log-uniform flux prior 

# ----------------------
# analyse with 1 gaussian

def loglike1(cube, ndim, nparams):
    pos1, width1, height1 = cube[0], cube[1], cube[2]
    ymodel1 = model1(pos1, width1, height1)
    loglikelihood =-0.5 * (((ymodel1 - ydata) / noise)**2).sum()
    return loglikelihood

# number of dimensions our problem has
parameters = ["pos1", "width1", "height1"]
n_params = len(parameters)

# run MultiNest
pymultinest.run(loglike1, prior, n_params, outputfiles_basename=outfile + '_1_', n_live_points = 200, multimodal = False, resume = False, verbose = False)

Answer 1

You could parallelize the processing inside loglike1.

But it is better to run your script with MPI. For example, "mpirun -np 4" will run your script 4 times. Multinest realises it is in MPI mode and dispatches the likelihood calls. pymultines loads the multinest MPI library automatically if you have mpi4py installed.