(UPDATED) I am building a module to distribute agent based models, the idea is to split the model over multiple processes and then when the agents reach a boundary they are passed to the processor handling that region. I can get the processes set up and working with no communication, but cannot get the data to pass through the pipes and update the model segment on the other processor.
I have tried the solutions on stackoverflow and built a simple version of the model. As soon as I put in a model object into the pipe the model hangs (it works with python standard data types). The simple version just passes agents back and forth.
from pathos.multiprocessing import ProcessPool
from pathos.helpers import mp
import copy
class TestAgent:
"Agent Class-- Schedule iterates through each agent and \
executes step function"
def __init__(self, unique_id, model):
self.unique_id = unique_id
self.model = model
self.type = "agent"
def step(self):
pass
#print (' ', self.unique_id, "I have stepped")
class TestModel:
"Model Class iterates through schedule and executes step function for \
each agent"
def __init__(self):
self.schedule = []
self.pipe = None
self.process = None
for i in range(1000):
a = TestAgent(i, self)
self.schedule.append(a)
def step(self):
for a in self.schedule:
a.step()
if __name__ == '__main__':
pool = ProcessPool(nodes=2)
#create instance of model
test_model = TestModel()
#create copies of model to be run on 2 processors
test1 = copy.deepcopy(test_model)
#clear schedule
test1.schedule = []
#Put in only half the schedule
for i in range(0,500):
test1.schedule.append(test_model.schedule[i])
#Give process tracker number
test1.process = 1
#repeat for other processor
test2= copy.deepcopy(test_model)
test2.schedule = []
for i in range(500,1000):
test2.schedule.append(test_model.schedule[i])
test2.process = 2
#create pipe
end1, end2 = mp.Pipe()
#Main run function for each process
def run(model, pipe):
for i in range(5):
print (model.process)#, [a.unique_id for a in model.schedule])
model.step() # IT HANGS AFTER INITIAL STEP
print ("send")
pipe.send(model.schedule)
print ("closed")
sched = pipe.recv()
print ("received")
model.schedule = sched
pool.map(run, [test1, test2], [end1,end2])
The agents should switch processors and execute their print functions. (My next problem will be synchronizing the processors so they stay on each step, but one thing at a time.)
I got it to work. I was exceeding the pipe buffer limit in python (8192). This is particularly true if the agent holds a copy of the model as an attribute. A working version of the above code, which passes the agents one at a time is below. It uses Pympler to get the size of all the agents.
from pathos.multiprocessing import ProcessPool
from pathos.helpers import mp
import copy
# do a blocking map on the chosen function
class TestAgent:
"Agent Class-- Schedule iterates through each agent and \
executes step function"
def __init__(self, unique_id, model):
self.unique_id = unique_id
self.type = "agent"
def step(self):
pass
class TestModel:
"Model Class iterates through schedule and executes step function for \
each agent"
def __init__(self):
from pympler import asizeof
self.schedule = []
self.pipe = None
self.process = None
self.size = asizeof.asizeof
for i in range(1000):
a = TestAgent(i, self)
self.schedule.append(a)
def step(self):
for a in self.schedule:
a.step()
if __name__ == '__main__':
pool = ProcessPool(nodes=2)
#create instance of model
test_model = TestModel()
#create copies of model to be run on 2 processors
test1 = copy.deepcopy(test_model)
#clear schedule
test1.schedule = []
#Put in only half the schedule
for i in range(0,500):
test1.schedule.append(test_model.schedule[i])
#Give process tracker number
test1.process = 1
#repeat for other processor
test2= copy.deepcopy(test_model)
test2.schedule = []
for i in range(500,1000):
test2.schedule.append(test_model.schedule[i])
test2.process = 2
#create pipe
end1, end2 = mp.Pipe()
#Main run function for each process
def run(model, pipe):
for i in range(5):
agents = []
print (model.process, model.size(model.schedule) )
model.step() # IT HANGS AFTER INITIAL STEP
#agent_num = list(model.schedule._agents.keys())
for agent in model.schedule[:]:
model.schedule.remove(agent)
pipe.send(agent)
agent = pipe.recv()
agents.append(agent)
print (model.process, "all agents received")
for agent in agents:
model.schedule.append(agent)
print (model.process, len(model.schedule))
pool.map(run, [test1, test2], [end1,end2])
Mike McKerns and Thomas Moreau --Thanks for the help you put me on the right path.
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.