python - Python 多处理一长串列表

import time,os,numpy,multiprocessing from math import radians, sin, cos, acos, sqrt from multiprocessing.managers import SyncManager manager = SyncManager() def distance(pos1, pos2): R = 6378137.0 if pos1 == pos2: return 0.0 lat1 = radians(pos1[0]) lon1 = radians(pos1[1]) lat2 = radians(pos2[0]) lon2 = radians(pos2[1]) a = sin(lat1) * sin(lat2) + cos(lat1) * cos(lat2) * cos(lon2 - lon1) if a > 1: return 0.0 return acos(a) * R def cluster(points, radius, maxClusterList, ms): global mpPoints, mpRadius, clustersList, mpMS mpPoints = points mpRadius = radius mpMS = ms clustersList = manager.list() pool = multiprocessing.Pool(processes=len(os.sched_getaffinity(0))) pool.map(getMpPoints, points) staticClustersList = clustersList._getvalue() pool.map(rmSmallClusters, staticClustersList) if len(clustersList) > 0: maxCluster = max(clustersList, key=len) print("The max cluster seen was {}.".format(len(maxCluster)-1)) done = 0 else: print("There were no clusters found that fit the minimal cluster: {}.".format(ms)) done = 1 # clustersList = clustersList._getvalue() while len(clustersList) > 0 and done == 0: start_time = time.time() print(len(clustersList)) longestList = max(clustersList, key=len) if len(longestList)-1 >= ms and len(longestList)-1 > 0: clustersList.remove(longestList) for item in longestList: if type(item[0]) is str: maxClusterList.append(item[1]) else: # rmLongestList(item, ms) staticClustersList = clustersList._getvalue() for cluster in staticClustersList: cluster.append(item) pool.map(rmMpLongestList, staticClustersList) else: done = 1 print('Completed one cluster in {} seconds.\n'.format(time.time() - start_time)) pool.close() pool.join() return maxClusterList def getMpPoints(point): global clustersList, mpPoints ii = 0 pointsList = [] while ii < len(mpPoints): dist = distance(mpPoints[ii], point) if dist <= mpRadius: pointsList.append(mpPoints[ii]) if dist == 0.0: pointsList.append(tuple(("center",mpPoints[ii]))) ii=ii+1 clustersList.append(pointsList) def rmSmallClusters(cluster): global clustersList if len(cluster)-1 < mpMS: clustersList.remove(cluster) def rmMpLongestList(cluster): global clustersList item = cluster.pop() if len(cluster)-1 < mpMS: clustersList.remove(cluster) else: for cpoint in cluster: if type(cpoint[0]) is not str and item == cpoint: clustersList.remove(cluster) cluster.remove(cpoint) clustersList.append(cluster) break def rmLongestList(item, ms): global clustersList for cluster in clustersList: if len(cluster)-1 < ms: clustersList.remove(cluster) else: for cpoint in cluster: if type(cpoint[0]) is not str and item == cpoint: cluster.remove(cpoint) break def main(): points = [(37.11200449167421, -76.4678314007634), (37.11207628412798, -76.46774103035378), (37.11214807639325, -76.4676506598814), (37.11222133212531, -76.46828325187043), (37.11228265985711, -76.46810251161587), (37.11235445226874, -76.46801214139445), (37.11254889752732, -76.46756028934628), (37.11261168911815, -76.46810251161587), (37.1126131495857, -76.46882547112826), (37.11262068901043, -76.46746991874844), (37.11264914325335, -76.4662950952672), (37.11265354860385, -76.46846399187402), (37.11266255073489, -76.4678314007634), (37.11266401327842, -76.4685543617817), (37.11268348112467, -76.46801214139445), (37.11276427141113, -76.46728917736449), (37.11277178921591, -76.46593360898622), (37.11278373209291, -76.466746951707), (37.11279713312621, -76.46828325187043), (37.11280466442346, -76.46692769384377), (37.11280613383346, -76.4676506598814), (37.11284652813964, -76.46728917736449), (37.11285404563149, -76.46593360898622), (37.11286451256181, -76.46602398065055), (37.11287038541584, -76.46891584078493), (37.11287645489082, -76.46683732280675), (37.11288084971368, -76.46900621037886), (37.11291684603231, -76.4664758380313), (37.11292731249018, -76.46656620931927), (37.11292878477536, -76.46728917736449), (37.11294217800486, -76.46882547112826), (37.11295723575319, -76.46611435225216), (37.1129587113913, -76.46683732280675), (37.11298257651604, -76.46846399187402), (37.11299010958372, -76.4671084357296), (37.11300809115821, -76.46584323725914), (37.11306483326211, -76.46846399187402), (37.11307236601687, -76.4671084357296), (37.1130932977684, -76.46728917736449), (37.11310523022596, -76.46810251161587), (37.11311275787527, -76.466746951707), (37.11313516032654, -76.4676506598814), (37.11314415635486, -76.46701806481805), (37.11319501408888, -76.466746951707), (37.11320400421574, -76.46611435225216), (37.11321594650432, -76.46692769384377), (37.11322641259393, -76.46701806481805), (37.11323540439991, -76.46638546668062), (37.11324587097047, -76.4664758380313), (37.11325781039028, -76.46728917736449), (37.11326974318622, -76.46810251161587), (37.11329967301633, -76.4676506598814), (37.11330113812231, -76.46837362190361), (37.11332812697303, -76.4664758380313), (37.11333859347334, -76.46656620931927), (37.11337750181237, -76.46548174972361), (37.11338339448871, -76.46837362190361), (37.11342525333505, -76.46873510140881), (37.11344472084971, -76.46819288177451), (37.11346873897293, -76.4648491441215), (37.11347170542896, -76.4662950952672), (37.11348511620944, -76.4678314007634), (37.11349411274831, -76.46719880657842), (37.11350310521705, -76.46656620931927), (37.11353890297604, -76.46900621037886), (37.11354790694312, -76.46837362190361), (37.11355690684015, -76.46774103035378), (37.11357341494634, -76.46575286546934), (37.11358536094971, -76.46656620931927), (37.11359730032908, -76.46737954808782), (37.11362869728236, -76.4676506598814), (37.11364962819081, -76.4678314007634), (37.11366155736039, -76.46864473162664), (37.1136705587843, -76.46801214139445), (37.11367955613814, -76.46737954808782), (37.11369002187647, -76.46746991874844), (37.11372288388999, -76.46846399187402), (37.11372893897239, -76.46638546668062), (37.11375886007858, -76.46593360898622), (37.11376033860066, -76.4666565805445), (37.11377374494825, -76.46819288177451), (37.11377979404726, -76.46611435225216), (37.1137977599593, -76.4648491441215), (37.11380220366922, -76.46701806481805), (37.11382460515409, -76.4679217711103), (37.11383360164382, -76.46728917736449), (37.11384406747786, -76.46737954808782), (37.11384699850201, -76.46882547112826), (37.1138620493663, -76.46611435225216), (37.11387546450753, -76.4676506598814), (37.11391438295241, -76.46656620931927), (37.11393825644075, -76.46819288177451), (37.11396965134432, -76.46846399187402), (37.11397865104001, -76.4678314007634), (37.11398617165286, -76.4664758380313), (37.11398911641007, -76.4679217711103), (37.11399058088631, -76.46864473162664), (37.11399515857844, -76.46584323725914), (37.11399811259538, -76.46728917736449), (37.11401904421867, -76.46746991874844), (37.11402950991221, -76.46756028934628), (37.11403243902538, -76.46900621037886), (37.11403702661768, -76.46620472379105), (37.11404896988098, -76.46701806481805), (37.11406842682027, -76.4664758380313), (37.11406990199372, -76.46719880657842), (37.11409982631415, -76.466746951707), (37.11410881476664, -76.46611435225216), (37.11411176527425, -76.46756028934628), (37.11415966851602, -76.46584323725914), (37.11416555676233, -76.46873510140881), (37.11417161498651, -76.4666565805445), (37.1141745578327, -76.46810251161587), (37.11420741465432, -76.46909657991004), (37.11422541720254, -76.4678314007634), (37.11423145611347, -76.46575286546934), (37.11423441237153, -76.46719880657842), (37.11423734755637, -76.46864473162664), (37.11424043795243, -76.46512026118445), (37.11426727831216, -76.46819288177451), (37.11428820839452, -76.46837362190361), (37.11430767240445, -76.4678314007634), (37.11432713338494, -76.46728917736449), (37.11435705770187, -76.46683732280675), (37.1143869798725, -76.46638546668062), (37.11439285408164, -76.46927731878414), (37.1144064327264, -76.46584323725914), (37.11445125145169, -76.4676506598814), (37.11446171703015, -76.46774103035378), (37.11446477311067, -76.46421653544611), (37.11447673350624, -76.46502988889284), (37.11448411339125, -76.46864473162664), (37.11449457809504, -76.46873510140881), (37.1145035785566, -76.46810251161587), (37.11451550726641, -76.46891584078493), (37.11451860461806, -76.4653913776829), (37.11452450884736, -76.46828325187043), (37.11454851981214, -76.46493951653854), (37.11456343201356, -76.46719880657842), (37.1145753682336, -76.46801214139445), (37.11459776238456, -76.46891584078493), (37.1146142882266, -76.46692769384377), (37.11461722884904, -76.46837362190361), (37.11462475446051, -76.46701806481805), (37.11465008397739, -76.46936768812706), (37.11465021843971, -76.46439728109553), (37.11466366329743, -76.46593360898622), (37.11471141062729, -76.46918694937848), (37.11472041358292, -76.4685543617817), (37.11474134312017, -76.46873510140881), (37.11476080833534, -76.46819288177451), (37.1147683309247, -76.46683732280675), (37.11477434584371, -76.46475877164175), (37.11477731871921, -76.46620472379105), (37.11479676989256, -76.46566249361682), (37.11480266838873, -76.4685543617817), (37.1148041298421, -76.46927731878414), (37.11481166712208, -76.4679217711103), (37.11481472624972, -76.46439728109553), (37.11482817176723, -76.46593360898622), (37.1148325977957, -76.46810251161587), (37.11483406260195, -76.46882547112826), (37.11483715366014, -76.46530100557946), (37.11483863890129, -76.46602398065055), (37.11485352815435, -76.46828325187043), (37.11487604278277, -76.46421653544611), (37.11489245017081, -76.46719880657842), (37.11489698001552, -76.46439728109553), (37.11490585267293, -76.46873510140881), (37.1149089398226, -76.46521063341332), (37.11491631734042, -76.46882547112826), (37.11493885353602, -76.46475877164175), (37.11495978982379, -76.46493951653854), (37.11496717772194, -76.4685543617817), (37.11497025784953, -76.46502988889284), (37.1150090365833, -76.46891584078493), (37.11500914351134, -76.46394541650163), (37.1150360263762, -76.46701806481805), (37.11505998973141, -76.46367429699274), (37.11506297953745, -76.46512026118445), (37.11506594826938, -76.46656620931927), (37.11506889592773, -76.46801214139445), (37.11507789069407, -76.46737954808782), (37.11508242422043, -76.46457802649407), (37.11511382905736, -76.4648491441215), (37.11513168668394, -76.4685543617817), (37.11515420927984, -76.46448765382617), (37.11515570107078, -76.46521063341332), (37.11515718759321, -76.46593360898622), (37.11516308099899, -76.46882547112826), (37.11517960182752, -76.46683732280675), (37.11518107653618, -76.46756028934628), (37.11518254597653, -76.46828325187043), (37.11518561437849, -76.46475877164175), (37.115188588861, -76.46620472379105), (37.11519301107167, -76.46837362190361), (37.11520200792462, -76.46774103035378), (37.11521552553978, -76.46430690830218), (37.11522440588465, -76.46864473162664), (37.11523795463457, -76.46521063341332), (37.11524842244955, -76.46530100557946), (37.11525286477821, -76.46746991874844), (37.11527232211021, -76.46692769384377), (37.11530076034318, -76.46575286546934), (37.11532169494939, -76.46593360898622), (37.11534558456908, -76.46756028934628), (37.11539791253446, -76.46801214139445), (37.11541143768115, -76.46457802649407), (37.11541441568176, -76.46602398065055), (37.11541884316939, -76.46819288177451), (37.11543087875958, -76.46403578954583), (37.11545479985601, -76.46566249361682), (37.11545776221189, -76.4671084357296), (37.11546228514133, -76.46430690830218), (37.11546822837317, -76.46719880657842), (37.11547424684998, -76.46512026118445), (37.11548322233541, -76.46448765382617), (37.11548620193438, -76.46593360898622), (37.1154996263845, -76.46746991874844), (37.11550861687935, -76.46683732280675), (37.11551760330401, -76.46620472379105), (37.11552807026973, -76.4662950952672), (37.1155310236871, -76.46774103035378), (37.11553705318384, -76.46566249361682), (37.11555798799848, -76.46584323725914), (37.11558188006766, -76.46746991874844), (37.11558491567729, -76.46394541650163), (37.11558790358085, -76.4653913776829), (37.11569857516704, -76.46421653544611), (37.11571053810049, -76.46502988889284), (37.11572397742697, -76.46656620931927), (37.11573296171584, -76.46593360898622), (37.11573444409466, -76.4666565805445), (37.11577483015303, -76.4662950952672), (37.11581223390617, -76.46448765382617), (37.1158241948571, -76.46530100557946), (37.11585261169393, -76.46412616252734), (37.11586606556713, -76.46566249361682), (37.11586755018276, -76.46638546668062), (37.11587653304777, -76.46575286546934), (37.11588401796675, -76.46439728109553), (37.11596776456258, -76.46512026118445), (37.11596925333045, -76.46584323725914), (37.11614124399208, -76.46448765382617), (37.11615171253891, -76.46457802649407), (37.11618311770684, -76.4648491441215), (37.11625340447803, -76.46403578954583), (37.11630574847823, -76.46448765382617)] filename = "outfile.txt" start_time = time.time() manager.start() print('Processing coordinates...') clusters = cluster(points, 70, [], 6) rowcount = 0 f = open(filename, 'w') for c in clusters: f.write(str(str(c[0]) + ',' + str(c[1]) +'\n')) rowcount += 1 f.close() print('{} clusters with {} or more coordinates in them.\n'.format(rowcount, 6)) end_time = time.time() print('Coordinates written to the {} file.'.format(filename)) print('Completed in {:.2f} seconds.\n'.format(end_time - start_time)) if __name__ == "__main__": main()