1

I am trying to take advantage of multiprocessing for what would otherwise be a very long run time for a loop execution.

The basic idea is to solve for the optimal route between a given Origin and Destination, then save the route, and then add the route back in as a restriction to then solve for the next best alternative using unique edges. I need to iterate over many OD pairs. My code is pasted below, I am not sure how to pass some arguments to the function correctly.

import subprocess
```# Description: Calculates the K fastest routes between an origin and destination, using 1st, ```2nd, and K-1 routes as line restrictions           
# Requirements: Network Analyst Extension

#Import system modules
import arcpy
from arcpy import env

import os
import os,sys,time, itertools
import pandas as pd
import numpy as np
from itertools import combinations
from scipy import sparse
import multiprocessing
import concurrent.futures

arcpy.CheckOutExtension("Network")
print "extensions checked out"

#Define Working Directory
working_dir = "C:/Users/"

# Load Network Database
network = working_dir+"roads_2010_diss_ND.nd"

# Load Centroid (Network Nodes) Shapefile
routeStops = working_dir+"centroids_2012_project_1.shp"
print "centroids loaded"


## Need to make a Geodatabase to write K-Fastest Routes to and import back in as restrictions, check to see if it already exists, if so, delete and make new database
## Need to make a Geodatabase for the Origin-Destination Shapefiles
if arcpy.Exists(working_dir+"optimalRoutes.gdb/"):
       arcpy.Delete_management(working_dir+"optimalRoutes.gdb")
       arcpy.Delete_management(working_dir+"odPoints.gdb")
       print "old geodatabase deleted"
       arcpy.management.CreateFileGDB(out_folder_path=working_dir, out_name="optimalRoutes.gdb")
       arcpy.management.CreateFileGDB(out_folder_path=working_dir, out_name="odPoints.gdb")
       edgepath = working_dir+"optimalRoutes.gdb/"
       print "Created Geodatabase"
else:         
       arcpy.management.CreateFileGDB(out_folder_path=working_dir, out_name="optimalRoutes.gdb")
       arcpy.management.CreateFileGDB(out_folder_path=working_dir, out_name="odPoints.gdb")
       edgepath = working_dir+"optimalRoutes.gdb/"
       print "Created Geodatabase"

## Make Centroid Shapefile a Layer 
centLayer = arcpy.MakeFeatureLayer_management(in_features=routeStops, out_layer="centroids_Layer")
centLayer = centLayer.getOutput(0)

# Number of Best Routes to Solve
kRoutes = [1,2,3]
 
# Number of Origins 
originFips= pd.read_csv(working_dir+"county_pairs_2010_1.csv", usecols = ['originfips'])
indexName = np.arange(len(originFips))
destinationFips= pd.read_csv(working_dir+"county_pairs_2010_1.csv", usecols = ['destinationfips'])
fips = pd.DataFrame(originFips, index= indexName)
fips['destinationFips']=destinationFips

##Initialize Where Clause and Path Names for Route Solver Output Create O-D Point Shapefiles
whereClause = [None]*len(originFips)
edgename = [None]*len(originFips)
junctionname = [None]*len(originFips)
turnname = [None]*len(originFips)
centName= [None]*len(originFips)
for index, row in fips.iterrows():
        whereClause[index] =  '"Fips"=' + str(row['originfips']) +" "+ "OR" +" "+'"Fips"='+str(row['destinationFips'])
        edgename[index]= "Edges"+"_"+str(row['originfips'])+"_"+str(row['destinationFips'])
        junctionname[index] = "Junctions"+"_"+str(row['originfips'])+"_"+str(row['destinationFips'])
        turnname[index] = "Turns"+"_"+str(row['originfips'])+"_"+str(row['destinationFips'])
        centName[index] = working_dir+"odPoints.gdb/"+"cent_"+str(row['originfips'])+"_"+str(row['destinationFips'])
        arcpy.SelectLayerByAttribute_management(in_layer_or_view=centLayer, selection_type="NEW_SELECTION", where_clause=whereClause[index])
        arcpy.CopyFeatures_management(centLayer,centName[index])


fips['edgename']=edgename
fips['junctionname']=junctionname
fips['turnname']=turnname

print turnname
print fips.loc[2,'originfips']

##
##Create Function to solve for 3 best routes so I can use multi processing
def solveMYRoutes(network, fips, working_dir, edgepath):
       index = fips.index
       print index
       ## Create Route Layer to Solve path between given origin and destination pair 
       outRouteLayer = arcpy.MakeRouteLayer_na(in_network_dataset=network,
                                               out_network_analysis_layer="Route",
                                               impedance_attribute="cost2cong",
                                               find_best_order="USE_INPUT_ORDER",
                                               ordering_type="PRESERVE_BOTH",
                                               time_windows="NO_TIMEWINDOWS",
                                               accumulate_attribute_name="",
                                               UTurn_policy="ALLOW_UTURNS",
                                               restriction_attribute_name="",
                                               hierarchy="NO_HIERARCHY",
                                               hierarchy_settings="",
                                               output_path_shape="TRUE_LINES_WITH_MEASURES",
                                               start_date_time="")
       outRouteLayer = outRouteLayer.getOutput(0) 

       ## Check to see if the O-D pair is the same or different
       if fips.loc[index,'originfips']==fips.loc[index,'destinationFips']:
              return
       else:
              
       ## Add Locations to the Route Layer - Origins and Destinations
              arcpy.AddLocations_na(in_network_analysis_layer = outRouteLayer,
                                    sub_layer="Stops",
                                    in_table= working_dir+"odPoints.gdb/"+"cent_"+str(fips.loc[index,'originfips'])+"_"+str(fips.loc[index,'destinationFips']),
                                    field_mappings="",
                                    search_tolerance="5000 Meters",
                                    sort_field="",
                                    search_criteria="lineEdges SHAPE;lineEdges_ND_Junctions NONE",
                                    match_type="MATCH_TO_CLOSEST",
                                    append="APPEND",
                                    snap_to_position_along_network="NO_SNAP",
                                    snap_offset="5 Meters",
                                    exclude_restricted_elements="INCLUDE",
                                    search_query="lineEdges #;lineEdges_ND_Junctions #")

              for k in kRoutes:

                     ## Solve for the Optimal Route
                     arcpy.Solve_na(outRouteLayer)

                     edges = str(fips.loc[index,'edgename'])+"_"+str(k)
                     junction =  str(fips.loc[index,'junctionname'])+"_"+str(k)
                     turn = str(fips.loc[index,'turnname'])+"_"+str(k)
                                       
                     ## Export Route and Edges to Geodatabase - Naming convention Edge_OriginID_DestinationID_RouteK
                     arcpy.CopyTraversedSourceFeatures_na(outRouteLayer, output_location=edgepath, edge_feature_class_name=edges, junction_feature_class_name=junction, turn_table_name=turn)
                     barrier =edgepath+"/"+edges
                     arcpy.Delete_management(edgepath + "/" + junction)
                     arcpy.Delete_management(edgepath + "/" + turn)
                       
                     ## Add Optimal Route to Route File as a Line Restriction with Sclaed Cost
                     if k<3:
                        arcpy.AddLocations_na(outRouteLayer, sub_layer="Line Barriers", in_table=barrier, field_mappings="Name Name #;BarrierType # 1;Attr_cost2cong # 20", search_tolerance="5000 Meters", sort_field="", search_criteria="roads2010_cong SHAPE;roads2010_cong_ND1_Junctions NONE", match_type="MATCH_TO_CLOSEST", append="APPEND", snap_to_position_along_network="NO_SNAP", snap_offset="5 Meters", exclude_restricted_elements="INCLUDE", search_query="roads2010_cong #;roads2010_cong_ND1_Junctions #") 
                     else:
                            continue
                       
       del barrier
       return index
   
#### Initialize Multiprocessing number of cores====================================================================================================================================

pool = multiprocessing.Pool(processes=3)
pool.map(solveMYRoutes, fips)
pool.close()
pool.join()
if __name__ == "__main__":
       main()
0

1 Answer 1

0

concurrent.futures is a lot nicer to use, and I see you already have the Python 2 backport installed.

I can't really tell how to set up the args without studying all of your code, but it would look something like this:

from concurrent.futures import ProcessPoolExecutor, as_completed
from multiprocessing import current_process
from time import sleep

def solveMYRoutes(network, fips, working_dir, edge_path):
    sleep(2)  # pretend to do some work
    pid = current_process().pid
    return 'hello from PID {}! my args were: {}, {}, {}, {}'.format(
        pid, network, fips, working_dir, edge_path)


def main():
    list_of_fips = list(range(10))
    network='nickelodeon'
    working_dir=r'C:\temp'
    edge_path='PCT'

    with ProcessPoolExecutor() as executor:
        fs = []

        print('submitting jobs')
        for fips in list_of_fips:
            f = executor.submit(solveMYRoutes,
                                network=network,
                                fips=fips,
                                working_dir=working_dir,
                                edge_path=edge_path)
            fs.append(f)  # keep a list of in-progress futures

        print('waiting for results')
        for f in as_completed(fs):
            print(f.result())  # retrieve the results (or raise errors) as they finish


if __name__ == '__main__':
    main()

I suggest making sure you can run some dummy code like the above before you attempt to add ArcPy into the equation. ArcPy is always difficult when it comes to multiprocessing, in my experience, so for anything performance critical I always try to avoid using ArcPy altogether and lean on open-source GIS libraries instead.

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.