I've made created a script that uses multiprocessing to break up a very large point dataset into pieces and extract values from 2 different rasters to them. When I tested it using the command line with the code not set up to work as an ArcMap script tool yet it runs just fine and completes the process. However, now I'm trying to get it to work as a script tool. I'm not getting any syntax errors when I run it. The code starts running like it would from the command line but the problem is none of the children will run the processes that are suppose to get passed to them by the parent. For example, the code never makes is inside the getfloats function. The tool just keeps running endlessly without ever passing anything to the children. I've already read through some of the posts about multiprocessing with script tools and I haven't came across anyone else having this issue. Anyone have any ideas?
Here is the script my script tool uses as a source code:
import arcpy
import floatpoint_getter
watershedpoints = arcpy.GetParameter(0)
workspace = arcpy.GetParameter(1)
def main():
arcpy.AddMessage("Calling multiprocessing code...")
floatpoint_getter.execute(watershedpoints,workspace)
if __name__ == '__main__':
main()
Here is my script that my script tool imports:
import arcpy
import os
from arcpy import env
from multiprocessing import Process, Lock, Array, Value
import time
import sys
import atexit
from time import clock
from itertools import repeat
from arcpy.sa import *
arcpy.CheckOutExtension("Spatial")
arcpy.env.overwriteOutput = True
CHILD_GO = 0
PARENT_GO = 1
PARENT_BUSY = 3
NO_MORE_DATA = 4
RESULTS_NOT_READY = 10
RESULTS_READY = 11
NO_RESULTS = 12
PROCS_GOING = 44
NO_PROCS_GOING = 45
NUM_PROCS = 5
SECTION_SIZE=500000
unitedPoints=os.path.join(env.workspace,"united_points")
minID=0
maxID=SECTION_SIZE
inRasterList=["N:\Wortmr\Bankfull_tool\Rasters\slope100float",
"N:\Wortmr\Bankfull_tool\Rasters\dem100float"]
mergelist=[]
def getfloats(Flag, Results, MinID, MaxID, printLock, layerLock,
totalPoints,
ID,):
#code never makes it inside this function
in_memory=r"in_memory\points"+str(MaxID.value)
in_permanent=os.path.join(env.workspace,"points"+str(MaxID.value))
mem_point=arcpy.MakeFeatureLayer_management(watershedPoints,
"pointlayer")
while True:
while Flag.value != CHILD_GO and Flag.value != NO_MORE_DATA:
arcpy.AddMessage("In while loop 1")
pass
if Flag.value == NO_MORE_DATA:
arcpy.Delete_management(mem_point)
break
minID = MinID.value+1
maxID = MaxID.value
while Results.value != RESULTS_NOT_READY:
arcpy.AddMessage("In while loop 2")
pass
layerLock.acquire()
arcpy.SelectLayerByAttribute_management(mem_point,
"CLEAR_SELECTION")
print "OBJECTID MAX IS "+str(maxID)
print "OBJECTID MIN IS "+str(minID)
memm_point=arcpy.SelectLayerByAttribute_management(mem_point,
"NEW_SELECTION", "OBJECTID <= {} AND OBJECTID >=
{}".format(maxID,minID))
arcpy.CopyFeatures_management(memm_point,in_memory)
ExtractMultiValuesToPoints(in_memory, inRasterList, "NONE")
arcpy.CopyFeatures_management(in_memory,in_permanent)
mergelist.append(in_permanent)
arcpy.SelectLayerByAttribute_management(mem_point,
"CLEAR_SELECTION")
layerLock.release()
Flag.value = PARENT_GO
def execute(watershedPoints,workspace):
env.workspace=workspace
import floatpoint_getter
mem_point=arcpy.MakeFeatureLayer_management(watershedPoints,
"pointlayer")
Flags = [Value('i', PARENT_BUSY, lock=False) for i in repeat(None,
NUM_PROCS)]
Results = [Value('i', RESULTS_NOT_READY, lock=False) for i in
repeat(None, NUM_PROCS)]
minIDs = [Value('i', 0) for i in repeat(None, NUM_PROCS)]
maxIDs = [Value('i', 0) for i in repeat(None, NUM_PROCS)]
printLock = Lock()
layerLock = Lock()
total_points=arcpy.GetCount_management(mem_point).getOutput(0)
arcpy.AddMessage("Starting Workers")
for i in xrange(NUM_PROCS):
p = Process(target=getfloats, args=(Flags[i], Results[i], minIDs[i],
maxIDs[i], printLock, layerLock, total_points, i,))
p.start()
minIDs[0].value = 0
maxIDs[0].value = SECTION_SIZE
test = PROCS_GOING
init = 0
doneFlag = 0
newMax = -1
while True:
test = NO_PROCS_GOING
if init == 0:
for i in xrange(NUM_PROCS):
if i == 0:
Flags[i].value = CHILD_GO
continue
minIDs[i].value = maxIDs[i-1].value #+1
maxIDs[i].value = minIDs[i].value + SECTION_SIZE
Flags[i].value = CHILD_GO
init = 1
else:
for i in xrange(NUM_PROCS):
if Flags[i].value == NO_MORE_DATA:
continue
else:
test = PROCS_GOING
if Flags[i].value == PARENT_GO:
if newMax == -1:
minIDs[i].value =
maxIDs[NUM_PROCS - 1].value + 1
else:
minIDs[i].value = newMax + 1
maxIDs[i].value = minIDs[i].value + SECTION_SIZE
newMax = maxIDs[i].value
if int(maxIDs[i].value) > int(total_points) +
int(SECTION_SIZE):
Flags[i].value = NO_MORE_DATA
doneFlag = 1
else:
if doneFlag == 1:
Flags[i].value = NO_MORE_DATA
else:
Flags[i].value = CHILD_GO
if test == NO_PROCS_GOING:
break
arcpy.Merge_management(mergelist, unitedPoints)
for i in xrange(NUM_PROCS)
multiprocessing.set_executable('path/to/pythonw.exe')
in your script (otherwise multiprocessing will try to use arcmap.exe to run the subprocesses) .