0

I've made created a script that uses multiprocessing to break up a very large point dataset into pieces and extract values from 2 different rasters to them. When I tested it using the command line with the code not set up to work as an ArcMap script tool yet it runs just fine and completes the process. However, now I'm trying to get it to work as a script tool. I'm not getting any syntax errors when I run it. The code starts running like it would from the command line but the problem is none of the children will run the processes that are suppose to get passed to them by the parent. For example, the code never makes is inside the getfloats function. The tool just keeps running endlessly without ever passing anything to the children. I've already read through some of the posts about multiprocessing with script tools and I haven't came across anyone else having this issue. Anyone have any ideas?

Here is the script my script tool uses as a source code:

import arcpy  
import floatpoint_getter

watershedpoints = arcpy.GetParameter(0)  
workspace = arcpy.GetParameter(1)  

def main():  
    arcpy.AddMessage("Calling multiprocessing code...")  
    floatpoint_getter.execute(watershedpoints,workspace)  

if __name__ == '__main__':  
    main()  

Here is my script that my script tool imports:

import arcpy
import os
from arcpy import env
from multiprocessing import Process, Lock, Array, Value
import time
import sys
import atexit
from time import clock
from itertools import repeat
from arcpy.sa import *

arcpy.CheckOutExtension("Spatial")
arcpy.env.overwriteOutput = True

CHILD_GO = 0
PARENT_GO = 1
PARENT_BUSY = 3
NO_MORE_DATA = 4
RESULTS_NOT_READY = 10
RESULTS_READY = 11
NO_RESULTS = 12
PROCS_GOING = 44
NO_PROCS_GOING = 45
NUM_PROCS = 5
SECTION_SIZE=500000

unitedPoints=os.path.join(env.workspace,"united_points")

minID=0
maxID=SECTION_SIZE

inRasterList=["N:\Wortmr\Bankfull_tool\Rasters\slope100float",
"N:\Wortmr\Bankfull_tool\Rasters\dem100float"]
mergelist=[]

def getfloats(Flag, Results, MinID, MaxID, printLock, layerLock, 
totalPoints, 
ID,):

    #code never makes it inside this function

    in_memory=r"in_memory\points"+str(MaxID.value)
    in_permanent=os.path.join(env.workspace,"points"+str(MaxID.value))

    mem_point=arcpy.MakeFeatureLayer_management(watershedPoints,
    "pointlayer")

    while True:

        while Flag.value != CHILD_GO and Flag.value != NO_MORE_DATA:
            arcpy.AddMessage("In while loop 1")
            pass


        if Flag.value == NO_MORE_DATA:
            arcpy.Delete_management(mem_point)  
            break


        minID = MinID.value+1
        maxID = MaxID.value

        while Results.value != RESULTS_NOT_READY:
            arcpy.AddMessage("In while loop 2")
            pass

        layerLock.acquire()
        arcpy.SelectLayerByAttribute_management(mem_point, 
        "CLEAR_SELECTION")
        print "OBJECTID MAX IS "+str(maxID)
        print "OBJECTID MIN IS "+str(minID)
        memm_point=arcpy.SelectLayerByAttribute_management(mem_point, 
        "NEW_SELECTION", "OBJECTID <= {} AND OBJECTID >= 
        {}".format(maxID,minID))
        arcpy.CopyFeatures_management(memm_point,in_memory)
        ExtractMultiValuesToPoints(in_memory, inRasterList, "NONE")
        arcpy.CopyFeatures_management(in_memory,in_permanent)
        mergelist.append(in_permanent)
        arcpy.SelectLayerByAttribute_management(mem_point, 
        "CLEAR_SELECTION")
        layerLock.release()


        Flag.value = PARENT_GO

def execute(watershedPoints,workspace):

    env.workspace=workspace

    import floatpoint_getter

    mem_point=arcpy.MakeFeatureLayer_management(watershedPoints,
    "pointlayer")

    Flags = [Value('i', PARENT_BUSY, lock=False) for i in repeat(None, 
    NUM_PROCS)]

    Results = [Value('i', RESULTS_NOT_READY, lock=False) for i in 
    repeat(None, NUM_PROCS)]

    minIDs = [Value('i', 0) for i in repeat(None, NUM_PROCS)]
    maxIDs = [Value('i', 0) for i in repeat(None, NUM_PROCS)]

    printLock = Lock() 
    layerLock = Lock()

    total_points=arcpy.GetCount_management(mem_point).getOutput(0)

    arcpy.AddMessage("Starting Workers")
    for i in xrange(NUM_PROCS):
        p = Process(target=getfloats, args=(Flags[i], Results[i], minIDs[i], 
        maxIDs[i], printLock, layerLock, total_points, i,))
        p.start()

        minIDs[0].value = 0
        maxIDs[0].value = SECTION_SIZE

    test = PROCS_GOING
    init = 0
    doneFlag = 0
    newMax = -1

    while True:
        test = NO_PROCS_GOING
        if init == 0:
            for i in xrange(NUM_PROCS):
                if i == 0:
                    Flags[i].value = CHILD_GO
                    continue

                minIDs[i].value = maxIDs[i-1].value #+1
                maxIDs[i].value = minIDs[i].value + SECTION_SIZE
                Flags[i].value = CHILD_GO
             init = 1

        else:

             for i in xrange(NUM_PROCS):
                if Flags[i].value == NO_MORE_DATA:
                    continue

                else:
                     test = PROCS_GOING
                     if Flags[i].value == PARENT_GO:

                        if newMax == -1:
                            minIDs[i].value = 
                            maxIDs[NUM_PROCS - 1].value + 1
                        else:
                            minIDs[i].value = newMax + 1

                        maxIDs[i].value = minIDs[i].value + SECTION_SIZE
                        newMax = maxIDs[i].value

                        if int(maxIDs[i].value) > int(total_points) + 
                        int(SECTION_SIZE):
                            Flags[i].value = NO_MORE_DATA

                            doneFlag = 1
                        else:
                            if doneFlag == 1:
                                Flags[i].value = NO_MORE_DATA
                            else:
                                Flags[i].value = CHILD_GO

            if test == NO_PROCS_GOING:
                 break

    arcpy.Merge_management(mergelist, unitedPoints)
  • 1
    Have you added in print statements to find out where it gets to, or where it's getting stuck? – Midavalo Jul 13 '17 at 17:26
  • Yes I put one at the beginning of the getfloats function and it never reaches it. I also put on at the beginning of the while True loop in the execute function and that one keeps getting printed repeatedly. So for whatever reason it won't call the getfloats function now, but it would when I had the code set up to run from the cmd line. – Rdubya54 Jul 13 '17 at 17:39
  • 2
    Add more print statements. Find out the values it is using at certain parts of your script, particularly right before you expect it to run parts that aren't running. Put them in and around your for i in xrange(NUM_PROCS) – Midavalo Jul 13 '17 at 17:43
  • Some multiprocessing tips and tricks: gis.stackexchange.com/a/140777/2856 Also you need to escape your backslashes in your file paths or you will get bit some day when python interprets \t as a tab character. – user2856 Jul 15 '17 at 11:08
  • Sounds like your script is running "in process" i.e it's not being run by python.exe but by the embedded python interpreter in arcmap.exe. You need to uncheck the run script in process box in the script tool properties in the tool box. Or use multiprocessing.set_executable('path/to/pythonw.exe') in your script (otherwise multiprocessing will try to use arcmap.exe to run the subprocesses) . – user2856 Jul 15 '17 at 11:21

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.