I created some arcpy script which actually runs slow on huge dataset. such as 100000 feature in it. I read from below path that multiprocessing can boost up arcpy script script performance. http://blogs.esri.com/esri/arcgis/2011/08/29/multiprocessing/ I tried it myself and applied it in my code.
It is giving an error "Could not find file: from multiprocessing.forking import main;main().mxd"
I could not understand what does this error means, and was unable to get any reference or help from any Internet source about the error.
Below is the code which actually find duplicates in a neaby buffer area(defined by user input)
import arcpy
import os
import multiprocessing
# defining input
inputFeatureClass = arcpy.GetParameterAsText(0)
InputField = arcpy.GetParameterAsText(1)
Distance = arcpy.GetParameterAsText(2)
#checking whether UNIID and dup field exist or not, if not than adding it
to_add = ["UNIID","Dup"]
#Create a list of existing field names
fieldList = arcpy.ListFields(inputFeatureClass)
fieldName = [f.name for f in fieldList]
for field in to_add:
if not field in fieldName:
arcpy.AddField_management(inputFeatureClass,"UNIID","TEXT")
arcpy.AddField_management(inputFeatureClass,"Dup","SHORT")
#updating squence number into UNIID Field
rows = arcpy.UpdateCursor(inputFeatureClass)
counter = 1
for row in rows:
row.UNIID = counter
rows.updateRow(row)
counter += 1
#setting up progressor
RowCount = arcpy.GetCount_management(inputFeatureClass)
PACnt = int(RowCount.getOutput(0))
arcpy.SetProgressor("step", "Finding Duplicate...",0, PACnt, 1)
def Find_dup(oid_range):
i = oid_range[0]
j = oid_range[1]
lyr = arcpy.management.MakeFeatureLayer(inputFeatureClass, 'layer{0}'.format(i),"OID >= {0} AND OID <= {1}".format(i, j))
#selecting neaby Point for every point inside searchcursor
with arcpy.da.SearchCursor(lyr, ["UNIID", InputField]) as cursor:
for row in cursor:
value=row[0]
field="UNIID"
exp=field + "='" +value+"'"
point_lyr=arcpy.SelectLayerByAttribute_management(lyr,"NEW_SELECTION",exp)
arcpy.SelectLayerByLocation_management(point_lyr, overlap_type="WITHIN_A_DISTANCE", select_features=point_lyr, search_distance=Distance + " Meters", selection_type="NEW_SELECTION")
#pushing selected points into array
occursTwice = []
cursor = arcpy.da.SearchCursor (lyr, [InputField])
for row in cursor:
occursTwice.append (row[0])
del cursor
#counting occurance of every Field Value inside Array and updating into dup field
cursor = arcpy.da.UpdateCursor (lyr, [InputField, 'Dup'])
for row in cursor:
# check whether dup contains already a duplicate value, if yes than do not update lower value
if occursTwice.count(row[0]) > row[1]:
row[1] = occursTwice.count(row[0])
cursor.updateRow(row)
del cursor
del point_lyr
arcpy.SetProgressorPosition()
def main():
oid_ranges = [[0, 1000],[1001, 2000], [2001, 3000], [3001, 4000], [4001, 5000],[5001, 6000], [6001, 7000], [7001, 8000], [8001, 9000], [9001, 10001]]
arcpy.env.overwriteOutput = True
pool = multiprocessing.Pool(processes=4)
pool.map(Find_dup, oid_ranges)
pool.close()
pool.join()
if __name__ == '__main__':
main()
#Deleting UNIID Field and Unselect everything
arcpy.DeleteField_management(inputFeatureClass,"UNIID")
arcpy.SelectLayerByAttribute_management(inputFeatureClass,"CLEAR_SELECTION")
arcpy.ResetProgressor()