Arcgis ModelBuilder - OD cost matrix on network with iteration on attributes

Question

I am totally new to ArcGis' ModelBuilder, and I need some advice as how to build an OD cost matrix for large sets of origins and destinations.

I want to compute an OD matrix where I can have as many as 20,000 points as origins and as destinations. Using the normal procedure with Network Analyst results in an "out of memory error". That's why I turned to modelbuilder.

I managed to (re-)build the core model that computes the distance between OD pairs with the network dataset (niger_ND) and the points (p100). The output is written in the database reseau.mdb. This works fine.

Now, I want to add some features to my model, but I do not know exactly where I should put them:

1. Each point in the layer containing the points has a unique identifier. As origins, I want to select only 1000 of them at a time. When the analysis is done, I want to relaunch the process on the next 1000 points until every point has been used. How should I design the model to use my points one batch at a time?
2. With each batch of origins, I save the result in a database. How can I avoid overwriting the previous result by giving a different name to the output at each iteration?

I guess it is not too complicated when you know ModelBuilder, but I do not fully understand its logic yet.

Answer 1

The answer by this question's asker has been moved from within the question to be an answer:

Here is the script I used to work around the problem. However, it does not strictly answer the questions.

# Import modules
import arcpy
import numpy
import multiprocessing
import time
import csv
start = time.time()
# Check out any necessary licenses
arcpy.CheckOutExtension("Network")
# Input arguments
print "Importing data"
Arguments_Array = numpy.loadtxt('C:\Users\ddeltenre\Desktop\data.csv',dtype='string',delimiter=',',skiprows=0)
processes = len(Arguments_Array)
query = '"OBJECTID" > {} AND "OBJECTID" <= {}'
for x in xrange(0,processes):
    # Script arguments
    network = Arguments_Array[x][0]
    Origins = Arguments_Array[x][1]
    Sort_Field_Origins = Arguments_Array[x][2]
    Destinations = Arguments_Array[x][3]
    Sort_Field_Destinations = Arguments_Array[x][4]
    Output_Feature = Arguments_Array[x][5]
    Output_Location = Arguments_Array[x][6] 
    ODMatrix = 'OD_Cost_Matrix'
    print "Computing OD cost matrix for network "
    print network
    # Benchmarking
    #print "Checking benchmarks"
    nOrigins = int(arcpy.GetCount_management(Origins).getOutput(0))
    nDestinations = int(arcpy.GetCount_management(Destinations).getOutput(0))
    if nOrigins*nDestinations > 100000000:
        print "Too many points! The dataset will be split"
        xrow = (10000000//nDestinations)+1
        loops = nOrigins/xrow
        tmpOrigins = "{}\\tmp_Origins.shp".format(Output_Location)
        for x in xrange(0, loops):
            loopstart = time.time()
            print "Processing... {} / {}".format(x+1,loops)
            tmpOutput_Feature = "{}_{}{}".format(x*xrow,(x+1)*xrow,Output_Feature)
            # Process: Make OD Cost Matrix Layer
            arcpy.MakeODCostMatrixLayer_na(network, ODMatrix, "Length", "", "", "Length", "ALLOW_UTURNS", "", "NO_HIERARCHY", "", "NO_LINES", "")
            # Process: Add Destinations
            arcpy.AddLocations_na(ODMatrix, "Destinations", Destinations, "Name ID #", "5000 Meters", Sort_Field_Destinations, "", "MATCH_TO_CLOSEST", "CLEAR", "NO_SNAP", "5000 Meters", "INCLUDE", "")
            #print "Destinations added: {} s".format(time.time()-loopstart)
            # Select part of the origins
            #selectstart = time.time()
            arcpy.MakeFeatureLayer_management(Origins,tmpOrigins,query.format(x*xrow,(x+1)*xrow))
            #print "Selecting points: {} s".format(time.time()-selectstart)
            #arcpy.Select_analysis(Origins, tmpOrigins, query)
            # Process: Add Origins
            #originsstart = time.time()
            arcpy.AddLocations_na(ODMatrix, "Origins", tmpOrigins, "Name ID #", "5000 Meters", Sort_Field_Origins, "", "MATCH_TO_CLOSEST", "CLEAR", "NO_SNAP", "5000 Meters", "INCLUDE", "")
            #print "Loading origins: {} s".format(time.time()-originsstart)
            # Process: Solve
            #solvestart = time.time()
            arcpy.Solve_na(ODMatrix, "SKIP", "TERMINATE")
            #print "Solving: {} s".format(time.time()-solvestart)
            # Process: Feature Class to Feature Class
            #exportstart = time.time()
            Selection = arcpy.SelectData_management(ODMatrix, "Lines")
            fields = arcpy.ListFields(Selection)
            field_names = [field.name for field in fields]
            field_names = [field_names[2], field_names[6]]
            with open("{}\\{}".format(Output_Location,tmpOutput_Feature),'wb') as f:
                dw = csv.DictWriter(f,field_names)
                #--write all field names to the output file
                dw.writeheader()
                #--now we make the search cursor that will iterate through the rows of the table
                with arcpy.da.SearchCursor(Selection,field_names) as cursor:
                    for row in cursor:
                        dw.writerow(dict(zip(field_names,row)))
            #print "Data exported: {} s".format(time.time()-exportstart)
            #pool.close()
            #Delete useless select points
            arcpy.Delete_management(tmpOrigins)
            arcpy.Delete_management(ODMatrix)
            print "Time in loop: {} s".format(time.time()-loopstart)            
    else:
        # Process: Make OD Cost Matrix Layer
        arcpy.MakeODCostMatrixLayer_na(network, ODMatrix, "Length", "", "", "Length", "ALLOW_UTURNS", "", "NO_HIERARCHY", "", "NO_LINES", "")
        # Process: Add Origins
        arcpy.AddLocations_na(ODMatrix, "Origins", Origins, "Name ID #", "5000 Meters", Sort_Field_Origins, "", "MATCH_TO_CLOSEST", "CLEAR", "NO_SNAP", "5000 Meters", "INCLUDE", "")
        # Process: Add Destinations
        arcpy.AddLocations_na(ODMatrix, "Destinations", Destinations, "Name ID #", "5000 Meters", Sort_Field_Destinations, "", "MATCH_TO_CLOSEST", "CLEAR", "NO_SNAP", "5000 Meters", "INCLUDE", "")
        # Process: Solve
        arcpy.Solve_na(ODMatrix, "SKIP", "TERMINATE")
        # Process: Feature Class to Feature Class
        Selection = arcpy.SelectData_management(ODMatrix, "Lines")
        fields = arcpy.ListFields(Selection)
        field_names = [field.name for field in fields]
        field_names = [field_names[2], field_names[6]]
        with open("{}\\{}".format(Output_Location,Output_Feature),'wb') as f:
            dw = csv.DictWriter(f,field_names)
            #--write all field names to the output file
            dw.writeheader()
            #--now we make the search cursor that will iterate through the rows of the table
            with arcpy.da.SearchCursor(Selection,field_names) as cursor:
                for row in cursor:
                    dw.writerow(dict(zip(field_names,row)))
        #print "Data exported: {} s".format(time.time()-exportstart)
        print "Done ({}s)".format(time.time()-start)
        arcpy.Delete_management(ODMatrix)
print "Done"
raw_input()