Take the 2-minute tour ×
Geographic Information Systems Stack Exchange is a question and answer site for cartographers, geographers and GIS professionals. It's 100% free, no registration required.

I have a python tool which investigates mxd files and clips/projects and exports all visible datasets. Recently I've found that the record counts (even on non-clipped data) is different. It's ok if I right-click save data.

Not sure why this is.

enter image description here

OutFile should be 716k records and not just 101k records as reported here. I am running a difference process in QGIS to see where the issue is in the file but I don't want to have to do this if the export works correctly.

### Copies, clips and creates symbology of all visibile layers in mxd's in the current directory. Also creates a text file with metadata for use.
# Author: George Corea, Atherton Tablelands GIS
# georgec@atgis.com.au; info@atgis.com.au

# Licence:Creative Commons

import arcpy, string, datetime, shutil, os, glob
import arcpy.mapping as MAP
from arcpy import env

arcpy.env.workspace = os.getcwd()
arcpy.env.overwriteOutput = True

MXDList=string.split(arcpy.GetParameterAsText(0), ";")
ProjectPath=arcpy.GetParameterAsText(1)
CreateData=arcpy.GetParameterAsText(2)
clip_features=arcpy.GetParameterAsText(3) # polygon to clip data to AOI
AOI=arcpy.GetParameterAsText(4) #appended as suffix to dataset
arcpy.env.outputCoordinateSystem=arcpy.GetParameterAsText(5) #to maintain all datasets in the same projection
createdir=arcpy.GetParameterAsText(6)

rootPath=ProjectPath

#MXDList=glob.glob('*.mxd')
#ProjectPath=r'P:\2012\183_TownPlanning_Symbology\Working' # root output directory
#clip_features=r'L:\Vector_Data\Administrative\Boundaries\Local_Govt\TRC\trc_boundary_Polygon.shp' # polygon to clip data to AOI
#AOI='_trc' #appended as suffix to dataset
#arcpy.env.outputCoordinateSystem=r'L:\Vector_Data\Administrative\Boundaries\Local_Govt\TRC\trc_boundary_Polygon.prj' #to maintain all datasets in the same projection

xy_tolerance=1 #clip tolerance

# No edits should be required below this line.

VisibleLyrList = []
count=1


def layer_details(outPath, outFileN, lyrFile, type): #Generates the metadata

    descLayer = arcpy.Describe(lyrFile.dataSource)
    ReviewLog=outPath+'\\'+type+'_'+outFileN+'_log.txt'
    f = open(ReviewLog, 'a')
    try:
        f.write(str(lyrFile.name)+': name{'+str(lyrFile.datasetName)+\
    '}; query{'+str(lyrFile.definitionQuery)+\
    '}; source{'+str(lyrFile.dataSource)+\
    '}; description{'+str(lyrFile.description)+\
    '}; symbology{'+ str(lyrFile.symbologyType)+\
    '}; original projection{'+str(descLayer.spatialReference.name)+\
    '}; extent(x,y){'+str(descLayer.extent.XMax)+','+str(descLayer.extent.XMin)+','+str(descLayer.extent.YMax)+','+str(descLayer.extent.YMin)+\
    '}; format{'+str(descLayer.shapeType)+\
    '}; size(bytes) ~{'+str(os.path.getsize(lyrFile.dataSource))+\
    '} @{'+str(datetime.datetime.now())+'}'\
    )
        f.close()
    except:
        pass

print 'starting...'+str(MXDList)

for MXDFile in MXDList:

    mxd=arcpy.mapping.MapDocument(MXDFile)
    if createdir=="false":
        outPath = ProjectPath
    else:
        outPath = ProjectPath+'\\'+mxd.filePath[mxd.filePath.rfind('\\')+1:mxd.filePath.rfind('.')]

    arcpy.AddMessage ('Working on file #' + str(count) +' ...'+str(mxd.filePath))
    try:
        os.mkdir(outPath)
    except:
        pass
    dfList = arcpy.mapping.ListDataFrames(mxd)
    #print arcpy.GetMessages()
    #print 'working on 2...'+str(dfList)+str(outPath)

    for df in dfList:
    #msd = outPath+'.msd'
    #arcpy.mapping.ConvertToMSD(mxd, msd, df, "NORMAL", "NORMAL")
    #print (str(count)+'...' +str(outPath)+'...'+'\n')
        arcpy.AddMessage ('Working on dataframe ... ' +str(df.name))
        lyrList=arcpy.mapping.ListLayers(mxd, "", df)
        outPath = ProjectPath+'\\'+mxd.filePath[mxd.filePath.rfind('\\')+1:mxd.filePath.rfind('.')]+'\\'+str(df.name)
        try:
            os.mkdir(outPath)
        except:
            pass
        print ('working on 3...in: '+str(outPath))
        for lyrFile in lyrList:
            #print (str(lyrFile))
            if lyrFile.isFeatureLayer == True:
                if lyrFile.visible == True:
                    if lyrFile.name not in VisibleLyrList:
                        VisibleLyrList.append(lyrFile.name)
                        arcpy.AddMessage (str(lyrFile)+' is visible PROCESSING...')
                        outFileN=str(arcpy.ValidateTableName(lyrFile.longName[lyrFile.longName.rfind('\\')+1:]))
                        try:
                            rows = arcpy.SearchCursor(lyrFile.dataSource)
                            row = rows.next()
                            if row:
                                if CreateData=="true":
                                    arcpy.FeatureClassToFeatureClass_conversion(lyrFile.dataSource, outPath, outFileN)
                                    #arcpy.Copy_management(lyrFile.dataSource, outPath+'//'+outFileN, "")
                                    arcpy.Clip_analysis(outPath+'\\'+outFileN+'.shp', clip_features, outPath+'\\'+outFileN+AOI+'.shp', xy_tolerance)
                                    #updateLayer = outPath+'\\'+outFileN+'.lyr'
                                    sourceLayer = arcpy.mapping.Layer(outPath+'\\'+outFileN+'.shp')
                                    sourceLayer_AOI = arcpy.mapping.Layer(outPath+'\\'+outFileN+AOI+'.shp')
                                    #arcpy.mapping.UpdateLayer(df, updateLayer, sourceLayer, True)
                                    arcpy.SaveToLayerFile_management(lyrFile,outPath+'\\'+outFileN+'_sym.lyr', "ABSOLUTE")
                                    arcpy.SaveToLayerFile_management(sourceLayer_AOI,outPath+'\\'+outFileN+AOI+'.lyr', "ABSOLUTE")
                                    arcpy.ApplySymbologyFromLayer_management (outPath+'\\'+outFileN+AOI+'.lyr', outPath+'\\'+outFileN+'_sym.lyr')
                                    descLayer = arcpy.Describe(sourceLayer)
                                    #QA Row Count
                                    QA_rowCount_InFile = arcpy.GetCount_management(lyrFile)
                                    outLyrFile=arcpy.mapping.Layer(outPath+'\\'+outFileN+AOI+'.shp')
                                    QA_rowCount_OutFile = arcpy.GetCount_management(outLyrFile)
                                    if int(str(QA_rowCount_InFile))==int(str(QA_rowCount_OutFile)):
                                        arcpy.AddMessage("Count same - QA should not be Required..."+str(QA_rowCount_InFile)+" Out="+str(QA_rowCount_OutFile))
                                    else:
                                        arcpy.AddMessage("Count NOT same - QA IS Required...In="+str(QA_rowCount_InFile)+" Out="+str(QA_rowCount_OutFile))
                                        layer_details(outPath, outFileN,lyrFile,"Row Count ISSUE --- Count NOT same - QA IS Required...In="+str(QA_rowCount_InFile)+" Out="+str(QA_rowCount_OutFile))
                                        IssueLog=rootPath+'\\'+'Issue_log.txt'
                                        f = open(IssueLog, 'a')
                                        f.write(str(lyrFile.name)+': name{'+str(lyrFile.datasetName)+ '}; source{'+str(lyrFile.dataSource)+'}'+'\n')
                                        f.close()
                                    #QA Row Count process
                                    layer_details(outPath, outFileN, lyrFile,"COMPLETED")
                                    arcpy.Delete_management(outPath+'\\'+outFileN+'_sym.lyr')
                                    arcpy.Delete_management(outPath+'\\'+outFileN+'.shp')
                                else:
                                    layer_details(outPath, outFileN, lyrFile,"SIMULATED")
                                #arcpy.LayerToKML_conversion(outPath+'\\'+outFileN+'.lyr', outPath+'\\'+outFileN+'.kmz')
                            else:
                                arcpy.AddMessage ("!!!Datasource Issue!!!...continuing")
                                layer_details(outPath, outFileN,lyrFile,"ISSUE")
                                IssueLog=rootPath+'\\'+'Issue_log.txt'
                                f = open(IssueLog, 'a')
                                f.write(str(lyrFile.name)+': name{'+str(lyrFile.datasetName)+ '}; source{'+str(lyrFile.dataSource)+'}'+'\n')
                                f.close()
                        except:
                            errorm=arcpy.GetMessages()
                            arcpy.AddMessage ('!!! ERROR !!!...'+str(errorm)+' ...continuing')
                            try:
                                layer_details(outPath, outFileN, lyrFile,"ERROR")
                            except:
                                pass
                            #break
                            ErrorLog=rootPath+'\\'+'Error_log.txt'
                            f = open(ErrorLog, 'a')
                            f.write(str(lyrFile.name)+': name{'+str(lyrFile.datasetName)+ '}; source{'+str(lyrFile.dataSource)+'}'+'\n')
                            f.close()


                        else:
                            pass
                    else:
                        arcpy.AddMessage (str(lyrFile)+' Is NOT Visible. Not Processing')
                else:
                    arcpy.AddMessage (str(lyrFile)+' Is NOT Feature Layer. Not Processing')
        count=count+1
            #print str(lyrFile)+' is not visible'

    #Remove variable reference to file
    del mxd, outPath, lyrFile

See code from rows = arcpy.SearchCursor(lyrFile.dataSource) onwards

share|improve this question
    
I think you are only using Feature Class To Feature Class and Clip to "export" layers so is it possible that some layers have a Definition Query that the underlying Feature Class does not know about? If not, are you able to cut down your code to just those bits necessary to show where the observed count differs from the expected? I suspect that if you isolated one layer with a wrong count and re-ran it with just the "exporting" tool the cause would be much easier to trace. –  PolyGeo Jan 24 '13 at 6:58
    
Tks. These don't have a definition query. There are only 3 commands related to the data - arcpy.FeatureClassToFeatureClass_conversion(lyrFile.dataSource, outPath, outFileN) , QA_rowCount_InFile = arcpy.GetCount_management(lyrFile) and then an if command to compare the two. I wonder if I should use #arcpy.Copy_management(lyrFile.dataSource, outPath+'//'+outFileN, "") instead? –  GeorgeC Jan 24 '13 at 7:13
2  
I'd run a test using one layer/FC that is not adding up to see if you can reproduce outside your large code. GetCount can be used on a layer or FC so then I think how you "export all features" from a layer comes down to a choice of FCtoFC, CopyFeatures and Copy. FCtoFC works on feature class (not layer) so will not know about selections, Def Queries, etc. Copy does likewise so expect the same result. CopyFeatures will do the same on feature classes but respects selections on layers. –  PolyGeo Jan 24 '13 at 7:24

Your Answer

 
discard

By posting your answer, you agree to the privacy policy and terms of service.

Browse other questions tagged or ask your own question.