I have a python tool which investigates mxd files and clips/projects and exports all visible datasets. Recently I've found that the record counts (even on non-clipped data) is different. It's ok if I right-click save data.
Not sure why this is.
OutFile should be 716k records and not just 101k records as reported here. I am running a difference process in QGIS to see where the issue is in the file but I don't want to have to do this if the export works correctly.
### Copies, clips and creates symbology of all visibile layers in mxd's in the current directory. Also creates a text file with metadata for use.
# Author: George Corea, Atherton Tablelands GIS
# georgec@atgis.com.au; info@atgis.com.au
# Licence:Creative Commons
import arcpy, string, datetime, shutil, os, glob
import arcpy.mapping as MAP
from arcpy import env
arcpy.env.workspace = os.getcwd()
arcpy.env.overwriteOutput = True
MXDList=string.split(arcpy.GetParameterAsText(0), ";")
ProjectPath=arcpy.GetParameterAsText(1)
CreateData=arcpy.GetParameterAsText(2)
clip_features=arcpy.GetParameterAsText(3) # polygon to clip data to AOI
AOI=arcpy.GetParameterAsText(4) #appended as suffix to dataset
arcpy.env.outputCoordinateSystem=arcpy.GetParameterAsText(5) #to maintain all datasets in the same projection
createdir=arcpy.GetParameterAsText(6)
rootPath=ProjectPath
#MXDList=glob.glob('*.mxd')
#ProjectPath=r'P:\2012\183_TownPlanning_Symbology\Working' # root output directory
#clip_features=r'L:\Vector_Data\Administrative\Boundaries\Local_Govt\TRC\trc_boundary_Polygon.shp' # polygon to clip data to AOI
#AOI='_trc' #appended as suffix to dataset
#arcpy.env.outputCoordinateSystem=r'L:\Vector_Data\Administrative\Boundaries\Local_Govt\TRC\trc_boundary_Polygon.prj' #to maintain all datasets in the same projection
xy_tolerance=1 #clip tolerance
# No edits should be required below this line.
VisibleLyrList = []
count=1
def layer_details(outPath, outFileN, lyrFile, type): #Generates the metadata
descLayer = arcpy.Describe(lyrFile.dataSource)
ReviewLog=outPath+'\\'+type+'_'+outFileN+'_log.txt'
f = open(ReviewLog, 'a')
try:
f.write(str(lyrFile.name)+': name{'+str(lyrFile.datasetName)+\
'}; query{'+str(lyrFile.definitionQuery)+\
'}; source{'+str(lyrFile.dataSource)+\
'}; description{'+str(lyrFile.description)+\
'}; symbology{'+ str(lyrFile.symbologyType)+\
'}; original projection{'+str(descLayer.spatialReference.name)+\
'}; extent(x,y){'+str(descLayer.extent.XMax)+','+str(descLayer.extent.XMin)+','+str(descLayer.extent.YMax)+','+str(descLayer.extent.YMin)+\
'}; format{'+str(descLayer.shapeType)+\
'}; size(bytes) ~{'+str(os.path.getsize(lyrFile.dataSource))+\
'} @{'+str(datetime.datetime.now())+'}'\
)
f.close()
except:
pass
print 'starting...'+str(MXDList)
for MXDFile in MXDList:
mxd=arcpy.mapping.MapDocument(MXDFile)
if createdir=="false":
outPath = ProjectPath
else:
outPath = ProjectPath+'\\'+mxd.filePath[mxd.filePath.rfind('\\')+1:mxd.filePath.rfind('.')]
arcpy.AddMessage ('Working on file #' + str(count) +' ...'+str(mxd.filePath))
try:
os.mkdir(outPath)
except:
pass
dfList = arcpy.mapping.ListDataFrames(mxd)
#print arcpy.GetMessages()
#print 'working on 2...'+str(dfList)+str(outPath)
for df in dfList:
#msd = outPath+'.msd'
#arcpy.mapping.ConvertToMSD(mxd, msd, df, "NORMAL", "NORMAL")
#print (str(count)+'...' +str(outPath)+'...'+'\n')
arcpy.AddMessage ('Working on dataframe ... ' +str(df.name))
lyrList=arcpy.mapping.ListLayers(mxd, "", df)
outPath = ProjectPath+'\\'+mxd.filePath[mxd.filePath.rfind('\\')+1:mxd.filePath.rfind('.')]+'\\'+str(df.name)
try:
os.mkdir(outPath)
except:
pass
print ('working on 3...in: '+str(outPath))
for lyrFile in lyrList:
#print (str(lyrFile))
if lyrFile.isFeatureLayer == True:
if lyrFile.visible == True:
if lyrFile.name not in VisibleLyrList:
VisibleLyrList.append(lyrFile.name)
arcpy.AddMessage (str(lyrFile)+' is visible PROCESSING...')
outFileN=str(arcpy.ValidateTableName(lyrFile.longName[lyrFile.longName.rfind('\\')+1:]))
try:
rows = arcpy.SearchCursor(lyrFile.dataSource)
row = rows.next()
if row:
if CreateData=="true":
arcpy.FeatureClassToFeatureClass_conversion(lyrFile.dataSource, outPath, outFileN)
#arcpy.Copy_management(lyrFile.dataSource, outPath+'//'+outFileN, "")
arcpy.Clip_analysis(outPath+'\\'+outFileN+'.shp', clip_features, outPath+'\\'+outFileN+AOI+'.shp', xy_tolerance)
#updateLayer = outPath+'\\'+outFileN+'.lyr'
sourceLayer = arcpy.mapping.Layer(outPath+'\\'+outFileN+'.shp')
sourceLayer_AOI = arcpy.mapping.Layer(outPath+'\\'+outFileN+AOI+'.shp')
#arcpy.mapping.UpdateLayer(df, updateLayer, sourceLayer, True)
arcpy.SaveToLayerFile_management(lyrFile,outPath+'\\'+outFileN+'_sym.lyr', "ABSOLUTE")
arcpy.SaveToLayerFile_management(sourceLayer_AOI,outPath+'\\'+outFileN+AOI+'.lyr', "ABSOLUTE")
arcpy.ApplySymbologyFromLayer_management (outPath+'\\'+outFileN+AOI+'.lyr', outPath+'\\'+outFileN+'_sym.lyr')
descLayer = arcpy.Describe(sourceLayer)
#QA Row Count
QA_rowCount_InFile = arcpy.GetCount_management(lyrFile)
outLyrFile=arcpy.mapping.Layer(outPath+'\\'+outFileN+AOI+'.shp')
QA_rowCount_OutFile = arcpy.GetCount_management(outLyrFile)
if int(str(QA_rowCount_InFile))==int(str(QA_rowCount_OutFile)):
arcpy.AddMessage("Count same - QA should not be Required..."+str(QA_rowCount_InFile)+" Out="+str(QA_rowCount_OutFile))
else:
arcpy.AddMessage("Count NOT same - QA IS Required...In="+str(QA_rowCount_InFile)+" Out="+str(QA_rowCount_OutFile))
layer_details(outPath, outFileN,lyrFile,"Row Count ISSUE --- Count NOT same - QA IS Required...In="+str(QA_rowCount_InFile)+" Out="+str(QA_rowCount_OutFile))
IssueLog=rootPath+'\\'+'Issue_log.txt'
f = open(IssueLog, 'a')
f.write(str(lyrFile.name)+': name{'+str(lyrFile.datasetName)+ '}; source{'+str(lyrFile.dataSource)+'}'+'\n')
f.close()
#QA Row Count process
layer_details(outPath, outFileN, lyrFile,"COMPLETED")
arcpy.Delete_management(outPath+'\\'+outFileN+'_sym.lyr')
arcpy.Delete_management(outPath+'\\'+outFileN+'.shp')
else:
layer_details(outPath, outFileN, lyrFile,"SIMULATED")
#arcpy.LayerToKML_conversion(outPath+'\\'+outFileN+'.lyr', outPath+'\\'+outFileN+'.kmz')
else:
arcpy.AddMessage ("!!!Datasource Issue!!!...continuing")
layer_details(outPath, outFileN,lyrFile,"ISSUE")
IssueLog=rootPath+'\\'+'Issue_log.txt'
f = open(IssueLog, 'a')
f.write(str(lyrFile.name)+': name{'+str(lyrFile.datasetName)+ '}; source{'+str(lyrFile.dataSource)+'}'+'\n')
f.close()
except:
errorm=arcpy.GetMessages()
arcpy.AddMessage ('!!! ERROR !!!...'+str(errorm)+' ...continuing')
try:
layer_details(outPath, outFileN, lyrFile,"ERROR")
except:
pass
#break
ErrorLog=rootPath+'\\'+'Error_log.txt'
f = open(ErrorLog, 'a')
f.write(str(lyrFile.name)+': name{'+str(lyrFile.datasetName)+ '}; source{'+str(lyrFile.dataSource)+'}'+'\n')
f.close()
else:
pass
else:
arcpy.AddMessage (str(lyrFile)+' Is NOT Visible. Not Processing')
else:
arcpy.AddMessage (str(lyrFile)+' Is NOT Feature Layer. Not Processing')
count=count+1
#print str(lyrFile)+' is not visible'
#Remove variable reference to file
del mxd, outPath, lyrFile
See code from rows = arcpy.SearchCursor(lyrFile.dataSource) onwards
