Our MapInfo and ArcGIS users have self managed file shares. I do my best to educate our users about windows naming conventions but years of violating basic conventions and personalities that don't like conventions have made things difficult.
I have written a python crawl script that crawls a parent folder and all subsequent children folders while looking for and summarizing vector and raster data; projection names, feature counts, repair geometry count, etc. The tool works perfectly in most locations.
Geoprocessing and crawls unfortunately error in deep deep folders that violate Microsoft Windows total character counts, and sometimes where characters break naming conventions.
Are there ArcPy / Python / Desktop workflows to account for violations in MS Windows naming conventions?
import arcpy
import os
import math
import numpy as np
import sys
from time import strftime
arcpy.env.overwriteOutput=True
ComputerName =os.getenv('COMPUTERNAME')
arcpy.AddMessage(" ")
arcpy.AddMessage("===================================================================")
arcpy.AddMessage(" ")
arcpy.AddMessage(" VERSION 5: SUMMARISE FOLDER: VECTOR & RASTERS")
arcpy.AddMessage(" Data Walk: Listings Rasters & Feature Classes")
arcpy.AddMessage(" ")
FolderOrWorkspace = arcpy.GetParameterAsText(0)
ReportFolder = arcpy.GetParameterAsText(1)
txtRasterFile = open(ReportFolder+"\\" +"FullRasterReport.csv","w")
txtVectorFile = open(ReportFolder+"\\" +"FullVectorReport.csv","w")
State =" "
txtFile2 = open(ReportFolder+"\\" +"SumReport.csv","w")
TempFolder = os.path.join(ReportFolder,"Temp")
TempFolderName ="Temp"
if arcpy.Exists(TempFolder):
print "ok"
else:
arcpy.CreateFolder_management(ReportFolder, TempFolderName)
#txtFile.write ('\n')
#txtFile.write ("===================================================================")
#txtFile.write ('\n')
#txtFile.close()
txtRasterFile.write ("Time,Folder,Type,BandCount,RasterCompression,RasterFormat,SpatialRef,ReferenceType,Count")
txtVectorFile.write ("Time,FullPath, FeatureClass,DataType, ShapeType, GDBFD, GDBFDN, SpatialRef,SpatialType,FeatureCount, RepairGeometry,GeomErrorCount,FieldCount,FeatureClassCount, ComputerName")
txtFile2.write ("Time,RasterCount,VectorCount")
gdbignore = ".gdb"
RasterCount = 0
FeatureClassCount = 0
FeatureDataSetCount = 0
DirList = []
DirList.append(FolderOrWorkspace) #ADDS PARENT FOLDER TO LIST
DataList = []
VectorList = []
for dirname, dirnames, filenames in os.walk(FolderOrWorkspace):
#arcpy.AddMessage(" PREPROCESSING WALK: "+str(dirname))
for subdirname in dirnames:
if gdbignore in str(subdirname):
print "ignore"
#arcpy.AddMessage(" " + str(subdirname) +" is .gdb")
else:
#arcpy.AddMessage(" " + str(subdirname) +" is not .gdb")
DirVar = os.path.join(dirname, subdirname)
DirList.append(DirVar)
for a in DirList:
arcpy.AddMessage(" Processing Folder: "+str(a))
del a
DirListLen = len(DirList)
arcpy.AddMessage(" ")
arcpy.AddMessage(" -----------------------------------------------------")
arcpy.AddMessage(" ")
wsl = 0
for fdir in DirList:
wsl = wsl+1
arcpy.AddMessage(" ")
arcpy.AddMessage(" -----------------------------------------------------")
arcpy.AddMessage(" WORKSPACE LOOP: "+str(wsl) + " of " +str(DirListLen))
arcpy.AddMessage(" ")
arcpy.env.workspace = fdir
arcpy.AddMessage(" RASTER PROCESSING")
RasterList = arcpy.ListRasters("*","")
Rlen = len(RasterList)
if Rlen == 0:
arcpy.AddMessage( " There are no rasters in: "+str(fdir))
else:
for raster in RasterList:
RasterCount = (RasterCount +1)
txtRasterFile.write ('\n')
arcpy.AddMessage(" - RASTER:" + str(RasterCount) +" Name: "+raster +" Within: "+subdirname)
txtRasterFile.write ((strftime("%d")+ " "+strftime("%b")+ " "+strftime("20%y")+ " "+strftime('%H:%M:%S')))
txtRasterFile.write (","+subdirname + ",Raster,"+raster+",")
desc = arcpy.Describe(raster)
SR = desc.spatialReference
try:
arcpy.AddMessage(" - Band Count: %d" % desc.bandCount)
txtRasterFile.write (desc.bandCount+",")
except:
arcpy.AddMessage(" - Does Not Support Bands")
txtRasterFile.write ("Unsupported,")
try:
arcpy.AddMessage(" - Compression Type: %s" % desc.compressionType)
txtRasterFile.write (desc.compressionType+",")
except:
arcpy.AddMessage(" - Does Not Support Compression")
txtRasterFile.write ("Unsupported,")
try:
arcpy.AddMessage(" - Raster Format: "+desc.format)
txtRasterFile.write (desc.format+",")
except:
arcpy.AddError(" - Raster Format: Error") #Error Message
txtRasterFile.write ("Error,")
arcpy.AddMessage(" - Spatial Reference: "+SR.name)
txtRasterFile.write (SR.name+",")
arcpy.AddMessage(" - Reference Type: "+SR.type)
txtRasterFile.write (SR.type+","+str(RasterCount))
arcpy.AddMessage(" ")
arcpy.AddMessage(" ")
arcpy.AddMessage(" VECTOR PROCESSING")
######## VECTOR
ShpFcList = []
### FOR SHAPEFILES IN FOLDERS
arcpy.AddMessage(" SHAPEFILE PROCESSING")
Shapefilevectorlist = arcpy.ListFeatureClasses('','All','')
ShapeLen = len(Shapefilevectorlist)
if ShapeLen == 0:
arcpy.AddMessage( " There are no shapefiles in: "+str(fdir))
else:
arcpy.AddMessage( " There are " +str(ShapeLen) +" shapefiles in: "+str(fdir))
for Shapefilevector in Shapefilevectorlist:
DesShapefilevector = arcpy.Describe(Shapefilevector)
DesShapefilevectorPath = DesShapefilevector.catalogPath
arcpy.AddMessage( " Feature class name: "+str(Shapefilevector))
ShpFcList.append(DesShapefilevectorPath)
### FOR GEODATABASE IN FOLDERS
eGDBList = []
eGDBsfc =[] # to store fc
arcpy.AddMessage(" GDB WORKSPACE PROCESSING")
AcGDBList = arcpy.ListWorkspaces("", "Access")
AcGDBListLen = len(AcGDBList)
if AcGDBListLen == 0:
arcpy.AddMessage( " There are no access geodatabase in: "+str(fdir))
else:
arcpy.AddMessage( " There are "+str(AcGDBListLen)+ " access geodatabase in: "+str(fdir))
for AcGDB in AcGDBList:
eGDBList.append(AcGDB)
FiGDBList = arcpy.ListWorkspaces("", "FileGDB")
FiGDBListLen = len(FiGDBList)
if FiGDBListLen == 0:
arcpy.AddMessage( " There are no file geodatabase in: "+str(fdir))
else:
arcpy.AddMessage( " There are "+str(FiGDBListLen)+ " file geodatabase in: "+str(fdir))
for FiGDB in FiGDBList:
eGDBList.append(FiGDB)
eGDBLen = len(eGDBList)
if eGDBLen == 0:
arcpy.AddMessage( " There are no geodatabase in: "+str(fdir))
arcpy.AddMessage(" ")
else:
arcpy.AddMessage( " There are " +str(eGDBLen) +" geodatabase in: "+str(fdir))
arcpy.AddMessage(" ")
###################### MASTER LOOP
for Gdbstet in eGDBList:
arcpy.AddMessage(" " +str(Gdbstet))
arcpy.AddMessage(" ")
del Gdbstet
WorkSpaceCnt = 0
for Gdbs in eGDBList:
WorkSpaceCnt = WorkSpaceCnt+1
arcpy.AddMessage(" WORKSPACE ######## "+str(WorkSpaceCnt) + " of " +str(eGDBLen))
try:
DenvGDB = arcpy.Describe(Gdbs) #
envGDB = DenvGDB.catalogPath
arcpy.AddMessage(" " +str(envGDB))
except Exception as e:
arcpy.AddError(" "+e.message)
if WorkSpaceCnt > eGDBLen:
arcpy.AddMessage( "CRITCAL " +str( WorkSpaceCnt) +" is > "+ str(eGDBLen))
### FOR STANDALONE FEATURE CLASS
arcpy.AddMessage(" GDB STANDALONE FEATURE CLASS PROCESSING")
envHold = arcpy.env.workspace
arcpy.env.workspace = envGDB
StandaloneFeatureClassList = arcpy.ListFeatureClasses('','All','')
StandaloneFeatureClassListLen = len(StandaloneFeatureClassList)
if StandaloneFeatureClassListLen == 0:
arcpy.AddMessage( " There are no stand alone feature class in: "+str(Gdbs))
else:
arcpy.AddMessage( " There are "+str(StandaloneFeatureClassListLen) +" stand alone feature class in: "+str(Gdbs))
for safc in StandaloneFeatureClassList:
Dessafc = arcpy.Describe(safc)
DessafcPath = Dessafc.catalogPath
arcpy.AddMessage( " Feature class name: "+str(DessafcPath))
eGDBsfc.append(DessafcPath)
del safc, Dessafc, DessafcPath
del StandaloneFeatureClassList
### FOR FEATURE DATASET FEATURE CLASS
arcpy.AddMessage(" GDB FEATURE DATASET PROCESSING")
FDS = arcpy.ListDatasets('','Feature')
FDSTLen = len(FDS)
if FDSTLen == 0:
arcpy.AddMessage(" There are no Geodatabase Feature Dataset in: "+str(envGDB))
else:
arcpy.AddMessage(" There are " +str(FDSTLen) +" in Geodatabase Feature Dataset in: "+str(envGDB))
arcpy.AddMessage(" GDB FEATURE DATASET FEATURE CLASS PROCESSING")
FeatureDataSetCount = (FeatureDataSetCount+1)
for FDST in FDS:# FOR EVERY FD
FDFeatureClassList = arcpy.ListFeatureClasses('','ALL',FDST) #ONLY X FEATURE DATA SET
FDFeatureClassListLen = len(FDFeatureClassList)
if FDFeatureClassListLen == 0:
arcpy.AddMessage( " There are no Feature Class in the Feature Dataset in: "+str(FDST))
else:
arcpy.AddMessage( " There are " +str(FDFeatureClassListLen) +" Feature Class in the Feature Dataset in: "+str(FDST))
for FDFC in FDFeatureClassList:
DesFDFC = arcpy.Describe(FDFC)
fdfcpath = os.path.join(DesFDFC.path, FDST, DesFDFC.BaseName) #idea
arcpy.AddMessage( " Feature class name: "+str(FDST +"/"+ DesFDFC.BaseName))
eGDBsfc.append(fdfcpath)
del FDFC,DesFDFC,fdfcpath
del FDFeatureClassList
del FDST
#arcpy.AddMessage( " -")
del FDS
#arcpy.AddMessage( " --")
# End of gdb loop
envHold = arcpy.env.workspace
#arcpy.AddMessage( " ---")
##### THE MAIN VECTOR PROCESS
arcpy.AddMessage( " ")
arcpy.AddMessage(" FEATURE CLASS PROCESSING: A")
mainvList = []
eGDBsfclen = len(eGDBsfc)
if eGDBsfclen == 0:
arcpy.AddMessage( " There are no gdb Feature Class")
else:
for x in eGDBsfc:
mainvList.append(x)
arcpy.AddMessage( " There are "+ str(eGDBsfclen)+" gdb Feature Class")
del x
ShpFcListLen = len(ShpFcList)
if ShpFcListLen == 0:
arcpy.AddMessage( " There are no Shapefile Feature Class")
else:
for x in ShpFcList:
mainvList.append(x)
arcpy.AddMessage( " There are "+ str(ShpFcListLen)+" Shapefile Feature Class")
del x
FullFCCnt = len(mainvList)
if FullFCCnt == 0:
arcpy.AddMessage( " There are "+ str(FullFCCnt)+" Feature Class")
else:
arcpy.AddMessage(" FEATURE CLASS PROCESSING: B")
arcpy.AddMessage( " There are "+ str(FullFCCnt)+" Feature Class")
arcpy.AddMessage(" ")
## for Featuret in mainvList:
## arcpy.AddMessage(" -: "+str(Featuret))
## del Featuret
arcpy.AddMessage(" ")
for Feature in mainvList:
try:
desc = arcpy.Describe(Feature)
BaseName = desc.baseName
catalogPath = desc.catalogPath
path = desc.path
CheckOutTableName = os.path.join(TempFolder,"Check.dbf") #the repair reports
SR = desc.spatialReference
FeatureCount = arcpy.GetCount_management(Feature)
DataType = (desc.dataType)
FDNameT = "Stand Alone Feature Class"
FDSNE = 0
FeatureDataSetNameA = str(desc.path)
gdbignore2 = str(gdbignore+"\\")
if gdbignore2 in FeatureDataSetNameA:
FDNameT ="Feature Dataset Feature Class"
FeatureDataSetNameB = FeatureDataSetNameA.split(".gdb\\")[1]
FDSNE = 1
if FDSNE == 0:
FeatureDataSetNameB = "NA"
except Exception as e:
arcpy.AddError(" "+e.message)
try:
fieldsList = arcpy.ListFields(catalogPath,"","All")
fieldsListLen = len(fieldsList)
except Exception as e:
arcpy.AddError(" "+e.message)
############## WRITE TO CSV FILE
FeatureClassCount = (FeatureClassCount +1)
txtVectorFile.write ('\n')
arcpy.AddMessage(" ======== ")
arcpy.AddMessage(" -FEATURE CLASS COUNT:"+str(FeatureClassCount) )
arcpy.AddMessage(" -"+str(catalogPath))
try: ### TIME
arcpy.AddMessage(" - TIME--------------- " +str((strftime("%d")+ " "+strftime("%b")+ " "+strftime("20%y")+ " "+strftime('%H:%M:%S'))))
txtVectorFile.write ((strftime("%d")+ " "+strftime("%b")+ " "+strftime("20%y")+ " "+strftime('%H:%M:%S')))
except Exception as e:
arcpy.AddError(" "+e.message)
try: ### WORKSPACE ### NAME
#arcpy.AddMessage(" - PATH " + str(catalogPath))
arcpy.AddMessage(" - FEATURE NAME------- " + str(BaseName))
txtVectorFile.write (","+catalogPath + ","+BaseName+",")
except Exception as e:
arcpy.AddError(" "+e.message)
try: ### DATA TYPE
arcpy.AddMessage(" - DATA TYPE---------- "+DataType)
txtVectorFile.write (DataType +",")
except Exception as e:
txtVectorFile.write (DataType +",")
try: ### SHAPE TYPE
arcpy.AddMessage(" - SHAPE TYPE--------- "+desc.shapeType)
txtVectorFile.write (desc.shapeType+",")
except Exception as e:
arcpy.AddMessage(" - SHAPE TYPE--------- NON-STANDARD")
arcpy.AddError(" - : "+e.message)
txtVectorFile.write ("Non Standard,") #for the lack of a better word
try: ### STANDALONE?
arcpy.AddMessage(" - STANDALONE--------- "+FDNameT)
txtVectorFile.write (FDNameT +",")
except Exception as e:
txtVectorFile.write (FDNameT +",")
try: ### STANDALONE NAME?
arcpy.AddMessage(" - FEATURE DATASET---- "+FeatureDataSetNameB)
txtVectorFile.write (FeatureDataSetNameB +",")
except Exception as e:
txtVectorFile.write (FeatureDataSetNameB +",")
try: ### SPATIAL REFERENCE
arcpy.AddMessage(" - SPATIAL REF-------- "+SR.name)
txtVectorFile.write (SR.name+",")
except Exception as e:
arcpy.AddMessage(" - SPATIAL REF-------- ERROR")
arcpy.AddError(" - "+e.message)
txtVectorFile.write ("error,")#for the lack of a better word
try:
arcpy.AddMessage(" - REFERENCE TYPE----- " +SR.type)
txtVectorFile.write (SR.type+",")
except Exception as e:
arcpy.AddMessage(" - REFERENCE TYPE----- ERROR")
arcpy.AddError(" - : "+e.message)
txtVectorFile.write ("error,")#for the lack of a better word
try:
arcpy.AddMessage(" - FEATURE COUNT------ "+str(FeatureCount))
arcpy.AddMessage(" - FIELD COUNT-------- " +str(fieldsListLen))
txtVectorFile.write (str(FeatureCount)+",")
except Exception as e:
arcpy.AddError(" "+e.message)
txtVectorFile.write ("error,")#for the lack of a better word
try: #ISSUES
#arcpy.RepairGeometry_management(catalogPath, "DELETE_NULL" )
arcpy.CheckGeometry_management (catalogPath, CheckOutTableName)
MesseageText = arcpy.GetMessages(1)
ErrorCount = arcpy.GetCount_management(CheckOutTableName)
if str(ErrorCount) != "0":
arcpy.AddMessage(" - GEOMETRY CHECK----- There are : "+str(ErrorCount) +" errors" )
txtVectorFile.write ("geometry had errors ,"+str(ErrorCount)+",")
arcpy.AddMessage(" ")
arcpy.AddMessage(str(MesseageText))
arcpy.AddMessage(" ")
else:
arcpy.AddMessage(" - GEOMETRY CHECK----- Geometry is good" )
txtVectorFile.write ("geometry is good,"+str(ErrorCount)+",")
arcpy.AddMessage(" ")
except:
arcpy.AddMessage(" ")
#arcpy.AddError(e.message) # prints the error messages
arcpy.AddMessage(" - GEOMETRY CHECK----- Non standard geometry issue " )
txtVectorFile.write ("Non standard issue, #,")
try:
txtVectorFile.write (str(fieldsListLen)+",")
txtVectorFile.write (str(FeatureClassCount)+",")
txtVectorFile.write (str(ComputerName))
#txtVectorFile.write ('\n')
except Exception as e:
arcpy.AddMessage(" ")
arcpy.AddError(e.message) # prints the error messages
arcpy.AddMessage(" ")
arcpy.AddMessage("===================================================================")
txtRasterFile.close()
txtVectorFile.close()
txtFile2.write ('\n')
txtFile2.write ((strftime("%d")+ " "+strftime("%b")+ " "+strftime("20%y")+ " "+strftime('%H:%M:%S')))
txtFile2.write (","+str(RasterCount)+","+str(FeatureClassCount))
txtFile2.close()
if arcpy.Exists(TempFolder):
arcpy.Delete_management(TempFolder)
I'm aware that when folders are too deep and the ArcGIS Desktop applications ArcMap and ArcCatalog can't work with the data; one method to address the issue is to use "Map Network Drive". However I can't conceptualize a way to use this in ArcPy.
How are you working with your GIS desktops clients and with code when you are hampered by naming convention violations?
