I have a ArcPy script (10.2) which connects to an ArcGIS server (10.0) versioned geodatabase, via an .sde connection file in the Database connections directory through direct connect on a Microsoft SQL server.
The tool at a very simplified level updates one featureclass with an updatecursor to populate a single field, and appends data to another featureclass from a local FGDB containing buffers generated from the first featureclass. It also runs an updatecursor on the second featureclass to delete certain rows. Editor tracking is enabled on these featureclasses. The script is attached to a standard toolbox (*.tbx).
Everything works fine in a local FGDB. When I change the connection to a server and run the tool all the changes appear in the attribution tables as expected -- however as soon as I close my ArcMap session and re-open, the changes do not seem to have persisted. Both the update cursor functions and appends appear not to have worked. I do not enter an editing session or anything of the like.
Interestingly running the append management manually in Python window with identical parameters appears to make the changes stick.
Some other de-bugging methods I've tried:
- - unticking run script as process
- - constructing the .sde connection path with os.path.join
- - setting env.workspace = "" at the end of the script to release any holds
- - connecting through an application connect (sde)
- - running in a test 10.2 server environment
Is there something I am missing? I can't seem to figure out an obvious reason for the changes appearing to work and then not saving when opening another session.
Code below:
import arcpy
import os
import time
import uuid
from datetime import datetime, timedelta
from configuration import *
# Custom subclass of Python dict to allow easy extended nesting for assigning RES_ID's properties
class NestedDict(dict):
def __getitem__(self, key):
if key in self: return self.get(key)
return self.setdefault(key, NestedDict())
def checkResourcesUpdate():
"""
Checks for new or updated shapes.
will select out those shapes RES_IDs to be re-generated (buffered) in the resource protection layer,
and collects a list of RES_IDs to be deleted in the existing resources protection layer.
"""
rowsResources = arcpy.UpdateCursor(resourcesLayer)
rowsProtection = arcpy.UpdateCursor(resourcesProtectionLayer)
resourcesProtectionDelete = []
resourcesSelectList = []
resourceIds = []
protectionIds = []
resourceInfo = NestedDict()
for rowResources in rowsResources:
if rowResources.RES_ID == "" or rowResources.RES_ID is None:
rowResources.RES_ID = "R" + time.strftime("%Y%m%d") + str(uuid.uuid4().get_hex().upper()[0:6])
rowsResources.updateRow(rowResources)
arcpy.AddMessage("Res ID {0} assigned to new resource".format(rowResources.RES_ID))
if rowResources.PROTECT_TERM in protectConfig.keys():
if rowResources.SHAPE.area == 0:
arcpy.AddWarning("Skipping {0}, spatial extent missing, no protection area(s) generated".format(rowResources.RES_ID))
else:
resourceInfo[str(rowResources.RES_ID)]['protectTerm'] = rowResources.PROTECT_TERM
resourceIds.append(rowResources.RES_ID)
if rowResources.EDITED_DATE is not None or rowResources.EDITED_DATE != "":
resourceInfo[str(rowResources.RES_ID)]['recentDate'] = rowResources.EDITED_DATE
if rowResources.CREATED_DATE is not None or rowResources.CREATED_DATE != "":
if resourceInfo[str(rowResources.RES_ID)]['recentDate']:
if rowResources.CREATED_DATE > resourceInfo[str(rowResources.RES_ID)]['recentDate']:
resourceInfo[str(rowResources.RES_ID)]['recentDate'] = rowResources.CREATED_DATE
for rowProtection in rowsProtection:
if rowProtection.PROTECT_TERM in protectConfig.keys():
protectionIds.append(rowProtection.RES_ID)
if resourceInfo[str(rowProtection.RES_ID)]['recentDate']:
if rowProtection.CREATED_DATE < resourceInfo[str(rowProtection.RES_ID)]['recentDate']:
if rowProtection.PROTECT_TERM in protectConfig.keys():
arcpy.AddMessage("Creating new {4} area for modified Res ID {0} ({3}) modified on {2}, previous protection area(s) dated {1}".format(rowProtection.RES_ID,rowProtection.CREATED_DATE,resourceInfo[str(rowProtection.RES_ID)]['recentDate'],resourceInfo[str(rowProtection.RES_ID)]['protectTerm'],rowProtection.PROTECT_TYPE))
resourcesSelectList.append(rowProtection.RES_ID)
for resId in resourceIds:
if resId not in protectionIds:
resourcesSelectList.append(resId)
arcpy.AddMessage("Res ID {0} not found in protection layer, new protection area(s) will be generated".format(resId))
# list set for uniqueness as most will have both notification and protection area
for resId in list(set(protectionIds)):
if resId not in resourceIds:
arcpy.AddWarning("Possible error: Res ID {0} exists in the protection layer with no corresponding resource shape".format(resId))
del rowsResources, rowsProtection
if resourcesSelectList:
selectResources(resourcesSelectList)
deleteResourceProtect(resourcesSelectList)
else:
arcpy.AddMessage("No resources have been modified or are new.")
raise SystemExit
# bufferResources(resourcesUpdateList)
def deleteResourceProtect(resourcesSelectList):
"""
Expects a list of RES_IDs to delete from the resources protection layer,
deleted features are expected to have an updated protection areas.
"""
rowsProtectionDelete = arcpy.UpdateCursor(resourcesProtectionLayer)
for rowProtectionDelete in rowsProtectionDelete:
if rowProtectionDelete.RES_ID in resourcesSelectList:
arcpy.AddMessage("Res ID {0}: Deleting previous {1}, resource new or updated".format(rowProtectionDelete.RES_ID,rowProtectionDelete.PROTECT_TYPE))
rowsProtection.deleteRow(rowProtectionDelete)
try:
del rowsProtectionDelete
except Exception as e:
arcpy.AddMessage("No records in protection layer.")
def selectResources(selectedResources):
"""
Takes a list of resource Object IDs, parses them into a query to create
a in memory selection layer for buffering
"""
whereClause = "RES_ID in ({0})".format(str(selectedResources)[1:-1]).replace("u'","'")
arcpy.Select_analysis(resourcesLayer, selectOutLayer, whereClause)
parseResourcesLayer()
def parseResourcesLayer():
"""
Parses resources to be buffered based on PROTECT_TERM
"""
# project to WA Albers and store in processing_data.gdb (output cannot be to memory)
arcpy.Project_management(selectOutLayer, selectOutLayerAlbers, albersProjFile)
rows = arcpy.UpdateCursor(selectOutLayerAlbers)
protectTermsProcessing = {}
for row in rows:
if row.SHAPE_Area == 0:
arcpy.AddWarning("Skipping {0}, spatial extent missing, no protection area(s) generated".format(row.RES_ID))
if row.PROTECT_TERM in protectConfig.keys():
protectTermsProcessing[str(row.PROTECT_TERM)] = 1
arcpy.AddField_management(selectOutLayerAlbers,"PROTECT_TYPE","TEXT", "", "", 25)
arcpy.AddField_management(selectOutLayerAlbers,"PROTECT_DISTANCE","SHORT")
arcpy.DeleteField_management(selectOutLayerAlbers,["CREATED_DATE", "CREATED_BY","EDITED_DATE","EDITED_BY"])
for protectTerm in protectTermsProcessing.keys():
if 'separationDist' and 'notificationDist' in protectConfig[protectTerm]:
bufferProtection(protectTerm,selectOutLayerAlbers,"separation")
bufferProtection(protectTerm,selectOutLayerAlbers,"notification")
elif 'separationDist' in protectConfig[protectTerm]:
bufferProtection(protectTerm,selectOutLayerAlbers,"separation")
# would be unexpected, currently no protect terms with only notficiaton dist
elif 'notificationDist' in protectConfig[protectTerm]:
bufferProtection(protectTerm,selectOutLayerAlbers,"notication")
#del rows, row
appendBuffers()
def bufferProtection(protectTerm,selectOutLayerAlbers,bufferType):
"""
Subsets on PROTECT_TERM and buffers separation or notification area into WA Albers
"""
protectTermLayerBuffered = processingDir + "\\{0}_buffer\{1}_{0}_albers_buffered".format(bufferType,protectTerm)
selectOutLayerProtectTerm = processingDir + "\\{0}_nobuffer\{1}_{0}_albers".format(bufferType,protectTerm)
whereClause = '"PROTECT_TERM" = \'{0}\''.format(protectTerm)
arcpy.Select_analysis(selectOutLayerAlbers, selectOutLayerProtectTerm, whereClause)
if bufferType == 'separation':
sideType = 'OUTSIDE_ONLY'
else:
sideType = 'FULL'
if protectConfig[protectTerm][bufferType + 'Dist']:
arcpy.Buffer_analysis(selectOutLayerProtectTerm, protectTermLayerBuffered, str(protectConfig[protectTerm][bufferType + 'Dist']) + " Meters", sideType, "ROUND")
arcpy.CalculateField_management(protectTermLayerBuffered,'PROTECT_TYPE',"'" + bufferType.title() + " area'",'PYTHON')
arcpy.CalculateField_management(protectTermLayerBuffered,'PROTECT_DISTANCE',protectConfig[protectTerm][bufferType + 'Dist'],'PYTHON')
arcpy.AddMessage("Generating {0} {1}m {2} area buffers in GDA94 WA Albers, from new or updated resources".format(protectConfig[protectTerm]['protectTerm'],protectConfig[protectTerm][bufferType + 'Dist'],bufferType))
def appendBuffers():
arcpy.env.workspace = processingDir
featureDatasets = arcpy.ListDatasets()
arcpy.AddMessage("Merging new separation area and notification area buffers...")
for dataSet in featureDatasets:
if dataSet == 'notification_buffer' or dataSet == 'separation_buffer':
for featureClass in arcpy.ListFeatureClasses("*", "ALL", str(dataSet)):
appendAll(str(dataSet) + "\\" + featureClass)
#arcpy.env.workspace = resourceDataLocation
def appendAll(mergeFeature):
arcpy.env.workspace = processingDir
mergeFeature = processingDir + "\\" + mergeFeature
arcpy.AddWarning("Merging {0}".format(mergeFeature))
arcpy.Append_management(mergeFeature, protectionTemplate, "NO_TEST")
def cleanProcessingData():
arcpy.env.workspace = processingDir
featureDatasets = arcpy.ListDatasets()
arcpy.AddMessage("Removing temporary processing data from previous execution...")
for dataSet in featureDatasets:
for processFeature in arcpy.ListFeatureClasses("*", "ALL", dataSet):
arcpy.Delete_management(processFeature)
arcpy.AddWarning("Deleted {0}".format(processFeature))
arcpy.Delete_management(selectOutLayer)
arcpy.Delete_management(selectOutLayerAlbers)
arcpy.Delete_management(protectionTemplateDegrees)
arcpy.DeleteFeatures_management(protectionTemplate)
arcpy.AddMessage("Previous processing data successfully removed")
#arcpy.env.workspace = resourceDataLocation
def pushUpdates():
arcpy.env.workspace = ""
arcpy.AddMessage("Projecting to GDA94 Latitude/Longitude..")
arcpy.Project_management(protectionTemplate, protectionTemplateDegrees, degreesProjFile)
arcpy.AddMessage("Appending new protection area(s) to SDIDIV..")
arcpy.AddWarning("Appending: {0},{1}".format(protectionTemplateDegrees,resourcesProtectionLayer))
arcpy.Append_management(protectionTemplateDegrees,resourcesProtectionLayer, "NO_TEST")
def protectConfigDisplay(d, indent=0):
arcpy.AddWarning("{0}".format("*"*45))
for key, value in d.iteritems():
if key != 'protectTerm':
arcpy.AddWarning("{0}".format(str(key).replace("Dist"," distance:")))
if isinstance(value, dict):
protectConfigDisplay(value, indent+1)
else:
arcpy.AddWarning("{0}{1} meters".format("\t"*5,str(value)))
arcpy.AddWarning("{0}".format("*"*45))
def main():
arcpy.env.workspace = processingDir
initMsg = "\n\n{0}\n{1} Landuse Geoscience - Process Protection Areas\n{0}".format('*'*90, time.asctime())
arcpy.AddMessage(initMsg)
arcpy.AddMessage("ArcGIS {3} version {0}, build {1} (installed {2})".format(arcpy.GetInstallInfo()["Version"],arcpy.GetInstallInfo()["BuildNumber"],arcpy.GetInstallInfo()["InstallDate"],arcpy.GetInstallInfo()["ProductName"]))
arcpy.AddMessage("\nLoading following configuration (changes can be made configuration.py)..")
protectConfigDisplay(protectConfig)
cleanProcessingData()
checkResourcesUpdate()
pushUpdates()
if __name__ == "__main__":
main()
arcpy.AddMessage("Completed successfully")
And the configuration file:
protectConfig = {
'blast1' : {
'protectTerm' : 'blast1',
'separationDist' : '1000',
'notificationDist' : '2000' },
'blast2' : {
'protectTerm' : 'blast2',
'separationDist' : '1200',
'notificationDist' : '1200' },
'noblast1' : {
'protectTerm' : 'noblast1',
'separationDist' : '500',
'notificationDist' : '500' },
'noblast2' : {
'protectTerm' : 'noblast2',
'separationDist' : '500',
'notificationDist' : '1000' },
'noblast3' : {
'protectTerm' : 'noblast3',
'separationDist' : '500' }
}
# SDIDIV LUG feature dataset
arcpy.env.overwriteOutput = 1
currentDir = os.path.dirname(os.path.abspath(__file__))
processingDir = currentDir + r"\processing_data.gdb"
protectionTemplate = processingDir + r"\LUG_resources_protection_template"
protectionTemplateDegrees = protectionTemplate + "_degrees"
# Processing data
albersProjFile = currentDir + r"\projections\wa_alb94.prj"
degreesProjFile = currentDir + r"\projections\gda94.prj"
selectOutLayerAlbers = processingDir + r"\select_resources_processing"
selectOutLayer = processingDir + r"\select_resources"
# SDIDIV LUG feature classes
resourcesProtectionLayer = r"Database Connections\SDIDIV_DEV.sde\SDIDIV.GSD.LUG\SDIDIV.GSD.LUG_resources_protection"
resourcesLayer = r"Database Connections\SDIDIV_DEV.sde\SDIDIV.GSD.LUG\SDIDIV.GSD.LUG_resources"
