For anyone interested, I ended up writing a fairly involved function to do the job, so not exactly an "easy way". Broadly speaking, the function:
- Creates the ArcGIS query url
- Retrieves the data and stores it in a GeoPackage (GPKG)
- Edits the GPKG fields to replace codes with the name, for specified fields, by
- creating a new field
- doing the mapping
- deleting the old field
- renaming the new field to the old field
It hasn't been extensively tested so probably has some edge cases that haven't been teased out. There's also little to no error handling. However, it should be possible to run the two code sections below via the Python Console and have QGIS generate a layer.
Function code:
def addArcGisLayer (url, layer, crs, params):
# PARAMETERS
# url (str): URL of the FeatureServer or MapServer
# layer (str): ArcGIS name of the layer
# outfile (str): full filepath of the output layer (GPKG only)
# crs (QgsCoordinateReferenceSystem): spatial reference for the output layer
#
# params - a dict which can contain any of:
# where (str): a WHERE clause for the query filter - default is none
# extent (dict): spatial filter - in {"xmin":<xmin>,"ymin":<ymin>,"xmax":<xmax>,"ymax":<ymax>} format - default is none
# filterCrs (str): spatial reference for the spatial filter (in WKT format) - default is none (if extent is specfied, the extent is assumed to be in the SR of the ArcGIS source layer)
# outputFields (str) - comma-separated string of output fields - default is "*" (ALL)
# outputCodeValueFields (str) - comma-separated string of output fields for which code/value mapping translation will be applied - default is none
# outputFormat (str) - output format - defaults to "json" (not sure if anything else will actually work for PyQGIS)
# NOTE: no checking is carried out on the parameters passed
# RETURNS
# outfile - location of the output file (GeoPackage)
import urllib.request, json, subprocess
arcGisParams = {
"where": "1=1",
"outfields":"*",
"f":"json"
}
# Set defaults
if 'where' in params:
arcGisParams['where'] = params['where']
if 'extent' in params:
arcGisParams['geometry'] = params['extent']
if 'filterCrs' in params:
arcGisParams['inSR'] = {"wkt": params['filterCrs'] }
if 'outputFields' in params:
arcGisParams['outfields'] = params['outputFields']
if 'outputFormat' in params:
arcGisParams['f'] = params['outputFormat']
outfile = QgsProcessingUtils.generateTempFilename(layer + '.gpkg')
# Get layers from service
layersUrl = url + "/layers?f=pjson"
# Download the layers json
with urllib.request.urlopen(layersUrl) as u:
data = json.loads(u.read().decode())
layersArcGis = data['layers']
availableLayers = dict()
for l in layersArcGis:
availableLayers[l['name']] = l['id']
layerUrl = url + '/' + str(availableLayers[layer])
# Build query string component of URL
queryStr='?'
for key, value in arcGisParams.items():
queryStr += key
queryStr += '='
if isinstance(value,str):
queryStr += urllib.parse.quote(value)
if isinstance(value,list):
queryStr += urllib.parse.quote(','.join(value))
if isinstance(value,dict):
queryStr += urllib.parse.quote(json.dumps(value))
queryStr += '&'
queryStr = queryStr.rstrip('&')
# Extract data from ArcGIS Server
cmd = 'ogr2ogr -f GPKG '+outfile+' "'+layerUrl+'/query'+queryStr+'" -t_srs '+crs.authid()
print(cmd)
subprocess.run (cmd)
# Replace code value fields
if 'outputCodeValueFields' in params:
layerJsonUrl = url + "/" + str(availableLayers[layer]) + "?f=pjson"
with urllib.request.urlopen(layerJsonUrl) as u:
dataLayer = json.loads(u.read().decode())
fields = dataLayer['fields']
for f in fields:
if f['name'] in params['outputCodeValueFields']:
if f['domain'] is not None:
if f['domain']['type'] == 'codedValue':
mapping = {}
for cv in f['domain']['codedValues']:
mapping[cv['code']] = cv['name']
vl = QgsVectorLayer(outfile, 'temp', 'ogr')
pr = vl.dataProvider()
fieldName = f['name']
# Create new field
tempFieldName = fieldName + '_placeholder'
pr.addAttributes([QgsField(tempFieldName, QVariant.String)])
vl.updateFields()
# Update new field with mapping (code translation)
tempFieldIndex = pr.fieldNameIndex(tempFieldName)
with edit(vl):
for ft in vl.getFeatures():
vl.changeAttributeValue(ft.id(), tempFieldIndex, mapping[ft[fieldName]])
# Remove original field (with code values)
fieldIndex = pr.fieldNameIndex(fieldName)
pr.deleteAttributes([fieldIndex])
# Rename new field (with code translation) to original
tempFieldIndex = pr.fieldNameIndex(tempFieldName)
pr.renameAttributes({tempFieldIndex: fieldName})
vl.updateFields()
return outfile
The function is then called by passing a series of parameters, as follows:
extent = QgsRectangle(231224, 6235012, 236480, 6239368)
crs = QgsCoordinateReferenceSystem("EPSG:28356")
layerParams = {
"layerName": "geographic-line-labels",
"url": "https://maps.six.nsw.gov.au/arcgis/rest/services/sixmaps/LPIMap/MapServer",
"layer": "Ridge_Beach",
"where": "generalname <> 'GREAT DIVIDING RANGE'",
"outputFields": ["fuzzylinefeaturetype", "generalname"],
"outputCodeValueFields": ["fuzzylinefeaturetype"],
}
arcGisExtent = {
"xmin":extent.xMinimum(),
"ymin":extent.yMinimum(),
"xmax":extent.xMaximum(),
"ymax":extent.yMaximum()
}
arcGisParams = {
"where": layerParams['where'],
"filterCrs": crs.toWkt(),
"extent": arcGisExtent,
"outputFields": layerParams['outputFields'],
"outputCodeValueFields": layerParams['outputCodeValueFields'],
}
result = addArcGisLayer (layerParams['url'], layerParams['layer'], crs, arcGisParams)
layer = QgsVectorLayer(result, layerParams['layerName'], 'ogr')
QgsProject.instance().addMapLayer(layer)
