2

Back again with a similar issue, but this time with commented code! I'm trying to perform table joins iteratively for each field in each CSV I'm using for each of vector layers in the project. The problem is at the bottom of the code sample starting with the comment, "#joins the shape layer to the proper CSV field". I feel like I've tried almost everything I can think of and now I'm just spinning my wheels.

I've tried the following construction as well (with and without "set" in the method name):

#joins the shape layer to the proper CSV field
shpField = 'puma'
csvField = 'puma'
joinCsvName = shpSavedLayer.name().split("-")[0]
for csvSavedLayer in csvSavedLayerList:
    csvSavedLayerName = csvSavedLayer.name()
    if csvSavedLayerName == joinCsvName:
        joinCsvLayer = csvSavedLayer
joinCsvField = []
joinCsvField.append(shpSavedLayer.name().split("-")[1])
joinObject = QgsVectorLayerJoinInfo()
joinObject.setJoinLayerId = joinCsvLayer.id()
joinObject.setJoinFieldName = csvField
joinObject.setTargetFieldName = shpField
joinObject.setUsingMemoryCache = True
joinObject.setJoinFieldNamesSubset = joinCsvField
shpSavedLayer.addJoin(joinObject)

I have no clue what's happening here. The script executes, but there are no table joins in the project when the result is generated. I'm using QGIS 3.10.

If people want the project files to diagnose what's happening, I'll talk to the researcher I'm working with and see if he's willing to release his data. I don't think it will be a problem.

If there are suggestions on how to diagnose the problem, I would be interested in hearing those as well.

from qgis.core import QgsProject, QgsVectorLayer, QgsLayerTreeGroup, QgsLayerTreeLayer, QgsLayoutNodesItem
from PyQt5 import QtGui
import csv
import os

#creates the project directories and reformats for compatibility with all methods used in script
projectDirRawUri = r"E:\Jason\Documents\2020-01\mapping project\gis\test_automate_all"
projectDirUri = projectDirRawUri.replace("\\", "/")
layerDirName = "layers"
#layerDirPath = os.path.join(projectPathUri, layerDirName)
layerDirUri = projectDirUri + "/" + layerDirName
if not os.path.isdir(layerDirUri):
    os.mkdir(layerDirUri)
else:
    print("Directory already exists. Please specify another.")
dataLayerDirName = "datasets"
dataLayerDirUri = layerDirUri + "/" + dataLayerDirName
if not os.path.isdir(dataLayerDirUri):
    os.mkdir(dataLayerDirUri)
else:
    print("Directory already exists. Please specify another.")
    
#deselects all nodes in the "layers panel"/"map legend"
QgsLayoutNodesItem.deselectNode

#creates the root variable
root = QgsProject.instance().layerTreeRoot()

#creates the group for storing the CSV layers
#dataGroup = root.addGroup("")
dataGroup = QgsLayerTreeGroup("data_layers")
root.addChildNode(dataGroup)

#assign/initiate CSV layer variables
csvSrcRawUri = r"E:\Jason\Documents\2020-01\mapping project\gis\test_automate_all\sources\datasetsrekeithsstudy"
csvSrcUri = csvSrcRawUri.replace("\\", "/")
#print(csvSrcUri)
csvSrcUriList = []
csvSrcUriIndex = 0
csvLayerDirUri = dataLayerDirUri
csvLayerUriList = []
csvLayerUriIndex = 0
csvLayerNameList = []
csvLayerNameIndex = 0
csvLayerIndex = 0
csvSavedLayerList = []
wktRawUri = r"E:\Jason\Documents\2020-01\mapping project\gis\test_automate_all\sources\Public Use Microdata Areas (PUMA)\geo_export_0ca223b7-207e-4bc2-bf55-9947aa4998c5.prj"
wktUri = wktRawUri.replace("\\", "/")

## reading prj file 
#with open(wktUri, 'r') as wkt: 
#    # creating a wkt reader object 
#    wktInfo = wkt.read() 
#    # extracting field names through first row 
#    crs = QgsCoordinateReferenceSystem(wkt)
#    assert crs.isValid()
#print(crs)

#populates the CSV source URI and layer name lists
for root, dirs, files in os.walk(csvSrcUri):
    for file in files:
        if(file.endswith(".csv")):
            csvSrcUriList.append(os.path.join(root,file))
            csvLayerNameList.append(os.path.join(file))

#sorts the CSV source URI and layer name lists alphabetically to match eachother
csvSrcUriList.sort()
csvLayerNameList.sort()

#formats the CSV source URI list so the items in it can be used by the QgsVectorLayer() method
for file in csvSrcUriList:
    filePrefix = 'file:///'
    fileSuffix = '?delimeter=,'
    fileLocation = filePrefix + file.replace("\\", "/") + fileSuffix
    csvSrcUriList[csvSrcUriIndex] = fileLocation
    csvSrcUriIndex += 1

#populates the CSV layer URI list
while csvLayerUriIndex < csvSrcUriIndex:
    csvLayerUri = csvLayerDirUri + "/" + csvLayerNameList[csvLayerUriIndex]
    csvLayerUriList.append(csvLayerUri)
    csvLayerUriIndex += 1
#print(csvLayerUriList)

#removes the .csv extention from the items in the CSV layer name list
for file in csvLayerNameList:
    layerName =  file.replace(".csv", "")
    csvLayerNameList[csvLayerNameIndex] = layerName
    csvLayerNameIndex += 1
#print(csvSrcUriList)
#print(csvLayerNameList)

#loop to create and perform all operations on all CSV layers including grouping
for csvLayerName in csvLayerNameList:
    csvSrcUri = csvSrcUriList[csvLayerIndex]
    #creates the CSV layer name from the list of csv names and fields
    csvLayerName = csvLayerNameList[csvLayerIndex]
    #adds the initial CSV layer to the project and checks the layer validity
    csvLayer = QgsVectorLayer(csvSrcUri, csvLayerName, 'delimitedtext')
    QgsProject.instance().addMapLayer(csvLayer)
    if not csvLayer or not csvLayer.isValid():
        csvAddLayerErrorMsg = csvLayerName + " failed to load!"
        print(csvAddLayerErrorMsg)
    #copies the shape layer and places the copy in the correct group
    root = QgsProject.instance().layerTreeRoot()
    csvGroup = root.findGroup(dataGroup.name())
    csvOriginalLayer = root.findLayer(csvLayer.id())
    csvGroupedLayer = csvOriginalLayer.clone()
    csvGroup.insertChildNode(-1, csvGroupedLayer)
    csvSaveOptions = QgsVectorFileWriter.SaveVectorOptions()
    csvSaveOptions.driverName = "CSV"
    csvSaveOptions.fileEncoding = "UTF-8"
    csvSavedLayer = csvGroupedLayer.layer()
    QgsVectorFileWriter.writeAsVectorFormatV2(csvSavedLayer, csvLayerUriList[csvLayerIndex], QgsCoordinateTransformContext(), csvSaveOptions)
    if not csvGroupedLayer.layer() or not csvGroupedLayer.layer().isValid():
        csvSaveLayerErrorMsg = csvLayerName + " failed to save!"
        print(csvAddLayerErrorMsg)
    #removes the ungrouped shape layer
    parent = csvOriginalLayer.parent()
    parent.removeChildNode(root.findLayer(csvLayer.id()))
    #populates in the CSV saved layer list for use in table joins
    csvSavedLayerList.append(csvSavedLayer)
    
    csvLayerIndex +=1

#deselects all nodes in the "layers panel"/"map legend"
QgsLayoutNodesItem.deselectNode

#assign/initiate SHP layer variables
shpGroupNameList = csvLayerNameList
shpGroupNameIndex = 0
shpSrcRawUri = r"E:\Jason\Documents\2020-01\mapping project\gis\test_automate_all\sources\Public Use Microdata Areas (PUMA)\geo_export_0ca223b7-207e-4bc2-bf55-9947aa4998c5.shp"
shpSrcUri = shpSrcRawUri.replace("\\", "/")
#print(shpSrcUri)
shpLayerDirUri = layerDirUri
shpLayerUriList = []
shpLayerUriIndex0 = 0
shpLayerUriIndex1 = 0
shpLayerNameList = []
shpLayerNameIndex0 = 0
shpLayerNameIndex1 = 0
shpLayerIndex0 = 0
shpLayerIndex1 = 0
#print(csvLayerNameList)

#creates the groups for storing the SHP layers
for shpGroup in shpGroupNameList:
    shpGroup = QgsLayerTreeGroup(shpGroupNameList[shpGroupNameIndex])
    root.addChildNode(shpGroup)
    shpGroupNameIndex +=1


#populates the SHP layer name list in nested list consistant with the SHP group name list
for file in csvLayerUriList:
    fields0 = []
    shpLayerNameIndex1 = 0
    with open(csvLayerUriList[shpLayerNameIndex0], 'r') as csvfile:
        csvreader = csv.reader(csvfile)
        fields0 = next(csvreader)
        fields0.sort()
        rItem = "puma"
        for field0 in fields0:
            if(field0 == rItem):
                fields0.remove(rItem)
    for field0 in fields0:
        field0 = "{}-{}".format(shpGroupNameList[shpLayerNameIndex0], fields0[shpLayerNameIndex1])
        fields0[shpLayerNameIndex1] = field0
        shpLayerNameIndex1 += 1
    shpLayerNameList.append(fields0)
    shpLayerNameIndex0 += 1

#populates the SHP layer URI list in nested list consistant with the SHP layer name list
for shpLayerNameList1 in shpLayerNameList:
    shpLayerUriList1 = []
    shpLayerUriIndex1 = 0
    for shpLayerName in shpLayerNameList1:
        shpLayerUri = shpLayerDirUri + "/" + shpLayerNameList[shpLayerUriIndex0][shpLayerUriIndex1] + '.shp'
        shpLayerUriList1.append(shpLayerUri)
        shpLayerUriIndex1 +=1
    shpLayerUriList.append(shpLayerUriList1)
    shpLayerUriIndex0 +=1

#loop to create and perform all operations on all SHP layers including grouping
for shpLayerNameList1 in shpLayerNameList:
    shpLayerIndex1 = 0
    for shpLayerName in shpLayerNameList1:
        #creates the shape layer name from the list of csv names and fields
        shpLayerName = shpLayerNameList[shpLayerIndex0][shpLayerIndex1]
        #adds the initial shape layer to the project and checks the layer validity
        shpLayer = QgsVectorLayer(shpSrcUri, shpLayerName, "ogr")
        QgsProject.instance().addMapLayer(shpLayer)
        if not shpLayer or not shpLayer.isValid():
            shpAddLayerErrorMsg = shpLayerName + " failed to load!"
            print(shpAddLayerErrorMsg)
        #copies the shape layer and places the copy in the correct group
        root = QgsProject.instance().layerTreeRoot()
        shpGroupName = shpGroupNameList[shpLayerIndex0]
        shpGroup = root.findGroup(shpGroupName)
        shpOriginalLayer = root.findLayer(shpLayer.id())
        shpGroupedLayer = shpOriginalLayer.clone()
        shpGroup.insertChildNode(-1, shpGroupedLayer)
        #saves the grouped shape layer and checks the layer validity
        shpSaveOptions = QgsVectorFileWriter.SaveVectorOptions()
        shpSaveOptions.driverName = "ESRI Shapefile"
        shpSaveOptions.fileEncoding = "UTF-8"
        shpSavedLayer = shpGroupedLayer.layer()
        QgsVectorFileWriter.writeAsVectorFormatV2(shpSavedLayer, shpLayerUriList[shpLayerIndex0][shpLayerIndex1], QgsCoordinateTransformContext(), shpSaveOptions)
        if not shpGroupedLayer.layer() or not shpGroupedLayer.layer().isValid():
            shpSaveLayerErrorMsg = shpLayerName + " failed to save!"
            print(shpAddLayerErrorMsg)
        #joins the shape layer to the proper CSV field
        shpField = 'puma'
        csvField = 'puma'
        joinCsvName = shpSavedLayer.name().split("-")[0]
        for csvSavedLayer in csvSavedLayerList:
            csvSavedLayerName = csvSavedLayer.name()
            if csvSavedLayerName == joinCsvName:
                joinCsvLayer = csvSavedLayer
        joinCsvField = []
        joinCsvField.append(shpSavedLayer.name().split("-")[1])
        joinObject = QgsVectorLayerJoinInfo()
        joinObject.setJoinLayerId(joinCsvLayer.id())
        joinObject.setJoinFieldName(csvField)
        joinObject.setTargetFieldName(shpField)
        joinObject.setUsingMemoryCache(True)
        joinObject.setJoinFieldNamesSubset(joinCsvField)
        shpSavedLayer.addJoin(joinObject)
        #removes the ungrouped shape layer
        parent = shpOriginalLayer.parent()
        parent.removeChildNode(root.findLayer(shpLayer.id()))
        
        shpLayerIndex1 += 1
        
    shpLayerIndex0 +=1

#print(vectorLayerNameIndex0)
#print(vectorLayerNameIndex1)
#print(fields0)
#print(fields1)
#print(vectorLayerNameList)
#print(vectorLayerUriList)

"""
layers = list(QgsProject.instance().mapLayers().values())

myColumn = ''
myRangeList = []
myOpacity = 1

ranges = []

gradientIndex = 0
gradientLevels = 20

while gradientIndex < gradientLevels:
    myMin = gradientIndex * 5
    myMax = (gradientIndex + 1) * 5
    myLabel = "{}-{}".format(myMin, myMax)
    colorRGB = '{:x}'.format(255-(gradientIndex * (255 // gradientLevels)))
    myColor = QtGui.QColor("#{}{}{}".format(colorRGB, colorRGB, colorRGB))
    ranges.append((myMin, myMax, myLabel, myColor))
    gradientIndex += 1

for myMin, myMax, myLabel, myColor in ranges:
    mySymbol = QgsSymbol.defaultSymbol(2)
    mySymbol.setColor(myColor)
    mySymbol.setOpacity(myOpacity)
    myRange = QgsRendererRange(myMin, myMax, mySymbol, myLabel)
    myRangeList.append(myRange)

myRenderer = QgsGraduatedSymbolRenderer('', myRangeList)
myRenderer.setMode(QgsGraduatedSymbolRenderer.Quantile)
myRenderer.setClassAttribute(myColumn)

for layer in layers:
    layerType = layer.type()
    if layerType == QgsMapLayerType.VectorLayer:
        layer.setRenderer(myRenderer)
"""

1 Answer 1

1

for this block of code:

#joins the shape layer to the proper CSV field
shpField = 'puma'
csvField = 'puma'
joinCsvName = shpSavedLayer.name().split("-")[0]
for csvSavedLayer in csvSavedLayerList:
    csvSavedLayerName = csvSavedLayer.name()
    if csvSavedLayerName == joinCsvName:
        joinCsvLayer = csvSavedLayer
joinCsvField = []
joinCsvField.append(shpSavedLayer.name().split("-")[1])
joinObject = QgsVectorLayerJoinInfo()
joinObject.setJoinLayerId(joinCsvLayer.id())
joinObject.setJoinFieldName(csvField)
joinObject.setTargetFieldName(shpField)
joinObject.setUsingMemoryCache(True)
joinObject.setJoinFieldNamesSubset(joinCsvField)
shpSavedLayer.addJoin(joinObject)
#removes the ungrouped shape layer
parent = shpOriginalLayer.parent()
parent.removeChildNode(root.findLayer(shpLayer.id()))

joinObject.setJoinLayerId(joinCsvLayer.id()) should be joinObject.setJoinLayer(joinCsvLayer). For some reason, the setJoinLayerId() method did not work in this instance and the setJoinLayer() method did.

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.