PyQGIS script to add multiple delimited text layers from folder to project

Question

I have a folder of ~120 csv files.

I would like to add them all to my QGIS project.

I know this is accomplished with the 'Data Source Manager' and 'Delimited Text' option. However, instead of adding them one by one I would like a script to run in the Python console to add every one of them in the folder.

I have started the potential frame for this code:

import os

#Define the source folder
src_folder = 'C:\Users\Rollins\Documents\PVQ_Stuff\General_Election'

#Iterate through the files in the folder
for file in os.scandir(src_folder):
    if file.name.endswith('.csv'):
        #Define the csv name when imported as a layer
        layer_name = file.name[-6:-1]
        #This part I made up! I have no idea if any of this syntax is correct, but it's my
        # best guess for calling the 'delimited text' tool multiple times and defining the 
        # parameters based on the current file in the folder
        params = {'FILE_NAME':file.name,'LAYER_NAME':layer_name,
                    'ENCODING':'latin9','FILE_FORMAT':custom_delimiters[semicolon],
                    'GEOMETRY_DEFINITION':no_geometry}
        processing.run("qgis:datasourcemanager:delimitedtext", params)

As I write in the above code, I don't know how to select the 'delimited text' tool and input it's parameters using a script.

Answer 1

Try using os.walk like this:

import os

basefolder = '/home/bera/GIS/Data/testdata/csvfiles'
epsg_code=3006
xfield, yfield = 'CoordinateX', 'CoordinateY'
delimiter = ','

layers=[]
for root, folder, files in os.walk(basefolder):
    for file in files:
        fullPath = os.path.join(root, file)
        if os.path.isfile(fullPath) and fullPath.endswith('.csv'):
            uri = "file://{}?delimiter='{}'&xField={}&yField={}&crs=epsg:{}".format(fullPath, delimiter, xfield, yfield, epsg_code)
            vlayer = QgsVectorLayer(uri, os.path.basename(file).split('.')[0] , "delimitedtext")
            layers.append(vlayer)

QgsProject.instance().addMapLayers(layers)

Answer 2

A self contained solution by downloading a 3rd party CSV on the net, chunking it into part, then looping to add each CSV files created from these chunks. You should be able to adapt this working solution for your case

import csv
from pathlib import Path
from io import BytesIO, TextIOWrapper
from zipfile import ZipFile
from urllib.request import urlopen

from qgis.core import QgsVectorLayer

# path to destination for the demo
outpath = '/tmp'

# Remove in your sample until we say to keep code

def gen_chunks(reader, chunksize=100):
    """ 
    Chunk generator. Take a CSV `reader` and yield
    `chunksize` sized slices.
    """
    chunk = []
    for i, line in enumerate(reader):
        if (i % chunksize == 0 and i > 0):
            yield chunk
            del chunk[:]
        chunk.append(line)
    yield chunk

# Download CSV
url = "https://www.douane.gouv.fr/sites/default/files/openData/files/annuaire-des-debits-de-tabac-2018.zip"
resp = urlopen(url)

# Read from CSV
zipfile = ZipFile(BytesIO(resp.read()))
files_list = zipfile.namelist()
print(files_list)

# Read CSV file input and chunk it into small parts to reproduce the case
with zipfile.open(files_list[0], 'r') as infile:
    reader = csv.reader(TextIOWrapper(infile, 'iso-8859-1'))
    for i, chunk in enumerate(gen_chunks(reader)):
        with open(Path(outpath) / f'filename_{i}.csv', 'w') as csvinfile:
            if i == 0:
                headers = chunk[0]
            else:
                chunk.insert(0, headers)
            writer = csv.writer(csvinfile)
            # import ipdb;ipdb.set_trace()
            writer.writerows(chunk) # process chunk

# Above code removable for your case
# Later, keep and change outpath to your input path

outpath = '/tmp'
current_dir = Path(outpath)
current_absolute_dir = Path.resolve(current_dir)
delimiter = ';'

project = QgsProject.instance()
layers = []
# Loop on files using pathlib glob
for i, file in enumerate(current_dir.glob('filename*.csv')):
    # Limitation to only append 10 files otherwise around 245 files
    # Remove condition and deindent
    if i < 10:
        uri = f'file://{ current_absolute_dir / file }?type=csv&delimiter={delimiter}&maxFields=10000&detectTypes=yes&geomType=none&subsetIndex=no&watchFile=no'
        lyr = QgsVectorLayer(uri, Path(file).stem, "delimitedtext")
        layers.append(lyr)

project.addMapLayers(layers)

Answer 3

I figured out another way in the end, but this method from the other responses:

QgsProject.instance().addMapLayers(layers)

Was the key component I was missing. This is how I got it to work:

#Declare source folder 
src_folder = 'C:/Users/Rollins/Documents/PVQ_Stuff/General Election by Polling Subdivision 2018-10-01'

#Declare open project as an object
project = QgsProject.instance()

#Iterate through the src folder to add all the csv files as vector layers.
for file in os.scandir(src_folder):
    #Select proper files
    if file.name.startswith('DGE'):
        #Turn csv file into a vector layer
        vlayer = QgsVectorLayer(file.path, file.name, 'ogr')
        #Add files as layers to the project
        project.addMapLayer(vlayer)