2

I have a QGIS 3 processing script that I'm working on which has the following workflow:

  1. Get xy coordinates from a grid layer and construct a list.
  2. Make calls to the Google Distance Matrix API using the list of coordinates.
  3. Parse output json file and extract distance and time values, then write to a CSV file.
  4. Add CSV file to project as point layer.

I'm stuck at step 4 because I'm not sure how I can create a feature sink for the output, or if that is even required in this case since my desired output is in csv format.

Here's my processing script which I adapted from the template.

# -*- coding: utf-8 -*-

"""
***************************************************************************
*                                                                         *
*   This program is free software; you can redistribute it and/or modify  *
*   it under the terms of the GNU General Public License as published by  *
*   the Free Software Foundation; either version 2 of the License, or     *
*   (at your option) any later version.                                   *
*                                                                         *
***************************************************************************
"""

import requests
import csv
import itertools
import os
import time
from urllib.parse import *
from PyQt5.QtCore import QCoreApplication, QVariant
from qgis.core import (QgsProcessing,
                       QgsVectorLayer,
                       QgsProject,
                       QgsProcessingException,
                       QgsProcessingAlgorithm,
                       QgsProcessingParameterString,
                       QgsVectorFileWriter
                       )
import processing


class DriveTime(QgsProcessingAlgorithm):
    INPUT = 'centre'
    MODE = 'travel_mode'

    def tr(self, string):
        return QCoreApplication.translate('Processing', string)

    def createInstance(self):
        return DriveTime()

    def name(self):
        return 'drive_time'

    def displayName(self):
        return self.tr('Drive Time')

    def group(self):
        return self.tr('base')

    def groupId(self):
        return 'base'

    def shortHelpString(self):
        return self.tr("blahblahblah")

    def initAlgorithm(self, config=None):
        self.addParameter(QgsProcessingParameterString(self.INPUT,'Centre Coordinates'))
        self.addParameter(QgsProcessingParameterString(self.MODE,"Travel Mode: Input 'driving', 'transit', or 'walking'."))

    def processAlgorithm(self, parameters, context, feedback):
        centre = self.parameterAsString(parameters,self.INPUT,context)
        travel_mode = self.parameterAsString(parameters,self.MODE,context)

        MY_API_KEY = "AAAAAAAA"
        TRY_ATTEMPT_MAX = 5
        GROUP_N = 40
        SLEEP_DELAY = 0.5  # 1000 per 10 seconds; 100 per second; 50 per 0.5 seconds

        def mygrouper(n, iterable):
            """
            Cut a list into chunks of (n)
            """
            args = [iter(iterable)] * n
            return ([e for e in t if e is not None] for t in itertools.zip_longest(*args))

        def build_url(private_key,
                      travel_mode,
                      origin,
                      centre):
            """
            Create URL to send to Google API given origin and destination
            """
            if origin == '':
                raise Exception('origin cannot be blank.')
            elif isinstance(origin, str):
                origin_str = quote(origin)
            elif isinstance(origin, list):
                origin_str = ''
                for element in origin:
                    if isinstance(element, str):
                        origin_str = '{0}|{1}'.format(origin_str, quote(element))
                    elif isinstance(element, list) and len(element) == 2:
                        origin_str = '{0}|{1}'.format(origin_str, ','.join(map(str, element)))
                    else:
                        raise Exception('origin must be a list of lists [lat, lng] or a list of strings.')
                origin_str = origin_str.strip('|')
            else:
                raise Exception('origin must be a a list of lists [lat, lng] or a list of strings.')
            # '|' not correctly handled so use '%7C' instead
            origin_str = origin_str.replace('|', '%7C')
            # print('Building URL...')
            prefix = 'https://maps.googleapis.com/maps/api/distancematrix/json?mode=%s' % travel_mode
            # prefix = 'https://maps.googleapis.com/maps/api/geocode/json'
            url = urlparse('{0}&origins={1}&destinations={2}&key={3}'.format(prefix,
                                                                      origin_str,
                                                                      centre,
                                                                      private_key))
            full_url = url.scheme + '://' + url.netloc + url.path + '?' + url.query
            return full_url

        def parse_json(url):
            response = requests.get(url)
            d = response.json()
            # print('Parsing JSON - Status: %s' % d['status'])
            if not d['status'] == 'OK':
                raise Exception('Error. Google Maps API return status: {}'.format(d['status']))
            addresses = d['origin_addresses']
            i = 0
            durations = [0] * len(addresses)
            distance = [0] * len(addresses)
            for row in d['rows']:
                if not row['elements'][0]['status'] == 'OK':
                    durations[i] = -9999
                else:
                    durations[i] = row['elements'][0]['duration']['value'] / 60
                    distance[i] = row['elements'][0]['distance']['value'] / 1000
                i += 1
            return [addresses, durations, distance]

        def runsearch(centre,travel_mode):
            # Initiate variables
            end_coordinates = []
            counter = 0
            name = '%s_%s' % (centre, travel_mode)
            produce_csv_file = name + '.csv'
            if os.path.isfile(produce_csv_file):
                raise Exception('Please delete: %s' % produce_csv_file)
            else:
                print("Producing: %s" % produce_csv_file)
            #Extracting coordinates from centroids layer.
            layer = QgsProject.instance().mapLayersByName('GridWGS84')[0]
            features = layer.getFeatures()
            attributes = []
            for feature in features:
                attributes.append(feature.attributes())
            coords = []
            for attribute in attributes:
                coords.append(attribute[5:7])
            # Break up coordinates into groups of 50 (process in bulk):
            group_coordinates = mygrouper(GROUP_N, coords)
            for one_coord in group_coordinates:
                url = build_url(MY_API_KEY, travel_mode, one_coord, centre)
                if len(url) >= 2048:
                    raise Exception('URL length %d exceeds maximum of 2048' % (len(url)))
                print(url)
                time.sleep(SLEEP_DELAY)
                try_count = 0
                while try_count < TRY_ATTEMPT_MAX:
                    try:
                        data = parse_json(url)
                        break
                    except Exception as err:
                        time.sleep(5)
                        print('Waiting 5 seconds ... retry: %s' % err)
                        try_count += 1
                if try_count == TRY_ATTEMPT_MAX:
                    raise Exception('Failed to calculate distances - possibly API LIMIT REACHED')
                for i in range(0, len(one_coord)):
                    assert len(one_coord[i]) == 2
                    counter += 1
                    # Save all points (latitude, longitude, minutes)
                    row_append = [one_coord[i][0], one_coord[i][1], data[1][i], data[2][i], travel_mode]
                    end_coordinates.append(row_append)
            # OUTPUT RESULTS to CSV
            myDir = QgsProject.instance().readPath("./")
            f = open(myDir + '/' + produce_csv_file, 'w', newline='')
            fieldnames = ['lat', 'long', 'time', 'distance', 'mode']
            w = csv.writer(f)
            w.writerow(fieldnames)
            for endcord in end_coordinates:
                w.writerow(endcord)
            f.close()

        runsearch(centre, travel_mode)
        myDir = QgsProject.instance().readPath("./")
        uri = 'file:///'+ myDir + '/' + centre + '_' + travel_mode + '.csv?type=csv&xField=long&yField=lat&crs=EPSG:4326'
        csv_layer = QgsVectorLayer(uri, 'Drive Time Points', 'delimitedtext')
        QgsProject.instance().addMapLayer(csv_layer)

        return {}

The runsearch function does its thing, and creates the csv file correctly. However, it doesn't load the csv layer at the end. How should I fix this?

  • 1
    Did u check the validity of the csv_layer generated : csv_layer.isValid() ? – snaileater Mar 7 at 9:21

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.