I'm currently in the works of creating a pythonic way of extracting street names from an OSM dump for the Province of Quebec. Originally, I wanted to do this in Python to make it easier to integrate the process in future code, but I'm failing miserably, and after weeks of on and off research without getting anywhere, so now I'll honestly take what I can get.
What's the best way to extract all the street names from a dump such as these? http://download.geofabrik.de/north-america/canada/quebec.html
Here's my original code if anyone can make it work. At the moment this code maxs my 16GB of ram and 32GB of swap in about a minute before it evens begins going through each element.
from lxml import etree import xmltodict, sys, gc from pymongo import MongoClient #Ultimate fix of life. reload(sys) sys.setdefaultencoding("utf-8") client = MongoClient() db = client.re streetsDB = db.streets hwTypes = ['motorway', 'trunk', 'primary', 'secondary', 'tertiary', 'pedestrian', 'unclassified', 'service'] #Enable Garbadge Collection gc.enable() def fast_iter(context, func): print 'Begun processing' placement = 1 # http://www.ibm.com/developerworks/xml/library/x-hiperfparse/ # Author: Liza Daly for event, elem in context: placement += 1 print placement func(elem) elem.clear() while elem.getprevious() is not None: del elem.getparent() del context def process_element(elem): data = etree.tostring(elem) data = xmltodict.parse(data) keys = data['way'].keys() if 'tag' in keys: if isinstance(data['way']['tag'], dict): if data['way']['tag']['@k'] == 'highway': if data['way']['tag']['@v'] in hwTypes: streetsDB.insert(data) else: for y in data['way']['tag']: if y['@k'] == 'highway': if y['@v'] in hwTypes: streetsDB.insert(data) break del data del keys gc.collect() context = etree.iterparse('quebec-latest.osm', tag='way' ) fast_iter(context,process_element)