I'm currently in the works of creating a pythonic way of extracting street names from an OSM dump for the Province of Quebec. Originally, I wanted to do this in Python to make it easier to integrate the process in future code, but I'm failing miserably, and after weeks of on and off research without getting anywhere, so now I'll honestly take what I can get.
What's the best way to extract all the street names from a dump such as these? http://download.geofabrik.de/north-america/canada/quebec.html
Here's my original code if anyone can make it work. At the moment this code maxs my 16GB of ram and 32GB of swap in about a minute before it evens begins going through each element.
from lxml import etree
import xmltodict, sys, gc
from pymongo import MongoClient
#Ultimate fix of life.
reload(sys)
sys.setdefaultencoding("utf-8")
client = MongoClient()
db = client.re
streetsDB = db.streets
hwTypes = ['motorway', 'trunk', 'primary', 'secondary', 'tertiary', 'pedestrian', 'unclassified', 'service']
#Enable Garbadge Collection
gc.enable()
def fast_iter(context, func):
print 'Begun processing'
placement = 1
# http://www.ibm.com/developerworks/xml/library/x-hiperfparse/
# Author: Liza Daly
for event, elem in context:
placement += 1
print placement
func(elem)
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
del context
def process_element(elem):
data = etree.tostring(elem)
data = xmltodict.parse(data)
keys = data['way'].keys()
if 'tag' in keys:
if isinstance(data['way']['tag'], dict):
if data['way']['tag']['@k'] == 'highway':
if data['way']['tag']['@v'] in hwTypes:
streetsDB.insert(data)
else:
for y in data['way']['tag']:
if y['@k'] == 'highway':
if y['@v'] in hwTypes:
streetsDB.insert(data)
break
del data
del keys
gc.collect()
context = etree.iterparse('quebec-latest.osm', tag='way' )
fast_iter(context,process_element)