EDIT: In case the previous links become unavailable, here is another version of the same blog posts:
- Importing BANO dataset with Logstash
- Using Logstash to lookup for addresses in BANO index
- Using Logstash to enrich an existing dataset with BANO
Also here are some content extracted from those sources:
A logstash configuration to read a CSV file using filebeat (but you can replace the input part with whatever source you have):
input {
beats {
port => 5044
}
}
filter {
csv {
separator => ","
columns => [
"id","number","street_name","zipcode","city","source","latitude","longitude"
]
remove_field => [ "message", "host", "@timestamp", "@version", "input", "log", "ecs", "agent", "tags" ]
}
mutate {
convert => { "longitude" => "float" }
convert => { "latitude" => "float" }
}
mutate {
rename => {
"longitude" => "[location][lon]"
"latitude" => "[location][lat]"
"number" => "[address][number]"
"street_name" => "[address][street_name]"
"zipcode" => "[address][zipcode]"
"city" => "[address][city]"
}
}
}
output {
elasticsearch {
hosts => ["http://elasticsearch:9200"]
index => "bano"
user => "elastic"
password => "changeme"
document_id => "%{[id]}"
}
}
Mapping template you can use when creating the index:
PUT _template/bano
{
"index_patterns": "bano*",
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 0,
"index.analysis": {
"analyzer": {
"bano_city_analyzer": {
"type": "custom",
"tokenizer": "standard",
"filter": [
"lowercase",
"asciifolding"
]
},
"bano_street_analyzer": {
"type": "custom",
"tokenizer": "standard",
"filter": [
"elision",
"lowercase",
"asciifolding",
"bano_synonym"
]
}
},
"filter": {
"bano_synonym": {
"type": "synonym",
"synonyms": [
"bd => boulevard",
"av => avenue",
"r => rue",
"rte => route"
]
}
}
}
},
"mappings": {
"properties": {
"address": {
"properties": {
"city": {
"type": "text",
"analyzer": "bano_city_analyzer",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"number": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"street_name": {
"type": "text",
"analyzer": "bano_street_analyzer"
},
"zipcode": {
"type": "keyword"
}
}
},
"id": {
"type": "keyword"
},
"source": {
"type": "keyword"
},
"location": {
"type": "geo_point"
}
}
}
}
Typical request to find a geo point from a postal address:
GET bano/_search
{
"size": 1,
"query": {
"multi_match": {
"query": "6 allée des myrtilles cergy",
"fields": [
"address.city",
"address.street_name",
"address.number"
],
"type": "most_fields"
}
}
}
Typical request to find the closest postal address (and its actual coordinates):
GET bano/_search
{
"size": 1,
"query": {
"bool": {
"filter": {
"geo_distance": {
"distance": "1km",
"location": {
"lat": 49.0409,
"lon": 2.0178
}
}
}
}
},
"sort": [
{
"_geo_distance": {
"location": {
"lat": 49.0409,
"lon": 2.0178
}
}
}
]
}