I have a Python script that hits a query endpoint for a map service on an unfederated server. I am avoiding the arcgis
library because it is not performant enough and its compatibility with geopandas
is awful. So I'm resorting to using the requests
and concurrent.futures
libraries to query the API in parallel.
import concurrent.futures
import json
import requests
import geopandas as gpd
def from_api(url: str, endpoint: str = None, params: dict = {}) -> dict:
api_url = f"{url}/{endpoint}" if endpoint else url
resp = requests.get(api_url, params=params)
try:
txt = json.loads(resp.text)
except json.JSONDecodeError:
txt = {}
return txt
# open data url
bfr_url = "https://maps.bouldercolorado.gov/arcgis/rest/services/fire/FireResponseTimesOpenData/MapServer/0"
# Get a count of the total number of records in the where clause, along with the max step allowed for pagination
where = "RESPONSEYEAR = 2022"
params = {
"where": where,
"returnCountOnly":True,
"f": "geojson"
}
tot_records = from_api(url=bfr_url, endpoint="query", params=params)["count"] # 15,463
step = from_api(url=bfr_url, params={'f':'json'})["maxRecordCount"] # 1,000
I make sure to paginate the results so that I can get all records for a given where clause, and I also make sure to order the results so that each page theoretically consists of no duplicates.
# Query endpoint parameters (note that I order fields, and iteratively specify offsets)
params = {
"where": where,
"outFields": '*',
"outSr": 4326,
"f": "geojson",
"orderByFields": "OBJECTID",
"resultRecordCount": step
}
with concurrent.futures.ThreadPoolExecutor() as ex:
futures = []
for offset in range(0, tot_records, step):
params["resultOffset"] = offset
futures.append(ex.submit(from_api, bfr_url, "query", params))
results = [f.result() for f in concurrent.futures.as_completed(futures)]
features = [feat for r in results for feat in r["features"]]
gdf = gpd.GeoDataFrame.from_features(features, crs=4326)
I say "theoretically" because it doesn't do this at all: many of the concurrent threads return duplicated results despite carefully designating offsets. A quick gdf.OBJECTID.is_unique
proves false.
Is there a problem with my code, or is this a server-side bug?
EDIT 1:
Attempting this API call serially does give the results I expect, as below:
params = {
"where": where,
"outFields": '*',
"outSr": 4326,
"f": "geojson",
"orderByFields": "OBJECTID"
}
features = []
for offset in range(0, 3000, 1000): # first three pages
params["resultOffset"] = offset
gjson = from_api(bfr_url, "query", params)
features += gjson["features"]
gdf2 = gpd.GeoDataFrame.from_features(features, crs=4326)
gdf2.OBJECTID.is_unique # returns True
len(gdf2) == tot_records # returns True
concurrent.futures
? I know the goal is to speed it up by running concurrently, but checking whether the code runs serially will help isolate the issue.