Reading shape files : UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc3 in position 9: unexpected end of data

Question

I am trying to read shapefiles. I have handled the error that could occur when reading a shapefile by writing a function read_gdf (see code below) but I am still getting an error UnicodeDecodeError: 'utf-8' codec can't...

from pathlib import Path

def read_gdf(file:str,crs:str,current_crs:str)->gpd.GeoDataFrame:
    #Read data
    collection = list(fiona.open(file,'r'))
    df1 = pd.DataFrame(collection)

    
    df1['isvalid'] = df1['geometry'].apply(lambda x: isvalid(x))
    df1 = df1[df1['isvalid'] == 1]
    collection = json.loads(df1.to_json(orient='records'))

    #Convert to geodataframe
    return gpd.GeoDataFrame.from_features(collection, crs=current_crs).to_crs(crs)

gdfs = []
for file in Path(rootdir).rglob("*.shp"):
    print('file is', file)            
    gdf = read_gdf(file, crs,current_crs=current_crs)
    gdf.columns = map(str.lower, gdf.columns)                
    gdfs.append(gdf)

How can I ignore the line that has the error and continue reading the rest of the shapefile?

Answer 1

To solve the issue, change the encoding to encoding='iso-8859-1'

def read_gdf(file:str,crs:str,current_crs:str)->gpd.GeoDataFrame:
    #Read data
    collection = list(fiona.open(file,'r',  encoding='iso-8859-1'))
    df1 = pd.DataFrame(collection)

    
    df1['isvalid'] = df1['geometry'].apply(lambda x: isvalid(x))
    df1 = df1[df1['isvalid'] == 1]
    collection = json.loads(df1.to_json(orient='records'))

    return gpd.GeoDataFrame.from_features(collection, crs=current_crs).to_crs(crs)

Answer 2

Try directly using GeoPandas, without Fiona (GeoPandas use Fiona) and Pandas.

import geopandas as gpd
for file in pathlib.Path(directory).rglob('**/*.shp'):
     print('file is', file)   
     # open the shapefile
     try:
        df1 = gpd.read_file(file)
        # all changes in df1
     except:
        print("error")