How to increase shapefile creation speed from CSV file in GeoTools

Question

I want create shapefile from CSV. Here is my code

 File file = JFileDataStoreChooser.showOpenFile("csv", null);
    if (file == null) {
        return;
    }

    final SimpleFeatureType TYPE =
            DataUtilities.createType(
                    "Location",
                    "the_geom:Point:srid=4326,"
                            + // <- the geometry attribute: Point type
                            "name:String,"
                            + // <- a String attribute
                            "number:Float" // a number attribute
            );
    System.out.println("TYPE:" + TYPE);

     /*
     * A list to collect features as we create them.
     */
    List<SimpleFeature> features = new ArrayList<>();

    /*
     * GeometryFactory will be used to create the geometry attribute of each feature,
     * using a Point object for the location.
     */
    GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory();

    SimpleFeatureBuilder featureBuilder = new SimpleFeatureBuilder(TYPE);

    try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
        /* First line of the data file is the header */
        String line = reader.readLine();
        System.out.println("Header: " + line);

        double latitude;
        double longitude;
        double number;
        String name;
        int index=0;

        for (line = reader.readLine(); line != null; line = reader.readLine()) {
            if (line.trim().length() > 0) { // skip blank lines
                String[] tokens = line.split(";");
                try {
                    latitude = Double.parseDouble(tokens[9].replace(',', '.').replace(" ",""));
                    longitude = Double.parseDouble(tokens[8].replace(',', '.').replace(" ",""));
                    name = tokens[1].trim();
                    number = Double.parseDouble(tokens[5].trim().replace(",",".").replace(" ",""));
                }catch (Exception e){
                    System.out.print("ERROR on line:"+e.toString()+(index++)+"=>");
                    for (String tkn:tokens)
                        System.out.print( tkn + ";");
                    System.out.println();
                    continue;
                }

                /* Longitude (= x coord) first ! */
                Point point = geometryFactory.createPoint(new Coordinate(longitude, latitude));

                featureBuilder.add(point);
                featureBuilder.add(name);
                featureBuilder.add(number);
                SimpleFeature feature = featureBuilder.buildFeature(null);
                features.add(feature);
            }
        }
    }

     /*
     * Get an output file name and create the new shapefile
     */
    File newFile = getNewShapeFile(file);

    ShapefileDataStoreFactory dataStoreFactory = new ShapefileDataStoreFactory();

    Map<String, Serializable> params = new HashMap<>();
    params.put("url", newFile.toURI().toURL());
    params.put("create spatial index", Boolean.TRUE);

    ShapefileDataStore newDataStore =
            (ShapefileDataStore) dataStoreFactory.createNewDataStore(params);

    /*
     * TYPE is used as a template to describe the file contents
     */
    newDataStore.createSchema(TYPE);

     /*
     * Write the features to the shapefile
     */
    Transaction transaction = new DefaultTransaction("create");

    String typeName = newDataStore.getTypeNames()[0];
    SimpleFeatureSource featureSource = newDataStore.getFeatureSource(typeName);
    SimpleFeatureType SHAPE_TYPE = featureSource.getSchema();
    /*
     * The Shapefile format has a couple limitations:
     * - "the_geom" is always first, and used for the geometry attribute name
     * - "the_geom" must be of type Point, MultiPoint, MuiltiLineString, MultiPolygon
     * - Attribute names are limited in length
     * - Not all data types are supported (example Timestamp represented as Date)
     *
     * Each data store has different limitations so check the resulting SimpleFeatureType.
     */
    System.out.println("SHAPE:" + SHAPE_TYPE);

    if (featureSource instanceof SimpleFeatureStore) {
        SimpleFeatureStore featureStore = (SimpleFeatureStore) featureSource;
        /*
         * SimpleFeatureStore has a method to add features from a
         * SimpleFeatureCollection object, so we use the ListFeatureCollection
         * class to wrap our list of features.
         */
        SimpleFeatureCollection collection = new ListFeatureCollection(TYPE, features);
        featureStore.setTransaction(transaction);
        try {
            featureStore.addFeatures(collection);
            transaction.commit();
        } catch (Exception problem) {
            problem.printStackTrace();
            transaction.rollback();
        } finally {
            transaction.close();
        }
        System.exit(0); // success!
    } else {
        System.out.println(typeName + " does not support read/write access");
        System.exit(1);
    }

I got this code from GeoTools tutorial. It is working perfect for 500 000 points. But I have 8000000 points. featureStore.addFeatures(collection); is working so slowly. How can I increase the performance?

Answer 1

You can make life easier by using a CSVDatastore to read in your data and then if you insist on writing a shapefile, you need to retype the features which will be slow and will probably use up a lot of memory, so you need to chunk up the input and output. You'll probably end up with something like:

File inFile = new File("states.csv");
File outFile = new File("states.shp");
outFile.createNewFile();
// Read
Map<String, Object> params = new HashMap<>();
params.put(CSVDataStoreFactory.FILE_PARAM.key, inFile);
params.put(CSVDataStoreFactory.STRATEGYP.key, CSVDataStoreFactory.GUESS_STRATEGY);
params.put(CSVDataStoreFactory.LATFIELDP.key, "lat");
params.put(CSVDataStoreFactory.LnGFIELDP.key, "lng");

FileDataStore csvDataStore = (FileDataStore) DataStoreFinder.getDataStore(params);
String inputTypeName = csvDataStore.getTypeNames()[0];
// Write

FeatureSource<SimpleFeatureType, SimpleFeature> source = csvDataStore.getFeatureSource(inputTypeName);

FeatureCollection<SimpleFeatureType, SimpleFeature> inputFeatureCollection = source.getFeatures();

DataStore shpDataStore = DataStoreFinder.getDataStore(Collections.singletonMap("url", URLs.fileToUrl(outFile)));
SimpleFeatureType schema = csvDataStore.getSchema();
SimpleFeatureType newSchema = retypeForFormat(schema);

shpDataStore.createSchema(newSchema);

String typeName = shpDataStore.getTypeNames()[0];

SimpleFeatureStore featureStore = (SimpleFeatureStore) shpDataStore.getFeatureSource(typeName);

List<SimpleFeature> feats = new ArrayList<>(MAXCHUNK);
int count = 0;
try (FeatureIterator<SimpleFeature> features2 = inputFeatureCollection.features()) {
  while (features2.hasNext()) {
    while (count++ < MAXCHUNK) {
      SimpleFeature f = features2.next();
      SimpleFeature reType = DataUtilities.reType(newSchema, f, true);

      reType.setAttribute(newSchema.getGeometryDescriptor().getName(),
          f.getAttribute(schema.getGeometryDescriptor().getName()));

      feats.add(reType);
    }
    ListFeatureCollection collection = new ListFeatureCollection(newSchema, feats);
    try (Transaction transaction = new DefaultTransaction("create")) {
      featureStore.setTransaction(transaction);
      try {
        List<FeatureId> ids = featureStore.addFeatures(collection);

        transaction.commit();
      } catch (Exception problem) {
        problem.printStackTrace();
        transaction.rollback();
      }
    }
  }
}

with the schema reformatting done in:

private static SimpleFeatureType retypeForFormat(SimpleFeatureType schema) {
    SimpleFeatureType ret = schema;

    GeometryDescriptor geom = schema.getGeometryDescriptor();

    List<AttributeDescriptor> attributes = schema.getAttributeDescriptors();
    GeometryType geomType = null;
    List<AttributeDescriptor> attribs = new ArrayList<>();
    for (AttributeDescriptor attrib : attributes) {
      AttributeType type = attrib.getType();
      if (type instanceof GeometryType) {
        geomType = (GeometryType) type;
      } else {
        attribs.add(attrib);
      }
    }

    GeometryTypeImpl gt = new GeometryTypeImpl(new NameImpl("the_geom"), geomType.getBinding(),
        geom.getCoordinateReferenceSystem(), geomType.isIdentified(), geomType.isAbstract(), geomType.getRestrictions(),
        geomType.getSuper(), geomType.getDescription());

    GeometryDescriptor geomDesc = new GeometryDescriptorImpl(gt, new NameImpl("the_geom"), geom.getMinOccurs(),
        geom.getMaxOccurs(), geom.isNillable(), geom.getDefaultValue());

    attribs.add(0, geomDesc);

    ret = new SimpleFeatureTypeImpl(schema.getName(), attribs, geomDesc, schema.isAbstract(), schema.getRestrictions(),
        schema.getSuper(), schema.getDescription());

    return ret;
  }

As @Vince points out in the comments, you will need to watch that your shapefile stays with in the bounds of 2Gb. So for easy of use and speed of copying I would go for a GeoPackage instead which makes the code much simpler.

File inFile = new File("states.csv");
File outFile = new File("states.gpkg");
outFile.createNewFile();
// Read
Map<String, Object> params = new HashMap<>();
params.put(CSVDataStoreFactory.FILE_PARAM.key, inFile);
params.put(CSVDataStoreFactory.STRATEGYP.key, CSVDataStoreFactory.GUESS_STRATEGY);
params.put(CSVDataStoreFactory.LATFIELDP.key, "lat");
params.put(CSVDataStoreFactory.LnGFIELDP.key, "lng");

FileDataStore csvDataStore = (FileDataStore) DataStoreFinder.getDataStore(params);
String inputTypeName = csvDataStore.getTypeNames()[0];
// Write

FeatureSource<SimpleFeatureType, SimpleFeature> source = csvDataStore.getFeatureSource(inputTypeName);

FeatureCollection<SimpleFeatureType, SimpleFeature> inputFeatureCollection = source.getFeatures();

Map<String, Object> outParams = new HashMap<>();
outParams.put(GeoPkgDataStoreFactory.DBTYPE.key, GeoPkgDataStoreFactory.DBTYPE.sample);
outParams.put(GeoPkgDataStoreFactory.DATABASE.key, outFile);
DataStore gpkgDataStore = DataStoreFinder.getDataStore(outParams);
SimpleFeatureType schema = csvDataStore.getSchema();


gpkgDataStore.createSchema(schema);

String typeName = gpkgDataStore.getTypeNames()[0];

SimpleFeatureStore featureStore = (SimpleFeatureStore) gpkgDataStore.getFeatureSource(typeName);

featureStore.addFeatures(inputFeatureCollection);