2

I want create shapefile from CSV. Here is my code

 File file = JFileDataStoreChooser.showOpenFile("csv", null);
    if (file == null) {
        return;
    }

    final SimpleFeatureType TYPE =
            DataUtilities.createType(
                    "Location",
                    "the_geom:Point:srid=4326,"
                            + // <- the geometry attribute: Point type
                            "name:String,"
                            + // <- a String attribute
                            "number:Float" // a number attribute
            );
    System.out.println("TYPE:" + TYPE);

     /*
     * A list to collect features as we create them.
     */
    List<SimpleFeature> features = new ArrayList<>();

    /*
     * GeometryFactory will be used to create the geometry attribute of each feature,
     * using a Point object for the location.
     */
    GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory();

    SimpleFeatureBuilder featureBuilder = new SimpleFeatureBuilder(TYPE);

    try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
        /* First line of the data file is the header */
        String line = reader.readLine();
        System.out.println("Header: " + line);

        double latitude;
        double longitude;
        double number;
        String name;
        int index=0;

        for (line = reader.readLine(); line != null; line = reader.readLine()) {
            if (line.trim().length() > 0) { // skip blank lines
                String[] tokens = line.split(";");
                try {
                    latitude = Double.parseDouble(tokens[9].replace(',', '.').replace(" ",""));
                    longitude = Double.parseDouble(tokens[8].replace(',', '.').replace(" ",""));
                    name = tokens[1].trim();
                    number = Double.parseDouble(tokens[5].trim().replace(",",".").replace(" ",""));
                }catch (Exception e){
                    System.out.print("ERROR on line:"+e.toString()+(index++)+"=>");
                    for (String tkn:tokens)
                        System.out.print( tkn + ";");
                    System.out.println();
                    continue;
                }

                /* Longitude (= x coord) first ! */
                Point point = geometryFactory.createPoint(new Coordinate(longitude, latitude));

                featureBuilder.add(point);
                featureBuilder.add(name);
                featureBuilder.add(number);
                SimpleFeature feature = featureBuilder.buildFeature(null);
                features.add(feature);
            }
        }
    }

     /*
     * Get an output file name and create the new shapefile
     */
    File newFile = getNewShapeFile(file);

    ShapefileDataStoreFactory dataStoreFactory = new ShapefileDataStoreFactory();

    Map<String, Serializable> params = new HashMap<>();
    params.put("url", newFile.toURI().toURL());
    params.put("create spatial index", Boolean.TRUE);

    ShapefileDataStore newDataStore =
            (ShapefileDataStore) dataStoreFactory.createNewDataStore(params);

    /*
     * TYPE is used as a template to describe the file contents
     */
    newDataStore.createSchema(TYPE);

     /*
     * Write the features to the shapefile
     */
    Transaction transaction = new DefaultTransaction("create");

    String typeName = newDataStore.getTypeNames()[0];
    SimpleFeatureSource featureSource = newDataStore.getFeatureSource(typeName);
    SimpleFeatureType SHAPE_TYPE = featureSource.getSchema();
    /*
     * The Shapefile format has a couple limitations:
     * - "the_geom" is always first, and used for the geometry attribute name
     * - "the_geom" must be of type Point, MultiPoint, MuiltiLineString, MultiPolygon
     * - Attribute names are limited in length
     * - Not all data types are supported (example Timestamp represented as Date)
     *
     * Each data store has different limitations so check the resulting SimpleFeatureType.
     */
    System.out.println("SHAPE:" + SHAPE_TYPE);

    if (featureSource instanceof SimpleFeatureStore) {
        SimpleFeatureStore featureStore = (SimpleFeatureStore) featureSource;
        /*
         * SimpleFeatureStore has a method to add features from a
         * SimpleFeatureCollection object, so we use the ListFeatureCollection
         * class to wrap our list of features.
         */
        SimpleFeatureCollection collection = new ListFeatureCollection(TYPE, features);
        featureStore.setTransaction(transaction);
        try {
            featureStore.addFeatures(collection);
            transaction.commit();
        } catch (Exception problem) {
            problem.printStackTrace();
            transaction.rollback();
        } finally {
            transaction.close();
        }
        System.exit(0); // success!
    } else {
        System.out.println(typeName + " does not support read/write access");
        System.exit(1);
    }

I got this code from GeoTools tutorial. It is working perfect for 500 000 points. But I have 8000000 points. featureStore.addFeatures(collection); is working so slowly. How can I increase the performance?

3
  • Please Edit the question to give performance timing on what "slowly" means and information about the type of device to which you are writing (e.g., local SSD, network share, thumb drive,...)
    – Vince
    Jun 30, 2021 at 13:08
  • more then 15 minutes counting. Jun 30, 2021 at 13:17
  • Please Edit the question to contain the details on the system and performance characteristics for both the 500K and 8M row counts. Note that shapefile has various size limitations that may make 8M features impossible to generate.
    – Vince
    Jun 30, 2021 at 13:25

1 Answer 1

3

You can make life easier by using a CSVDatastore to read in your data and then if you insist on writing a shapefile, you need to retype the features which will be slow and will probably use up a lot of memory, so you need to chunk up the input and output. You'll probably end up with something like:

File inFile = new File("states.csv");
File outFile = new File("states.shp");
outFile.createNewFile();
// Read
Map<String, Object> params = new HashMap<>();
params.put(CSVDataStoreFactory.FILE_PARAM.key, inFile);
params.put(CSVDataStoreFactory.STRATEGYP.key, CSVDataStoreFactory.GUESS_STRATEGY);
params.put(CSVDataStoreFactory.LATFIELDP.key, "lat");
params.put(CSVDataStoreFactory.LnGFIELDP.key, "lng");

FileDataStore csvDataStore = (FileDataStore) DataStoreFinder.getDataStore(params);
String inputTypeName = csvDataStore.getTypeNames()[0];
// Write

FeatureSource<SimpleFeatureType, SimpleFeature> source = csvDataStore.getFeatureSource(inputTypeName);

FeatureCollection<SimpleFeatureType, SimpleFeature> inputFeatureCollection = source.getFeatures();

DataStore shpDataStore = DataStoreFinder.getDataStore(Collections.singletonMap("url", URLs.fileToUrl(outFile)));
SimpleFeatureType schema = csvDataStore.getSchema();
SimpleFeatureType newSchema = retypeForFormat(schema);

shpDataStore.createSchema(newSchema);

String typeName = shpDataStore.getTypeNames()[0];

SimpleFeatureStore featureStore = (SimpleFeatureStore) shpDataStore.getFeatureSource(typeName);

List<SimpleFeature> feats = new ArrayList<>(MAXCHUNK);
int count = 0;
try (FeatureIterator<SimpleFeature> features2 = inputFeatureCollection.features()) {
  while (features2.hasNext()) {
    while (count++ < MAXCHUNK) {
      SimpleFeature f = features2.next();
      SimpleFeature reType = DataUtilities.reType(newSchema, f, true);

      reType.setAttribute(newSchema.getGeometryDescriptor().getName(),
          f.getAttribute(schema.getGeometryDescriptor().getName()));

      feats.add(reType);
    }
    ListFeatureCollection collection = new ListFeatureCollection(newSchema, feats);
    try (Transaction transaction = new DefaultTransaction("create")) {
      featureStore.setTransaction(transaction);
      try {
        List<FeatureId> ids = featureStore.addFeatures(collection);

        transaction.commit();
      } catch (Exception problem) {
        problem.printStackTrace();
        transaction.rollback();
      }
    }
  }
}

with the schema reformatting done in:

private static SimpleFeatureType retypeForFormat(SimpleFeatureType schema) {
    SimpleFeatureType ret = schema;

    GeometryDescriptor geom = schema.getGeometryDescriptor();

    List<AttributeDescriptor> attributes = schema.getAttributeDescriptors();
    GeometryType geomType = null;
    List<AttributeDescriptor> attribs = new ArrayList<>();
    for (AttributeDescriptor attrib : attributes) {
      AttributeType type = attrib.getType();
      if (type instanceof GeometryType) {
        geomType = (GeometryType) type;
      } else {
        attribs.add(attrib);
      }
    }

    GeometryTypeImpl gt = new GeometryTypeImpl(new NameImpl("the_geom"), geomType.getBinding(),
        geom.getCoordinateReferenceSystem(), geomType.isIdentified(), geomType.isAbstract(), geomType.getRestrictions(),
        geomType.getSuper(), geomType.getDescription());

    GeometryDescriptor geomDesc = new GeometryDescriptorImpl(gt, new NameImpl("the_geom"), geom.getMinOccurs(),
        geom.getMaxOccurs(), geom.isNillable(), geom.getDefaultValue());

    attribs.add(0, geomDesc);

    ret = new SimpleFeatureTypeImpl(schema.getName(), attribs, geomDesc, schema.isAbstract(), schema.getRestrictions(),
        schema.getSuper(), schema.getDescription());

    return ret;
  }

As @Vince points out in the comments, you will need to watch that your shapefile stays with in the bounds of 2Gb. So for easy of use and speed of copying I would go for a GeoPackage instead which makes the code much simpler.

File inFile = new File("states.csv");
File outFile = new File("states.gpkg");
outFile.createNewFile();
// Read
Map<String, Object> params = new HashMap<>();
params.put(CSVDataStoreFactory.FILE_PARAM.key, inFile);
params.put(CSVDataStoreFactory.STRATEGYP.key, CSVDataStoreFactory.GUESS_STRATEGY);
params.put(CSVDataStoreFactory.LATFIELDP.key, "lat");
params.put(CSVDataStoreFactory.LnGFIELDP.key, "lng");

FileDataStore csvDataStore = (FileDataStore) DataStoreFinder.getDataStore(params);
String inputTypeName = csvDataStore.getTypeNames()[0];
// Write

FeatureSource<SimpleFeatureType, SimpleFeature> source = csvDataStore.getFeatureSource(inputTypeName);

FeatureCollection<SimpleFeatureType, SimpleFeature> inputFeatureCollection = source.getFeatures();

Map<String, Object> outParams = new HashMap<>();
outParams.put(GeoPkgDataStoreFactory.DBTYPE.key, GeoPkgDataStoreFactory.DBTYPE.sample);
outParams.put(GeoPkgDataStoreFactory.DATABASE.key, outFile);
DataStore gpkgDataStore = DataStoreFinder.getDataStore(outParams);
SimpleFeatureType schema = csvDataStore.getSchema();


gpkgDataStore.createSchema(schema);

String typeName = gpkgDataStore.getTypeNames()[0];

SimpleFeatureStore featureStore = (SimpleFeatureStore) gpkgDataStore.getFeatureSource(typeName);

featureStore.addFeatures(inputFeatureCollection);

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.