Calculating max shardSize and fileDimensions to avoid tiling large raster exports in Earth Engine

Question

I am exporting a very large raster in Google Earth Engine to my Google Drive. I am trying to keep my exported raster to 1 file to avoid additional processing to merge the tiles (e.g. GDAL merge in QGIS). I know you can specify larger shardSize and fileDimensions to increase the size of your exported raster tiles.

For example, based on this post, I've been setting my shardSize to 1024 and playing around with my fileDimensions by trial and error. So I arbitrarily set my fileDimensions to 1024*17 = 17408 but return an error on the following...

  Export.image.toDrive({
    image: large_raster
    scale: 30,
    maxPixels: 1e13,
    region: aoi,
    shardSize: 1024, // must be no greater than 1024; larger shardSize --> larger tile
    fileDimensions: 17408 // must be a multiple of 1024 and no greater than 131072
  });

Error: File dimensions are too large for the number of bands and data type: the total file size requested is 32728154112 bytes before compression (108 bytes per pixel), but must be less than 17179869184 bytes.

...then I will adjust my fileDimensions to the next lowest multiple of 1024 (e.g. 1024*16=16384), try to export, and see if I return an error.

But, I want to calculate my max fileDimensions from the get-go without this trial and error. Does anyone know how to do that?

Answer 1

There are three limitations on fileDimensions. It need to be a multiple of shardSize, it cannot be larger than 131072, and the size of the file before compression cannot exceed 17179869183 bytes. To come up with a way to find the maximum valid fileDimensions is a tricky little problem.

My take on this require you to specify the number of bits the pixels in each layer have. I couldn't come up with a way to automatically figure this value out based on an image. It would be great if someone else have an idea how to do that.

var image = ee.Image(0).float()
var region = ee.Geometry(Map.getBounds(true))
var scale = 30
var shardSize = 256 // Will not really change your tile size

var size = imageSize({
    image: image,
    region: ee.Geometry(Map.getBounds(true)),
    scale: scale, 
    bits: 32, 
    shardSize: shardSize
  })

print(size)

// Export require client-side objects, so we evaluate to
// get the fileDimensions as a client-side object.
size.evaluate(function (size) {
  try {
    Export.image.toDrive({
      image: image,
      scale: scale,
      maxPixels: 1e13,
      region: region,
      shardSize: shardSize,
      fileDimensions: size.fileDimensions
    })
  } catch(e) {
    print('ERROR: ', e)
  }
})  

/**
 * Calculates information about the image size.
 * 
 * Arguments:
 * 
 *   params - an object with:
 * 
 *     image - the ee.Image to get the size for
 *     region - the ee.Geometry to include
 *     scale - resolution in meters per pixel
 *     bits - the number of bits for each band in the image
 *       float (32-bit), double (64-bit), int (16-bit)
 *       int8, uint8, int16, uint16, int32, uint32 
 *       are 8-, 16-, and 32-bit, as their name hints
 *     shardSize - size in pixels of the shards Google split your processing work into
 * 
 * Returns an ee.Dictionary with:
 * 
 *   fileDimensions - largest possible fileDimensions when exporting
 *   fileSize - the uncompressed file size of each tile
 *   imageDimensions - total image dimensions
 *   imageSize - the uncompressed file size of the whole image
 */ 
function imageSize(params) {
  var image = params.image
  var region = params.region
  var scale = params.scale || 30
  var bits = params.bits || 64
  var shardSize = params.shardSize || 256

  var numberOfBands = image.bandNames().size()
  var bytesPerPixel = numberOfBands.multiply(bits).divide(8)
  var MAX_BYTES = 17179869183
  var MAX_SIDE = 131072
  var maxMultipleSide = prevMultiple(MAX_SIDE, shardSize)
  var maxPixels = ee.Number(MAX_BYTES).divide(bytesPerPixel).floor()

  var coordinates = ee.List(region.bounds().coordinates().get(0))
  var sw = ee.Geometry.Point(coordinates.get(0))
  var se = ee.Geometry.Point(coordinates.get(1))
  var ne = ee.Geometry.Point(coordinates.get(2))
  var totalWidth = sw.distance(se).divide(scale).int()
  var totalHeight = se.distance(ne).divide(scale).int()
  var clampedWidth = clampUp(totalWidth)
  var clampedHeight = clampUp(totalHeight)
  var dimensionsOptions = ee.List([
    [clampedWidth, clampDown(maxPixels.divide(clampedWidth))],
    [clampDown(maxPixels.divide(clampedHeight)), clampedHeight]
  ])
  var optionSizes = dimensionsOptions.map(function (dimensions) {
    return ee.List(dimensions).reduce(ee.Reducer.product())
  })
  var dimensions = ee.List(dimensionsOptions.sort(optionSizes).get(1))

  return ee.Dictionary({
    imageDimensions: ee.List([totalWidth, totalHeight]),
    imageSize: bytesPerPixel.multiply(totalWidth).multiply(totalHeight),
    fileDimensions: dimensions,
    fileSize: bytesPerPixel.multiply(dimensions.reduce(ee.Reducer.product()))
  })

  function clampUp(side) {
    return nextMultiple(side).min(maxMultipleSide)
  }

  function clampDown(side) {
    return prevMultiple(side).min(maxMultipleSide)
  }

  function nextMultiple(side) {
    return ee.Number(side)
      .divide(shardSize).ceil()
      .multiply(shardSize)
  }

  function prevMultiple(side) {
    return ee.Number(side)
      .divide(shardSize).floor()
      .multiply(shardSize)
  } 
}

https://code.earthengine.google.com/eea7d691959a7c25a9206a107cea638d

Answer 2

The Python equivalent to get the file size would be

import ee

# Initialize Earth Engine
ee.Initialize()

# Define the image and region
image = ee.Image(0).float()
region = ee.Geometry(ee.ImageCollection("LANDSAT/LC08/C01/T1_SR").first().geometry())

# Define scale and shardSize
scale = 30
shardSize = 256

# Define the size function
def imageSize(params):
    image = params['image']
    region = params['region']
    scale = params.get('scale', 30)
    bits = params.get('bits', 64)
    shardSize = params.get('shardSize', 256)
    
    numberOfBands = image.bandNames().size()
    bytesPerPixel = numberOfBands.multiply(bits).divide(8)
    MAX_BYTES = 17179869183
    MAX_SIDE = 131072
    maxMultipleSide = prevMultiple(MAX_SIDE)
    maxPixels = ee.Number(MAX_BYTES).divide(bytesPerPixel).floor()
    
    coordinates = ee.List(region.bounds().coordinates().get(0))
    sw = ee.Geometry.Point(coordinates.get(0))
    se = ee.Geometry.Point(coordinates.get(1))
    ne = ee.Geometry.Point(coordinates.get(2))
    totalWidth = sw.distance(se).divide(scale).int()
    totalHeight = se.distance(ne).divide(scale).int()
    clampedWidth = clampUp(totalWidth, maxMultipleSide)
    clampedHeight = clampUp(totalHeight, maxMultipleSide)
    dimensionsOptions = ee.List([
        [clampedWidth, clampDown(maxPixels.divide(clampedWidth), maxMultipleSide)],
        [clampDown(maxPixels.divide(clampedHeight), maxMultipleSide), clampedHeight]
    ])
    optionSizes = dimensionsOptions.map(lambda dimensions: ee.List(dimensions).reduce(ee.Reducer.product()))
    dimensions = ee.List(dimensionsOptions.sort(optionSizes).get(1))
    
    return ee.Dictionary({
        'imageDimensions': ee.List([totalWidth, totalHeight]),
        'imageSize': bytesPerPixel.multiply(totalWidth).multiply(totalHeight),
        'fileDimensions': dimensions,
        'fileSize': bytesPerPixel.multiply(dimensions.reduce(ee.Reducer.product()))
    })

def clampUp(side, maxMultipleSide):
    return nextMultiple(side).min(maxMultipleSide)

def clampDown(side, maxMultipleSide):
    return prevMultiple(side).min(maxMultipleSide)

def nextMultiple(side):
    return ee.Number(side).divide(shardSize).ceil().multiply(shardSize)

def prevMultiple(side):
    return ee.Number(side).divide(shardSize).floor().multiply(shardSize)

# Create the params dictionary
params = {
    'image': image,
    'region': region,
    'scale': scale,
    'bits': 32,
    'shardSize': shardSize
}

# Call the size function and print the result
size = imageSize(params)
print(size.getInfo())