0

I have a complicated code (sorry for that, tried to make it simpler but having trouble with that. Pseudocode:

  1. Make an image collection of all thee Sentinel2 images from start to finish date. Calculate NDVI for every image.

  2. Make 5 imageCollections of Sentinel3 images that match every image in the Sen2 collection, with offset (1st collection - images from the exact same dates as the Sen2 collection, 2nd collection - images from +1 days from every image in the Sen2 collection, 3rd - +2 days... etc).

  3. Calculate NDVI for every image in the 5 Sen3 imageCollections.

  4. Export the data to CSV files so that each Sentinel2 image is exported with it's corresponding 5 days images (For example - Sen2 image time T with Sen3 image time T, T+1, T+2, T+3, T+4).

Should look something like this:

enter image description here

For some reason, I get only the SEN2 NDVI values output in the CSV files.. The SEN3 values do not go well.

This is the code, but I think it is better to just run it via this link:

//Dates of Interest
var start = ee.Date("2018-04-01");
var finish = ee.Date("2018-05-01");

var region = ee.Geometry.Polygon([
  [[34.433411922574145, 31.163674639521915], [34.433663918367166, 31.162771680415535], [34.43562164221252, 31.162749860616337], [34.43557366428209, 31.163708694862848], [34.433411922574145, 31.163674639521915]]])
Map.addLayer(region)
Map.centerObject(region)

//------------------- Data bases ---------------------//
var sen2 = ee.ImageCollection('COPERNICUS/S2_SR')
  .filterDate(start,finish)
  .filterBounds(region)
  .map(function(image){
    return image
    .clip(region)
    .normalizedDifference(['B8','B4'])
    .rename('ndvi')
  })


var sen3 = ee.ImageCollection('COPERNICUS/S3/OLCI')
  .filterDate(start, finish)
  .filterBounds(region)
  .select(['Oa17_radiance', 'Oa08_radiance'])
  .map(function(image){
  //   // Convert to radiance units.
     return image
     .clip(region)
  //   .divide(ee.Image([0.00493004, 0.00876539]))
   })

var sen2_for_dates = ee.ImageCollection('COPERNICUS/S2_SR')
  .filterDate(start, finish)
  .filterBounds(region)
  .map(function(image){
    return image
    .clipToCollection(region)
    .updateMask(image.select('QA60').not())
    .addBands(image.normalizedDifference(['B8','B4']).rename('ndvi'))
  })
  .select('ndvi')

//------------------ Unique dates ----------------------//
var sen2_dates = sen2_for_dates.map(function(image){
  return image.set('simpleDateMillis', ee.Date(ee.Date(image.date().format('YYYY-MM-dd'))))
});
var listMillisSimple = ee.List(sen2_dates.aggregate_array('simpleDateMillis'));
var uniqueDatesSimple = listMillisSimple.distinct();
var SEN2_day0 = uniqueDatesSimple

// make another list with (e.g.) 5 days added to all unique dates
var SEN3_day4 = uniqueDatesSimple.map(function(date){
  return ee.Date(date).advance(4, 'day');
});
var SEN3_day3 = uniqueDatesSimple.map(function(date){
  return ee.Date(date).advance(3, 'day');
});
var SEN3_day2 = uniqueDatesSimple.map(function(date){
  return ee.Date(date).advance(2, 'day');
});
var SEN3_day1 = uniqueDatesSimple.map(function(date){
  return ee.Date(date).advance(1, 'day');
});
var SEN3_day0 = uniqueDatesSimple.map(function(date){
  return ee.Date(date).advance(0, 'day');
});

var sen3_dates = sen3.map(function(image){
  return image.set('simpleDateMillis', ee.Date(ee.Date(image.date().format('YYYY-MM-dd'))))
});

var allImagesInRangeS2 = sen2_dates.filter(ee.Filter.inList("simpleDateMillis", SEN2_day0))
var allImagesInRangeS30 = sen3_dates.filter(ee.Filter.inList("simpleDateMillis", SEN3_day0))
var allImagesInRangeS31 = sen3_dates.filter(ee.Filter.inList("simpleDateMillis", SEN3_day1))
var allImagesInRangeS32 = sen3_dates.filter(ee.Filter.inList("simpleDateMillis", SEN3_day2))
var allImagesInRangeS33 = sen3_dates.filter(ee.Filter.inList("simpleDateMillis", SEN3_day3))
var allImagesInRangeS34 = sen3_dates.filter(ee.Filter.inList("simpleDateMillis", SEN3_day4))

Map.addLayer(allImagesInRangeS30.map(function (image){return image}))

allImagesInRangeS30 = allImagesInRangeS30.map(function(image){
  return image
  .divide(ee.Image([0.00493004, 0.00876539]))
  .addBands(image.normalizedDifference(['Oa17_radiance', 'Oa08_radiance']).rename('ndvi'))
})
.select('ndvi')

allImagesInRangeS31 = allImagesInRangeS31.map(function(image){
  return image
  .divide(ee.Image([0.00493004, 0.00876539]))
  .addBands(image.normalizedDifference(['Oa17_radiance', 'Oa08_radiance']).rename('ndvi'))
})
.select('ndvi')

allImagesInRangeS32 = allImagesInRangeS32.map(function(image){
  return image
  .divide(ee.Image([0.00493004, 0.00876539]))
  .addBands(image.normalizedDifference(['Oa17_radiance', 'Oa08_radiance']).rename('ndvi'))
})
.select('ndvi')

allImagesInRangeS33 = allImagesInRangeS33.map(function(image){
  return image
  .divide(ee.Image([0.00493004, 0.00876539]))
  .addBands(image.normalizedDifference(['Oa17_radiance', 'Oa08_radiance']).rename('ndvi'))
})
.select('ndvi')

allImagesInRangeS34 = allImagesInRangeS34.map(function(image){
  return image
  .divide(ee.Image([0.00493004, 0.00876539]))
  .addBands(image.normalizedDifference(['Oa17_radiance', 'Oa08_radiance']).rename('ndvi'))
})
.select('ndvi')


//------------------ Creating the feature collection for export --------------//  
function get_List(sngl_sen2, sngl_sen30, sngl_sen31, sngl_sen32, sngl_sen33, sngl_sen34){
   sngl_sen2 = ee.Image(sngl_sen2)
   var list2 = sngl_sen2.reduceRegion({
    reducer: ee.Reducer.toList(),
    geometry: region,
    maxPixels: 1e8,
    scale: 10
  });


   sngl_sen30 = ee.Image(sngl_sen30)
  var list30 = sngl_sen30.reduceRegion({
    reducer: ee.Reducer.toList(),
    geometry: region,
    maxPixels: 1e8,
    scale: 10
  });

  sngl_sen31 = ee.Image(sngl_sen31)
  var list31 = sngl_sen31.reduceRegion({
    reducer: ee.Reducer.toList(),
    geometry: region,
    maxPixels: 1e8,
    scale: 10
  });

 sngl_sen32 = ee.Image(sngl_sen32)
  var list32 = sngl_sen32.reduceRegion({
    reducer: ee.Reducer.toList(),
    geometry: region,
    maxPixels: 1e8,
    scale: 10
  });

 sngl_sen33 = ee.Image(sngl_sen33)
  var list33 = sngl_sen33.reduceRegion({
    reducer: ee.Reducer.toList(),
    geometry: region,
    maxPixels: 1e8,
    scale: 10
  });

  sngl_sen34 = ee.Image(sngl_sen34)
  var list34 = sngl_sen34.reduceRegion({
    reducer: ee.Reducer.toList(),
    geometry: region,
    maxPixels: 1e8,
    scale: 10
  });

  // Make a feature without geometry and set the properties to the dictionary of means.
  var feature2 = ee.Feature(null, list2);
  var feature30 = ee.Feature(null, list30);
  var feature31 = ee.Feature(null, list31);
  var feature32 = ee.Feature(null, list32);
  var feature33 = ee.Feature(null, list33);
  var feature34 = ee.Feature(null, list34);

  // Wrap the Feature in a FeatureCollection for export.
  var featureCollection = ee.FeatureCollection([feature2, feature30, feature31, feature32, feature33, feature34]); 
  return featureCollection
}

var S2_list = allImagesInRangeS2.toList(allImagesInRangeS2.size())
var S30_list = allImagesInRangeS30.toList(allImagesInRangeS30.size())
var S31_list = allImagesInRangeS31.toList(allImagesInRangeS31.size())
var S32_list = allImagesInRangeS32.toList(allImagesInRangeS32.size())
var S33_list = allImagesInRangeS33.toList(allImagesInRangeS33.size())
var S34_list = allImagesInRangeS34.toList(allImagesInRangeS34.size())

print(S30_list)

var main_fold = 'AI_data'

//Export the FeatureCollection.
for(var i=0 ; i<5 ; i++){
  print(S30_list.get(i))
  Export.table.toDrive({
  collection: get_List(ee.Image(S2_list.get(i)), ee.Image(S30_list.get(i)), ee.Image(S31_list.get(i)), ee.Image(S32_list.get(i)), ee.Image(S33_list.get(i)), ee.Image(S34_list.get(i))),
  fileNamePrefix: 'List'+i,
  folder: 'Die',
  selectors: ['ndvi'],
  fileFormat: 'CSV'
  });
}

Edit - clarification: 1. The purpose of the whole process is to build a dataset for a deep neural network that will use T0 SEN2 and T0-T4 SEN3, and will try to estimate SEN2 at T4. The network will use the next SEN2 in the imageCollection as a validation to the estimation done by using SEN2(T0) and SEN3(T0-T4). My training data is from 2017 until December 2019, and the test data will be that of 2020.

  1. The values for each row in the data set are Per-Pixel values of the image at the specific time (T0\T1\T2\T3\T4) and the specific satellite (SEN2\SEN3).

  2. Every CSV file should have 7 rows - a. SEN2(T0), b. SEN3(T0)...., f. SEN3(T4), g. SEN2(Validation, The next one in the imageColl, hopefully at T4). Each row is the NDVI values for the specific image in the specific time by the specific satellite.

  3. Missing data will be filled by null.

Any ideas why I get this output?

enter image description here

2
  • Dear @DanielWiell, thank you for your comment. I will address all the issues you have listed and will update the question as well: 1. The purpose of the whole process is to build a dataset for a deep neural network that will use T0 SEN2 and T0-T4 SEN3, and will try to estimate SEN2 at T4. The network will use the next SEN2 in the imageCollection as an validation to the estimation done by using SEN2(T0) and SEN3(T0-T4). My training data is from 2017 until December 2019, and the test data will be that of 2020. 2. Per-Pixel values. (To be continued in the next comment). – Sahar Attia May 18 '20 at 12:42
  • @DanielWiell following the previous comment: 3. Every CSV file should have 7 rows - a. SEN2(T0), b. SEN3(T0)...., f. SEN3(T4), g. SEN2(The next one in the imageColl, hopefully at T4). Each row is the NDVI values for the specific image in the specific time by the specific satellite. 4. Missing data will be filled by null. – Sahar Attia May 18 '20 at 12:42
1

It's hard to debug your script, but the below should pretty much do what youwant. It's sensitive to picking startDate where you have S2 imagery. You also should make sure to pick a date that get you cloud-free imagery.

To do this in a larger areas, you will probably need to export the imagery itself, and use Python or R to turn it into a format suitable for further processing.

var startDate = ee.Date("2020-04-10")
var numberOfDays = 6
var region = ee.Geometry.Polygon([[
  [34.433411922574145, 31.163674639521915],
  [34.433663918367166, 31.162771680415535],
  [34.43562164221252, 31.162749860616337],
  [34.43557366428209, 31.163708694862848],
  [34.433411922574145, 31.163674639521915]
]])

var dayOffsets = ee.List.sequence(0, numberOfDays - 1)
var allDaily = ee.ImageCollection(dayOffsets.map(function (dayOffset) {
  var start = startDate.advance(dayOffset, 'days')
  var end = start.advance(1, 'days')
  var filter = ee.Filter.and(
    ee.Filter.date(start, end),
    ee.Filter.bounds(region)
  )
  var s2 = ee.ImageCollection('COPERNICUS/S2_SR')
    .filter(filter)
    .map(function (image) {
      return image
        .normalizedDifference(['B8', 'B4'])
        .updateMask(image.select('QA60').not())
        .rename('S2')
    })
    .median()

  var s3 = ee.ImageCollection('COPERNICUS/S3/OLCI') 
    .filter(filter)
    .map(function (image) {
      return image
        .select(['Oa17_radiance', 'Oa08_radiance'])
        .divide(ee.Image([0.00493004, 0.00876539]))
        .normalizedDifference(['Oa17_radiance', 'Oa08_radiance'])
        .rename('S3')
    })
    .median()
  return s2
    .addBands(s3)
    .set('date', start.format('yyyy-MM-dd'))
}))

var completeDaily = allDaily
  .sort('date')
  .toList(numberOfDays)

print('Make sure there are ' + numberOfDays + ' images:', completeDaily.size())

var firstS2Image = completeDaily.get(0)
var lastS2Image = completeDaily.get(numberOfDays - 1)

var fistS2Feature = toFeature('', 'S2', 'T0', firstS2Image)
var lastS2Feature = toFeature('Validation', 'S2', 'T' + (numberOfDays - 1), lastS2Image)
var allS3Features = completeDaily
      .zip(ee.List.sequence(0, completeDaily.size().subtract(1)))
      .map(function (tuple) {
        tuple = ee.List(tuple)
        var time = ee.String('T').cat(tuple.getNumber(1).format('%d'))
        var image = tuple.get(0)
        return toFeature('', 'S3', time, image)
      })

var collection = ee.FeatureCollection(fistS2Feature)
  .merge(allS3Features)
  .merge(ee.FeatureCollection(lastS2Feature))

Export.table.toDrive({
  collection: collection,
  description: 'ndvi',
  selectors: ['VALIDATION', 'SENSOR', 'TIME', 'NDVI']
})

print(collection)

function toFeature(validation, sensor, time, ndvi) {
  ndvi = ee.Image(ndvi)
  var pixels =  ee.Number(ndvi
    .select(sensor)
    .reduceRegion({
      reducer: ee.Reducer.toList(),
      geometry: region,
      scale: 10
    })
    .values()
    .get(0))
  return ee.Feature(null, {
    VALIDATION: validation,
    SENSOR: sensor, 
    TIME: time,
    NDVI: pixels
  })
}

https://code.earthengine.google.com/e35f0424554ad44e16eff53c9e8045e0

2
  • First - @DanielWiell, I must say it amazes me the way you solved it. Exactly as I needed. Thank you. May I ask how could do this to every image of Sentinal2 from "start date" to "finish" so that for every image I will get an output of CSV file in the manner that you have done it here for a single image with a specific date? – Sahar Attia May 18 '20 at 16:58
  • Dear @daniel, if you could please take a moment to try and solve my follow-up question found here - gis.stackexchange.com/questions/362793/… – Sahar Attia May 24 '20 at 4:57

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.