Removing clouds from Sentinel 2 Surface Reflectance in Google Earth Engine?

Question

I am trying to get cloud masking to work properly with the 'COPERNICUS/S2_SR' surface reflectance dataset.

I can get it to somewhat work with my code below using the TOA dataset 'COPERNICUS/S2'.

Additionally, there seems to be duplicate dates which I had with this code using LandSat 8, that was solved by locking rows and paths. From my research I think I need to lock it to granules with Sentinel Data which I believe the area to be 'RN'

What is the correct way to cloud mask Sentinel 2 Surface Reflectance for my areas of interest and remove duplicate values?

var ROI = ee.Geometry.Point([141.041807, -34.033391]);

//Center the Map
Map.setCenter(141.041807, -34.033391, 15);

var kulkurna_A = ee.Geometry.Polygon([
  [
    [141.045513, -34.031637], [141.045252, -34.031597], [141.045002, -34.031647], 
    [141.044686, -34.031915], [141.044520, -34.032018], [141.040777, -34.035606], 
    [141.040349, -34.036386], [141.041069, -34.036233], [141.041372, -34.035775], 
    [141.042721, -34.034827], [141.043463, -34.034567], [141.045337, -34.031958]
    ]
    ]);

Map.addLayer(kulkurna_A, {color: 'blue'}, 'Kulkurna A');


var kulkurna_B = ee.Geometry.Polygon([
  [
    [141.041761, -34.031699], [141.041590, -34.031794], [141.039482, -34.033245], 
    [141.038768, -34.034018], [141.039511, -34.035584], [141.039796, -34.034948], 
    [141.039535, -34.033698], [141.040783, -34.032723], [141.041335, -34.032393], 
    [141.041793, -34.031935]
    ]
    ]);

Map.addLayer(kulkurna_B, {color: 'red'}, 'Kulkurna B');



var ens = [
ee.Feature(kulkurna_A, {label : 'Kulkurna A'}),
ee.Feature(kulkurna_B, {label : 'Kulkurna B'})
];

// Create image collection of S-2 imagery for the perdiod 2016-2018
var S2 = ee.ImageCollection('COPERNICUS/S2')

      //filter start and end date
      .filter(ee.Filter.calendarRange(2013,2019,'year'))
      .filter(ee.Filter.calendarRange(1,12,'month'))

      //filter according to drawn boundary
      .filterBounds(ROI);

// Function to mask cloud from built-in quality band
// information on cloud
var maskcloud1 = function(image) {
var QA60 = image.select(['QA60']);
return image.updateMask(QA60.lt(1));
};

// Function to calculate and add an NDVI band
var addNDVI = function(image) {
return image.addBands(image.normalizedDifference(['B8', 'B4']));
};

// Add NDVI band to image collection
var S2 = S2.map(addNDVI);

// Extract NDVI band and create NDVI median composite image
var NDVI = S2.select(['nd']);
var NDVI = NDVI.median();


//------------------------------------------------
//------------------------------------------------
//Start graphing results
//------------------------------------------------

// Create an empty panel in which to arrange widgets.
// The layout is vertical flow by default.
var panel = ui.Panel({style: {width: '400px'}})
    .add(ui.Label('NDVI Charts - Sentinel 2'));


//Graph all regions on same chart
var all_regions_graph = ui.Chart.image.seriesByRegion({
  imageCollection: S2, 
  regions: ens, 
  band: 'nd',
  reducer: ee.Reducer.mean(),
  scale: 30,
  seriesProperty: 'label'
      })
      .setChartType('ScatterChart')
      .setOptions({
  title: 'Kulkurna A & B',
  trendlines: {0: {color: 'purple'}},
  hAxis: {title: 'Time'},
  vAxis: {title: 'NDVI'},
  lineWidth: 2,
  pointSize: 3,
  });
panel.widgets().set(2, all_regions_graph);

//Graph individual region on its own chart
var kulkurna_A_graph = ui.Chart.image.seriesByRegion({
  imageCollection: S2, 
  regions: kulkurna_A, 
  band: 'nd',
  reducer: ee.Reducer.mean(),
  scale: 30,
  seriesProperty: 'kulkurna_A',
 })
      .setChartType('ScatterChart')
      .setOptions({
  title: 'Kulkurna A',
  trendlines: {0: {color: 'purple'}},
  hAxis: {title: 'Time'},
  vAxis: {title: 'NDVI'},
  lineWidth: 2,
  pointSize: 3,
  series: {0: {color: 'blue'}},
  });
panel.widgets().set(3, kulkurna_A_graph);

//Graph individual region on its own chart
var kulkurna_B_graph = ui.Chart.image.seriesByRegion({
        imageCollection: S2, 
        regions: kulkurna_B,
        band: 'nd',
        reducer: ee.Reducer.mean(),
        scale: 30,
        seriesProperty: 'Kulkurna_B'
 })
      .setChartType('ScatterChart')
      .setOptions({
  title: 'Kulkurna B',
  trendlines: {0: {color: 'purple'}},
  hAxis: {title: 'Time'},
  vAxis: {title: 'NDVI'},
  lineWidth: 2,
  pointSize: 3,
  series: {0: {color: 'red'}},
  });
panel.widgets().set(4, kulkurna_B_graph);


// Add the panel to the ui.root.
ui.root.add(panel);

Answer 1

Sentinel 2 Surface Reflectance (SR) dataset comes with 2 ways for removing clouds (and the rest of the "bad bits" like cloud shadows, dark pixels, etc). As well as Sentinel 2 TOA dataset it comes with QA60 bit band, but also comes with the "Scene Classification Map" band (SCM) which is not bit encoded but just a classified data (see SCL Class Table here)

Here is a way of applying it:

var cld = require('users/fitoprincipe/geetools:cloud_masks')

var s2SR = ee.ImageCollection('COPERNICUS/S2_SR')
              //filter start and end date
             .filter(ee.Filter.calendarRange(2013,2019,'year'))
             .filter(ee.Filter.calendarRange(1,12,'month'))
             //filter according to drawn boundary
             .filterBounds(ROI)
             .filterMetadata('CLOUD_COVERAGE_ASSESSMENT', 'greater_than', 50)

var test_image = s2SR.first()

Map.addLayer(test_image, {bands:['B8', 'B11', 'B4'], min:0, max:5000}, 'test image')

var masked = cld.sclMask(['cloud_low', 'cloud_medium', 'cloud_high', 'shadow'])(test_image)
Map.addLayer(masked, {bands:['B8', 'B11', 'B4'], min:0, max:5000}, 'masked')

Argument options for cld.sclMask are: 'cloud_low', 'cloud_medium', 'cloud_high', 'shadow', 'saturated', 'dark', 'cirrus', 'snow' and 'water' (you can also mask out 'vegetation' and 'bare_soil', but take in count that your are masking this out)

You can look at the source code of geetools:cloud_masks here

Answer 2

For Sentinel 2 the QA60 band contains info on whether the pixel is cloudy or not in 10th and 11th bit for opaque and cirrus clouds. So we can check that by checking values that have 1 on 10th and 11th bit or we can use bitwiseAnd to achieve the same.

// Function to mask cloud from built-in quality band
// information on cloud      
var maskcloud1 = function(image) {
  var QA60 = image.select(['QA60']);
  var clouds = QA60.bitwiseAnd(1<<10).or(QA60.bitwiseAnd(1<<11));// this gives us cloudy pixels
  return image.updateMask(clouds.not()); // remove the clouds from image
};