Skip to main content
deleted 2 characters in body
Source Link

Thanks a lot @JeffreyEvans and @mdsumner for your comments. I was able to rework my function the following way to enable it to vectorize over all the files:

hrrr_processing_fn <- function(hrrr_file,vars){
  file_date <- str_extract(tail(unlist(str_split(hrrr_file,"/")),1),"[^\\D$]{8}\\.[^\\D$]{2}")
  
  message(hrrr_file)
  
  # 1) Identify the layer number of each of the variables
  index.file <- varfiles[grepl(file_date,varfiles)]
  
  hrrr.idx <- fread(index.file, sep = ":", header = F)
  hrrr.idx[,V5 := str_replace_all(V5, " ","_")]
  var_num <- hrrr.idx[V5 %in% c("surface", "2_m_above_ground") & V4 %in% vars,V1]
  colnames <- hrrr.idx[var_num,paste0(V4,"_",V5)]
  
  # 2) Read the grib file as a RasterStack
  hrrr_stack <- stack(hrrr_file, bands = var_num)
  
  # 3) Mask the rasterstack to the shapefile
  hrrr_stack_masked <- mask(hrrr_stack, mask = st_transform(suffolk_county,crs = crs(hrrr_stack)))
  
  # 4) Extracts the centroid and data of the masked raster as a data.table
  hrrr_mask_dt <- as.data.frame(hrrr_stack_masked,na.rm = T,xy = T)
  setDT(hrrr_mask_dt)
  setnames(hrrr_mask_dt,c("lon","lat",colnames))
  hrrr_mask_dt[,ID := .I]
  file_date <- str_replace(file_date, "\\.","_")
  hrrr_mask_dt[,date := ymd_h(file_date, tz = "America/New_York")]
  
}

hrrr_2017_dt <- lapply(grib.2017,function(hrrr_file, vars = "TMP"){
  tryCatch(hrrr_processing_fn(hrrr_file, vars = "TMP"),
           error = function(err){
             message( hrrr_file ," could not be processed. Error: ",err)
             write(hrrr_file,file = file.path(hrrr_path,"hrrr_error_files.txt\n"txt"), 
                   append = TRUE)})})

Thanks a lot @JeffreyEvans and @mdsumner for your comments. I was able to rework my function the following way to enable it to vectorize over all the files:

hrrr_processing_fn <- function(hrrr_file,vars){
  file_date <- str_extract(tail(unlist(str_split(hrrr_file,"/")),1),"[^\\D$]{8}\\.[^\\D$]{2}")
  
  message(hrrr_file)
  
  # 1) Identify the layer number of each of the variables
  index.file <- varfiles[grepl(file_date,varfiles)]
  
  hrrr.idx <- fread(index.file, sep = ":", header = F)
  hrrr.idx[,V5 := str_replace_all(V5, " ","_")]
  var_num <- hrrr.idx[V5 %in% c("surface", "2_m_above_ground") & V4 %in% vars,V1]
  colnames <- hrrr.idx[var_num,paste0(V4,"_",V5)]
  
  # 2) Read the grib file as a RasterStack
  hrrr_stack <- stack(hrrr_file, bands = var_num)
  
  # 3) Mask the rasterstack to the shapefile
  hrrr_stack_masked <- mask(hrrr_stack, mask = st_transform(suffolk_county,crs = crs(hrrr_stack)))
  
  # 4) Extracts the centroid and data of the masked raster as a data.table
  hrrr_mask_dt <- as.data.frame(hrrr_stack_masked,na.rm = T,xy = T)
  setDT(hrrr_mask_dt)
  setnames(hrrr_mask_dt,c("lon","lat",colnames))
  hrrr_mask_dt[,ID := .I]
  file_date <- str_replace(file_date, "\\.","_")
  hrrr_mask_dt[,date := ymd_h(file_date, tz = "America/New_York")]
  
}

hrrr_2017_dt <- lapply(grib.2017,function(hrrr_file, vars = "TMP"){
  tryCatch(hrrr_processing_fn(hrrr_file, vars = "TMP"),
           error = function(err){
             message( hrrr_file ," could not be processed. Error: ",err)
             write(hrrr_file,file = file.path(hrrr_path,"hrrr_error_files.txt\n"), 
                   append = TRUE)})})

Thanks a lot @JeffreyEvans and @mdsumner for your comments. I was able to rework my function the following way to enable it to vectorize over all the files:

hrrr_processing_fn <- function(hrrr_file,vars){
  file_date <- str_extract(tail(unlist(str_split(hrrr_file,"/")),1),"[^\\D$]{8}\\.[^\\D$]{2}")
  
  message(hrrr_file)
  
  # 1) Identify the layer number of each of the variables
  index.file <- varfiles[grepl(file_date,varfiles)]
  
  hrrr.idx <- fread(index.file, sep = ":", header = F)
  hrrr.idx[,V5 := str_replace_all(V5, " ","_")]
  var_num <- hrrr.idx[V5 %in% c("surface", "2_m_above_ground") & V4 %in% vars,V1]
  colnames <- hrrr.idx[var_num,paste0(V4,"_",V5)]
  
  # 2) Read the grib file as a RasterStack
  hrrr_stack <- stack(hrrr_file, bands = var_num)
  
  # 3) Mask the rasterstack to the shapefile
  hrrr_stack_masked <- mask(hrrr_stack, mask = st_transform(suffolk_county,crs = crs(hrrr_stack)))
  
  # 4) Extracts the centroid and data of the masked raster as a data.table
  hrrr_mask_dt <- as.data.frame(hrrr_stack_masked,na.rm = T,xy = T)
  setDT(hrrr_mask_dt)
  setnames(hrrr_mask_dt,c("lon","lat",colnames))
  hrrr_mask_dt[,ID := .I]
  file_date <- str_replace(file_date, "\\.","_")
  hrrr_mask_dt[,date := ymd_h(file_date, tz = "America/New_York")]
  
}

hrrr_2017_dt <- lapply(grib.2017,function(hrrr_file, vars = "TMP"){
  tryCatch(hrrr_processing_fn(hrrr_file, vars = "TMP"),
           error = function(err){
             message( hrrr_file ," could not be processed. Error: ",err)
             write(hrrr_file,file = file.path(hrrr_path,"hrrr_error_files.txt"), 
                   append = TRUE)})})
Source Link

Thanks a lot @JeffreyEvans and @mdsumner for your comments. I was able to rework my function the following way to enable it to vectorize over all the files:

hrrr_processing_fn <- function(hrrr_file,vars){
  file_date <- str_extract(tail(unlist(str_split(hrrr_file,"/")),1),"[^\\D$]{8}\\.[^\\D$]{2}")
  
  message(hrrr_file)
  
  # 1) Identify the layer number of each of the variables
  index.file <- varfiles[grepl(file_date,varfiles)]
  
  hrrr.idx <- fread(index.file, sep = ":", header = F)
  hrrr.idx[,V5 := str_replace_all(V5, " ","_")]
  var_num <- hrrr.idx[V5 %in% c("surface", "2_m_above_ground") & V4 %in% vars,V1]
  colnames <- hrrr.idx[var_num,paste0(V4,"_",V5)]
  
  # 2) Read the grib file as a RasterStack
  hrrr_stack <- stack(hrrr_file, bands = var_num)
  
  # 3) Mask the rasterstack to the shapefile
  hrrr_stack_masked <- mask(hrrr_stack, mask = st_transform(suffolk_county,crs = crs(hrrr_stack)))
  
  # 4) Extracts the centroid and data of the masked raster as a data.table
  hrrr_mask_dt <- as.data.frame(hrrr_stack_masked,na.rm = T,xy = T)
  setDT(hrrr_mask_dt)
  setnames(hrrr_mask_dt,c("lon","lat",colnames))
  hrrr_mask_dt[,ID := .I]
  file_date <- str_replace(file_date, "\\.","_")
  hrrr_mask_dt[,date := ymd_h(file_date, tz = "America/New_York")]
  
}

hrrr_2017_dt <- lapply(grib.2017,function(hrrr_file, vars = "TMP"){
  tryCatch(hrrr_processing_fn(hrrr_file, vars = "TMP"),
           error = function(err){
             message( hrrr_file ," could not be processed. Error: ",err)
             write(hrrr_file,file = file.path(hrrr_path,"hrrr_error_files.txt\n"), 
                   append = TRUE)})})