Optimization for Large Files - #39
Conversation
![github-actions[bot]](https://avatars.githubusercontent.com/in/15368?s=60&v=4){kind=small}
There was a problem hiding this comment.
Remaining comments which cannot be posted as a review comment to avoid GitHub Rate Limit
air
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/crop_nc_2d.R
Lines 76 to 78 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/crop_nc_2d.R
Lines 80 to 85 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/crop_nc_2d.R
Line 87 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/crop_nc_2d.R
Lines 89 to 90 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/crop_nc_2d.R
Line 96 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/crop_nc_2d.R
Lines 100 to 102 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/crop_nc_2d.R
Lines 104 to 108 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/crop_nc_2d.R
Line 110 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/crop_nc_2d.R
Lines 113 to 116 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/crop_nc_2d.R
Line 118 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/crop_nc_2d.R
Line 120 in 6cae0bc
[air] reported by reviewdog 🐶
[air] reported by reviewdog 🐶
[air] reported by reviewdog 🐶
[air] reported by reviewdog 🐶
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/make_2d_anomaly_gridded.R
Lines 72 to 73 in 6cae0bc
[air] reported by reviewdog 🐶
[air] reported by reviewdog 🐶
[air] reported by reviewdog 🐶
[air] reported by reviewdog 🐶
[air] reported by reviewdog 🐶
[air] reported by reviewdog 🐶
[air] reported by reviewdog 🐶
[air] reported by reviewdog 🐶
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/make_2d_summary_ts.R
Line 122 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/make_2d_summary_ts.R
Lines 125 to 126 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/mask_nc_2d.R
Line 63 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/mask_nc_2d.R
Line 65 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/mask_nc_2d.R
Line 67 in 6cae0bc
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/mask_nc_2d.R
Line 69 in 6cae0bc
| convert_2d_longitude_gridded <- function(data.in, write.out = FALSE, output.files = NA) { | ||
|
|
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| convert_2d_longitude_gridded <- function(data.in, write.out = FALSE, output.files = NA) { | |
| convert_2d_longitude_gridded <- function( | |
| data.in, | |
| write.out = FALSE, | |
| output.files = NA | |
| ) { |
|
|
||
| # Boost terra memory limit for in-memory processing fallback | ||
| terra::terraOptions(memfrac = 0.8) | ||
|
|
| data.ls = EDABUtilities:::import_data(data.in) | ||
|
|
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| data.ls = EDABUtilities:::import_data(data.in) | |
| data.ls <- EDABUtilities:::import_data(data.in) | |
| @@ -23,18 +26,59 @@ convert_2d_longitude_gridded <- function(data.in, write.out = FALSE, output.file | |||
| out.ls <- lapply(seq_along(data.ls), function(i) { | |||
|
|
|||
|
|
||
| current_data <- data.ls[[i]] | ||
| is_file_input <- is.character(current_data) | ||
|
|
| target_col <- col | ||
| break |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| target_col <- col | |
| break | |
| target_col <- col | |
| break |
|
|
||
| if (is.null(target_col)) { | ||
| stop("None of the attributes in shp.file contain all specified area.names.") | ||
| if (is.null(target_col)) { | ||
| stop("None of the attributes in shp.file contain all specified area.names.") |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| if (is.null(target_col)) { | |
| stop("None of the attributes in shp.file contain all specified area.names.") | |
| if (is.null(target_col)) { | |
| stop("None of the attributes in shp.file contain all specified area.names.") | |
| if (is.null(target_col)) { | |
| stop( | |
| "None of the attributes in shp.file contain all specified area.names." | |
| ) |
|
|
||
| # Subset vector directly using terra logic | ||
| shp.vect <- shp.vect[shp.vect[[target_col]] %in% area.names, ] | ||
| shp.vect <- shp.vect[shp.vect[[target_col]][,1] %in% area.names , ] |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| # Subset vector directly using terra logic | |
| shp.vect <- shp.vect[shp.vect[[target_col]] %in% area.names, ] | |
| shp.vect <- shp.vect[shp.vect[[target_col]][,1] %in% area.names , ] | |
| shp.vect <- shp.vect[shp.vect[[target_col]][, 1] %in% area.names, ] |
|
|
||
| out.ls <- list() | ||
| out.ls <- list() | ||
|
|
||
| for (i in seq_along(data.ls)) { | ||
| for (i in seq_along(data.ls)) { | ||
|
|
||
| if (is.character(data.ls[[i]])) { | ||
| if (!file.exists(data.ls[[i]])) stop(sprintf("File does not exist: %s", data.ls[[i]])) | ||
| data.orig <- terra::rast(data.ls[[i]]) | ||
| if (is.character(data.ls[[i]])) { | ||
| if (!file.exists(data.ls[[i]])) stop(sprintf("File does not exist: %s", data.ls[[i]])) | ||
| data.orig <- terra::rast(data.ls[[i]]) |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| out.ls <- list() | |
| out.ls <- list() | |
| for (i in seq_along(data.ls)) { | |
| for (i in seq_along(data.ls)) { | |
| if (is.character(data.ls[[i]])) { | |
| if (!file.exists(data.ls[[i]])) stop(sprintf("File does not exist: %s", data.ls[[i]])) | |
| data.orig <- terra::rast(data.ls[[i]]) | |
| if (is.character(data.ls[[i]])) { | |
| if (!file.exists(data.ls[[i]])) stop(sprintf("File does not exist: %s", data.ls[[i]])) | |
| data.orig <- terra::rast(data.ls[[i]]) | |
| out.ls <- list() | |
| for (i in seq_along(data.ls)) { | |
| if (is.character(data.ls[[i]])) { | |
| if (!file.exists(data.ls[[i]])) { | |
| stop(sprintf("File does not exist: %s", data.ls[[i]])) | |
| } | |
| data.orig <- terra::rast(data.ls[[i]]) |
| data.orig <- terra::rast(data.ls[[i]]) | ||
| } else { | ||
| data.orig <- data.ls[[i]] | ||
| data.orig <- data.ls[[i]] |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| data.orig <- data.ls[[i]] | |
| data.orig <- data.ls[[i]] |
atyrell3
left a comment
atyrell3
left a comment
There was a problem hiding this comment.
Two thoughts...
(1) coordinate reference system: I think we should have a check to make sure that the input nc is in the same CRS as shp.file. We could add a CRS input and project both files to the same CRS. To do it more efficiently: extract the CRS from the nc --> project shp.file to the same as the nc file --> crop nc --> project to the output CRS
(2) rotate vs shift: I think it's safer to use terra::rotate rather than terra::shift ; this is the behavior we want, right?:
changed from shift to rotate
There was a problem hiding this comment.
Remaining comments which cannot be posted as a review comment to avoid GitHub Rate Limit
air
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/make_2d_summary_ts.R
Line 122 in 866e255
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/make_2d_summary_ts.R
Lines 125 to 126 in 866e255
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/mask_nc_2d.R
Line 63 in 866e255
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/mask_nc_2d.R
Line 65 in 866e255
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/mask_nc_2d.R
Line 67 in 866e255
[air] reported by reviewdog 🐶
READ_EDAB_Utilities/R/mask_nc_2d.R
Line 69 in 866e255
| data.crs = terra::crs(data.orig) | ||
| shp.crs = terra::crs(shp.vect) | ||
|
|
||
| data.orig <- EDABUtilities::convert_2d_longitude_gridded(data.orig)[[1]] | ||
| if(!identical(data.crs,shp.crs)){ | ||
| shp.vect = terra::project(shp.vect,data.crs) |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| data.crs = terra::crs(data.orig) | |
| shp.crs = terra::crs(shp.vect) | |
| data.orig <- EDABUtilities::convert_2d_longitude_gridded(data.orig)[[1]] | |
| if(!identical(data.crs,shp.crs)){ | |
| shp.vect = terra::project(shp.vect,data.crs) | |
| data.crs <- terra::crs(data.orig) | |
| shp.crs <- terra::crs(shp.vect) | |
| if (!identical(data.crs, shp.crs)) { | |
| shp.vect <- terra::project(shp.vect, data.crs) |
| if(!identical(data.crs,shp.crs)){ | ||
| shp.vect = terra::project(shp.vect,data.crs) | ||
| } | ||
|
|
| e_rast <- as.vector(terra::ext(data.orig)) | ||
| shp_crop <- shp.vect | ||
|
|
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| e_rast <- as.vector(terra::ext(data.orig)) | |
| shp_crop <- shp.vect | |
| e_rast <- as.vector(terra::ext(data.orig)) | |
| shp_crop <- shp.vect | |
| # If raster is 0-360 but shapefile has negative longitudes | ||
| if (e_rast["xmax"] > 180.001 && any(terra::ext(shp_crop)[1:2] < 0)) { | ||
| # Shift the shapefile geometry 360 degrees East so it overlays on the 0-360 raster | ||
| shp_crop <- terra::rotate(shp_crop, long = 0,split = T, left = F) |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| shp_crop <- terra::rotate(shp_crop, long = 0,split = T, left = F) | |
| shp_crop <- terra::rotate(shp_crop, long = 0, split = T, left = F) |
| # Shift the shapefile geometry 360 degrees East so it overlays on the 0-360 raster | ||
| shp_crop <- terra::rotate(shp_crop, long = 0,split = T, left = F) | ||
| #If raster is -180 to 180 but shapefile is 0-360 | ||
| } else if (e_rast["xmax"] <= 180.001 && any(terra::ext(shp_crop)[1:2] > 180)) { |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| } else if (e_rast["xmax"] <= 180.001 && any(terra::ext(shp_crop)[1:2] > 180)) { | |
| } else if ( | |
| e_rast["xmax"] <= 180.001 && any(terra::ext(shp_crop)[1:2] > 180) | |
| ) { |
| @@ -97,7 +97,7 @@ make_2d_deg_day_gridded <- function(data.in, var.name, metric, ref.value, type, | |||
|
|
|||
| data.out <- terra::mask(data.out, shp.vect) | ||
| data.out <- EDABUtilities::crop_nc_2d(data.out, shp.vect)[[1]] | ||
| } | ||
|
|
| @@ -72,7 +72,8 @@ make_2d_summary_gridded <- function(data.in, var.name, statistics, agg.time, fil | |||
|
|
|||
| # OPTIMIZATION: Pre-crop to bounding box once to reduce spatial memory footprint immediately | ||
| if (use.shp) { | ||
| data <- terra::crop(data, shp.vect) | ||
| data <- EDABUtilities::crop_nc_2d(data, shp.file = shp.vect,area.names = area.names,var.name = var.name )[[1]] |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| data <- EDABUtilities::crop_nc_2d(data, shp.file = shp.vect,area.names = area.names,var.name = var.name )[[1]] | |
| data <- EDABUtilities::crop_nc_2d( | |
| data, | |
| shp.file = shp.vect, | |
| area.names = area.names, | |
| var.name = var.name | |
| )[[1]] |
| data <- EDABUtilities::crop_nc_2d(data, shp.file = shp.vect,area.names = area.names,var.name = var.name )[[1]] | ||
| # data <- terra::crop(data, shp.vect) | ||
| } | ||
|
|
|
@atyrell3 Good suggestions. I made those changes. still is relatively faster. |
| #' @param var.name character. Name of variable to be subseted from dataset | ||
| #' | ||
| #' @return list of spatRasters | ||
| #' |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| #' | |
| #' |
| import_data = function(data.in, var.name = NULL){ | ||
|
|
||
| if (inherits(data.in, "SpatRaster")|inherits(data.in, "SpatRasterDataset")) { |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| import_data = function(data.in, var.name = NULL){ | |
| if (inherits(data.in, "SpatRaster")|inherits(data.in, "SpatRasterDataset")) { | |
| import_data <- function(data.in, var.name = NULL) { | |
| if ( | |
| inherits(data.in, "SpatRaster") | inherits(data.in, "SpatRasterDataset") | |
| ) { |
| @@ -23,5 +24,25 @@ import_data = function(data.in){ | |||
| stop("data.in must be a file path, a vector of file paths, a single SpatRaster, or a list of SpatRasters.") | |||
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| stop("data.in must be a file path, a vector of file paths, a single SpatRaster, or a list of SpatRasters.") | |
| stop( | |
| "data.in must be a file path, a vector of file paths, a single SpatRaster, or a list of SpatRasters." | |
| ) |
|
|
||
| if(!is.null(var.name)){ | ||
| data.ls = lapply(data.ls,function(x){ | ||
| data.varname = terra::varnames(x) | ||
| if(length(data.varname > 1)){ | ||
| data = x[[terra::varnames(x) == var.name]] | ||
| }else{ | ||
|
|
||
| data.names = terra::names(data) | ||
| data.names = data.names[grepl(var.name,data.names)] | ||
|
|
||
| if(length(data.names) ==0){ | ||
| warning('NetCDF file does not contain any fields with var.name=',var.name) | ||
| }else{ | ||
|
|
||
| data = subset(x,data.names) |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| if(!is.null(var.name)){ | |
| data.ls = lapply(data.ls,function(x){ | |
| data.varname = terra::varnames(x) | |
| if(length(data.varname > 1)){ | |
| data = x[[terra::varnames(x) == var.name]] | |
| }else{ | |
| data.names = terra::names(data) | |
| data.names = data.names[grepl(var.name,data.names)] | |
| if(length(data.names) ==0){ | |
| warning('NetCDF file does not contain any fields with var.name=',var.name) | |
| }else{ | |
| data = subset(x,data.names) | |
| if (!is.null(var.name)) { | |
| data.ls <- lapply(data.ls, function(x) { | |
| data.varname <- terra::varnames(x) | |
| if (length(data.varname > 1)) { | |
| data <- x[[terra::varnames(x) == var.name]] | |
| } else { | |
| data.names <- terra::names(data) | |
| data.names <- data.names[grepl(var.name, data.names)] | |
| if (length(data.names) == 0) { | |
| warning( | |
| 'NetCDF file does not contain any fields with var.name=', | |
| var.name | |
| ) | |
| } else { | |
| data <- subset(x, data.names) |
| } | ||
| }) | ||
| } | ||
|
|
|
|
||
| # OPTIMIZATION: Pull terra::tapp entirely out of the area loop. |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| # OPTIMIZATION: Pull terra::tapp entirely out of the area loop. | |
| # OPTIMIZATION: Pull terra::tapp entirely out of the area loop. |
| @@ -62,7 +62,8 @@ mask_nc_2d <- function(data.in, var.name, min.value, max.value, write.out = FALS | |||
| data <- if (is.character(item)) terra::rast(item) else item | |||
|
|
|||
|
|
||
| if (use.shp) { | ||
| data <- terra::crop(data, shp.vect) | ||
| data <- EDABUtilities::crop_nc_2d(data, shp.file = shp.vect,area.names = area.names,var.name = var.name )[[1]] |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| data <- EDABUtilities::crop_nc_2d(data, shp.file = shp.vect,area.names = area.names,var.name = var.name )[[1]] | |
| data <- EDABUtilities::crop_nc_2d( | |
| data, | |
| shp.file = shp.vect, | |
| area.names = area.names, | |
| var.name = var.name | |
| )[[1]] |
| data <- terra::crop(data, shp.vect) | ||
| data <- EDABUtilities::crop_nc_2d(data, shp.file = shp.vect,area.names = area.names,var.name = var.name )[[1]] | ||
| # data <- terra::crop(data, shp.vect) | ||
| data <- terra::mask(data, shp.vect) |
There was a problem hiding this comment.
[air] reported by reviewdog 🐶
| data <- terra::mask(data, shp.vect) | |
| data <- terra::mask(data, shp.vect) |
| # data <- terra::crop(data, shp.vect) | ||
| data <- terra::mask(data, shp.vect) | ||
| } | ||
|
|
2 participants
Improved memory management for global files