Add in met dropbox pull

Author: stephpenn1

Lysimeter/01_lysimeter_teros_vwc_comparison.Rmd

@@ -29,6 +29,8 @@ knitr::opts_chunk$set(echo = F,
                       warning = F)
 
 common_tz = "Etc/GMT+5"
+
+source("dropbox copy.R")
 ```
 
 Change this chunk to eval = TRUE if you need a dropbox token.
@@ -42,7 +44,7 @@ saveRDS(token, file = "token.rds")
 Run this chunk if you already have your token:
 ```{r load dropbox token, include=FALSE, eval=TRUE}
 
-read_rds("token.rds")
+read_rds("token.rds") -> token
 ```
 
 # 1. Set timeframe of interest
@@ -77,9 +79,9 @@ teros_inv <- readxl::read_excel("TEROSinventory.xlsx", sheet="Sheet1") %>%
 # 3. Bring in dropbox data, merge with inventory sheet
 ```{r dropbox}
 
-datadir <- "TEMPEST_PNNL_Data/Current_Data"
+datadir_teros <- "TEMPEST_PNNL_Data/Current_Data"
 
-teros_primitive <- compasstools::process_teros_dir(datadir, tz = "EST", dropbox_token = token)
+teros_primitive <- compasstools::process_teros_dir(datadir_teros, tz = "EST", dropbox_token = token)
 
 teros_primitive %>%
   left_join(teros_inv,
@@ -110,16 +112,10 @@ teros %>%
 # 6. Bring in GCREW loggernet rain data
 
 ```{r gcrew rain}
-#temp dataset
-
-read_csv("sample_data/GCREW_MET_GCREW_MET_15min.dat", skip = 1, col_types = "Tdcd") %>% 
-  select(TIMESTAMP, Rain_in_Tot) -> rain1
-
-read_csv("sample_data/GCREW_MET_GCREW_MET_15min_20250415005900.dat", skip = 1, col_types = "Tdcd") %>% 
-  select(TIMESTAMP, Rain_in_Tot) %>% 
-  rbind(rain1) %>% 
-  filter(!is.na(Rain_in_Tot)) %>% 
-  rename(Timestamp = TIMESTAMP) -> rain
+datadir_met <- "GCREW_LOGGERNET_DATA/current_data"
+rain <- process_dir(datadir = datadir_met, pattern = "GCREW_MET_GCREW_MET_15min", read_datalogger_file, dropbox_token = token) %>% 
+  mutate(Timestamp = as_datetime(TIMESTAMP)) %>% 
+  select(Timestamp, Rain_in_Tot)
 
 rain %>% 
   ggplot(aes(x = Timestamp, y = Rain_in_Tot)) +

Lysimeter/dropbox copy.R

@@ -0,0 +1,81 @@
+
+#' Download and read a data file from Dropbox
+#'
+#' @param filename A Dropbox filename, e.g. returned by \code{drop_dir}
+#' @param token A dropbox token
+#' @param read_function A function to read the downloaded file with
+#' @param ... Other parameters to be passed to \code{read_function}
+#' @description This function downloads and reads (via a caller-supplied
+#' function) a file on Dropbox.
+#' @return A \code{\link[tibble]{tibble}} with the data.
+#' @export
+#' @author Ben Bond-Lamberty
+read_file_dropbox <- function(filename, token, read_function, ...) {
+    # We don't want users to need rdrop2 to use this package (i.e. we don't
+    # want to put it in DESCRIPTION's Imports:), so check for availability
+    if(requireNamespace("rdrop2", quietly = TRUE)) {
+        # download to temp file
+        tf <- tempfile()
+        rdrop2::drop_download(filename, local_path = tf,
+                              dtoken = token, overwrite = TRUE)
+        read_function(tf, ...)
+    } else {
+        stop("rdrop2 package is not available")
+    }
+}
+
+
+#' Read a directory of files
+#'
+#' @param datadir A directory, either in Dropbox or local
+#' @param pattern Filename regex pattern
+#' @param read_function The file-read function to use
+#' @param dropbox_token Optional Dropbox token
+#' @param progress_bar Optional progress bar to call while reading
+#' @param ... Other parameters to be passed to \code{read_function}
+#' @description A general-purpose function to read data files, either from
+#' Dropbox or locally. The function identifies files based on a pattern,
+#' reads data from each file, and binds them together Callers
+#' pass a `read_function` that handles the actual file read.
+#' @return All data files in directory, read and concatenated
+#' in a \code{\link[tibble]{tibble}}.
+#' @importFrom dplyr bind_rows
+#' @note This is not intended to be called directly by users; use
+#' \code{\link{process_sapflow_dir}}, etc., instead.
+#' @note We use \code{\link[dplyr]{bind_rows}} for its much greater efficiency
+#' over \code{\link{rbind}}
+#' @author Ben Bond-Lamberty
+process_dir <- function(datadir, pattern, read_function,
+                        dropbox_token = NULL,
+                        progress_bar = NULL, ...) {
+
+    local <- is.null(dropbox_token)
+
+    # Get our file list, either locally or in Dropbox
+    if(local) {
+        s_files <- list.files(datadir, pattern = pattern, full.names = TRUE)
+    } else {
+        # We don't want users to need rdrop2 to use this package (i.e. we don't
+        # want to put it in DESCRIPTION's Imports:), so check for availability
+        if(requireNamespace("rdrop2", quietly = TRUE)) {
+            # Generate list of 'current' (based on token) files
+            s_dir <- rdrop2::drop_dir(datadir, dtoken = dropbox_token)
+            s_files <- grep(s_dir$path_display, pattern = pattern, value = TRUE)
+        } else {
+            stop("rdrop2 package is not available")
+        }
+    }
+
+    # Function called by lapply below; handles progress bar and calls file reader
+    f <- function(filename, read_function, token, total_files) {
+        if(!is.null(progress_bar)) progress_bar(1 / total_files)
+        # Read file, either locally or from Dropbox
+        if(local) {
+            read_function(filename, ...)
+        } else {
+            read_file_dropbox(filename, dropbox_token, read_function, ...)
+        }
+    }
+    x <- lapply(s_files, f, read_function, dropbox_token, length(s_files))
+    bind_rows(x)
+}