Merge pull request #842 from fastverse/development

Development

Author: SebKrantz

.Rbuildignore

@@ -6,6 +6,9 @@
 ^CONTRIBUTING\.md$
 ^README\.md$
 ^README\.Rmd$
+^CLAUDE\.md$
+^\.positai$
+^\.claude$
 ^cran-comments\.md$
 ^experimental$
 ^testing$
@@ -27,13 +30,9 @@ man/figures
 _cache$
 _snaps
 ^CITATION\.cff$
-^\.DS_Store$
 ^revdep$
 \.orig$
-\.DS_store$
-.DS_store
+(^|/)\.DS_Store$
 testthat-problems.rds
 vignettes/figure
 vignettes/cache
-
-

.gitignore

@@ -15,3 +15,4 @@ Meta
 testing
 .Rproj.user
 *.db
+.positai

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: collapse
 Title: Advanced and Fast Data Transformation
-Version: 2.1.6
-Date: 2025-12-21
+Version: 2.1.7
+Date: 2026-05-17
 Authors@R: c(
            person("Sebastian", "Krantz", role = c("aut", "cre"), 
                   email = "sebastian.krantz@graduateinstitute.ch", 
@@ -18,7 +18,7 @@ Authors@R: c(
            person("Martyn", "Plummer", role = "cph"),
            person("1999-2016 The R Core Team", role = "cph")
            )
-Description: A large C/C++-based package for advanced data transformation and 
+Description: A C/C++-based package for advanced data transformation and 
     statistical computing in R that is extremely fast, class-agnostic, robust, and 
     programmer friendly. Core functionality includes a rich set of S3 generic grouped 
     and weighted statistical functions for vectors, matrices and data frames, which 
@@ -31,7 +31,8 @@ Description: A large C/C++-based package for advanced data transformation and
     fast data object conversions, functions for memory efficient R programming, and 
     helpers to effectively deal with variable labels, attributes, and missing data. It 
     seamlessly supports base R objects/classes as well as 'units', 'integer64', 'xts'/
-    'zoo', 'tibble', 'grouped_df', 'data.table', 'sf', and 'pseries'/'pdata.frame'.
+    'zoo', 'tibble', 'grouped_df', 'data.table', 'sf', and 'pseries'/'pdata.frame'. 
+    See Krantz (2026) <doi:10.18637/jss.v116.i01>.
 URL: https://fastverse.org/collapse/, https://github.com/fastverse/collapse
 BugReports: https://github.com/fastverse/collapse/issues
 License: GPL (>= 2) | file LICENSE

NEWS.md

@@ -1,7 +1,26 @@
+# collapse 2.1.7
+
+* The *collapse* article is now published in the Journal of Statistical Software: https://doi.org/10.18637/jss.v116.i01. This article is now the primary citation for academic use of *collapse*. It is also a great reference to quickly and thoroughly understand the package. `citation("collapse")` was also updated in this regard. The APA-style citation is:
+
+  Krantz, S. (2026). **collapse**: Advanced and fast statistical computing and data transformation in R. *Journal of Statistical Software, 116*(1), 1–38. [https://doi.org/10.18637/jss.v116.i01](https://doi.org/10.18637/jss.v116.i01)
+
+
+* Performance improvements to `fsum()` and `fmean()` in the non-grouped case through multiple-accumulator SIMD optimizations, particularly benefiting systems without OpenMP support. `fsum()` sees ~2x speedup and `fmean()` ~7x speedup on such systems, with smaller but notable gains on systems with OpenMP. Thanks @TylerSagendorf for the implementation and benchmarking (#824, #828, #832, #833).
+
+* `GRP.default()` gains a `drop = TRUE` argument. Setting `drop = FALSE` (and providing at least one factor among the grouping columns) retains all combinations of factor levels with the observed unique values of non-factor grouping columns---the full Cartesian product---similar to `dplyr::group_by(.drop = FALSE)`. Unobserved combinations get `group.sizes` of `0` and `group.starts` of `0L`. Correspondingly, `fgroup_by()`/`gby()` gain a `.drop` argument, and `fcount()`/`fcountv()`/`collap()`/`collapv()` gain a `drop` argument, enabling counts and aggregations that retain empty groups. Thanks @egoipse for the feature request (#820, #839).
+
+* Fixed a bug in `descr()` that caused R to crash with a segmentation fault when called on zero-row data frames. Thanks @hatschibratschi for reporting (#831).
+
+* Consistency with internal updates to *data.table* regarding growable vectors. Thanks @aitap (#809).
+
+* Force C locale sorting in internal operations for consistency. Thanks @MichaelChirico (#815).
+
+* Added contributor Ivan Krylov (@aitap) to package authors.
+
 # collapse 2.1.6
 
 * The repo has moved to `fastverse/collapse` and the website to [fastverse.org/collapse](https://fastverse.org/collapse/)---for better visibility and maintenance. Appropriate redirects from the old repo/site have been implemented.
-  Selected people now have access to the repo through the organization account and may respond to issues or submit fixes. 
+  Selected people now have access to the repo through the organization account and may respond to issues or submit fixes.
 
 * Added new AI-generated interactive/chattable [DeepWiki documentation](https://deepwiki.com/fastverse/collapse).
 

R/GRP.R

@@ -95,21 +95,21 @@ GRP.GRP <- function(X, ...) X
 
 GRP.default <- function(X, by = NULL, sort = .op[["sort"]], decreasing = FALSE, na.last = TRUE,
                         return.groups = TRUE, return.order = sort, method = "auto",
-                        call = TRUE, ...) {
+                        drop = TRUE, call = TRUE, ...) {
 
   use.group <- switch(method, auto = !sort, hash = TRUE, radix = FALSE, stop("method needs to be 'auto', 'hash' or 'radix'."))
 
   if(is.na(na.last)) stop("here na.last needs to be TRUE or FALSE, otherwise the GRP object does not match the data dimensions.")
 
   if(is.list(X)) {
     if(inherits(X, "GRP")) return(X)
-    if(is.null(by)) {
+    by_null <- is.null(by)
+    if(by_null) {
       by <- seq_along(unclass(X))
       # # This is so that fgroup_by(iris, Species = GRP(Species)) is possible.
       # if(length(by) == 1L && is.list(.subset2(X, 1L)) && inherits(.subset2(X, 1L), "GRP")) return(.subset2(X, 1L))
       namby <- attr(X, "names")
       if(is.null(namby)) attr(X, "names") <- namby <- paste0("Group.", by)
-      o <- switchGRP(X, na.last, decreasing, return.groups || !use.group, TRUE, sort, use.group)
     } else {
       if(is.call(by)) {
         namby <- all.vars(by, unique = FALSE)
@@ -126,8 +126,26 @@ GRP.default <- function(X, by = NULL, sort = .op[["sort"]], decreasing = FALSE,
           attr(X, "names") <- paste0("Group.", seq_along(unclass(X))) # best ?
         }
       }
-      o <- switchGRP(.subset(X, by), na.last, decreasing, return.groups || !use.group, TRUE, sort, use.group)
     }
+    # drop = FALSE: full Cartesian product of (factor) levels, computed in C.
+    # Falls through to the existing path when no grouping column is a factor (matches dplyr).
+    if(!drop) {
+      cols <- if(by_null) unclass(X) else .subset(unclass(X), by)
+      if(any(.Call(C_vtypes, cols, 2L))) {
+        res <- .Call(C_GRP_default_drop, X, cols, namby, return.groups)
+        return(`oldClass<-`(list(N.groups = res[[1L]],
+                              group.id = res[[2L]],
+                              group.sizes = res[[3L]],
+                              groups = if(return.groups) res[[5L]] else NULL,
+                              group.vars = namby,
+                              ordered = c(ordered = NA, sorted = NA),
+                              order = NULL,
+                              group.starts = res[[4L]],
+                              call = if(call) match.call() else NULL), "GRP"))
+      }
+    }
+    o <- switchGRP(if(by_null) X else .subset(X, by),
+                   na.last, decreasing, return.groups || !use.group, TRUE, sort, use.group)
   } else {
    if(length(by)) stop("by can only be used to subset list / data.frame columns")
    namby <- l1orlst(as.character(substitute(X))) # paste(all.vars(call), collapse = ".") # good in all circumstances ?
@@ -382,7 +400,7 @@ GRP.pseries <- function(X, effect = 1L, ..., group.sizes = TRUE, return.groups =
 GRP.pdata.frame <- function(X, effect = 1L, ..., group.sizes = TRUE, return.groups = TRUE, call = TRUE)
   GRP.pseries(X, effect, ..., group.sizes = group.sizes, return.groups = return.groups, call = call)
 
-fgroup_by <- function(.X, ..., sort = .op[["sort"]], decreasing = FALSE, na.last = TRUE, return.groups = TRUE, return.order = sort, method = "auto") {          #   e <- substitute(list(...)) # faster but does not preserve attributes of unique groups !
+fgroup_by <- function(.X, ..., sort = .op[["sort"]], decreasing = FALSE, na.last = TRUE, return.groups = TRUE, return.order = sort, method = "auto", .drop = TRUE) {          #   e <- substitute(list(...)) # faster but does not preserve attributes of unique groups !
   clx <- oldClass(.X)
   oldClass(.X) <- NULL
   m <- match(c("GRP_df", "grouped_df", "data.frame"), clx, nomatch = 0L)
@@ -416,7 +434,7 @@ fgroup_by <- function(.X, ..., sort = .op[["sort"]], decreasing = FALSE, na.last
       } else names(e) <- vars
     }
   }
-  attr(.X, "groups") <- GRP.default(e, NULL, sort, decreasing, na.last, return.groups, return.order, method, FALSE)
+  attr(.X, "groups") <- GRP.default(e, NULL, sort, decreasing, na.last, return.groups, return.order, method, .drop, FALSE)
   # if(any(clx == "sf")) oldClass(.X) <- clx[clx != "sf"]
   # attr(.X, "groups") <- GRP.default(fselect(if(m[2L]) fungroup(.X) else .X, ...), NULL, sort, decreasing, na.last, TRUE, return.order, method, FALSE)
     # Needed: wlddev %>% fgroup_by(country) gives error if dplyr is loaded. Also sf objects etc..

R/collap.R

@@ -95,7 +95,7 @@ rbindlist_factor <- function(l, idcol = "Function") {
 collap <- function(X, by, FUN = fmean, catFUN = fmode, cols = NULL, w = NULL, wFUN = fsum, custom = NULL,
                    ...,
                    keep.by = TRUE, keep.w = TRUE, keep.col.order = TRUE, sort = .op[["sort"]], decreasing = FALSE,
-                   na.last = TRUE, return.order = sort, method = "auto", parallel = FALSE, mc.cores = 2L,
+                   na.last = TRUE, return.order = sort, method = "auto", drop = TRUE, parallel = FALSE, mc.cores = 2L,
                    return = c("wide","list","long","long_dupl"), give.names = "auto") {
 
   return <- switch(return[1L], wide = 1L, list = 2L, long = 3L, long_dupl = 4L, stop("Unknown return output option"))
@@ -127,14 +127,14 @@ collap <- function(X, by, FUN = fmean, catFUN = fmode, cols = NULL, w = NULL, wF
       numby <- ckmatch(all.vars(by), nam)
       if(ncustoml) v <- if(is.null(cols)) seq_along(X)[-numby] else cols2int(cols, X, nam)
     }
-    by <- GRP.default(X, numby, sort, decreasing, na.last, keep.by, return.order, method, call = FALSE)
+    by <- GRP.default(X, numby, sort, decreasing, na.last, keep.by, return.order, method, drop = drop, call = FALSE)
   } else if(is.atomic(by)) {
     numby <- 0L
     if(ncustoml) if(is.null(cols)) vl <- FALSE else v <- cols2int(cols, X, nam)
-    by <- GRP.default(`names<-`(list(by), l1orlst(as.character(substitute(by)))), NULL, sort, decreasing, na.last, keep.by, return.order, method, call = FALSE)
+    by <- GRP.default(`names<-`(list(by), l1orlst(as.character(substitute(by)))), NULL, sort, decreasing, na.last, keep.by, return.order, method, drop = drop, call = FALSE)
   } else {
     if(ncustoml) if(is.null(cols)) vl <- FALSE else v <- cols2int(cols, X, nam)
-    if(!is_GRP(by)) by <- GRP.default(by, NULL, sort, decreasing, na.last, keep.by, return.order, method, call = FALSE)
+    if(!is_GRP(by)) by <- GRP.default(by, NULL, sort, decreasing, na.last, keep.by, return.order, method, drop = drop, call = FALSE)
     numby <- rep(0L, length(by[[5L]]))
     if(keep.by && !vl && any(m <- nam %in% by[[5L]])) {
       v <- whichv(m, FALSE)
@@ -322,7 +322,7 @@ collap <- function(X, by, FUN = fmean, catFUN = fmode, cols = NULL, w = NULL, wF
 # collapv: allows vector input to by and w
 collapv <- function(X, by, FUN = fmean, catFUN = fmode, cols = NULL, w = NULL, wFUN = fsum, custom = NULL, ...,
                     keep.by = TRUE, keep.w = TRUE, keep.col.order = TRUE, sort = .op[["sort"]], decreasing = FALSE,
-                    na.last = TRUE, return.order = sort, method = "auto", parallel = FALSE, mc.cores = 2L,
+                    na.last = TRUE, return.order = sort, method = "auto", drop = TRUE, parallel = FALSE, mc.cores = 2L,
                     return = c("wide","list","long","long_dupl"), give.names = "auto") {
 
   return <- switch(return[1L], wide = 1L, list = 2L, long = 3L, long_dupl = 4L, stop("Unknown return output option"))
@@ -343,7 +343,7 @@ collapv <- function(X, by, FUN = fmean, catFUN = fmode, cols = NULL, w = NULL, w
 
   # identifying by
   numby <- cols2int(by, X, nam)
-  by <- GRP.default(X, numby, sort, decreasing, na.last, keep.by, return.order, method, call = FALSE)
+  by <- GRP.default(X, numby, sort, decreasing, na.last, keep.by, return.order, method, drop = drop, call = FALSE)
   if(ncustoml) v <- if(is.null(cols)) seq_along(X)[-numby] else cols2int(cols, X, nam)
 
 

R/fcount.R

@@ -34,18 +34,20 @@ fcount_core <- function(x, g, w = NULL, name = "N", add = FALSE) {
   condalc(copyMostAttributes(c(res, `names<-`(list(g$group.sizes), name[1L])), res), inherits(x, "data.table"))
 }
 
-fcount <- function(x, ..., w = NULL, name = "N", add = FALSE, sort = FALSE, decreasing = FALSE) {
+fcount <- function(x, ..., w = NULL, name = "N", add = FALSE, sort = FALSE, decreasing = FALSE, drop = TRUE) {
   if(is.list(x)) w <- eval(substitute(w), x, parent.frame())
   else x <- qDF(x)
   if(is.character(add)) add <- switch(add, gv =, group_vars = 2L, stop("add must be TRUE, FALSE or group_vars (gv)")) # add = "g", "groups" or "group_vars"
   # Note: this code duplication with GRP() is needed for GRP() to capture x (using substitute) if x is atomic.
   # if(is.atomic(x)) `names<-`(list(x), l1orlst(as.character(substitute(x)))) else
-  g <- if(missing(...)) GRP(x, sort = sort, decreasing = decreasing, return.groups = !add, return.order = FALSE, call = FALSE) else
-    GRP.default(fselect(x, ...), sort = sort, decreasing = decreasing, return.groups = !add, return.order = FALSE, call = FALSE)
+  g <- if(missing(...)) {
+         if(inherits(x, "grouped_df")) GRP(x, sort = sort, decreasing = decreasing, return.groups = !add, return.order = FALSE, call = FALSE)
+         else GRP.default(x, sort = sort, decreasing = decreasing, return.groups = !add, return.order = FALSE, drop = drop, call = FALSE)
+       } else GRP.default(fselect(x, ...), sort = sort, decreasing = decreasing, return.groups = !add, return.order = FALSE, drop = drop, call = FALSE)
   fcount_core(x, g, w, name, add)
 }
 
-fcountv <- function(x, cols = NULL, w = NULL, name = "N", add = FALSE, sort = FALSE, ...) {
+fcountv <- function(x, cols = NULL, w = NULL, name = "N", add = FALSE, sort = FALSE, drop = TRUE, ...) {
   # Safe enough ? or only allow character ? what about collapv() ?, extra option ?
   # if(length(w) == 1L && is.list(x) && length(unclass(x)) > 1L && (is.character(w) || is.integer(w) || (is.numeric(w) && w %% 1 < 1e-6)))
   if(is.atomic(x)) x <- qDF(x)
@@ -54,7 +56,9 @@ fcountv <- function(x, cols = NULL, w = NULL, name = "N", add = FALSE, sort = FA
     if(is.null(w)) stop("Unknown column: ", w)
   }
   if(is.character(add)) add <- switch(add, gv =, group_vars = 2L, stop("add must be TRUE, FALSE or group_vars (gv)")) # add = "g", "groups" or "group_vars"
-  g <- if(is.null(cols)) GRP(x, sort = sort, return.groups = !add, return.order = FALSE, call = FALSE, ...) else
-    GRP.default(colsubset(x, cols), sort = sort, return.groups = !add, return.order = FALSE, call = FALSE, ...)
+  g <- if(is.null(cols)) {
+         if(inherits(x, "grouped_df")) GRP(x, sort = sort, return.groups = !add, return.order = FALSE, call = FALSE, ...)
+         else GRP.default(x, sort = sort, return.groups = !add, return.order = FALSE, drop = drop, call = FALSE, ...)
+       } else GRP.default(colsubset(x, cols), sort = sort, return.groups = !add, return.order = FALSE, drop = drop, call = FALSE, ...)
   fcount_core(x, g, w, name, add)
 }

README.md

@@ -13,7 +13,7 @@
 [![minimal R version](https://img.shields.io/badge/R%3E%3D-3.5.0-6666ff.svg)](https://cran.r-project.org/)
 [![dependencies](https://tinyverse.netlify.app/badge/collapse)](https://CRAN.R-project.org/package=collapse)
 [![DOI](https://zenodo.org/badge/172910283.svg)](https://zenodo.org/badge/latestdoi/172910283)
-[![arXiv](https://img.shields.io/badge/arXiv-2403.05038-0969DA.svg)](https://arxiv.org/abs/2403.05038)
+[![JSS](https://img.shields.io/badge/JSS-10.18637%2Fjss.v116.i01-blue)](https://doi.org/10.18637/jss.v116.i01)
 [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/fastverse/collapse)
 <!-- badges: end -->
 
@@ -79,9 +79,9 @@ In addition there are several [vignettes](<https://fastverse.org/collapse/articl
 <a href="https://raw.githubusercontent.com/fastverse/collapse/master/misc/collapse%20cheat%20sheet/collapse_cheat_sheet.pdf"><img src="https://raw.githubusercontent.com/fastverse/collapse/master/misc/collapse%20cheat%20sheet/preview/page1.png" width="330"/></a>  <!-- height="227" 294 -->
 <a href="https://raw.githubusercontent.com/fastverse/collapse/master/misc/collapse%20cheat%20sheet/collapse_cheat_sheet.pdf"><img src="https://raw.githubusercontent.com/fastverse/collapse/master/misc/collapse%20cheat%20sheet/preview/page2.png" width="330"/></a> 
 
-### Article on arXiv
+### JSS Article
 
-An [**article**](https://arxiv.org/abs/2403.05038) on *collapse* is forthcoming at [Journal of Statistical Software](https://www.jstatsoft.org/). 
+The [**collapse** article](https://doi.org/10.18637/jss.v116.i01) is published in the [Journal of Statistical Software](https://www.jstatsoft.org/) (volume 116, issue 1). It is the primary reference for academic use of the package and a concise introduction to its design and capabilities.
 
 ### Presentation at [useR 2022](https://user2022.r-project.org)
 

inst/CITATION

@@ -1,15 +1,18 @@
 citHeader("To cite collapse in publications, please use:")
 
 bibentry(bibtype = "misc",
-         key = "krantz2024collapse",
-         title = "collapse: Advanced and Fast Statistical Computing and Data Transformation in R",
-         author = person("Sebastian", "Krantz"),
-         year = "2024",
-         eprint="2403.05038",
-         archivePrefix="arXiv",
-         primaryClass="stat.CO",
-         url = "https://arxiv.org/abs/2403.05038",
-         textVersion = "Krantz, S. (2024). collapse: Advanced and Fast Statistical Computing and Data Transformation in R [Preprint]. arXiv. https://arxiv.org/abs/2403.05038")
+         key = "krantz2026collapse",
+         title        = "{collapse}: Advanced and Fast Statistical Computing and Data Transformation in {R}",
+         author       = person(given = "Sebastian",
+                               family = "Krantz",
+                               email = "sebastian.krantz@kielinstitut.de"),
+         journal      = "Journal of Statistical Software",
+         year         = "2026",
+         volume       = "116",
+         number       = "1",
+         pages        = "1--38",
+         doi          = "10.18637/jss.v116.i01",
+         textVersion = "Krantz, S. (2026). collapse: Advanced and fast statistical computing and data transformation in R. Journal of Statistical Software, 116(1), 1–38. https://doi.org/10.18637/jss.v116.i01")
 
 year <- sub("-.*", "", meta$Date)
 note <- sprintf("R package version %s", meta$Version)