diff --git a/R/version.R b/R/version.R index 683b644b9..37e62d28c 100644 --- a/R/version.R +++ b/R/version.R @@ -1,7 +1,7 @@ # Generated by rconfigure.py, do not edit by hand # DuckDB version information -duckdb_version <- "1.5.0-dev2455" +duckdb_version <- "1.5.0-dev2471" # Function to get DuckDB version without establishing a connection get_duckdb_version <- function() { diff --git a/src/duckdb/src/function/table/version/pragma_version.cpp b/src/duckdb/src/function/table/version/pragma_version.cpp index 7f4d4a97f..06121ba4a 100644 --- a/src/duckdb/src/function/table/version/pragma_version.cpp +++ b/src/duckdb/src/function/table/version/pragma_version.cpp @@ -1,5 +1,5 @@ #ifndef DUCKDB_PATCH_VERSION -#define DUCKDB_PATCH_VERSION "0-dev2455" +#define DUCKDB_PATCH_VERSION "0-dev2471" #endif #ifndef DUCKDB_MINOR_VERSION #define DUCKDB_MINOR_VERSION 5 @@ -8,10 +8,10 @@ #define DUCKDB_MAJOR_VERSION 1 #endif #ifndef DUCKDB_VERSION -#define DUCKDB_VERSION "v1.5.0-dev2455" +#define DUCKDB_VERSION "v1.5.0-dev2471" #endif #ifndef DUCKDB_SOURCE_ID -#define DUCKDB_SOURCE_ID "287ac426ad" +#define DUCKDB_SOURCE_ID "6c5e16c2fb" #endif #include "duckdb/function/table/system_functions.hpp" #include "duckdb/main/database.hpp" diff --git a/src/duckdb/src/include/duckdb/function/table_function.hpp b/src/duckdb/src/include/duckdb/function/table_function.hpp index 6bdd980c0..4f67d2a7e 100644 --- a/src/duckdb/src/include/duckdb/function/table_function.hpp +++ b/src/duckdb/src/include/duckdb/function/table_function.hpp @@ -114,14 +114,16 @@ struct TableFunctionBindInput { struct RowGroupOrderOptions { RowGroupOrderOptions(column_t column_idx_p, OrderByStatistics order_by_p, RowGroupOrderType order_type_p, - OrderByColumnType column_type_p) - : column_idx(column_idx_p), order_by(order_by_p), order_type(order_type_p), column_type(column_type_p) { + OrderByColumnType column_type_p, optional_idx row_limit_p = optional_idx()) + : column_idx(column_idx_p), order_by(order_by_p), order_type(order_type_p), column_type(column_type_p), + row_limit(row_limit_p) { } const column_t column_idx; const OrderByStatistics order_by; const RowGroupOrderType order_type; const OrderByColumnType column_type; + const optional_idx row_limit; }; struct TableFunctionInitInput { diff --git a/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp b/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp index 4d6f9448d..fd8896c2d 100644 --- a/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp +++ b/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp @@ -194,18 +194,18 @@ class RowGroupReorderer { optional_ptr> GetRootSegment(RowGroupSegmentTree &row_groups); optional_ptr> GetNextRowGroup(SegmentNode &row_group); + static Value RetrieveStat(const BaseStatistics &stats, OrderByStatistics order_by, OrderByColumnType column_type); + private: const column_t column_idx; const OrderByStatistics order_by; const RowGroupOrderType order_type; const OrderByColumnType column_type; + const optional_idx row_limit; idx_t offset; bool initialized; vector>> ordered_row_groups; - -private: - static Value RetrieveStat(const BaseStatistics &stats, OrderByStatistics order_by, OrderByColumnType column_type); }; class CollectionScanState { diff --git a/src/duckdb/src/optimizer/topn_optimizer.cpp b/src/duckdb/src/optimizer/topn_optimizer.cpp index a48faafbe..6b8b68ae6 100644 --- a/src/duckdb/src/optimizer/topn_optimizer.cpp +++ b/src/duckdb/src/optimizer/topn_optimizer.cpp @@ -17,18 +17,39 @@ namespace duckdb { namespace { -bool CanReorderRowGroups(LogicalTopN &op) { +bool CanReorderRowGroups(LogicalTopN &op, bool &use_limit) { + use_limit = true; + for (const auto &order : op.orders) { + // We do not support any null-first orders as this requires unimplemented logic in the row group reorderer + if (order.null_order == OrderByNullType::NULLS_FIRST) { + use_limit = false; + break; + } + } + // Only reorder row groups if there are no additional limit operators since they could modify the order reference current_op = op; + while (!current_op.get().children.empty()) { if (current_op.get().children.size() > 1) { return false; } - if (current_op.get().type == LogicalOperatorType::LOGICAL_LIMIT) { + const auto op_type = current_op.get().type; + if (op_type == LogicalOperatorType::LOGICAL_LIMIT) { return false; } + if (op_type == LogicalOperatorType::LOGICAL_FILTER || + op_type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + use_limit = false; + } current_op = *current_op.get().children[0]; } + D_ASSERT(current_op.get().type == LogicalOperatorType::LOGICAL_GET); + auto &logical_get = current_op.get().Cast(); + if (!logical_get.table_filters.filters.empty()) { + use_limit = false; + } + return true; } @@ -133,8 +154,9 @@ void TopN::PushdownDynamicFilters(LogicalTopN &op) { // put the filter into the Top-N clause op.dynamic_filter = filter_data; + bool use_limit = false; bool use_custom_rowgroup_order = - CanReorderRowGroups(op) && (colref.return_type.IsNumeric() || colref.return_type.IsTemporal()); + CanReorderRowGroups(op, use_limit) && (colref.return_type.IsNumeric() || colref.return_type.IsTemporal()); for (auto &target : pushdown_targets) { auto &get = target.get; @@ -156,8 +178,9 @@ void TopN::PushdownDynamicFilters(LogicalTopN &op) { auto order_type = op.orders[0].type == OrderType::ASCENDING ? RowGroupOrderType::ASC : RowGroupOrderType::DESC; auto order_by = order_type == RowGroupOrderType::ASC ? OrderByStatistics::MIN : OrderByStatistics::MAX; - auto order_options = - make_uniq(column_index.GetPrimaryIndex(), order_by, order_type, column_type); + auto row_limit = use_limit ? op.limit + op.offset : optional_idx(); + auto order_options = make_uniq(column_index.GetPrimaryIndex(), order_by, order_type, + column_type, row_limit); get.function.set_scan_order(std::move(order_options), get.bind_data.get()); } } diff --git a/src/duckdb/src/storage/table/scan_state.cpp b/src/duckdb/src/storage/table/scan_state.cpp index a5da219e7..cd6869c25 100644 --- a/src/duckdb/src/storage/table/scan_state.cpp +++ b/src/duckdb/src/storage/table/scan_state.cpp @@ -10,6 +10,87 @@ namespace duckdb { +namespace { + +struct RowGroupMapEntry { + reference> row_group; + unique_ptr stats; +}; + +bool CompareValues(const Value &v1, const Value &v2, const OrderByStatistics order) { + return (order == OrderByStatistics::MAX && v1 < v2) || (order == OrderByStatistics::MIN && v1 > v2); +} + +idx_t GetQualifyingTupleCount(RowGroup &row_group, BaseStatistics &stats, const OrderByColumnType type) { + if (!stats.CanHaveNull()) { + return row_group.count; + } + + if (type == OrderByColumnType::NUMERIC) { + if (!NumericStats::HasMinMax(stats)) { + return 0; + } + if (NumericStats::IsConstant(stats)) { + return 1; + } + return 2; + } + // We cannot check if the min/max for StringStats have actually been set. As the strings may be truncated, we + // also cannot assume that min and max are the same + return 0; +} + +template +void AddRowGroups(It it, End end, vector>> &ordered_row_groups, const idx_t row_limit, + const OrderByColumnType column_type, const OrderByStatistics stat_type) { + const auto opposite_stat_type = + stat_type == OrderByStatistics::MAX ? OrderByStatistics::MIN : OrderByStatistics::MAX; + + idx_t qualifying_tuples = 0; + idx_t qualify_later = 0; + + auto last_unresolved_entry = it; + auto &last_stats = it->second.stats; + idx_t last_unresolved_row_group_sum = + GetQualifyingTupleCount(*it->second.row_group.get().node, *last_stats, column_type); + auto last_unresolved_boundary = RowGroupReorderer::RetrieveStat(*last_stats, opposite_stat_type, column_type); + + for (; it != end; ++it) { + auto ¤t_key = it->first; + auto &row_group = it->second.row_group; + + while (last_unresolved_entry != it) { + if (!CompareValues(current_key, last_unresolved_boundary, stat_type)) { + if (current_key != std::prev(it)->first) { + // Row groups overlap: we can only guarantee one additional qualifying tuple + qualifying_tuples += qualify_later; + qualify_later = 0; + qualifying_tuples++; + } else { + // Row groups have the same order value, we can only guarantee a qualifying tuple later + qualify_later++; + } + + break; + } + // Row groups do not overlap: we can guarantee that the tuples qualify + qualifying_tuples = last_unresolved_row_group_sum; + ++last_unresolved_entry; + auto &upcoming_row_group = *last_unresolved_entry->second.row_group.get().node; + auto &upcoming_stats = *last_unresolved_entry->second.stats; + + last_unresolved_row_group_sum += GetQualifyingTupleCount(upcoming_row_group, upcoming_stats, column_type); + last_unresolved_boundary = RowGroupReorderer::RetrieveStat(upcoming_stats, opposite_stat_type, column_type); + } + if (qualifying_tuples >= row_limit) { + return; + } + ordered_row_groups.emplace_back(row_group); + } +} + +} // namespace + TableScanState::TableScanState() : table_state(*this), local_state(*this) { } @@ -112,7 +193,7 @@ void ScanFilterInfo::SetFilterAlwaysTrue(idx_t filter_idx) { RowGroupReorderer::RowGroupReorderer(const RowGroupOrderOptions &options) : column_idx(options.column_idx), order_by(options.order_by), order_type(options.order_type), - column_type(options.column_type), offset(0), initialized(false) { + column_type(options.column_type), row_limit(options.row_limit), offset(0), initialized(false) { } optional_ptr> RowGroupReorderer::GetNextRowGroup(SegmentNode &row_group) { @@ -134,6 +215,26 @@ Value RowGroupReorderer::RetrieveStat(const BaseStatistics &stats, OrderByStatis return Value(); } +void SetRowGroupVectorWithLimit(const multimap &row_group_map, const optional_idx row_limit, + const RowGroupOrderType order_type, const OrderByColumnType column_type, + vector>> &ordered_row_groups) { + D_ASSERT(row_limit.IsValid()); + + const auto stat_type = order_type == RowGroupOrderType::ASC ? OrderByStatistics::MIN : OrderByStatistics::MAX; + ordered_row_groups.reserve(row_group_map.size()); + + Value previous_key; + if (order_type == RowGroupOrderType::ASC) { + auto it = row_group_map.begin(); + auto end = row_group_map.end(); + AddRowGroups(it, end, ordered_row_groups, row_limit.GetIndex(), column_type, stat_type); + } else { + auto it = row_group_map.rbegin(); + auto end = row_group_map.rend(); + AddRowGroups(it, end, ordered_row_groups, row_limit.GetIndex(), column_type, stat_type); + } +} + optional_ptr> RowGroupReorderer::GetRootSegment(RowGroupSegmentTree &row_groups) { if (initialized) { if (ordered_row_groups.empty()) { @@ -144,27 +245,30 @@ optional_ptr> RowGroupReorderer::GetRootSegment(RowGroupSe initialized = true; - multimap>> row_group_map; - for (auto &node : row_groups.SegmentNodes()) { - auto &row_group = *node.node; - auto stats = row_group.GetStatistics(column_idx); + multimap row_group_map; + for (auto &row_group : row_groups.SegmentNodes()) { + auto stats = row_group.node->GetStatistics(column_idx); Value comparison_value = RetrieveStat(*stats, order_by, column_type); - - row_group_map.emplace(comparison_value, reference>(node)); + auto entry = RowGroupMapEntry {row_group, std::move(stats)}; + row_group_map.emplace(comparison_value, std::move(entry)); } if (row_group_map.empty()) { return nullptr; } - ordered_row_groups.reserve(row_group_map.size()); - if (order_type == RowGroupOrderType::ASC) { - for (auto &row_group : row_group_map) { - ordered_row_groups.emplace_back(row_group.second); - } + if (row_limit.IsValid()) { + SetRowGroupVectorWithLimit(row_group_map, row_limit, order_type, column_type, ordered_row_groups); } else { - for (auto it = row_group_map.rbegin(); it != row_group_map.rend(); ++it) { - ordered_row_groups.emplace_back(it->second); + ordered_row_groups.reserve(row_group_map.size()); + if (order_type == RowGroupOrderType::ASC) { + for (auto &row_group : row_group_map) { + ordered_row_groups.emplace_back(row_group.second.row_group); + } + } else { + for (auto it = row_group_map.rbegin(); it != row_group_map.rend(); ++it) { + ordered_row_groups.emplace_back(it->second.row_group); + } } }