From 77d1854b0418d9b4222e88b9e4c604fa50b04066 Mon Sep 17 00:00:00 2001 From: erthalion <9erthalion6@gmail.com> Date: Tue, 2 Apr 2019 17:49:18 +0200 Subject: [PATCH] Group by optimization Make planner consider additional paths with reordered group by clauses, since some orders could potentially make sorting more efficient due to more generated groups, or different comparison costs. Such additional paths are created for normal and incremental sort. Originally proposed and implemented by Teodor Sigaev. --- src/backend/optimizer/path/costsize.c | 25 ++ src/backend/optimizer/path/equivclass.c | 13 +- src/backend/optimizer/path/pathkeys.c | 401 +++++++++++++++++++++++ src/backend/optimizer/plan/planner.c | 242 ++++++++++---- src/backend/optimizer/util/pathnode.c | 48 +++ src/backend/utils/misc/guc.c | 9 + src/include/nodes/pathnodes.h | 12 + src/include/optimizer/cost.h | 5 + src/include/optimizer/pathnode.h | 6 + src/include/optimizer/paths.h | 12 + src/test/regress/expected/aggregates.out | 263 +++++++++++++++ src/test/regress/expected/sysviews.out | 3 +- src/test/regress/sql/aggregates.sql | 109 ++++++ 13 files changed, 1086 insertions(+), 62 deletions(-) diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index fda4b2c6e8..1409def226 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -1911,6 +1911,31 @@ cost_incremental_sort(Path *path, path->total_cost = startup_cost + run_cost; } +void +cost_reordered_sort(Path *path, PlannerInfo *root, + List *pathkeys, Cost input_cost, double tuples, + int width, List *pathkeys_cost_details, int sort_mem, + double limit_tuples) +{ + Cost startup_cost; + Cost run_cost; + GroupCosts *group_costs = linitial(pathkeys_cost_details); + double group_ratio = group_costs->est_num_groups / tuples; + double tuples_ratio = 1.0 / group_costs->est_num_groups; + double default_cost = 2.0 * cpu_operator_cost; + + cost_tuplesort(&startup_cost, &run_cost, tuples, + group_costs->width + (width - group_costs->width) * tuples_ratio, + -default_cost + (default_cost * (1 - group_ratio)), + sort_mem, limit_tuples); + + startup_cost += input_cost; + + path->rows = tuples; + path->startup_cost = startup_cost; + path->total_cost = startup_cost + run_cost; +} + /* * cost_sort * Determines and returns the cost of sorting a relation, including diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index b68a5a0ec7..8cacea1a64 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -682,7 +682,18 @@ get_eclass_for_sort_expr(PlannerInfo *root, if (opcintype == cur_em->em_datatype && equal(expr, cur_em->em_expr)) - return cur_ec; /* Match! */ + { + /* + * Match! + * + * Copy sortref if it wasn't set yet, it's possible if ec was + * constructed from WHERE clause, ie it doesn't have target + * reference at all + */ + if (cur_ec->ec_sortref == 0 && sortref > 0) + cur_ec->ec_sortref = sortref; + return cur_ec; + } } } diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index ce9bf87e9b..7a269a4252 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -17,17 +17,25 @@ */ #include "postgres.h" +#include + +#include "access/htup_details.h" #include "access/stratnum.h" #include "catalog/pg_opfamily.h" +#include "catalog/pg_proc.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/plannodes.h" +#include "nodes/supportnodes.h" #include "optimizer/optimizer.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" #include "partitioning/partbounds.h" #include "utils/lsyscache.h" +#include "utils/syscache.h" +#include "utils/selfuncs.h" +#define LOG2(x) (log(x) / 0.693147180559945) static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); static bool matches_boolean_partition_clause(RestrictInfo *rinfo, @@ -37,6 +45,7 @@ static Var *find_var_for_subquery_tle(RelOptInfo *rel, TargetEntry *tle); static bool right_merge_direction(PlannerInfo *root, PathKey *pathkey); +bool enable_groupby_reorder = false; /**************************************************************************** * PATHKEY CONSTRUCTION AND REDUNDANCY TESTING ****************************************************************************/ @@ -388,6 +397,360 @@ pathkeys_count_contained_in(List *keys1, List *keys2, int *n_common) return (key1 == NULL); } +/* + * preorder_group_keys + * Reorder GROUP BY pathkeys and clauses to match order of pathkeys. + * + * Function returns length of the reordered part and new reordered lists via + * provided arguments, original GROUP BY lists stay untouched. + */ +int +preorder_group_keys(List *pathkeys, + List **group_pathkeys, + List **group_clauses) +{ + List *new_group_pathkeys= NIL, + *new_group_clauses = NIL; + ListCell *key; + int prefix; + + if (enable_groupby_reorder == false) + return 0; + + if (pathkeys == NIL || *group_pathkeys == NIL) + return 0; + + /* + * For each pathkey it tries to find corresponding GROUP BY pathkey and + * clause. + */ + foreach(key, pathkeys) + { + PathKey *pathkey = (PathKey *) lfirst(key); + SortGroupClause *sgc; + + /* + * Pathkey should use the same allocated struct, so, equiality of + * pointers is enough + */ + if (!list_member_ptr(*group_pathkeys, pathkey)) + break; + + new_group_pathkeys = lappend(new_group_pathkeys, pathkey); + + sgc = get_sortgroupref_clause(pathkey->pk_eclass->ec_sortref, + *group_clauses); + new_group_clauses = lappend(new_group_clauses, sgc); + } + + prefix = list_length(new_group_pathkeys); + + /* + * Just append the rest of pathkeys and clauses + */ + *group_pathkeys = list_concat_unique_ptr(new_group_pathkeys, + *group_pathkeys); + *group_clauses = list_concat_unique_ptr(new_group_clauses, + *group_clauses); + + return prefix; +} + +/* + * get_width_cost_multiplier + * Returns relative complexity of comparing two valyes based on it's width. + * + * The idea behind - long values have more expensive comparison. Return value is + * in cpu_operator_cost unit. + */ +static double +get_width_cost_multiplier(PlannerInfo *root, Expr *expr) +{ + double width = -1.0; + + if (IsA(expr, RelabelType)) + expr = (Expr *) ((RelabelType *) expr)->arg; + + /* Try to find actual stat in corresonding relation */ + if (IsA(expr, Var)) + { + Var *var = (Var *) expr; + + if (var->varno > 0 && var->varno < root->simple_rel_array_size) + { + RelOptInfo *rel = root->simple_rel_array[var->varno]; + + if (rel != NULL && + var->varattno >= rel->min_attr && + var->varattno <= rel->max_attr) + { + int ndx = var->varattno - rel->min_attr; + + if (rel->attr_widths[ndx] > 0) + width = rel->attr_widths[ndx]; + } + } + } + + /* Didn't find any actual stats, use estimation by type */ + if (width < 0.0) + { + Node *node = (Node*) expr; + + width = get_typavgwidth(exprType(node), exprTypmod(node)); + } + + /* + * Any value in pgsql is passed by Datum type, so any operation with value + * could not be cheaper than operation with Datum type + */ + if (width <= sizeof(Datum)) + return 1.0; + + /* + * Seems, cost of comparision is not directly proportional to args width, + * because comparing args could be differ width (we known only average over + * column) and difference often could be defined only by looking on first + * bytes. So, use log16(width) as estimation. + */ + return 1.0 + 0.125 * LOG2(width / sizeof(Datum)); +} + +/* + * get_func_cost + * Given procedure id, return the function's procost field. + */ +static Cost +get_func_cost(PlannerInfo *root, Oid funcid) +{ + HeapTuple proctup; + Form_pg_proc procform; + + proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); + if (!HeapTupleIsValid(proctup)) + elog(ERROR, "cache lookup failed for function %u", funcid); + + procform = (Form_pg_proc) GETSTRUCT(proctup); + + if (OidIsValid(procform->prosupport)) + { + SupportRequestCost req; + SupportRequestCost *sresult; + + req.type = T_SupportRequestCost; + req.root = root; + req.funcid = funcid; + + /* Initialize cost fields so that support function doesn't have to */ + req.startup = 0; + req.per_tuple = 0; + + sresult = (SupportRequestCost *) + DatumGetPointer(OidFunctionCall1(procform->prosupport, + PointerGetDatum(&req))); + + if (sresult == &req) + { + /* Success, so accumulate support function's estimate into *cost */ + ReleaseSysCache(proctup); + return req.per_tuple; + } + } + + /* No support function, or it failed, so rely on procost */ + ReleaseSysCache(proctup); + return procform->procost * cpu_operator_cost; +} + +/* + * get_cheapest_group_keys_order + * Order tail of list of group pathkeys by costs of sorting, taking into + * account uniqueness descendetly, width of values and comparison costs. + * + * It allows to speedup sorting. Returns newly allocated lists and the + * information required for the following costing (estimated number of groups, + * widths and comparison costs) for the chosen ordering, original lists stay + * untouched. n_preordered defines a head of list which order should be + * prevented. + */ +void +get_cheapest_group_keys_order(PlannerInfo *root, + double nrows, + List *target_list, + List **group_pathkeys, + List **group_clauses, + List **pathkeys_cost_details, + int n_preordered, int sort_mem) +{ + struct + { + PathKey *pathkey; + SortGroupClause *sgc; + Node *pathkeyExpr; + double est_num_groups; + double width; + Cost comparison_cost; + } + *keys, tmp; + int nkeys = list_length(*group_pathkeys) - n_preordered; + List *pathkeyExprList = NIL, + *new_group_pathkeys = NIL, + *new_group_clauses = NIL; + ListCell *cell, *nth_cell; + int i = 0, n_keys_to_est; + + if (!enable_groupby_reorder) + return; + + /* nothing to reorder */ + if (nkeys < 2) + return; + + /* + * Nothing to do here as well, since reordering of group clauses to match + * ORDER BY already performed in preprocess_groupclause + */ + if (n_preordered == 0 && root->sort_pathkeys) + return; + + keys = palloc(nkeys * sizeof(*keys)); + + /* + * Collect information about pathkey for subsequent usage + */ + for_each_cell(cell, *group_pathkeys, + list_nth_cell(*group_pathkeys, n_preordered)) + { + PathKey *pathkey = (PathKey *) lfirst(cell); + + keys[i].pathkey = pathkey; + keys[i].sgc = get_sortgroupref_clause(pathkey->pk_eclass->ec_sortref, + *group_clauses); + keys[i].pathkeyExpr = get_sortgroupclause_expr(keys[i].sgc, + target_list); + i++; + } + + /* + * Find the cheapest to sort order of columns. We will find a first column + * with bigger number of group, then pair (first column in pair is already + * defined in first step), them triple and so on. + */ + for(n_keys_to_est = 1; n_keys_to_est <= nkeys - 1; n_keys_to_est++) + { + ListCell *tail_cell; + int best_i = 0; + double best_coeff = -1; + + /* expand list of columns and remeber last cell */ + pathkeyExprList = lappend(pathkeyExprList, NULL); + tail_cell = list_tail(pathkeyExprList); + + /* + * Find the best last column - the best means bigger number of groups, + * previous columns are already choosen + */ + for(i = n_keys_to_est - 1; i < nkeys; i++) + { + double num_groups, width; + Cost comparison_cost = 1.0; + + PathKey *pathkey = keys[i].pathkey; + EquivalenceMember *em = (EquivalenceMember *) + linitial(pathkey->pk_eclass->ec_members); + + lfirst(tail_cell) = keys[i].pathkeyExpr; + num_groups = estimate_num_groups(root, pathkeyExprList, nrows, NULL); + width = get_width_cost_multiplier(root, (Expr *) keys[i].pathkeyExpr); + + if (em->em_datatype != InvalidOid) + { + Oid sortop; + + sortop = get_opfamily_member(pathkey->pk_opfamily, + em->em_datatype, + em->em_datatype, + pathkey->pk_strategy); + + comparison_cost = get_func_cost(root, get_opcode(sortop)); + } + + keys[i].est_num_groups = num_groups; + keys[i].width = width; + keys[i].comparison_cost = comparison_cost; + + /* + * Peform only estimations here to find a potentially good + * ordering, more precise costing will be done later. + */ + if (num_groups / (width * comparison_cost) > best_coeff) + { + best_coeff = num_groups / (width * comparison_cost); + best_i = i; + } + } + + /* Save the best choice */ + lfirst(tail_cell) = keys[best_i].pathkeyExpr; + if (best_i != n_keys_to_est - 1) + { + tmp = keys[n_keys_to_est - 1]; + keys[n_keys_to_est - 1] = keys[best_i]; + keys[best_i] = tmp; + } + } + list_free(pathkeyExprList); + + /* + * Construct result lists, keys array is already ordered to get a cheapest + * sort + */ + i = 0; + foreach(cell, *group_pathkeys) + { + PathKey *pathkey; + SortGroupClause *sgc; + + if (i < n_preordered) + { + pathkey = (PathKey *) lfirst(cell); + sgc = get_sortgroupref_clause(pathkey->pk_eclass->ec_sortref, + *group_clauses); + } + else + { + /* Save information necessary for the following costing */ + GroupCosts *details = (GroupCosts *) palloc(sizeof(GroupCosts)); + + details->comparison_cost = keys[i - n_preordered].comparison_cost; + details->width = keys[i - n_preordered].width; + details->est_num_groups = keys[i - n_preordered].est_num_groups; + *pathkeys_cost_details = lappend(*pathkeys_cost_details, details); + + pathkey = keys[i - n_preordered].pathkey; + sgc = keys[i - n_preordered].sgc; + } + + new_group_pathkeys = lappend(new_group_pathkeys, pathkey); + new_group_clauses = lappend(new_group_clauses, sgc); + + i++; + } + + pfree(keys); + + /* Just append the rest GROUP BY clauses */ + nth_cell = list_nth_cell(*group_clauses, n_preordered); + for_each_cell(cell, *group_clauses, nth_cell) + { + if (!list_member_ptr(new_group_clauses, lfirst(cell))) + new_group_clauses = lappend(new_group_clauses, lfirst(cell)); + } + + *group_pathkeys = new_group_pathkeys; + *group_clauses = new_group_clauses; +} + /* * get_cheapest_path_for_pathkeys * Find the cheapest path (according to the specified criterion) that @@ -1862,6 +2225,39 @@ pathkeys_useful_for_ordering(PlannerInfo *root, List *pathkeys) return n_common_pathkeys; } +/* + * pathkeys_useful_for_grouping + * Count the number of pathkeys that are useful for grouping (instead of + * explicit sort) + * + * Group pathkeys could be reordered, so we don't bother about actual order in + * pathkeys + */ +static int +pathkeys_useful_for_grouping(PlannerInfo *root, List *pathkeys) +{ + ListCell *key; + int n = 0; + + if (root->group_pathkeys == NIL) + return 0; /* no special ordering requested */ + + if (pathkeys == NIL) + return 0; /* unordered path */ + + foreach(key, pathkeys) + { + PathKey *pathkey = (PathKey *) lfirst(key); + + if (!list_member_ptr(root->group_pathkeys, pathkey)) + break; + + n++; + } + + return n; +} + /* * truncate_useless_pathkeys * Shorten the given pathkey list to just the useful pathkeys. @@ -1876,6 +2272,9 @@ truncate_useless_pathkeys(PlannerInfo *root, nuseful = pathkeys_useful_for_merging(root, rel, pathkeys); nuseful2 = pathkeys_useful_for_ordering(root, pathkeys); + if (nuseful2 > nuseful) + nuseful = nuseful2; + nuseful2 = pathkeys_useful_for_grouping(root, pathkeys); if (nuseful2 > nuseful) nuseful = nuseful2; @@ -1911,6 +2310,8 @@ has_useful_pathkeys(PlannerInfo *root, RelOptInfo *rel) { if (rel->joininfo != NIL || rel->has_eclass_joins) return true; /* might be able to use pathkeys for merging */ + if (root->group_pathkeys != NIL) + return true; /* might be able to use pathkeys for grouping */ if (root->query_pathkeys != NIL) return true; /* might be able to use them for ordering */ return false; /* definitely useless */ diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index b40a112c25..2a053bf373 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -6501,6 +6501,9 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, Path *path_original = path; bool is_sorted; int presorted_keys; + List *group_pathkeys = root->group_pathkeys, + *group_clauses = parse->groupClause; + int n_preordered_groups = 0; is_sorted = pathkeys_count_contained_in(root->group_pathkeys, path->pathkeys, @@ -6563,76 +6566,195 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel, } /* - * Now we may consider incremental sort on this path, but only - * when the path is not already sorted and when incremental sort - * is enabled. + * Now we may consider incremental sort on this path, but only when + * the path is not already sorted, incremental sort is enabled, + * and there is a shared prefix. */ - if (is_sorted || !enable_incremental_sort) - continue; - - /* Restore the input path (we might have added Sort on top). */ - path = path_original; - - /* no shared prefix, no point in building incremental sort */ - if (presorted_keys == 0) - continue; + if (!is_sorted && enable_incremental_sort && presorted_keys > 0) + { + /* Restore the input path (we might have added Sort on top). */ + path = path_original; - /* - * We should have already excluded pathkeys of length 1 because - * then presorted_keys > 0 would imply is_sorted was true. - */ - Assert(list_length(root->group_pathkeys) != 1); + /* + * We should have already excluded pathkeys of length 1 because + * then presorted_keys > 0 would imply is_sorted was true. + */ + Assert(list_length(group_pathkeys) != 1); - path = (Path *) create_incremental_sort_path(root, - grouped_rel, - path, - root->group_pathkeys, - presorted_keys, - -1.0); + path = (Path *) create_incremental_sort_path(root, + grouped_rel, + path, + root->group_pathkeys, + presorted_keys, + -1.0); - /* Now decide what to stick atop it */ - if (parse->groupingSets) - { - consider_groupingsets_paths(root, grouped_rel, - path, true, can_hash, - gd, agg_costs, dNumGroups); + /* Now decide what to stick atop it */ + if (parse->groupingSets) + consider_groupingsets_paths(root, grouped_rel, + path, true, can_hash, + gd, agg_costs, dNumGroups); + else if (parse->hasAggs) + /* + * We have aggregation, possibly with plain GROUP BY. Make an + * AggPath. + */ + add_path(grouped_rel, (Path *) + create_agg_path(root, + grouped_rel, + path, + grouped_rel->reltarget, + parse->groupClause ? AGG_SORTED : AGG_PLAIN, + AGGSPLIT_SIMPLE, + parse->groupClause, + havingQual, + agg_costs, + dNumGroups)); + else if (parse->groupClause) + /* + * We have GROUP BY without aggregation or grouping sets. Make + * a GroupPath. + */ + add_path(grouped_rel, (Path *) + create_group_path(root, + grouped_rel, + path, + parse->groupClause, + havingQual, + dNumGroups)); + else + /* Other cases should have been handled above */ + Assert(false); } - else if (parse->hasAggs) + + /* Now consider the same paths but for reordered grouping clauses */ + if (enable_groupby_reorder && !parse->groupingSets) { + List *cost_details = NIL; + + /* Restore the input path (we might have added Sort on top). */ + path = path_original; + + presorted_keys = n_preordered_groups = + preorder_group_keys(path->pathkeys, + &group_pathkeys, + &group_clauses); + is_sorted = (n_preordered_groups == list_length(group_pathkeys)); + + /* Sort the cheapest-total path if it isn't already sorted */ + get_cheapest_group_keys_order(root, + path->rows, + extra->targetList, + &group_pathkeys, + &group_clauses, + &cost_details, + 0, + work_mem); + + if (!is_sorted) + path = (Path *) create_reordered_sort_path(root, + grouped_rel, + path, + group_pathkeys, + cost_details, + -1.0); + + /* Now decide what to stick atop it */ + if (parse->groupingSets) + consider_groupingsets_paths(root, grouped_rel, + path, true, can_hash, + gd, agg_costs, dNumGroups); + else if (parse->hasAggs) + /* + * We have aggregation, possibly with plain GROUP BY. Make + * an AggPath. + */ + add_path(grouped_rel, (Path *) + create_agg_path(root, + grouped_rel, + path, + grouped_rel->reltarget, + group_clauses ? AGG_SORTED : AGG_PLAIN, + AGGSPLIT_SIMPLE, + group_clauses, + havingQual, + agg_costs, + dNumGroups)); + else if (parse->groupClause) + /* + * We have GROUP BY without aggregation or grouping sets. + * Make a GroupPath. + */ + add_path(grouped_rel, (Path *) + create_group_path(root, + grouped_rel, + path, + group_clauses, + havingQual, + dNumGroups)); + else + /* Other cases should have been handled above */ + Assert(false); + /* - * We have aggregation, possibly with plain GROUP BY. Make an - * AggPath. + * Incremental sort over reordered grouping clauses. First + * check if there is a shared prefix and it makes sense to + * considere incremental approach. */ - add_path(grouped_rel, (Path *) - create_agg_path(root, - grouped_rel, - path, - grouped_rel->reltarget, - parse->groupClause ? AGG_SORTED : AGG_PLAIN, - AGGSPLIT_SIMPLE, - parse->groupClause, - havingQual, - agg_costs, - dNumGroups)); - } - else if (parse->groupClause) - { + if (presorted_keys == 0) + continue; + + /* Restore the input path (we might have added Sort on top). */ + path = path_original; + /* - * We have GROUP BY without aggregation or grouping sets. Make - * a GroupPath. + * We should have already excluded pathkeys of length 1 because + * then presorted_keys > 0 would imply is_sorted was true. */ - add_path(grouped_rel, (Path *) - create_group_path(root, - grouped_rel, - path, - parse->groupClause, - havingQual, - dNumGroups)); - } - else - { - /* Other cases should have been handled above */ - Assert(false); + Assert(list_length(group_pathkeys) != 1); + + path = (Path *) create_incremental_sort_path(root, + grouped_rel, + path, + group_pathkeys, + presorted_keys, + -1.0); + + /* Now decide what to stick atop it */ + if (parse->groupingSets) + consider_groupingsets_paths(root, grouped_rel, + path, true, can_hash, + gd, agg_costs, dNumGroups); + else if (parse->hasAggs) + /* + * We have aggregation, possibly with plain GROUP BY. Make an + * AggPath. + */ + add_path(grouped_rel, (Path *) + create_agg_path(root, + grouped_rel, + path, + grouped_rel->reltarget, + group_clauses ? AGG_SORTED : AGG_PLAIN, + AGGSPLIT_SIMPLE, + group_clauses, + havingQual, + agg_costs, + dNumGroups)); + else if (parse->groupClause) + /* + * We have GROUP BY without aggregation or grouping sets. Make + * a GroupPath. + */ + add_path(grouped_rel, (Path *) + create_group_path(root, + grouped_rel, + path, + group_clauses, + havingQual, + dNumGroups)); + else + /* Other cases should have been handled above */ + Assert(false); } } diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index c1fc866cbf..c0602facef 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -2880,6 +2880,54 @@ create_sort_path(PlannerInfo *root, return pathnode; } +/* + * create_reordered_sort_path + * Similar to create_sort_path, but take into account grouping cost + * details, namely estimated number of groups, width and comparison costs + */ +SortPath * +create_reordered_sort_path(PlannerInfo *root, + RelOptInfo *rel, + Path *subpath, + List *pathkeys, + List *pathkeys_cost_details, + double limit_tuples) +{ + SortPath *pathnode = makeNode(SortPath); + + pathnode->path.pathtype = T_Sort; + pathnode->path.parent = rel; + /* Sort doesn't project, so use source path's pathtarget */ + pathnode->path.pathtarget = subpath->pathtarget; + /* For now, assume we are above any joins, so no parameterization */ + pathnode->path.param_info = NULL; + pathnode->path.parallel_aware = false; + pathnode->path.parallel_safe = rel->consider_parallel && + subpath->parallel_safe; + pathnode->path.parallel_workers = subpath->parallel_workers; + pathnode->path.pathkeys = pathkeys; + + pathnode->subpath = subpath; + + /* If cost details are not provided, turn it into a normal cost_sort */ + if (pathkeys_cost_details != NULL) + cost_reordered_sort(&pathnode->path, root, pathkeys, + subpath->total_cost, + subpath->rows, + subpath->pathtarget->width, + pathkeys_cost_details, + work_mem, limit_tuples); + else + cost_sort(&pathnode->path, root, pathkeys, + subpath->total_cost, + subpath->rows, + subpath->pathtarget->width, + 0.0, /* XXX comparison_cost shouldn't be 0? */ + work_mem, limit_tuples); + + return pathnode; +} + /* * create_group_path * Creates a pathnode that represents performing grouping of presorted input diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index de87ad6ef7..5b15155543 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -1108,6 +1108,15 @@ static struct config_bool ConfigureNamesBool[] = true, NULL, NULL, NULL }, + { + {"enable_groupby_reorder", PGC_USERSET, QUERY_TUNING_METHOD, + gettext_noop("enable reorder GROUP BY"), + NULL + }, + &enable_groupby_reorder, + false, + NULL, NULL, NULL + }, { {"geqo", PGC_USERSET, QUERY_TUNING_GEQO, gettext_noop("Enables genetic query optimization."), diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 485d1b06c9..3c95f99d03 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -2549,4 +2549,16 @@ typedef struct JoinCostWorkspace double inner_rows_total; } JoinCostWorkspace; +/* + * Describes extra information necessary for costing group by. In case if + * planner evaluates reordering of group by clauses, it will provide this + * information to calculate benefits of reordering. + */ +typedef struct GroupCosts +{ + Cost comparison_cost; + double width; + double est_num_groups; +} GroupCosts; + #endif /* PATHNODES_H */ diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index 6141654e47..bec27fa11d 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -65,6 +65,7 @@ extern PGDLLIMPORT bool enable_partitionwise_aggregate; extern PGDLLIMPORT bool enable_parallel_append; extern PGDLLIMPORT bool enable_parallel_hash; extern PGDLLIMPORT bool enable_partition_pruning; +extern PGDLLIMPORT bool enable_groupby_reorder; extern PGDLLIMPORT int constraint_exclusion; extern double index_pages_fetched(double tuples_fetched, BlockNumber pages, @@ -107,6 +108,10 @@ extern void cost_incremental_sort(Path *path, Cost input_startup_cost, Cost input_total_cost, double input_tuples, int width, Cost comparison_cost, int sort_mem, double limit_tuples); +extern void cost_reordered_sort(Path *path, PlannerInfo *root, + List *pathkeys, Cost input_cost, double tuples, + int width, List *pathkeys_cost_details, int sort_mem, + double limit_tuples); extern void cost_append(AppendPath *path); extern void cost_merge_append(Path *path, PlannerInfo *root, List *pathkeys, int n_streams, diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 715a24ad29..1262fd0922 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -195,6 +195,12 @@ extern SortPath *create_sort_path(PlannerInfo *root, Path *subpath, List *pathkeys, double limit_tuples); +extern SortPath *create_reordered_sort_path(PlannerInfo *root, + RelOptInfo *rel, + Path *subpath, + List *pathkeys, + List *pathkeys_cost_details, + double limit_tuples); extern GroupPath *create_group_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 10b6e81079..0ea6bed505 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -189,6 +189,18 @@ typedef enum extern PathKeysComparison compare_pathkeys(List *keys1, List *keys2); extern bool pathkeys_contained_in(List *keys1, List *keys2); extern bool pathkeys_count_contained_in(List *keys1, List *keys2, int *n_common); + +extern int preorder_group_keys(List *pathkeys, + List **group_pathkeys, + List **group_clauses); +extern void get_cheapest_group_keys_order(PlannerInfo *root, + double nrows, + List *target_list, + List **group_pathkeys, + List **group_clauses, + List **pathkeys_cost_details, + int n_preordered, + int sort_mem); extern Path *get_cheapest_path_for_pathkeys(List *paths, List *pathkeys, Relids required_outer, CostSelector cost_criterion, diff --git a/src/test/regress/expected/aggregates.out b/src/test/regress/expected/aggregates.out index 477fd1205c..027f931e92 100644 --- a/src/test/regress/expected/aggregates.out +++ b/src/test/regress/expected/aggregates.out @@ -2445,6 +2445,269 @@ SELECT balk(hundred) FROM tenk1; (1 row) ROLLBACK; +-- GROUP BY optimization by reorder columns +SELECT + i AS id, + i/2 AS p, + i%2 AS v, + format('%60s', i%2) AS s, + i/4 AS c, + i/8 AS d, + (random() * (10000/8))::int as e --the same as d but no correlation with p + INTO btg +FROM + generate_series(1, 10000) i; +VACUUM btg; +ANALYZE btg; +-- GROUP BY optimization by reorder columns by frequency +SET enable_groupby_reorder = on; +SET enable_hashagg = off; +SET max_parallel_workers = 0; +SET max_parallel_workers_per_gather = 0; +SET enable_indexscan = off; +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, v; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, v + -> Sort + Sort Key: p, v + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, v + -> Sort + Sort Key: p, v + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, v, c + -> Sort + Sort Key: p, v, c + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c ORDER BY v, p, c; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: v, p, c + -> Sort + Sort Key: v, p, c + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, d, c; + QUERY PLAN +------------------------------ + GroupAggregate + Group Key: p, v, d, c + -> Sort + Sort Key: p, v, d, c + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, d, c ORDER BY v, p, d ,c; + QUERY PLAN +------------------------------ + GroupAggregate + Group Key: v, p, d, c + -> Sort + Sort Key: v, p, d, c + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, d, c ORDER BY p, v, d ,c; + QUERY PLAN +------------------------------ + GroupAggregate + Group Key: p, v, d, c + -> Sort + Sort Key: p, v, d, c + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, d, e; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, d, e + -> Sort + Sort Key: p, d, e + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, e, d; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, e, d + -> Sort + Sort Key: p, e, d + -> Seq Scan on btg +(5 rows) + +CREATE STATISTICS btg_dep ON d, e, p FROM btg; +ANALYZE btg; +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, d, e; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, e, d + -> Sort + Sort Key: p, e, d + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, e, d; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, e, d + -> Sort + Sort Key: p, e, d + -> Seq Scan on btg +(5 rows) + +-- GROUP BY optimization by reorder columns by frequency +-- taking into account comparison costs +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, s; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: v, s + -> Sort + Sort Key: v, s + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY s, v; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: v, s + -> Sort + Sort Key: v, s + -> Seq Scan on btg +(5 rows) + +-- GROUP BY optimization by reorder columns by index scan +CREATE INDEX ON btg(p, v); +RESET enable_indexscan; +SET enable_seqscan = off; +SET enable_bitmapscan = off; +VACUUM btg; +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, v; + QUERY PLAN +------------------------------------------------ + GroupAggregate + Group Key: p, v + -> Index Only Scan using btg_p_v_idx on btg +(3 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, v ORDER BY p, v; + QUERY PLAN +------------------------------------------------ + GroupAggregate + Group Key: p, v + -> Index Only Scan using btg_p_v_idx on btg +(3 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p; + QUERY PLAN +------------------------------------------------ + GroupAggregate + Group Key: p, v + -> Index Only Scan using btg_p_v_idx on btg +(3 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p ORDER BY p, v; + QUERY PLAN +------------------------------------------------ + GroupAggregate + Group Key: p, v + -> Index Only Scan using btg_p_v_idx on btg +(3 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c; + QUERY PLAN +------------------------------------------------- + GroupAggregate + Group Key: p, v, c + -> Incremental Sort + Sort Key: p, v, c + Presorted Key: p, v + -> Index Scan using btg_p_v_idx on btg +(6 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c ORDER BY p, v; + QUERY PLAN +------------------------------------------------- + GroupAggregate + Group Key: p, v, c + -> Incremental Sort + Sort Key: p, v, c + Presorted Key: p, v + -> Index Scan using btg_p_v_idx on btg +(6 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, c, p, d; + QUERY PLAN +------------------------------------------------- + GroupAggregate + Group Key: p, v, c, d + -> Incremental Sort + Sort Key: p, v, c, d + Presorted Key: p, v + -> Index Scan using btg_p_v_idx on btg +(6 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, c, p, d ORDER BY p, v; + QUERY PLAN +------------------------------------------------- + GroupAggregate + Group Key: p, v, c, d + -> Incremental Sort + Sort Key: p, v, c, d + Presorted Key: p, v + -> Index Scan using btg_p_v_idx on btg +(6 rows) + +RESET enable_hashagg; +RESET max_parallel_workers; +RESET max_parallel_workers_per_gather; +RESET enable_seqscan; +RESET enable_bitmapscan; +RESET enable_groupby_reorder; -- test coverage for aggregate combine/serial/deserial functions BEGIN ISOLATION LEVEL REPEATABLE READ; SET parallel_setup_cost = 0; diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index 1cffc3349d..40c4482802 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -83,6 +83,7 @@ select name, setting from pg_settings where name like 'enable%'; --------------------------------+--------- enable_bitmapscan | on enable_gathermerge | on + enable_groupby_reorder | off enable_hashagg | on enable_hashjoin | on enable_incremental_sort | on @@ -99,7 +100,7 @@ select name, setting from pg_settings where name like 'enable%'; enable_seqscan | on enable_sort | on enable_tidscan | on -(18 rows) +(19 rows) -- Test that the pg_timezone_names and pg_timezone_abbrevs views are -- more-or-less working. We can't test their contents in any great detail diff --git a/src/test/regress/sql/aggregates.sql b/src/test/regress/sql/aggregates.sql index 54f5cf7ecc..74cca5a299 100644 --- a/src/test/regress/sql/aggregates.sql +++ b/src/test/regress/sql/aggregates.sql @@ -1026,6 +1026,115 @@ SELECT balk(hundred) FROM tenk1; ROLLBACK; +-- GROUP BY optimization by reorder columns + +SELECT + i AS id, + i/2 AS p, + i%2 AS v, + format('%60s', i%2) AS s, + i/4 AS c, + i/8 AS d, + (random() * (10000/8))::int as e --the same as d but no correlation with p + INTO btg +FROM + generate_series(1, 10000) i; + +VACUUM btg; +ANALYZE btg; + +-- GROUP BY optimization by reorder columns by frequency + +SET enable_groupby_reorder = on; +SET enable_hashagg = off; +SET max_parallel_workers = 0; +SET max_parallel_workers_per_gather = 0; +SET enable_indexscan = off; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, v; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c ORDER BY v, p, c; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, d, c; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, d, c ORDER BY v, p, d ,c; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, d, c ORDER BY p, v, d ,c; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, d, e; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, e, d; + +CREATE STATISTICS btg_dep ON d, e, p FROM btg; +ANALYZE btg; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, d, e; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, e, d; + +-- GROUP BY optimization by reorder columns by frequency +-- taking into account comparison costs + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, s; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY s, v; + +-- GROUP BY optimization by reorder columns by index scan + +CREATE INDEX ON btg(p, v); +RESET enable_indexscan; +SET enable_seqscan = off; +SET enable_bitmapscan = off; +VACUUM btg; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, v; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, v ORDER BY p, v; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p ORDER BY p, v; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c ORDER BY p, v; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, c, p, d; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, c, p, d ORDER BY p, v; + +RESET enable_hashagg; +RESET max_parallel_workers; +RESET max_parallel_workers_per_gather; +RESET enable_seqscan; +RESET enable_bitmapscan; +RESET enable_groupby_reorder; + -- test coverage for aggregate combine/serial/deserial functions BEGIN ISOLATION LEVEL REPEATABLE READ; -- 2.21.0