From 7f32d263f8b6e6aea95d6544fd10e46177346fb3 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Sun, 23 Oct 2016 17:35:47 +0200 Subject: [PATCH 2/9] PATCH: shared infrastructure and ndistinct coefficients Basic infrastructure shared by all kinds of multivariate stats, most importantly: - adds a new system catalog (pg_mv_statistic) - CREATE STATISTICS name ON (columns) FROM table - DROP STATISTICS name - ALTER STATISTICS ... OWNER TO / SET SCHEMA / RENAME - implementation of ndistinct coefficients (the simplest type of multivariate statistics) - computing ndistinct coefficients during ANALYZE - updates existing regression tests (new catalog etc.) - modifies estimate_num_groups() to use ndistinct if available The current implementation requires a valid 'ltopr' for the columns, so that we can sort the sample rows in various ways, both in this patch and other kinds of statistics. Maybe this restriction could be relaxed in the future, requiring just 'eqopr' in case of stats not sorting the data (e.g. functional dependencies and MCV lists). Some of the stats implemented in follow-up patches (e.g. functional dependencies and MCV list with limited functionality) might be made to work with hashes of the values. That would save a lot of space for storing the statistics, and it would be sufficient for estimating equality conditions. creating statistics ------------------- Statistics are created by CREATE STATISTICS command, with this syntax: CREATE STATISTICS statistics_name ON (columns) FROM table where 'statistics_name' may be a fully-qualified name (i.e. specifying a schema). It's expected that we'll eventually add support for join statistics, referencing tables that may be located in different schemas, so we can't make the name unique per-table (like constraints), and we can't just pick one of the table schemas. dropping statistics ------------------- The statistics may be dropped automatically using DROP STATISTICS. After ALTER TABLE ... DROP COLUMN, statistics referencing are: (a) dropped, if the statistics would reference only one column (b) retained, but modified on the next ANALYZE The goal of the lazy cleanup is not to disrupt the optimizer, but arguably this is over-engineering and it might also made work just like for indexes by simply dropping all dependent statistics on ALTER TABLE ... DROP COLUMN. If the user wants to minimize impact, the smaller statistics needs to be created explicitly in advance. This also adds a simple list of statistics to \d in psql. ndistinct coefficients ---------------------- The patch only implements a very simple type of statistics, tracking the number of groups for different combinations of columns. For example given columns (a,b,c) the statistics will estimate number of distinct combinations of values in (a,b), (a,c), (b,c) and (a,b,c). This is then used in estimate_num_groups() for estimating cardinality of GROUP BY and similar clauses. pg_ndistinct data type ---------------------- The patch introduces pg_ndistinct, a new varlena data type used for serialized version of ndistinct coefficients. Internally it's just a bytea value, but it allows us to control casting, input/output and so on. It's somewhat inspired by pg_node_tree. --- doc/src/sgml/catalogs.sgml | 108 +++++ doc/src/sgml/planstats.sgml | 141 +++++++ doc/src/sgml/ref/allfiles.sgml | 3 + doc/src/sgml/ref/alter_statistics.sgml | 115 ++++++ doc/src/sgml/ref/alter_table.sgml | 8 +- doc/src/sgml/ref/create_statistics.sgml | 152 +++++++ doc/src/sgml/ref/drop_statistics.sgml | 91 ++++ doc/src/sgml/reference.sgml | 3 + src/backend/catalog/Makefile | 1 + src/backend/catalog/aclchk.c | 27 ++ src/backend/catalog/dependency.c | 11 +- src/backend/catalog/heap.c | 101 +++++ src/backend/catalog/namespace.c | 56 +++ src/backend/catalog/objectaddress.c | 53 +++ src/backend/catalog/system_views.sql | 10 + src/backend/commands/Makefile | 6 +- src/backend/commands/alter.c | 3 + src/backend/commands/analyze.c | 8 + src/backend/commands/dropcmds.c | 4 + src/backend/commands/event_trigger.c | 3 + src/backend/commands/statscmds.c | 259 ++++++++++++ src/backend/nodes/copyfuncs.c | 16 + src/backend/nodes/outfuncs.c | 18 + src/backend/optimizer/util/plancat.c | 72 +++- src/backend/parser/gram.y | 58 ++- src/backend/tcop/utility.c | 12 + src/backend/utils/Makefile | 2 +- src/backend/utils/adt/selfuncs.c | 168 +++++++- src/backend/utils/cache/relcache.c | 78 ++++ src/backend/utils/cache/syscache.c | 23 ++ src/backend/utils/mvstats/Makefile | 17 + src/backend/utils/mvstats/README.ndistinct | 22 + src/backend/utils/mvstats/README.stats | 98 +++++ src/backend/utils/mvstats/common.c | 391 ++++++++++++++++++ src/backend/utils/mvstats/common.h | 80 ++++ src/backend/utils/mvstats/mvdist.c | 597 +++++++++++++++++++++++++++ src/bin/psql/describe.c | 44 ++ src/include/catalog/dependency.h | 5 +- src/include/catalog/heap.h | 1 + src/include/catalog/indexing.h | 7 + src/include/catalog/namespace.h | 2 + src/include/catalog/pg_cast.h | 4 + src/include/catalog/pg_mv_statistic.h | 78 ++++ src/include/catalog/pg_proc.h | 9 + src/include/catalog/pg_type.h | 4 + src/include/catalog/toasting.h | 1 + src/include/commands/defrem.h | 4 + src/include/nodes/nodes.h | 2 + src/include/nodes/parsenodes.h | 11 + src/include/nodes/relation.h | 27 ++ src/include/utils/acl.h | 1 + src/include/utils/builtins.h | 6 + src/include/utils/mvstats.h | 57 +++ src/include/utils/rel.h | 4 + src/include/utils/relcache.h | 1 + src/include/utils/syscache.h | 2 + src/test/regress/expected/mv_ndistinct.out | 117 ++++++ src/test/regress/expected/object_address.out | 7 +- src/test/regress/expected/opr_sanity.out | 3 +- src/test/regress/expected/rules.out | 8 + src/test/regress/expected/sanity_check.out | 1 + src/test/regress/expected/type_sanity.out | 11 +- src/test/regress/parallel_schedule | 3 + src/test/regress/serial_schedule | 1 + src/test/regress/sql/mv_ndistinct.sql | 68 +++ src/test/regress/sql/object_address.sql | 4 +- 66 files changed, 3286 insertions(+), 22 deletions(-) create mode 100644 doc/src/sgml/ref/alter_statistics.sgml create mode 100644 doc/src/sgml/ref/create_statistics.sgml create mode 100644 doc/src/sgml/ref/drop_statistics.sgml create mode 100644 src/backend/commands/statscmds.c create mode 100644 src/backend/utils/mvstats/Makefile create mode 100644 src/backend/utils/mvstats/README.ndistinct create mode 100644 src/backend/utils/mvstats/README.stats create mode 100644 src/backend/utils/mvstats/common.c create mode 100644 src/backend/utils/mvstats/common.h create mode 100644 src/backend/utils/mvstats/mvdist.c create mode 100644 src/include/catalog/pg_mv_statistic.h create mode 100644 src/include/utils/mvstats.h create mode 100644 src/test/regress/expected/mv_ndistinct.out create mode 100644 src/test/regress/sql/mv_ndistinct.sql diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 086fafc..2a7bd6c 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -201,6 +201,11 @@ + pg_mv_statistic + multivariate statistics + + + pg_namespace schemas @@ -4211,6 +4216,109 @@ + + <structname>pg_mv_statistic</structname> + + + pg_mv_statistic + + + + The catalog pg_mv_statistic + holds multivariate statistics about combinations of columns. + + + + <structname>pg_mv_statistic</> Columns + + + + + Name + Type + References + Description + + + + + + + starelid + oid + pg_class.oid + The table that the described columns belongs to + + + + staname + name + + Name of the statistic. + + + + stanamespace + oid + pg_namespace.oid + + The OID of the namespace that contains this statistic + + + + + staowner + oid + pg_authid.oid + Owner of the statistic + + + + ndist_enabled + bool + + + If true, ndistinct coefficients will be computed for the combination of + columns, covered by the statistics. This does not mean the coefficients + are already computed, though. + + + + + ndist_built + bool + + + If true, ndistinct coefficients are already computed and available for + use during query estimation. + + + + + stakeys + int2vector + pg_attribute.attnum + + This is an array of values that indicate which table columns this + statistic covers. For example a value of 1 3 would + mean that the first and the third table columns make up the statistic key. + + + + + standist + pg_ndistinct + + + Ndistict coefficients, serialized as pg_ndistinct type. + + + + + +
+ + <structname>pg_namespace</structname> diff --git a/doc/src/sgml/planstats.sgml b/doc/src/sgml/planstats.sgml index b73c66b..d5b975d 100644 --- a/doc/src/sgml/planstats.sgml +++ b/doc/src/sgml/planstats.sgml @@ -448,4 +448,145 @@ rows = (outer_cardinality * inner_cardinality) * selectivity + + Multivariate Statistics + + + multivariate statistics + planner + + + + The examples presented in used + statistics about individual columns to compute selectivity estimates. + When estimating conditions on multiple columns, the planner assumes + independence of the conditions and multiplies the selectivities. When the + columns are correlated, the independence assumption is violated, and the + estimates may be off by several orders of magnitude, resulting in poor + plan choices. + + + + The examples presented below demonstrate such estimation errors on simple + data sets, and also how to resolve them by creating multivariate statistics + using CREATE STATISTICS command. + + + + Let's start with a very simple data set - a table with two columns, + containing exactly the same values: + + +CREATE TABLE t (a INT, b INT); +INSERT INTO t SELECT i % 100, i % 100 FROM generate_series(1, 10000) s(i); +ANALYZE t; + + + As explained in , the planner can determine + cardinality of t using the number of pages and + rows is looked up in pg_class: + + +SELECT relpages, reltuples FROM pg_class WHERE relname = 't'; + + relpages | reltuples +----------+----------- + 45 | 10000 + + + The data distribution is very simple - there are only 100 distinct values + in each column, uniformly distributed. + + + + The following example shows the result of estimating a WHERE + condition on the a column: + + +EXPLAIN ANALYZE SELECT * FROM t WHERE a = 1; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Seq Scan on t (cost=0.00..170.00 rows=100 width=8) (actual time=0.031..2.870 rows=100 loops=1) + Filter: (a = 1) + Rows Removed by Filter: 9900 + Planning time: 0.092 ms + Execution time: 3.103 ms +(5 rows) + + + The planner examines the condition and computes the estimate using + eqsel, the selectivity function for =, and + statistics stored in the pg_stats table. In this case + the planner estimates the condition matches 1% rows, and by comparing + the estimated and actual number of rows, we see that the estimate is + very accurate (in fact exact, as the table is very small). + + + + Adding a condition on the second column results in the following plan: + + +EXPLAIN ANALYZE SELECT * FROM t WHERE a = 1 AND b = 1; + QUERY PLAN +----------------------------------------------------------------------------------------------- + Seq Scan on t (cost=0.00..195.00 rows=1 width=8) (actual time=0.033..3.006 rows=100 loops=1) + Filter: ((a = 1) AND (b = 1)) + Rows Removed by Filter: 9900 + Planning time: 0.121 ms + Execution time: 3.220 ms +(5 rows) + + + The planner estimates the selectivity for each condition individually, + arriving to the 1% estimates as above, and then multiplies them, getting + the final 0.01% estimate. The plan however shows that this results in + a significant underestimate, as the actual number of rows matching the + conditions is two orders of magnitude higher than estimated. + + + + Overestimates, i.e. errors in the opposite direction, are also possible. + Consider for example the following combination of range conditions, each + matching + + +EXPLAIN ANALYZE SELECT * FROM t WHERE a <= 49 AND b > 49; + QUERY PLAN +------------------------------------------------------------------------------------------------ + Seq Scan on t (cost=0.00..195.00 rows=2500 width=8) (actual time=1.607..1.607 rows=0 loops=1) + Filter: ((a <= 49) AND (b > 49)) + Rows Removed by Filter: 10000 + Planning time: 0.050 ms + Execution time: 1.623 ms +(5 rows) + + + The planner examines both WHERE clauses and estimates them + using the scalarltsel and scalargtsel functions, + specified as the selectivity functions matching the <= and + > operators. Both conditions match 50% of the + table, and assuming independence the planner multiplies them to compute + the total estimate of 25%. However as the explain output shows, the actual + number of rows is 0, because the columns are correlated and the conditions + contradict each other. + + + + Both estimation errors are caused by violation of the independence + assumption, as the two columns contain exactly the same values, and are + therefore perfectly correlated. Providing additional information about + correlation between columns is the purpose of multivariate statistics, + and the rest of this section explains in more detail how the planner + leverages them to improve estimates. + + + + For additional details about multivariate statistics, see + src/backend/utils/mvstats/README.stats. There are additional + READMEs for each type of statistics, mentioned in the following + sections. + + + + diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index 0d09f81..a49da6d 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -34,6 +34,7 @@ Complete list of usable sgml source files in this directory. + @@ -80,6 +81,7 @@ Complete list of usable sgml source files in this directory. + @@ -126,6 +128,7 @@ Complete list of usable sgml source files in this directory. + diff --git a/doc/src/sgml/ref/alter_statistics.sgml b/doc/src/sgml/ref/alter_statistics.sgml new file mode 100644 index 0000000..3f477cb --- /dev/null +++ b/doc/src/sgml/ref/alter_statistics.sgml @@ -0,0 +1,115 @@ + + + + + ALTER STATISTICS + + + + ALTER STATISTICS + 7 + SQL - Language Statements + + + + ALTER STATISTICS + + change the definition of a multivariate statistics + + + + + +ALTER STATISTICS name OWNER TO { new_owner | CURRENT_USER | SESSION_USER } +ALTER STATISTICS name RENAME TO new_name +ALTER STATISTICS name SET SCHEMA new_schema + + + + + Description + + + ALTER STATISTICS changes the parameters of an existing + multivariate statistics. Any parameters not specifically set in the + ALTER STATISTICS command retain their prior settings. + + + + You must own the statistics to use ALTER STATISTICS. + To change a statistics' schema, you must also have CREATE + privilege on the new schema. + To alter the owner, you must also be a direct or indirect member of the new + owning role, and that role must have CREATE privilege on + the statistics' schema. (These restrictions enforce that altering the owner + doesn't do anything you couldn't do by dropping and recreating the statistics. + However, a superuser can alter ownership of any statistics anyway.) + + + + + Parameters + + + + + name + + + The name (optionally schema-qualified) of a statistics to be altered. + + + + + + new_owner + + + The user name of the new owner of the statistics. + + + + + + new_name + + + The new name for the statistics. + + + + + + new_schema + + + The new schema for the statistics. + + + + + + + + + + Compatibility + + + There's no ALTER STATISTICS command in the SQL standard. + + + + + See Also + + + + + + + + diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index da431f8..9ce1079 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -119,9 +119,11 @@ ALTER TABLE [ IF EXISTS ] name This form drops a column from a table. Indexes and table constraints involving the column will be automatically - dropped as well. You will need to say CASCADE if - anything outside the table depends on the column, for example, - foreign key references or views. + dropped as well. Multivariate statistics referencing the column will + be dropped only if there would remain a single non-dropped column. + You will need to say CASCADE if anything outside the table + depends on the column, for example, foreign key references or views. + If IF EXISTS is specified and the column does not exist, no error is thrown. In this case a notice is issued instead. diff --git a/doc/src/sgml/ref/create_statistics.sgml b/doc/src/sgml/ref/create_statistics.sgml new file mode 100644 index 0000000..9f6a65c --- /dev/null +++ b/doc/src/sgml/ref/create_statistics.sgml @@ -0,0 +1,152 @@ + + + + + CREATE STATISTICS + + + + CREATE STATISTICS + 7 + SQL - Language Statements + + + + CREATE STATISTICS + define multivariate statistics + + + + +CREATE STATISTICS [ IF NOT EXISTS ] statistics_name ON ( + column_name, column_name [, ...]) + FROM table_name + + + + + + Description + + + CREATE STATISTICS will create a new multivariate + statistics on the table. The statistics will be created in the current + database and will be owned by the user issuing the command. + + + + If a schema name is given (for example, CREATE STATISTICS + myschema.mystat ...) then the statistics is created in the specified + schema. Otherwise it is created in the current schema. The name of + the table must be distinct from the name of any other statistics in the + same schema. + + + + To be able to create a table, you must have USAGE + privilege on all column types or the type in the OF + clause, respectively. + + + + + Parameters + + + + + IF NOT EXISTS + + + Do not throw an error if a statistics with the same name already exists. + A notice is issued in this case. Note that there is no guarantee that + the existing statistics is anything like the one that would have been + created. + + + + + + statistics_name + + + The name (optionally schema-qualified) of the statistics to be created. + + + + + + table_name + + + The name (optionally schema-qualified) of the table the statistics should + be created on. + + + + + + column_name + + + The name of a column to be included in the statistics. + + + + + + + + + + Examples + + + Create table t1 with two functionally dependent columns, i.e. + knowledge of a value in the first column is sufficient for determining the + value in the other column. Then functional dependencies are built on those + columns: + + +CREATE TABLE t1 ( + a int, + b int +); + +INSERT INTO t1 SELECT i/100, i/500 + FROM generate_series(1,1000000) s(i); + +CREATE STATISTICS s1 ON (a, b) FROM t1; + +ANALYZE t1; + +-- valid combination of values +EXPLAIN ANALYZE SELECT * FROM t1 WHERE (a = 1) AND (b = 0); + +-- invalid combination of values +EXPLAIN ANALYZE SELECT * FROM t1 WHERE (a = 1) AND (b = 1); + + + + + + + Compatibility + + + There's no CREATE STATISTICS command in the SQL standard. + + + + + See Also + + + + + + + diff --git a/doc/src/sgml/ref/drop_statistics.sgml b/doc/src/sgml/ref/drop_statistics.sgml new file mode 100644 index 0000000..3e73d10 --- /dev/null +++ b/doc/src/sgml/ref/drop_statistics.sgml @@ -0,0 +1,91 @@ + + + + + DROP STATISTICS + + + + DROP STATISTICS + 7 + SQL - Language Statements + + + + DROP STATISTICS + remove multivariate statistics + + + + +DROP STATISTICS [ IF EXISTS ] name [, ...] + + + + + Description + + + DROP STATISTICS removes statistics from the database. + Only the statistics owner, the schema owner, and superuser can drop a + statistics. + + + + + + Parameters + + + + IF EXISTS + + + Do not throw an error if the statistics do not exist. A notice is + issued in this case. + + + + + + name + + + The name (optionally schema-qualified) of the statistics to drop. + + + + + + + + + Examples + + + ... + + + + + + Compatibility + + + There's no DROP STATISTICS command in the SQL standard. + + + + + See Also + + + + + + + + diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index 34007d3..abc3f42 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -60,6 +60,7 @@ &alterSchema; &alterSequence; &alterServer; + &alterStatistics; &alterSubscription; &alterSystem; &alterTable; @@ -108,6 +109,7 @@ &createSchema; &createSequence; &createServer; + &createStatistics; &createSubscription; &createTable; &createTableAs; @@ -154,6 +156,7 @@ &dropSchema; &dropSequence; &dropServer; + &dropStatistics; &dropSubscription; &dropTable; &dropTableSpace; diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 3136858..5a891a0 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -33,6 +33,7 @@ POSTGRES_BKI_SRCS = $(addprefix $(top_srcdir)/src/include/catalog/,\ pg_attrdef.h pg_constraint.h pg_inherits.h pg_index.h pg_operator.h \ pg_opfamily.h pg_opclass.h pg_am.h pg_amop.h pg_amproc.h \ pg_language.h pg_largeobject_metadata.h pg_largeobject.h pg_aggregate.h \ + pg_mv_statistic.h \ pg_statistic.h pg_rewrite.h pg_trigger.h pg_event_trigger.h pg_description.h \ pg_cast.h pg_enum.h pg_namespace.h pg_conversion.h pg_depend.h \ pg_database.h pg_db_role_setting.h pg_tablespace.h pg_pltemplate.h \ diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index f4df6df..d0e5b2e 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -40,6 +40,7 @@ #include "catalog/pg_language.h" #include "catalog/pg_largeobject.h" #include "catalog/pg_largeobject_metadata.h" +#include "catalog/pg_mv_statistic.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" #include "catalog/pg_operator.h" @@ -5133,6 +5134,32 @@ pg_subscription_ownercheck(Oid sub_oid, Oid roleid) } /* + * Ownership check for a multivariate statistics (specified by OID). + */ +bool +pg_statistics_ownercheck(Oid stat_oid, Oid roleid) +{ + HeapTuple tuple; + Oid ownerId; + + /* Superusers bypass all permission checking. */ + if (superuser_arg(roleid)) + return true; + + tuple = SearchSysCache1(MVSTATOID, ObjectIdGetDatum(stat_oid)); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("statistics with OID %u do not exist", stat_oid))); + + ownerId = ((Form_pg_mv_statistic) GETSTRUCT(tuple))->staowner; + + ReleaseSysCache(tuple); + + return has_privs_of_role(roleid, ownerId); +} + +/* * Check whether specified role has CREATEROLE privilege (or is a superuser) * * Note: roles do not have owners per se; instead we use this test in diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index 1c43af6..7083270 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -42,6 +42,7 @@ #include "catalog/pg_init_privs.h" #include "catalog/pg_language.h" #include "catalog/pg_largeobject.h" +#include "catalog/pg_mv_statistic.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" #include "catalog/pg_operator.h" @@ -171,7 +172,8 @@ static const Oid object_classes[] = { PublicationRelationId, /* OCLASS_PUBLICATION */ PublicationRelRelationId, /* OCLASS_PUBLICATION_REL */ SubscriptionRelationId, /* OCLASS_SUBSCRIPTION */ - TransformRelationId /* OCLASS_TRANSFORM */ + TransformRelationId, /* OCLASS_TRANSFORM */ + MvStatisticRelationId /* OCLASS_STATISTICS */ }; @@ -1263,6 +1265,10 @@ doDeletion(const ObjectAddress *object, int flags) DropTransformById(object->objectId); break; + case OCLASS_STATISTICS: + RemoveStatisticsById(object->objectId); + break; + default: elog(ERROR, "unrecognized object class: %u", object->classId); @@ -2430,6 +2436,9 @@ getObjectClass(const ObjectAddress *object) case TransformRelationId: return OCLASS_TRANSFORM; + + case MvStatisticRelationId: + return OCLASS_STATISTICS; } /* shouldn't get here */ diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 7ce9115..470f7ad 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -48,6 +48,7 @@ #include "catalog/pg_constraint_fn.h" #include "catalog/pg_foreign_table.h" #include "catalog/pg_inherits.h" +#include "catalog/pg_mv_statistic.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" #include "catalog/pg_partitioned_table.h" @@ -1614,7 +1615,10 @@ RemoveAttributeById(Oid relid, AttrNumber attnum) heap_close(attr_rel, RowExclusiveLock); if (attnum > 0) + { RemoveStatistics(relid, attnum); + RemoveMVStatistics(relid, attnum); + } relation_close(rel, NoLock); } @@ -1865,6 +1869,11 @@ heap_drop_with_catalog(Oid relid) RemoveStatistics(relid, 0); /* + * delete multi-variate statistics + */ + RemoveMVStatistics(relid, 0); + + /* * delete attribute tuples */ DeleteAttributeTuples(relid); @@ -2783,6 +2792,98 @@ RemoveStatistics(Oid relid, AttrNumber attnum) /* + * RemoveMVStatistics --- remove entries in pg_mv_statistic for a rel + * + * If attnum is zero, remove all entries for rel; else remove only the one(s) + * for that column. + */ +void +RemoveMVStatistics(Oid relid, AttrNumber attnum) +{ + Relation pgmvstatistic; + TupleDesc tupdesc = NULL; + SysScanDesc scan; + ScanKeyData key; + HeapTuple tuple; + + /* + * When dropping a column, we'll drop statistics with a single remaining + * (undropped column). To do that, we need the tuple descriptor. + * + * We already have the relation locked (as we're running ALTER TABLE ... + * DROP COLUMN), so we'll just get the descriptor here. + */ + if (attnum != 0) + { + Relation rel = relation_open(relid, NoLock); + + /* multivariate stats are supported on tables and matviews */ + if (rel->rd_rel->relkind == RELKIND_RELATION || + rel->rd_rel->relkind == RELKIND_MATVIEW) + tupdesc = RelationGetDescr(rel); + + relation_close(rel, NoLock); + } + + if (tupdesc == NULL) + return; + + pgmvstatistic = heap_open(MvStatisticRelationId, RowExclusiveLock); + + ScanKeyInit(&key, + Anum_pg_mv_statistic_starelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + + scan = systable_beginscan(pgmvstatistic, + MvStatisticRelidIndexId, + true, NULL, 1, &key); + + /* we must loop even when attnum != 0, in case of inherited stats */ + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + bool delete = true; + + if (attnum != 0) + { + Datum adatum; + bool isnull; + int i; + int ncolumns = 0; + ArrayType *arr; + int16 *attnums; + + /* get the columns */ + adatum = SysCacheGetAttr(MVSTATOID, tuple, + Anum_pg_mv_statistic_stakeys, &isnull); + Assert(!isnull); + + arr = DatumGetArrayTypeP(adatum); + attnums = (int16 *) ARR_DATA_PTR(arr); + + for (i = 0; i < ARR_DIMS(arr)[0]; i++) + { + /* count the column unless it's has been / is being dropped */ + if ((!tupdesc->attrs[attnums[i] - 1]->attisdropped) && + (attnums[i] != attnum)) + ncolumns += 1; + } + + /* delete if there are less than two attributes */ + delete = (ncolumns < 2); + } + + if (delete) + simple_heap_delete(pgmvstatistic, &tuple->t_self); + } + + systable_endscan(scan); + + heap_close(pgmvstatistic, RowExclusiveLock); +} + + +/* * RelationTruncateIndexes - truncate all indexes associated * with the heap relation to zero tuples. * diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c index a38da30..6cd7d36 100644 --- a/src/backend/catalog/namespace.c +++ b/src/backend/catalog/namespace.c @@ -4236,3 +4236,59 @@ pg_is_other_temp_schema(PG_FUNCTION_ARGS) PG_RETURN_BOOL(isOtherTempNamespace(oid)); } + +/* + * get_statistics_oid - find a statistics by possibly qualified name + * + * If not found, returns InvalidOid if missing_ok, else throws error + */ +Oid +get_statistics_oid(List *names, bool missing_ok) +{ + char *schemaname; + char *stats_name; + Oid namespaceId; + Oid stats_oid = InvalidOid; + ListCell *l; + + /* deconstruct the name list */ + DeconstructQualifiedName(names, &schemaname, &stats_name); + + if (schemaname) + { + /* use exact schema given */ + namespaceId = LookupExplicitNamespace(schemaname, missing_ok); + if (missing_ok && !OidIsValid(namespaceId)) + stats_oid = InvalidOid; + else + stats_oid = GetSysCacheOid2(MVSTATNAMENSP, + PointerGetDatum(stats_name), + ObjectIdGetDatum(namespaceId)); + } + else + { + /* search for it in search path */ + recomputeNamespacePath(); + + foreach(l, activeSearchPath) + { + namespaceId = lfirst_oid(l); + + if (namespaceId == myTempNamespace) + continue; /* do not look in temp namespace */ + stats_oid = GetSysCacheOid2(MVSTATNAMENSP, + PointerGetDatum(stats_name), + ObjectIdGetDatum(namespaceId)); + if (OidIsValid(stats_oid)) + break; + } + } + + if (!OidIsValid(stats_oid) && !missing_ok) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("statistics \"%s\" do not exist", + NameListToString(names)))); + + return stats_oid; +} diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index 2a38792..d390dc1 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -39,6 +39,7 @@ #include "catalog/pg_language.h" #include "catalog/pg_largeobject.h" #include "catalog/pg_largeobject_metadata.h" +#include "catalog/pg_mv_statistic.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" #include "catalog/pg_opfamily.h" @@ -478,6 +479,18 @@ static const ObjectPropertyType ObjectProperty[] = InvalidAttrNumber, -1, true + }, + { + MvStatisticRelationId, + MvStatisticOidIndexId, + MVSTATOID, + MVSTATNAMENSP, + Anum_pg_mv_statistic_staname, + Anum_pg_mv_statistic_stanamespace, + Anum_pg_mv_statistic_staowner, + InvalidAttrNumber, /* no ACL (same as relation) */ + -1, /* no ACL */ + true } }; @@ -696,6 +709,10 @@ static const struct object_type_map /* OCLASS_TRANSFORM */ { "transform", OBJECT_TRANSFORM + }, + /* OBJECT_STATISTICS */ + { + "statistics", OBJECT_STATISTICS } }; @@ -980,6 +997,11 @@ get_object_address(ObjectType objtype, List *objname, List *objargs, address = get_object_address_defacl(objname, objargs, missing_ok); break; + case OBJECT_STATISTICS: + address.classId = MvStatisticRelationId; + address.objectId = get_statistics_oid(objname, missing_ok); + address.objectSubId = 0; + break; default: elog(ERROR, "unrecognized objtype: %d", (int) objtype); /* placate compiler, in case it thinks elog might return */ @@ -2361,6 +2383,10 @@ check_object_ownership(Oid roleid, ObjectType objtype, ObjectAddress address, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser"))); break; + case OBJECT_STATISTICS: + if (!pg_statistics_ownercheck(address.objectId, roleid)) + aclcheck_error_type(ACLCHECK_NOT_OWNER, address.objectId); + break; default: elog(ERROR, "unrecognized object type: %d", (int) objtype); @@ -3848,6 +3874,10 @@ getObjectTypeDescription(const ObjectAddress *object) appendStringInfoString(&buffer, "subscription"); break; + case OCLASS_STATISTICS: + appendStringInfoString(&buffer, "statistics"); + break; + default: appendStringInfo(&buffer, "unrecognized %u", object->classId); break; @@ -4871,6 +4901,29 @@ getObjectIdentityParts(const ObjectAddress *object, break; } + case OCLASS_STATISTICS: + { + HeapTuple tup; + Form_pg_mv_statistic formStatistic; + char *schema; + + tup = SearchSysCache1(MVSTATOID, + ObjectIdGetDatum(object->objectId)); + if (!HeapTupleIsValid(tup)) + elog(ERROR, "cache lookup failed for statistics %u", + object->objectId); + formStatistic = (Form_pg_mv_statistic) GETSTRUCT(tup); + schema = get_namespace_name_or_temp(formStatistic->stanamespace); + appendStringInfoString(&buffer, + quote_qualified_identifier(schema, + NameStr(formStatistic->staname))); + if (objname) + *objname = list_make2(schema, + pstrdup(NameStr(formStatistic->staname))); + ReleaseSysCache(tup); + } + break; + default: appendStringInfo(&buffer, "unrecognized object %u %u %d", object->classId, diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 4dfedf8..00ab440 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -181,6 +181,16 @@ CREATE OR REPLACE VIEW pg_sequences AS WHERE NOT pg_is_other_temp_schema(N.oid) AND relkind = 'S'; +CREATE VIEW pg_mv_stats AS + SELECT + N.nspname AS schemaname, + C.relname AS tablename, + S.staname AS staname, + S.stakeys AS attnums, + length(s.standist) AS ndistbytes + FROM (pg_mv_statistic S JOIN pg_class C ON (C.oid = S.starelid)) + LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace); + CREATE VIEW pg_stats WITH (security_barrier) AS SELECT nspname AS schemaname, diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile index e0fab38..4a6c99e 100644 --- a/src/backend/commands/Makefile +++ b/src/backend/commands/Makefile @@ -18,8 +18,8 @@ OBJS = amcmds.o aggregatecmds.o alter.o analyze.o async.o cluster.o comment.o \ event_trigger.o explain.o extension.o foreigncmds.o functioncmds.o \ indexcmds.o lockcmds.o matview.o operatorcmds.o opclasscmds.o \ policy.o portalcmds.o prepare.o proclang.o publicationcmds.o \ - schemacmds.o seclabel.o sequence.o subscriptioncmds.o tablecmds.o \ - tablespace.o trigger.o tsearchcmds.o typecmds.o user.o vacuum.o \ - vacuumlazy.o variable.o view.o + schemacmds.o seclabel.o sequence.o statscmds.o subscriptioncmds.o \ + tablecmds.o tablespace.o trigger.o tsearchcmds.o typecmds.o user.o \ + vacuum.o vacuumlazy.o variable.o view.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index 768fcc8..e2d1243 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -361,6 +361,7 @@ ExecRenameStmt(RenameStmt *stmt) case OBJECT_OPCLASS: case OBJECT_OPFAMILY: case OBJECT_LANGUAGE: + case OBJECT_STATISTICS: case OBJECT_TSCONFIGURATION: case OBJECT_TSDICTIONARY: case OBJECT_TSPARSER: @@ -475,6 +476,7 @@ ExecAlterObjectSchemaStmt(AlterObjectSchemaStmt *stmt, case OBJECT_OPERATOR: case OBJECT_OPCLASS: case OBJECT_OPFAMILY: + case OBJECT_STATISTICS: case OBJECT_TSCONFIGURATION: case OBJECT_TSDICTIONARY: case OBJECT_TSPARSER: @@ -790,6 +792,7 @@ ExecAlterOwnerStmt(AlterOwnerStmt *stmt) case OBJECT_OPERATOR: case OBJECT_OPCLASS: case OBJECT_OPFAMILY: + case OBJECT_STATISTICS: case OBJECT_TABLESPACE: case OBJECT_TSDICTIONARY: case OBJECT_TSCONFIGURATION: diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index c9f6afe..27eaf79 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -17,6 +17,7 @@ #include #include "access/multixact.h" +#include "access/sysattr.h" #include "access/transam.h" #include "access/tupconvert.h" #include "access/tuptoaster.h" @@ -27,6 +28,7 @@ #include "catalog/indexing.h" #include "catalog/pg_collation.h" #include "catalog/pg_inherits_fn.h" +#include "catalog/pg_mv_statistic.h" #include "catalog/pg_namespace.h" #include "commands/dbcommands.h" #include "commands/tablecmds.h" @@ -45,10 +47,13 @@ #include "storage/procarray.h" #include "utils/acl.h" #include "utils/attoptcache.h" +#include "utils/builtins.h" #include "utils/datum.h" +#include "utils/fmgroids.h" #include "utils/guc.h" #include "utils/lsyscache.h" #include "utils/memutils.h" +#include "utils/mvstats.h" #include "utils/pg_rusage.h" #include "utils/sampling.h" #include "utils/sortsupport.h" @@ -559,6 +564,9 @@ do_analyze_rel(Relation onerel, int options, VacuumParams *params, update_attstats(RelationGetRelid(Irel[ind]), false, thisdata->attr_cnt, thisdata->vacattrstats); } + + /* Build multivariate stats (if there are any). */ + build_mv_stats(onerel, totalrows, numrows, rows, attr_cnt, vacattrstats); } /* diff --git a/src/backend/commands/dropcmds.c b/src/backend/commands/dropcmds.c index ff3108c..8fd4269 100644 --- a/src/backend/commands/dropcmds.c +++ b/src/backend/commands/dropcmds.c @@ -294,6 +294,10 @@ does_not_exist_skipping(ObjectType objtype, List *objname, List *objargs) msg = gettext_noop("schema \"%s\" does not exist, skipping"); name = NameListToString(objname); break; + case OBJECT_STATISTICS: + msg = gettext_noop("statistics \"%s\" do not exist, skipping"); + name = NameListToString(objname); + break; case OBJECT_TSPARSER: if (!schema_does_not_exist_skipping(objname, &msg, &name)) { diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index 8125537..763a45a 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -112,6 +112,7 @@ static event_trigger_support_data event_trigger_support[] = { {"SCHEMA", true}, {"SEQUENCE", true}, {"SERVER", true}, + {"STATISTICS", true}, {"SUBSCRIPTION", true}, {"TABLE", true}, {"TABLESPACE", false}, @@ -1111,6 +1112,7 @@ EventTriggerSupportsObjectType(ObjectType obtype) case OBJECT_SCHEMA: case OBJECT_SEQUENCE: case OBJECT_SUBSCRIPTION: + case OBJECT_STATISTICS: case OBJECT_TABCONSTRAINT: case OBJECT_TABLE: case OBJECT_TRANSFORM: @@ -1176,6 +1178,7 @@ EventTriggerSupportsObjectClass(ObjectClass objclass) case OCLASS_PUBLICATION: case OCLASS_PUBLICATION_REL: case OCLASS_SUBSCRIPTION: + case OCLASS_STATISTICS: return true; } diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c new file mode 100644 index 0000000..bde7e4b --- /dev/null +++ b/src/backend/commands/statscmds.c @@ -0,0 +1,259 @@ +/*------------------------------------------------------------------------- + * + * statscmds.c + * Commands for creating and altering multivariate statistics + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/commands/statscmds.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/relscan.h" +#include "catalog/dependency.h" +#include "catalog/indexing.h" +#include "catalog/namespace.h" +#include "catalog/pg_mv_statistic.h" +#include "catalog/pg_namespace.h" +#include "commands/defrem.h" +#include "miscadmin.h" +#include "utils/builtins.h" +#include "utils/inval.h" +#include "utils/memutils.h" +#include "utils/mvstats.h" +#include "utils/rel.h" +#include "utils/syscache.h" + + +/* used for sorting the attnums in ExecCreateStatistics */ +static int +compare_int16(const void *a, const void *b) +{ + return memcmp(a, b, sizeof(int16)); +} + +/* + * Implements the CREATE STATISTICS name ON (columns) FROM table + * + * We do require that the types support sorting (ltopr), although some + * statistics might work with equality only. + */ +ObjectAddress +CreateStatistics(CreateStatsStmt *stmt) +{ + int i; + ListCell *l; + int16 attnums[MVSTATS_MAX_DIMENSIONS]; + int numcols = 0; + ObjectAddress address = InvalidObjectAddress; + char *namestr; + NameData staname; + Oid statoid; + Oid namespaceId; + + HeapTuple htup; + Datum values[Natts_pg_mv_statistic]; + bool nulls[Natts_pg_mv_statistic]; + int2vector *stakeys; + Relation mvstatrel; + Relation rel; + Oid relid; + ObjectAddress parentobject, + childobject; + + Assert(IsA(stmt, CreateStatsStmt)); + + /* resolve the pieces of the name (namespace etc.) */ + namespaceId = QualifiedNameGetCreationNamespace(stmt->defnames, &namestr); + namestrcpy(&staname, namestr); + + /* + * If if_not_exists was given and the statistics already exists, bail out. + */ + if (SearchSysCacheExists2(MVSTATNAMENSP, + PointerGetDatum(&staname), + ObjectIdGetDatum(namespaceId))) + { + if (stmt->if_not_exists) + { + ereport(NOTICE, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("statistics \"%s\" already exist, skipping", + namestr))); + return InvalidObjectAddress; + } + + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("statistics \"%s\" already exist", namestr))); + } + + rel = heap_openrv(stmt->relation, AccessExclusiveLock); + relid = RelationGetRelid(rel); + + /* + * Transform column names to array of attnums. While doing that, we + * also enforce the maximum number of keys. + */ + foreach(l, stmt->keys) + { + char *attname = strVal(lfirst(l)); + HeapTuple atttuple; + + atttuple = SearchSysCacheAttName(relid, attname); + + if (!HeapTupleIsValid(atttuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" referenced in statistics does not exist", + attname))); + + /* more than MVSTATS_MAX_DIMENSIONS columns not allowed */ + if (numcols >= MVSTATS_MAX_DIMENSIONS) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_COLUMNS), + errmsg("cannot have more than %d keys in statistics", + MVSTATS_MAX_DIMENSIONS))); + + attnums[numcols] = ((Form_pg_attribute) GETSTRUCT(atttuple))->attnum; + ReleaseSysCache(atttuple); + numcols++; + } + + /* + * Check that at least two columns were specified in the statement. + * The upper bound was already checked in the loop above. + */ + if (numcols < 2) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_COLUMNS), + errmsg("statistics require at least 2 columns"))); + + /* + * Sort the attnums, which makes detecting duplicies somewhat + * easier, and it does not hurt (it does not affect the efficiency, + * unlike for indexes, for example). + */ + qsort(attnums, numcols, sizeof(int16), compare_int16); + + /* + * Look for duplicities in the list of columns. The attnums are sorted + * so just check consecutive elements. + */ + for (i = 1; i < numcols; i++) + if (attnums[i] == attnums[i-1]) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("duplicate column name in statistics definition"))); + + stakeys = buildint2vector(attnums, numcols); + + /* + * Everything seems fine, so let's build the pg_mv_statistic entry. + * At this point we obviously only have the keys and options. + */ + + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + + /* metadata */ + values[Anum_pg_mv_statistic_starelid - 1] = ObjectIdGetDatum(relid); + values[Anum_pg_mv_statistic_staname - 1] = NameGetDatum(&staname); + values[Anum_pg_mv_statistic_stanamespace - 1] = ObjectIdGetDatum(namespaceId); + values[Anum_pg_mv_statistic_staowner - 1] = ObjectIdGetDatum(GetUserId()); + + values[Anum_pg_mv_statistic_stakeys - 1] = PointerGetDatum(stakeys); + + /* enabled statistics */ + values[Anum_pg_mv_statistic_ndist_enabled - 1] = BoolGetDatum(true); + + nulls[Anum_pg_mv_statistic_standist - 1] = true; + + /* insert the tuple into pg_mv_statistic */ + mvstatrel = heap_open(MvStatisticRelationId, RowExclusiveLock); + + htup = heap_form_tuple(mvstatrel->rd_att, values, nulls); + + simple_heap_insert(mvstatrel, htup); + + CatalogUpdateIndexes(mvstatrel, htup); + + statoid = HeapTupleGetOid(htup); + + heap_freetuple(htup); + + /* + * Add a dependency on a table, so that stats get dropped on DROP TABLE. + */ + ObjectAddressSet(parentobject, RelationRelationId, relid); + ObjectAddressSet(childobject, MvStatisticRelationId, statoid); + + recordDependencyOn(&childobject, &parentobject, DEPENDENCY_AUTO); + + /* + * Also add dependency on the schema (to drop statistics on DROP SCHEMA). + * This is not handled automatically by DROP TABLE because statistics have + * their own schema. + */ + ObjectAddressSet(parentobject, NamespaceRelationId, namespaceId); + + recordDependencyOn(&childobject, &parentobject, DEPENDENCY_AUTO); + + heap_close(mvstatrel, RowExclusiveLock); + + relation_close(rel, NoLock); + + /* + * Invalidate relcache so that others see the new statistics. + */ + CacheInvalidateRelcache(rel); + + ObjectAddressSet(address, MvStatisticRelationId, statoid); + + return address; +} + + +/* + * Implements the DROP STATISTICS + * + * DROP STATISTICS stats_name + */ +void +RemoveStatisticsById(Oid statsOid) +{ + Relation relation; + Oid relid; + Relation rel; + HeapTuple tup; + Form_pg_mv_statistic mvstat; + + /* + * Delete the pg_proc tuple. + */ + relation = heap_open(MvStatisticRelationId, RowExclusiveLock); + + tup = SearchSysCache1(MVSTATOID, ObjectIdGetDatum(statsOid)); + + if (!HeapTupleIsValid(tup)) /* should not happen */ + elog(ERROR, "cache lookup failed for statistics %u", statsOid); + + mvstat = (Form_pg_mv_statistic) GETSTRUCT(tup); + relid = mvstat->starelid; + + rel = heap_open(relid, AccessExclusiveLock); + + simple_heap_delete(relation, &tup->t_self); + + CacheInvalidateRelcache(rel); + + ReleaseSysCache(tup); + + heap_close(relation, RowExclusiveLock); + heap_close(rel, NoLock); +} diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 30d733e..dc42be0 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -4349,6 +4349,19 @@ _copyDropSubscriptionStmt(const DropSubscriptionStmt *from) return newnode; } +static CreateStatsStmt * +_copyCreateStatsStmt(const CreateStatsStmt *from) +{ + CreateStatsStmt *newnode = makeNode(CreateStatsStmt); + + COPY_NODE_FIELD(defnames); + COPY_NODE_FIELD(relation); + COPY_NODE_FIELD(keys); + COPY_SCALAR_FIELD(if_not_exists); + + return newnode; +} + /* **************************************************************** * pg_list.h copy functions * **************************************************************** @@ -5272,6 +5285,9 @@ copyObject(const void *from) case T_CommonTableExpr: retval = _copyCommonTableExpr(from); break; + case T_CreateStatsStmt: + retval = _copyCreateStatsStmt(from); + break; case T_FuncWithArgs: retval = _copyFuncWithArgs(from); break; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 1560ac3..57cc0b4 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2193,6 +2193,21 @@ _outForeignKeyOptInfo(StringInfo str, const ForeignKeyOptInfo *node) } static void +_outMVStatisticInfo(StringInfo str, const MVStatisticInfo *node) +{ + WRITE_NODE_TYPE("MVSTATISTICINFO"); + + /* NB: this isn't a complete set of fields */ + WRITE_OID_FIELD(mvoid); + + /* enabled statistics */ + WRITE_BOOL_FIELD(ndist_enabled); + + /* built/available statistics */ + WRITE_BOOL_FIELD(ndist_built); +} + +static void _outEquivalenceClass(StringInfo str, const EquivalenceClass *node) { /* @@ -3799,6 +3814,9 @@ outNode(StringInfo str, const void *obj) case T_PlannerParamItem: _outPlannerParamItem(str, obj); break; + case T_MVStatisticInfo: + _outMVStatisticInfo(str, obj); + break; case T_ExtensibleNode: _outExtensibleNode(str, obj); diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 7836e6b..fc9ad93 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -29,6 +29,7 @@ #include "catalog/heap.h" #include "catalog/partition.h" #include "catalog/pg_am.h" +#include "catalog/pg_mv_statistic.h" #include "foreign/fdwapi.h" #include "miscadmin.h" #include "nodes/makefuncs.h" @@ -41,7 +42,9 @@ #include "parser/parsetree.h" #include "rewrite/rewriteManip.h" #include "storage/bufmgr.h" +#include "utils/builtins.h" #include "utils/lsyscache.h" +#include "utils/syscache.h" #include "utils/rel.h" #include "utils/snapmgr.h" @@ -63,7 +66,7 @@ static List *get_relation_constraints(PlannerInfo *root, bool include_notnull); static List *build_index_tlist(PlannerInfo *root, IndexOptInfo *index, Relation heapRelation); - +static List *get_relation_statistics(RelOptInfo *rel, Relation relation); /* * get_relation_info - @@ -397,6 +400,8 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, rel->indexlist = indexinfos; + rel->mvstatlist = get_relation_statistics(rel, relation); + /* Grab foreign-table info using the relcache, while we have it */ if (relation->rd_rel->relkind == RELKIND_FOREIGN_TABLE) { @@ -1250,6 +1255,71 @@ get_relation_constraints(PlannerInfo *root, return result; } +/* + * get_relation_statistics + * + * Retrieve multivariate statistics defined on the table. + * + * Returns a List (possibly empty) of MVStatisticInfo objects describing + * the statistics. Only attributes needed for selecting statistics are + * retrieved (columns covered by the statistics, etc.). + */ +static List * +get_relation_statistics(RelOptInfo *rel, Relation relation) +{ + List *mvstatoidlist; + ListCell *l; + List *stainfos = NIL; + + mvstatoidlist = RelationGetMVStatList(relation); + + foreach(l, mvstatoidlist) + { + ArrayType *arr; + Datum adatum; + bool isnull; + Oid mvoid = lfirst_oid(l); + Form_pg_mv_statistic mvstat; + MVStatisticInfo *info; + + HeapTuple htup = SearchSysCache1(MVSTATOID, ObjectIdGetDatum(mvoid)); + + mvstat = (Form_pg_mv_statistic) GETSTRUCT(htup); + + /* unavailable stats are not interesting for the planner */ + if (mvstat->ndist_built) + { + info = makeNode(MVStatisticInfo); + + info->mvoid = mvoid; + info->rel = rel; + + /* enabled statistics */ + info->ndist_enabled = mvstat->ndist_enabled; + + /* built/available statistics */ + info->ndist_built = mvstat->ndist_built; + + /* stakeys */ + adatum = SysCacheGetAttr(MVSTATOID, htup, + Anum_pg_mv_statistic_stakeys, &isnull); + Assert(!isnull); + + arr = DatumGetArrayTypeP(adatum); + + info->stakeys = buildint2vector((int16 *) ARR_DATA_PTR(arr), + ARR_DIMS(arr)[0]); + + stainfos = lcons(info, stainfos); + } + + ReleaseSysCache(htup); + } + + list_free(mvstatoidlist); + + return stainfos; +} /* * relation_excluded_by_constraints diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index a4edea0..475a8a6 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -257,7 +257,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); ConstraintsSetStmt CopyStmt CreateAsStmt CreateCastStmt CreateDomainStmt CreateExtensionStmt CreateGroupStmt CreateOpClassStmt CreateOpFamilyStmt AlterOpFamilyStmt CreatePLangStmt - CreateSchemaStmt CreateSeqStmt CreateStmt CreateTableSpaceStmt + CreateSchemaStmt CreateSeqStmt CreateStmt CreateStatsStmt CreateTableSpaceStmt CreateFdwStmt CreateForeignServerStmt CreateForeignTableStmt CreateAssertStmt CreateTransformStmt CreateTrigStmt CreateEventTrigStmt CreateUserStmt CreateUserMappingStmt CreateRoleStmt CreatePolicyStmt @@ -867,6 +867,7 @@ stmt : | CreateSeqStmt | CreateStmt | CreateSubscriptionStmt + | CreateStatsStmt | CreateTableSpaceStmt | CreateTransformStmt | CreateTrigStmt @@ -3747,6 +3748,34 @@ OptConsTableSpace: USING INDEX TABLESPACE name { $$ = $4; } ExistingIndex: USING INDEX index_name { $$ = $3; } ; +/***************************************************************************** + * + * QUERY : + * CREATE STATISTICS stats_name ON relname (columns) WITH (options) + * + *****************************************************************************/ + + +CreateStatsStmt: CREATE STATISTICS any_name ON '(' columnList ')' FROM qualified_name + { + CreateStatsStmt *n = makeNode(CreateStatsStmt); + n->defnames = $3; + n->relation = $9; + n->keys = $6; + n->if_not_exists = false; + $$ = (Node *)n; + } + | CREATE STATISTICS IF_P NOT EXISTS any_name ON '(' columnList ')' FROM qualified_name + { + CreateStatsStmt *n = makeNode(CreateStatsStmt); + n->defnames = $6; + n->relation = $12; + n->keys = $9; + n->if_not_exists = true; + $$ = (Node *)n; + } + ; + /***************************************************************************** * @@ -6090,6 +6119,7 @@ drop_type: TABLE { $$ = OBJECT_TABLE; } | TEXT_P SEARCH TEMPLATE { $$ = OBJECT_TSTEMPLATE; } | TEXT_P SEARCH CONFIGURATION { $$ = OBJECT_TSCONFIGURATION; } | PUBLICATION { $$ = OBJECT_PUBLICATION; } + | STATISTICS { $$ = OBJECT_STATISTICS; } ; any_name_list: @@ -8475,6 +8505,15 @@ RenameStmt: ALTER AGGREGATE aggregate_with_argtypes RENAME TO name n->missing_ok = false; $$ = (Node *)n; } + | ALTER STATISTICS any_name RENAME TO name + { + RenameStmt *n = makeNode(RenameStmt); + n->renameType = OBJECT_STATISTICS; + n->object = $3; + n->newname = $6; + n->missing_ok = false; + $$ = (Node *)n; + } ; opt_column: COLUMN { $$ = COLUMN; } @@ -8760,6 +8799,15 @@ AlterObjectSchemaStmt: n->missing_ok = false; $$ = (Node *)n; } + | ALTER STATISTICS any_name SET SCHEMA name + { + AlterObjectSchemaStmt *n = makeNode(AlterObjectSchemaStmt); + n->objectType = OBJECT_STATISTICS; + n->object = $3; + n->newschema = $6; + n->missing_ok = false; + $$ = (Node *)n; + } ; /***************************************************************************** @@ -8966,6 +9014,14 @@ AlterOwnerStmt: ALTER AGGREGATE aggregate_with_argtypes OWNER TO RoleSpec n->newowner = $6; $$ = (Node *)n; } + | ALTER STATISTICS name OWNER TO RoleSpec + { + AlterOwnerStmt *n = makeNode(AlterOwnerStmt); + n->objectType = OBJECT_STATISTICS; + n->object = list_make1(makeString($3)); + n->newowner = $6; + $$ = (Node *)n; + } ; diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 5d3be38..6f2371c 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1621,6 +1621,10 @@ ProcessUtilitySlow(ParseState *pstate, commandCollected = true; break; + case T_CreateStatsStmt: /* CREATE STATISTICS */ + address = CreateStatistics((CreateStatsStmt *) parsetree); + break; + default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(parsetree)); @@ -1986,6 +1990,8 @@ AlterObjectTypeCommandTag(ObjectType objtype) break; case OBJECT_SUBSCRIPTION: tag = "ALTER SUBSCRIPTION"; + case OBJECT_STATISTICS: + tag = "ALTER STATISTICS"; break; default: tag = "???"; @@ -2280,6 +2286,8 @@ CreateCommandTag(Node *parsetree) break; case OBJECT_PUBLICATION: tag = "DROP PUBLICATION"; + case OBJECT_STATISTICS: + tag = "DROP STATISTICS"; break; default: tag = "???"; @@ -2679,6 +2687,10 @@ CreateCommandTag(Node *parsetree) tag = "EXECUTE"; break; + case T_CreateStatsStmt: + tag = "CREATE STATISTICS"; + break; + case T_DeallocateStmt: { DeallocateStmt *stmt = (DeallocateStmt *) parsetree; diff --git a/src/backend/utils/Makefile b/src/backend/utils/Makefile index 2e35ca5..0eb2331 100644 --- a/src/backend/utils/Makefile +++ b/src/backend/utils/Makefile @@ -9,7 +9,7 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global OBJS = fmgrtab.o -SUBDIRS = adt cache error fmgr hash init mb misc mmgr resowner sort time +SUBDIRS = adt cache error fmgr hash init mb misc mmgr mvstats resowner sort time # location of Catalog.pm catalogdir = $(top_srcdir)/src/backend/catalog diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index fa32e9e..7774c44 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -132,6 +132,7 @@ #include "utils/fmgroids.h" #include "utils/index_selfuncs.h" #include "utils/lsyscache.h" +#include "utils/mvstats.h" #include "utils/nabstime.h" #include "utils/pg_locale.h" #include "utils/rel.h" @@ -207,6 +208,8 @@ static Const *string_to_const(const char *str, Oid datatype); static Const *string_to_bytea_const(const char *str, size_t str_len); static List *add_predicate_to_quals(IndexOptInfo *index, List *indexQuals); +static double find_ndistinct(PlannerInfo *root, RelOptInfo *rel, List *varinfos, + bool *found); /* * eqsel - Selectivity of "=" for any data types. @@ -3436,12 +3439,26 @@ estimate_num_groups(PlannerInfo *root, List *groupExprs, double input_rows, * don't know by how much. We should never clamp to less than the * largest ndistinct value for any of the Vars, though, since * there will surely be at least that many groups. + * + * However we don't need to do this if we have ndistinct stats on + * the columns - in that case we can simply use the coefficient + * to get the (probably way more accurate) estimate. + * + * XXX Might benefit from some refactoring, mixing the ndistinct + * coefficients and clamp seems a bit unfortunate. */ double clamp = rel->tuples; if (relvarcount > 1) { - clamp *= 0.1; + bool found; + double ndist = find_ndistinct(root, rel, varinfos, &found); + + if (found) + reldistinct = ndist; + else + clamp *= 0.1; + if (clamp < relmaxndistinct) { clamp = relmaxndistinct; @@ -3450,6 +3467,7 @@ estimate_num_groups(PlannerInfo *root, List *groupExprs, double input_rows, clamp = rel->tuples; } } + if (reldistinct > clamp) reldistinct = clamp; @@ -7600,3 +7618,151 @@ brincostestimate(PlannerInfo *root, IndexPath *path, double loop_count, /* XXX what about pages_per_range? */ } + +/* + * Find applicable ndistinct statistics and compute the coefficient to + * correct the estimate (simply a product of per-column ndistincts). + * + * XXX Currently we only look for a perfect match, i.e. a single ndistinct + * estimate exactly matching all the columns of the statistics. This may be + * a bit problematic as adding a column (not covered by the ndistinct stats) + * will prevent us from using the stats entirely. So instead this needs to + * estimate the covered attributes, and then combine that with the extra + * attributes somehow (probably the old way). + */ +static double +find_ndistinct(PlannerInfo *root, RelOptInfo *rel, List *varinfos, bool *found) +{ + ListCell *lc; + Bitmapset *attnums = NULL; + VariableStatData vardata; + + /* assume we haven't found any suitable ndistinct statistics */ + *found = false; + + /* bail out immediately if the table has no multivariate statistics */ + if (!rel->mvstatlist) + return 0.0; + + foreach(lc, varinfos) + { + GroupVarInfo *varinfo = (GroupVarInfo *) lfirst(lc); + + if (varinfo->rel != rel) + continue; + + /* FIXME handle expressions in general only */ + + /* + * examine the variable (or expression) so that we know which + * attribute we're dealing with - we need this for matching the + * ndistinct coefficient + * + * FIXME probably might remember this from estimate_num_groups + */ + examine_variable(root, varinfo->var, 0, &vardata); + + if (HeapTupleIsValid(vardata.statsTuple)) + { + Form_pg_statistic stats + = (Form_pg_statistic) GETSTRUCT(vardata.statsTuple); + + attnums = bms_add_member(attnums, stats->staattnum); + + ReleaseVariableStats(vardata); + } + } + + /* look for a matching ndistinct statistics */ + foreach (lc, rel->mvstatlist) + { + int i, k; + bool matches; + MVStatisticInfo *info = (MVStatisticInfo *)lfirst(lc); + + /* skip statistics without ndistinct coefficient built */ + if (!info->ndist_built) + continue; + + /* + * Only ndistinct stats covering all Vars are acceptable, which can't + * happen if the statistics has fewer attributes than we have Vars. + */ + if (bms_num_members(attnums) > info->stakeys->dim1) + continue; + + /* check that all Vars are covered by the statistic */ + matches = true; /* assume match until we find unmatched attribute */ + k = -1; + while ((k = bms_next_member(attnums, k)) >= 0) + { + bool attr_found = false; + for (i = 0; i < info->stakeys->dim1; i++) + { + if (info->stakeys->values[i] == k) + { + attr_found = true; + break; + } + } + + /* found attribute not covered by this ndistinct stats, skip */ + if (!attr_found) + { + matches = false; + break; + } + } + + if (! matches) + continue; + + /* hey, this statistics matches! great, let's extract the value */ + *found = true; + + { + int j; + MVNDistinct stat = load_mv_ndistinct(info->mvoid); + + for (j = 0; j < stat->nitems; j++) + { + bool item_matches = true; + MVNDistinctItem * item = &stat->items[j]; + + /* not the right item (different number of attributes) */ + if (item->nattrs != bms_num_members(attnums)) + continue; + + /* check the attribute numbers */ + k = -1; + while ((k = bms_next_member(attnums, k)) >= 0) + { + bool attr_found = false; + for (i = 0; i < item->nattrs; i++) + { + if (info->stakeys->values[item->attrs[i]] == k) + { + attr_found = true; + break; + } + } + + if (! attr_found) + { + item_matches = false; + break; + } + } + + if (! item_matches) + continue; + + return item->ndistinct; + } + } + } + + Assert(!(*found)); + + return 0.0; +} diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 26ff7e1..1316104 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -49,6 +49,7 @@ #include "catalog/pg_auth_members.h" #include "catalog/pg_constraint.h" #include "catalog/pg_database.h" +#include "catalog/pg_mv_statistic.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" #include "catalog/pg_partitioned_table.h" @@ -4452,6 +4453,81 @@ RelationGetIndexList(Relation relation) } /* + * RelationGetMVStatList -- get a list of OIDs of statistics on this relation + * + * The statistics list is created only if someone requests it, in a way + * similar to RelationGetIndexList(). We scan pg_mv_statistic to find + * relevant statistics, and add the list to the relcache entry so that we + * won't have to compute it again. Note that shared cache inval of a + * relcache entry will delete the old list and set rd_mvstatvalid to 0, + * so that we must recompute the statistics list on next request. This + * handles creation or deletion of a statistic. + * + * The returned list is guaranteed to be sorted in order by OID, although + * this is not currently needed. + * + * Since shared cache inval causes the relcache's copy of the list to go away, + * we return a copy of the list palloc'd in the caller's context. The caller + * may list_free() the returned list after scanning it. This is necessary + * since the caller will typically be doing syscache lookups on the relevant + * statistics, and syscache lookup could cause SI messages to be processed! + */ +List * +RelationGetMVStatList(Relation relation) +{ + Relation indrel; + SysScanDesc indscan; + ScanKeyData skey; + HeapTuple htup; + List *result; + List *oldlist; + MemoryContext oldcxt; + + /* Quick exit if we already computed the list. */ + if (relation->rd_mvstatvalid != 0) + return list_copy(relation->rd_mvstatlist); + + /* + * We build the list we intend to return (in the caller's context) while + * doing the scan. After successfully completing the scan, we copy that + * list into the relcache entry. This avoids cache-context memory leakage + * if we get some sort of error partway through. + */ + result = NIL; + + /* Prepare to scan pg_index for entries having indrelid = this rel. */ + ScanKeyInit(&skey, + Anum_pg_mv_statistic_starelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationGetRelid(relation))); + + indrel = heap_open(MvStatisticRelationId, AccessShareLock); + indscan = systable_beginscan(indrel, MvStatisticRelidIndexId, true, + NULL, 1, &skey); + + while (HeapTupleIsValid(htup = systable_getnext(indscan))) + /* TODO maybe include only already built statistics? */ + result = insert_ordered_oid(result, HeapTupleGetOid(htup)); + + systable_endscan(indscan); + + heap_close(indrel, AccessShareLock); + + /* Now save a copy of the completed list in the relcache entry. */ + oldcxt = MemoryContextSwitchTo(CacheMemoryContext); + oldlist = relation->rd_mvstatlist; + relation->rd_mvstatlist = list_copy(result); + + relation->rd_mvstatvalid = true; + MemoryContextSwitchTo(oldcxt); + + /* Don't leak the old list, if there is one */ + list_free(oldlist); + + return result; +} + +/* * insert_ordered_oid * Insert a new Oid into a sorted list of Oids, preserving ordering * @@ -5531,6 +5607,8 @@ load_relcache_init_file(bool shared) rel->rd_pkattr = NULL; rel->rd_idattr = NULL; rel->rd_pubactions = NULL; + rel->rd_mvstatvalid = false; + rel->rd_mvstatlist = NIL; rel->rd_createSubid = InvalidSubTransactionId; rel->rd_newRelfilenodeSubid = InvalidSubTransactionId; rel->rd_amcache = NULL; diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c index bdfaa0c..fbd1885 100644 --- a/src/backend/utils/cache/syscache.c +++ b/src/backend/utils/cache/syscache.c @@ -44,6 +44,7 @@ #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" #include "catalog/pg_language.h" +#include "catalog/pg_mv_statistic.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" #include "catalog/pg_operator.h" @@ -507,6 +508,28 @@ static const struct cachedesc cacheinfo[] = { }, 4 }, + {MvStatisticRelationId, /* MVSTATNAMENSP */ + MvStatisticNameIndexId, + 2, + { + Anum_pg_mv_statistic_staname, + Anum_pg_mv_statistic_stanamespace, + 0, + 0 + }, + 4 + }, + {MvStatisticRelationId, /* MVSTATOID */ + MvStatisticOidIndexId, + 1, + { + ObjectIdAttributeNumber, + 0, + 0, + 0 + }, + 4 + }, {NamespaceRelationId, /* NAMESPACENAME */ NamespaceNameIndexId, 1, diff --git a/src/backend/utils/mvstats/Makefile b/src/backend/utils/mvstats/Makefile new file mode 100644 index 0000000..7295d46 --- /dev/null +++ b/src/backend/utils/mvstats/Makefile @@ -0,0 +1,17 @@ +#------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for utils/mvstats +# +# IDENTIFICATION +# src/backend/utils/mvstats/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/backend/utils/mvstats +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global + +OBJS = common.o mvdist.o + +include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/utils/mvstats/README.ndistinct b/src/backend/utils/mvstats/README.ndistinct new file mode 100644 index 0000000..9365b17 --- /dev/null +++ b/src/backend/utils/mvstats/README.ndistinct @@ -0,0 +1,22 @@ +ndistinct coefficients +====================== + +Estimating number of groups in a combination of columns (e.g. for GROUP BY) +is tricky, and the estimation error is often significant. + +The ndistinct coefficients address this by storing ndistinct estimates not +only for individual columns, but also for (all) combinations of columns. +So for example given three columns (a,b,c) the statistics will estimate +ndistinct for (a,b), (a,c), (b,c) and (a,b,c). The per-column estimates +are already available in pg_statistic. + + +GROUP BY estimation (estimate_num_groups) +----------------------------------------- + +Although ndistinct coefficient might be used for selectivity estimation +(of equality conditions in WHERE clause), that is not implemented at this +point. + +Instead, ndistinct coefficients are only used in estimate_num_groups() to +estimate grouped queries. diff --git a/src/backend/utils/mvstats/README.stats b/src/backend/utils/mvstats/README.stats new file mode 100644 index 0000000..30d60d6 --- /dev/null +++ b/src/backend/utils/mvstats/README.stats @@ -0,0 +1,98 @@ +Multivariate statistics +======================= + +When estimating various quantities (e.g. condition selectivities) the default +approach relies on the assumption of independence. In practice that's often +not true, resulting in estimation errors. + +Multivariate stats track different types of dependencies between the columns, +hopefully improving the estimates. + + +Types of statistics +------------------- + +Currently we only have two kinds of multivariate statistics + + (a) soft functional dependencies (README.dependencies) + + (b) ndistinct coefficients + + +Compatible clause types +----------------------- + +Each type of statistics may be used to estimate some subset of clause types. + + (a) functional dependencies - equality clauses (AND), possibly IS NULL + +Currently only simple operator clauses (Var op Const) are supported, but it's +possible to support more complex clause types, e.g. (Var op Var). + + +Complex clauses +--------------- + +We also support estimating more complex clauses - essentially AND/OR clauses +with (Var op Const) as leaves, as long as all the referenced attributes are +covered by a single statistics. + +For example this condition + + (a=1) AND ((b=2) OR ((c=3) AND (d=4))) + +may be estimated using statistics on (a,b,c,d). If we only have statistics on +(b,c,d) we may estimate the second part, and estimate (a=1) using simple stats. + +If we only have statistics on (a,b,c) we can't apply it at all at this point, +but it's worth pointing out clauselist_selectivity() works recursively and when +handling the second part (the OR-clause), we'll be able to apply the statistics. + +Note: The multi-statistics estimation patch also makes it possible to pass some +clauses as 'conditions' into the deeper parts of the expression tree. + + +Selectivity estimation +---------------------- + +When estimating selectivity, we aim to achieve several things: + + (a) maximize the estimate accuracy + + (b) minimize the overhead, especially when no suitable multivariate stats + exist (so if you are not using multivariate stats, there's no overhead) + +This clauselist_selectivity() performs several inexpensive checks first, before +even attempting to do the more expensive estimation. + + (1) check if there are multivariate stats on the relation + + (2) check there are at least two attributes referenced by clauses compatible + with multivariate statistics (equality clauses for func. dependencies) + + (3) perform reduction of equality clauses using func. dependencies + + (4) estimate the reduced list of clauses using regular statistics + +Whenever we find there are no suitable stats, we skip the expensive steps. + + +Size of sample in ANALYZE +------------------------- +When performing ANALYZE, the number of rows to sample is determined as + + (300 * statistics_target) + +That works reasonably well for statistics on individual columns, but perhaps +it's not enough for multivariate statistics. Papers analyzing estimation errors +all use samples proportional to the table (usually finding that 1-3% of the +table is enough to build accurate stats). + +The requested accuracy (number of MCV items or histogram bins) should also +be considered when determining the sample size, and in multivariate statistics +those are not necessarily limited by statistics_target. + +This however merits further discussion, because collecting the sample is quite +expensive and increasing it further would make ANALYZE even more painful. +Judging by the experiments with the current implementation, the fixed size +seems to work reasonably well for now, so we leave this as a future work. diff --git a/src/backend/utils/mvstats/common.c b/src/backend/utils/mvstats/common.c new file mode 100644 index 0000000..7d2f3f3 --- /dev/null +++ b/src/backend/utils/mvstats/common.c @@ -0,0 +1,391 @@ +/*------------------------------------------------------------------------- + * + * common.c + * POSTGRES multivariate statistics + * + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/utils/mvstats/common.c + * + *------------------------------------------------------------------------- + */ + +#include "common.h" + +static VacAttrStats **lookup_var_attr_stats(int2vector *attrs, + int natts, VacAttrStats **vacattrstats); + +static List *list_mv_stats(Oid relid); + +static void update_mv_stats(Oid relid, MVNDistinct ndistinct, + int2vector *attrs, VacAttrStats **stats); + + +/* + * Compute requested multivariate stats, using the rows sampled for the + * plain (single-column) stats. + * + * This fetches a list of stats from pg_mv_statistic, computes the stats + * and serializes them back into the catalog (as bytea values). + */ +void +build_mv_stats(Relation onerel, double totalrows, + int numrows, HeapTuple *rows, + int natts, VacAttrStats **vacattrstats) +{ + ListCell *lc; + List *mvstats; + + TupleDesc tupdesc = RelationGetDescr(onerel); + + /* + * Fetch defined MV groups from pg_mv_statistic, and then compute the MV + * statistics (histograms for now). + */ + mvstats = list_mv_stats(RelationGetRelid(onerel)); + + foreach(lc, mvstats) + { + int j; + MVStatisticInfo *stat = (MVStatisticInfo *) lfirst(lc); + MVNDistinct ndistinct = NULL; + + VacAttrStats **stats = NULL; + int numatts = 0; + + /* int2 vector of attnums the stats should be computed on */ + int2vector *attrs = stat->stakeys; + + /* see how many of the columns are not dropped */ + for (j = 0; j < attrs->dim1; j++) + if (!tupdesc->attrs[attrs->values[j] - 1]->attisdropped) + numatts += 1; + + /* if there are dropped attributes, build a filtered int2vector */ + if (numatts != attrs->dim1) + { + int16 *tmp = palloc0(numatts * sizeof(int16)); + int attnum = 0; + + for (j = 0; j < attrs->dim1; j++) + if (!tupdesc->attrs[attrs->values[j] - 1]->attisdropped) + tmp[attnum++] = attrs->values[j]; + + pfree(attrs); + attrs = buildint2vector(tmp, numatts); + } + + /* filter only the interesting vacattrstats records */ + stats = lookup_var_attr_stats(attrs, natts, vacattrstats); + + /* check allowed number of dimensions */ + Assert((attrs->dim1 >= 2) && (attrs->dim1 <= MVSTATS_MAX_DIMENSIONS)); + + /* compute ndistinct coefficients */ + if (stat->ndist_enabled) + ndistinct = build_mv_ndistinct(totalrows, numrows, rows, attrs, stats); + + /* store the statistics in the catalog */ + update_mv_stats(stat->mvoid, ndistinct, attrs, stats); + } +} + +/* + * Lookup the VacAttrStats info for the selected columns, with indexes + * matching the attrs vector (to make it easy to work with when + * computing multivariate stats). + */ +static VacAttrStats ** +lookup_var_attr_stats(int2vector *attrs, int natts, VacAttrStats **vacattrstats) +{ + int i, + j; + int numattrs = attrs->dim1; + VacAttrStats **stats = (VacAttrStats **) palloc0(numattrs * sizeof(VacAttrStats *)); + + /* lookup VacAttrStats info for the requested columns (same attnum) */ + for (i = 0; i < numattrs; i++) + { + stats[i] = NULL; + for (j = 0; j < natts; j++) + { + if (attrs->values[i] == vacattrstats[j]->tupattnum) + { + stats[i] = vacattrstats[j]; + break; + } + } + + /* + * Check that we found the info, that the attnum matches and that + * there's the requested 'lt' operator and that the type is + * 'passed-by-value'. + */ + Assert(stats[i] != NULL); + Assert(stats[i]->tupattnum == attrs->values[i]); + + /* + * FIXME This is rather ugly way to check for 'ltopr' (which is + * defined for 'scalar' attributes). + */ + Assert(((StdAnalyzeData *) stats[i]->extra_data)->ltopr != InvalidOid); + } + + return stats; +} + +/* + * Fetch list of MV stats defined on a table, without the actual data + * for histograms, MCV lists etc. + */ +static List * +list_mv_stats(Oid relid) +{ + Relation indrel; + SysScanDesc indscan; + ScanKeyData skey; + HeapTuple htup; + List *result = NIL; + + /* Prepare to scan pg_mv_statistic for entries having indrelid = this rel. */ + ScanKeyInit(&skey, + Anum_pg_mv_statistic_starelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + + indrel = heap_open(MvStatisticRelationId, AccessShareLock); + indscan = systable_beginscan(indrel, MvStatisticRelidIndexId, true, + NULL, 1, &skey); + + while (HeapTupleIsValid(htup = systable_getnext(indscan))) + { + MVStatisticInfo *info = makeNode(MVStatisticInfo); + Form_pg_mv_statistic stats = (Form_pg_mv_statistic) GETSTRUCT(htup); + + info->mvoid = HeapTupleGetOid(htup); + info->stakeys = buildint2vector(stats->stakeys.values, stats->stakeys.dim1); + info->ndist_enabled = stats->ndist_enabled; + info->ndist_built = stats->ndist_built; + + result = lappend(result, info); + } + + systable_endscan(indscan); + + heap_close(indrel, AccessShareLock); + + /* + * TODO maybe save the list into relcache, as in RelationGetIndexList + * (which was used as an inspiration of this one)?. + */ + + return result; +} + +/* + * update_mv_stats + * Serializes the statistics and stores them into the pg_mv_statistic tuple. + */ +static void +update_mv_stats(Oid mvoid, MVNDistinct ndistinct, + int2vector *attrs, VacAttrStats **stats) +{ + HeapTuple stup, + oldtup; + Datum values[Natts_pg_mv_statistic]; + bool nulls[Natts_pg_mv_statistic]; + bool replaces[Natts_pg_mv_statistic]; + + Relation sd = heap_open(MvStatisticRelationId, RowExclusiveLock); + + memset(nulls, 1, Natts_pg_mv_statistic * sizeof(bool)); + memset(replaces, 0, Natts_pg_mv_statistic * sizeof(bool)); + memset(values, 0, Natts_pg_mv_statistic * sizeof(Datum)); + + /* + * Construct a new pg_mv_statistic tuple - replace only the histogram and + * MCV list, depending whether it actually was computed. + */ + if (ndistinct != NULL) + { + bytea *data = serialize_mv_ndistinct(ndistinct); + + nulls[Anum_pg_mv_statistic_standist -1] = (data == NULL); + values[Anum_pg_mv_statistic_standist-1] = PointerGetDatum(data); + } + + /* always replace the value (either by bytea or NULL) */ + replaces[Anum_pg_mv_statistic_standist - 1] = true; + + /* always change the availability flags */ + nulls[Anum_pg_mv_statistic_ndist_built - 1] = false; + nulls[Anum_pg_mv_statistic_stakeys - 1] = false; + + /* use the new attnums, in case we removed some dropped ones */ + replaces[Anum_pg_mv_statistic_ndist_built - 1] = true; + replaces[Anum_pg_mv_statistic_stakeys - 1] = true; + + values[Anum_pg_mv_statistic_ndist_built - 1] = BoolGetDatum(ndistinct != NULL); + + values[Anum_pg_mv_statistic_stakeys - 1] = PointerGetDatum(attrs); + + /* Is there already a pg_mv_statistic tuple for this attribute? */ + oldtup = SearchSysCache1(MVSTATOID, + ObjectIdGetDatum(mvoid)); + + if (HeapTupleIsValid(oldtup)) + { + /* Yes, replace it */ + stup = heap_modify_tuple(oldtup, + RelationGetDescr(sd), + values, + nulls, + replaces); + ReleaseSysCache(oldtup); + simple_heap_update(sd, &stup->t_self, stup); + } + else + elog(ERROR, "invalid pg_mv_statistic record (oid=%d)", mvoid); + + /* update indexes too */ + CatalogUpdateIndexes(sd, stup); + + heap_freetuple(stup); + + heap_close(sd, RowExclusiveLock); +} + +/* multi-variate stats comparator */ + +/* + * qsort_arg comparator for sorting Datums (MV stats) + * + * This does not maintain the tupnoLink array. + */ +int +compare_scalars_simple(const void *a, const void *b, void *arg) +{ + Datum da = *(Datum *) a; + Datum db = *(Datum *) b; + SortSupport ssup = (SortSupport) arg; + + return ApplySortComparator(da, false, db, false, ssup); +} + +/* + * qsort_arg comparator for sorting data when partitioning a MV bucket + */ +int +compare_scalars_partition(const void *a, const void *b, void *arg) +{ + Datum da = ((ScalarItem *) a)->value; + Datum db = ((ScalarItem *) b)->value; + SortSupport ssup = (SortSupport) arg; + + return ApplySortComparator(da, false, db, false, ssup); +} + +/* initialize multi-dimensional sort */ +MultiSortSupport +multi_sort_init(int ndims) +{ + MultiSortSupport mss; + + Assert(ndims >= 2); + + mss = (MultiSortSupport) palloc0(offsetof(MultiSortSupportData, ssup) + +sizeof(SortSupportData) * ndims); + + mss->ndims = ndims; + + return mss; +} + +/* + * add sort into for dimension 'dim' (index into vacattrstats) to mss, + * at the position 'sortattr' + */ +void +multi_sort_add_dimension(MultiSortSupport mss, int sortdim, + int dim, VacAttrStats **vacattrstats) +{ + /* first, lookup StdAnalyzeData for the dimension (attribute) */ + SortSupportData ssup; + StdAnalyzeData *tmp = (StdAnalyzeData *) vacattrstats[dim]->extra_data; + + Assert(mss != NULL); + Assert(sortdim < mss->ndims); + + /* initialize sort support, etc. */ + memset(&ssup, 0, sizeof(ssup)); + ssup.ssup_cxt = CurrentMemoryContext; + + /* We always use the default collation for statistics */ + ssup.ssup_collation = DEFAULT_COLLATION_OID; + ssup.ssup_nulls_first = false; + + PrepareSortSupportFromOrderingOp(tmp->ltopr, &ssup); + + mss->ssup[sortdim] = ssup; +} + +/* compare all the dimensions in the selected order */ +int +multi_sort_compare(const void *a, const void *b, void *arg) +{ + int i; + SortItem *ia = (SortItem *) a; + SortItem *ib = (SortItem *) b; + + MultiSortSupport mss = (MultiSortSupport) arg; + + for (i = 0; i < mss->ndims; i++) + { + int compare; + + compare = ApplySortComparator(ia->values[i], ia->isnull[i], + ib->values[i], ib->isnull[i], + &mss->ssup[i]); + + if (compare != 0) + return compare; + + } + + /* equal by default */ + return 0; +} + +/* compare selected dimension */ +int +multi_sort_compare_dim(int dim, const SortItem *a, const SortItem *b, + MultiSortSupport mss) +{ + return ApplySortComparator(a->values[dim], a->isnull[dim], + b->values[dim], b->isnull[dim], + &mss->ssup[dim]); +} + +int +multi_sort_compare_dims(int start, int end, + const SortItem *a, const SortItem *b, + MultiSortSupport mss) +{ + int dim; + + for (dim = start; dim <= end; dim++) + { + int r = ApplySortComparator(a->values[dim], a->isnull[dim], + b->values[dim], b->isnull[dim], + &mss->ssup[dim]); + + if (r != 0) + return r; + } + + return 0; +} diff --git a/src/backend/utils/mvstats/common.h b/src/backend/utils/mvstats/common.h new file mode 100644 index 0000000..e471c88 --- /dev/null +++ b/src/backend/utils/mvstats/common.h @@ -0,0 +1,80 @@ +/*------------------------------------------------------------------------- + * + * common.h + * POSTGRES multivariate statistics + * + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/utils/mvstats/common.h + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/sysattr.h" +#include "access/tuptoaster.h" +#include "catalog/indexing.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_mv_statistic.h" +#include "foreign/fdwapi.h" +#include "postmaster/autovacuum.h" +#include "storage/lmgr.h" +#include "utils/builtins.h" +#include "utils/datum.h" +#include "utils/fmgroids.h" +#include "utils/mvstats.h" +#include "utils/sortsupport.h" +#include "utils/syscache.h" + + +/* FIXME private structure copied from analyze.c */ + +typedef struct +{ + Oid eqopr; /* '=' operator for datatype, if any */ + Oid eqfunc; /* and associated function */ + Oid ltopr; /* '<' operator for datatype, if any */ +} StdAnalyzeData; + +typedef struct +{ + Datum value; /* a data value */ + int tupno; /* position index for tuple it came from */ +} ScalarItem; + +/* multi-sort */ +typedef struct MultiSortSupportData +{ + int ndims; /* number of dimensions supported by the */ + SortSupportData ssup[1]; /* sort support data for each dimension */ +} MultiSortSupportData; + +typedef MultiSortSupportData *MultiSortSupport; + +typedef struct SortItem +{ + Datum *values; + bool *isnull; +} SortItem; + +MultiSortSupport multi_sort_init(int ndims); + +void multi_sort_add_dimension(MultiSortSupport mss, int sortdim, + int dim, VacAttrStats **vacattrstats); + +int multi_sort_compare(const void *a, const void *b, void *arg); + +int multi_sort_compare_dim(int dim, const SortItem *a, + const SortItem *b, MultiSortSupport mss); + +int multi_sort_compare_dims(int start, int end, const SortItem *a, + const SortItem *b, MultiSortSupport mss); + +/* comparators, used when constructing multivariate stats */ +int compare_scalars_simple(const void *a, const void *b, void *arg); +int compare_scalars_partition(const void *a, const void *b, void *arg); diff --git a/src/backend/utils/mvstats/mvdist.c b/src/backend/utils/mvstats/mvdist.c new file mode 100644 index 0000000..188bf99 --- /dev/null +++ b/src/backend/utils/mvstats/mvdist.c @@ -0,0 +1,597 @@ +/*------------------------------------------------------------------------- + * + * mvdist.c + * POSTGRES multivariate distinct coefficients + * + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/utils/mvstats/mvdist.c + * + *------------------------------------------------------------------------- + */ + +#include + +#include "common.h" +#include "utils/bytea.h" +#include "utils/lsyscache.h" + +static double estimate_ndistinct(double totalrows, int numrows, int d, int f1); + +/* internal state for generator of k-combinations of n elements */ +typedef struct CombinationGeneratorData +{ + + int k; /* size of the combination */ + int current; /* index of the next combination to return */ + + int ncombinations; /* number of combinations (size of array) */ + int *combinations; /* array of pre-built combinations */ + +} CombinationGeneratorData; + +typedef CombinationGeneratorData *CombinationGenerator; + +/* generator API */ +static CombinationGenerator generator_init(int2vector *attrs, int k); +static void generator_free(CombinationGenerator state); +static int *generator_next(CombinationGenerator state, int2vector *attrs); + +static int n_choose_k(int n, int k); +static int num_combinations(int n); +static double ndistinct_for_combination(double totalrows, int numrows, + HeapTuple *rows, int2vector *attrs, VacAttrStats **stats, + int k, int *combination); + +/* + * Compute ndistinct coefficient for the combination of attributes. This + * computes the ndistinct estimate using the same estimator used in analyze.c + * and then computes the coefficient. + */ +MVNDistinct +build_mv_ndistinct(double totalrows, int numrows, HeapTuple *rows, + int2vector *attrs, VacAttrStats **stats) +{ + int i, k; + int numattrs = attrs->dim1; + int numcombs = num_combinations(numattrs); + + MVNDistinct result; + + result = palloc0(offsetof(MVNDistinctData, items) + + numcombs * sizeof(MVNDistinctItem)); + + result->nitems = numcombs; + + i = 0; + for (k = 2; k <= numattrs; k++) + { + int * combination; + CombinationGenerator generator; + + generator = generator_init(attrs, k); + + while ((combination = generator_next(generator, attrs))) + { + MVNDistinctItem *item = &result->items[i++]; + + item->nattrs = k; + item->ndistinct = ndistinct_for_combination(totalrows, numrows, rows, + attrs, stats, k, combination); + + item->attrs = palloc(k * sizeof(int)); + memcpy(item->attrs, combination, k * sizeof(int)); + + /* must not overflow the output array */ + Assert(i <= result->nitems); + } + + generator_free(generator); + } + + /* must consume exactly the whole output array */ + Assert(i == result->nitems); + + return result; +} + +/* + * ndistinct_for_combination + * Estimates number of distinct values in a combination of columns. + * + * This uses the same ndistinct estimator as compute_scalar_stats() in + * ANALYZE, i.e. + * + * n*d / (n - f1 + f1*n/N) + * + * except that instead of values in a single column we are dealing with + * combination of multiple columns. + */ +static double +ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows, + int2vector *attrs, VacAttrStats **stats, + int k, int *combination) +{ + int i, j; + int f1, cnt, d; + int nmultiple, summultiple; + MultiSortSupport mss = multi_sort_init(k); + + /* + * It's possible to sort the sample rows directly, but this seemed + * somehow simpler / less error prone. Another option would be to + * allocate the arrays for each SortItem separately, but that'd be + * significant overhead (not just CPU, but especially memory bloat). + */ + SortItem * items = (SortItem*)palloc0(numrows * sizeof(SortItem)); + + Datum *values = (Datum*)palloc0(sizeof(Datum) * numrows * k); + bool *isnull = (bool*)palloc0(sizeof(bool) * numrows * k); + + Assert((k >= 2) && (k <= attrs->dim1)); + + for (i = 0; i < numrows; i++) + { + items[i].values = &values[i * k]; + items[i].isnull = &isnull[i * k]; + } + + for (i = 0; i < k; i++) + { + /* prepare the sort function for the first dimension */ + multi_sort_add_dimension(mss, i, combination[i], stats); + + /* accumulate all the data into the array and sort it */ + for (j = 0; j < numrows; j++) + { + items[j].values[i] + = heap_getattr(rows[j], attrs->values[combination[i]], + stats[combination[i]]->tupDesc, + &items[j].isnull[i]); + } + } + + qsort_arg((void *) items, numrows, sizeof(SortItem), + multi_sort_compare, mss); + + /* count number of distinct combinations */ + + f1 = 0; + cnt = 1; + d = 1; + for (i = 1; i < numrows; i++) + { + if (multi_sort_compare(&items[i], &items[i-1], mss) != 0) + { + if (cnt == 1) + f1 += 1; + else + { + nmultiple += 1; + summultiple += cnt; + } + + d++; + cnt = 0; + } + + cnt += 1; + } + + if (cnt == 1) + f1 += 1; + else + { + nmultiple += 1; + summultiple += cnt; + } + + return estimate_ndistinct(totalrows, numrows, d, f1); +} + +MVNDistinct +load_mv_ndistinct(Oid mvoid) +{ + bool isnull = false; + Datum ndist; + + /* Prepare to scan pg_mv_statistic for entries having indrelid = this rel. */ + HeapTuple htup = SearchSysCache1(MVSTATOID, ObjectIdGetDatum(mvoid)); + +#ifdef USE_ASSERT_CHECKING + Form_pg_mv_statistic mvstat = (Form_pg_mv_statistic) GETSTRUCT(htup); + Assert(mvstat->ndist_enabled && mvstat->ndist_built); +#endif + + ndist = SysCacheGetAttr(MVSTATOID, htup, + Anum_pg_mv_statistic_standist, &isnull); + + Assert(!isnull); + + ReleaseSysCache(htup); + + return deserialize_mv_ndistinct(DatumGetByteaP(ndist)); +} + +/* The Duj1 estimator (already used in analyze.c). */ +static double +estimate_ndistinct(double totalrows, int numrows, int d, int f1) +{ + double numer, + denom, + ndistinct; + + numer = (double) numrows *(double) d; + + denom = (double) (numrows - f1) + + (double) f1 * (double) numrows / totalrows; + + ndistinct = numer / denom; + + /* Clamp to sane range in case of roundoff error */ + if (ndistinct < (double) d) + ndistinct = (double) d; + + if (ndistinct > totalrows) + ndistinct = totalrows; + + return floor(ndistinct + 0.5); +} + + +/* + * pg_ndistinct_in - input routine for type pg_ndistinct. + * + * pg_ndistinct is real enough to be a table column, but it has no operations + * of its own, and disallows input too + * + * XXX This is inspired by what pg_node_tree does. + */ +Datum +pg_ndistinct_in(PG_FUNCTION_ARGS) +{ + /* + * pg_node_list stores the data in binary form and parsing text input is + * not needed, so disallow this. + */ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot accept a value of type %s", "pg_ndistinct"))); + + PG_RETURN_VOID(); /* keep compiler quiet */ +} + +/* + * pg_ndistinct - output routine for type pg_ndistinct. + * + * histograms are serialized into a bytea value, so we simply call byteaout() + * to serialize the value into text. But it'd be nice to serialize that into + * a meaningful representation (e.g. for inspection by people). + */ +Datum +pg_ndistinct_out(PG_FUNCTION_ARGS) +{ + int i, j; + char *ret; + StringInfoData str; + + bytea *data = PG_GETARG_BYTEA_PP(0); + + MVNDistinct ndist = deserialize_mv_ndistinct(data); + + initStringInfo(&str); + appendStringInfoString(&str, "["); + + for (i = 0; i < ndist->nitems; i++) + { + MVNDistinctItem item = ndist->items[i]; + + if (i > 0) + appendStringInfoString(&str, ", "); + + appendStringInfoString(&str, "{"); + + for (j = 0; j < item.nattrs; j++) + { + if (j > 0) + appendStringInfoString(&str, ", "); + + appendStringInfo(&str, "%d", item.attrs[j]); + } + + appendStringInfo(&str, ", %f", item.ndistinct); + + appendStringInfoString(&str, "}"); + } + + appendStringInfoString(&str, "]"); + + ret = pstrdup(str.data); + pfree(str.data); + + PG_RETURN_CSTRING(ret); +} + +/* + * pg_ndistinct_recv - binary input routine for type pg_ndistinct. + */ +Datum +pg_ndistinct_recv(PG_FUNCTION_ARGS) +{ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot accept a value of type %s", "pg_ndistinct"))); + + PG_RETURN_VOID(); /* keep compiler quiet */ +} + +/* + * pg_ndistinct_send - binary output routine for type pg_ndistinct. + * + * XXX Histograms are serialized into a bytea value, so let's just send that. + */ +Datum +pg_ndistinct_send(PG_FUNCTION_ARGS) +{ + return byteasend(fcinfo); +} + +static int +n_choose_k(int n, int k) +{ + int i, numer, denom; + + Assert((n > 0) && (k > 0) && (n >= k)); + + numer = denom = 1; + for (i = 1; i <= k; i++) + { + numer *= (n - i + 1); + denom *= i; + } + + Assert(numer % denom == 0); + + return numer / denom; +} + +static int +num_combinations(int n) +{ + int k; + int ncombs = 0; + + /* ignore combinations with a single column */ + for (k = 2; k <= n; k++) + ncombs += n_choose_k(n, k); + + return ncombs; +} + +/* + * generate all combinations (k elements from n) + */ +static void +generate_combinations_recurse(CombinationGenerator state, + int n, int index, int start, int *current) +{ + /* If we haven't filled all the elements, simply recurse. */ + if (index < state->k) + { + int i; + + /* + * The values have to be in ascending order, so make sure we start + * with the value passed by parameter. + */ + + for (i = start; i < n; i++) + { + current[index] = i; + generate_combinations_recurse(state, n, (index+1), (i+1), current); + } + + return; + } + else + { + /* we got a correct combination */ + state->combinations = (int*)repalloc(state->combinations, + state->k * (state->current + 1) * sizeof(int)); + memcpy(&state->combinations[(state->k * state->current)], + current, state->k * sizeof(int)); + state->current++; + } +} + +/* generate all k-combinations of n elements */ +static void +generate_combinations(CombinationGenerator state, int n) +{ + int *current = (int *) palloc0(sizeof(int) * state->k); + + generate_combinations_recurse(state, n, 0, 0, current); + + pfree(current); +} + +/* + * initialize the generator of combinations, and prebuild them. + * + * This pre-builds all the combinations. We could also generate them in + * generator_next(), but this seems simpler. + */ +static CombinationGenerator +generator_init(int2vector *attrs, int k) +{ + int n = attrs->dim1; + CombinationGenerator state; + + Assert((n >= k) && (k > 0)); + + /* allocate the generator state as a single chunk of memory */ + state = (CombinationGenerator) palloc0(sizeof(CombinationGeneratorData)); + state->combinations = (int*)palloc(k * sizeof(int)); + + state->ncombinations = n_choose_k(n, k); + state->current = 0; + state->k = k; + + /* now actually pre-generate all the combinations */ + generate_combinations(state, n); + + /* make sure we got the expected number of combinations */ + Assert(state->current == state->ncombinations); + + /* reset the number, so we start with the first one */ + state->current = 0; + + return state; +} + +/* free the generator state */ +static void +generator_free(CombinationGenerator state) +{ + /* we've allocated a single chunk, so just free it */ + pfree(state); +} + +/* generate next combination */ +static int * +generator_next(CombinationGenerator state, int2vector *attrs) +{ + if (state->current == state->ncombinations) + return NULL; + + return &state->combinations[state->k * state->current++]; +} + +/* + * serialize list of ndistinct items into a bytea + */ +bytea * +serialize_mv_ndistinct(MVNDistinct ndistinct) +{ + int i; + bytea *output; + char *tmp; + + /* we need to store nitems */ + Size len = VARHDRSZ + offsetof(MVNDistinctData, items) + + ndistinct->nitems * offsetof(MVNDistinctItem, attrs); + + /* and also include space for the actual attribute numbers */ + for (i = 0; i < ndistinct->nitems; i++) + len += (sizeof(int) * ndistinct->items[i].nattrs); + + output = (bytea *) palloc0(len); + SET_VARSIZE(output, len); + + tmp = VARDATA(output); + + ndistinct->magic = MVSTAT_NDISTINCT_MAGIC; + ndistinct->type = MVSTAT_NDISTINCT_TYPE_BASIC; + + /* first, store the number of items */ + memcpy(tmp, ndistinct, offsetof(MVNDistinctData, items)); + tmp += offsetof(MVNDistinctData, items); + + /* store number of attributes and attribute numbers for each ndistinct entry */ + for (i = 0; i < ndistinct->nitems; i++) + { + MVNDistinctItem item = ndistinct->items[i]; + + memcpy(tmp, &item, offsetof(MVNDistinctItem, attrs)); + tmp += offsetof(MVNDistinctItem, attrs); + + memcpy(tmp, item.attrs, sizeof(int) * item.nattrs); + tmp += sizeof(int) * item.nattrs; + + Assert(tmp <= ((char *) output + len)); + } + + return output; +} + +/* + * Reads serialized ndistinct into MVNDistinct structure. + */ +MVNDistinct +deserialize_mv_ndistinct(bytea *data) +{ + int i; + Size expected_size; + MVNDistinct ndistinct; + char *tmp; + + if (data == NULL) + return NULL; + + if (VARSIZE_ANY_EXHDR(data) < offsetof(MVNDistinctData, items)) + elog(ERROR, "invalid MVNDistinct size %ld (expected at least %ld)", + VARSIZE_ANY_EXHDR(data), offsetof(MVNDistinctData, items)); + + /* read the MVNDistinct header */ + ndistinct = (MVNDistinct) palloc0(sizeof(MVNDistinctData)); + + /* initialize pointer to the data part (skip the varlena header) */ + tmp = VARDATA_ANY(data); + + /* get the header and perform basic sanity checks */ + memcpy(ndistinct, tmp, offsetof(MVNDistinctData, items)); + tmp += offsetof(MVNDistinctData, items); + + if (ndistinct->magic != MVSTAT_NDISTINCT_MAGIC) + elog(ERROR, "invalid ndistinct magic %d (expected %dd)", + ndistinct->magic, MVSTAT_NDISTINCT_MAGIC); + + if (ndistinct->type != MVSTAT_NDISTINCT_TYPE_BASIC) + elog(ERROR, "invalid ndistinct type %d (expected %dd)", + ndistinct->type, MVSTAT_NDISTINCT_TYPE_BASIC); + + Assert(ndistinct->nitems > 0); + + /* what minimum bytea size do we expect for those parameters */ + expected_size = offsetof(MVNDistinctData, items) + + ndistinct->nitems * (offsetof(MVNDistinctItem, attrs) + sizeof(int) * 2); + + if (VARSIZE_ANY_EXHDR(data) < expected_size) + elog(ERROR, "invalid dependencies size %ld (expected at least %ld)", + VARSIZE_ANY_EXHDR(data), expected_size); + + /* allocate space for the ndistinct items */ + ndistinct = repalloc(ndistinct, offsetof(MVNDistinctData, items) + + (ndistinct->nitems * sizeof(MVNDistinctItem))); + + for (i = 0; i < ndistinct->nitems; i++) + { + MVNDistinctItem *item = &ndistinct->items[i]; + + /* number of attributes */ + memcpy(item, tmp, offsetof(MVNDistinctItem, attrs)); + tmp += offsetof(MVNDistinctItem, attrs); + + /* is the number of attributes valid? */ + Assert((item->nattrs >= 2) && (item->nattrs <= MVSTATS_MAX_DIMENSIONS)); + + /* now that we know the number of attributes, allocate the attribute */ + item->attrs = (int*)palloc0(item->nattrs * sizeof(int)); + + /* copy attribute numbers */ + memcpy(item->attrs, tmp, sizeof(int) * item->nattrs); + tmp += sizeof(int) * item->nattrs; + + /* still within the bytea */ + Assert(tmp <= ((char *) data + VARSIZE_ANY(data))); + } + + /* we should have consumed the whole bytea exactly */ + Assert(tmp == ((char *) data + VARSIZE_ANY(data))); + + return ndistinct; +} diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index c501168..e7d5b51 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -2293,6 +2293,50 @@ describeOneTableDetails(const char *schemaname, PQclear(result); } + /* print any multivariate statistics */ + if (pset.sversion >= 90600) + { + printfPQExpBuffer(&buf, + "SELECT oid, stanamespace::regnamespace AS nsp, staname, stakeys,\n" + " ndist_enabled,\n" + " ndist_built,\n" + " (SELECT string_agg(attname::text,', ')\n" + " FROM ((SELECT unnest(stakeys) AS attnum) s\n" + " JOIN pg_attribute a ON (starelid = a.attrelid and a.attnum = s.attnum))) AS attnums\n" + "FROM pg_mv_statistic stat WHERE starelid = '%s' ORDER BY 1;", + oid); + + result = PSQLexec(buf.data); + if (!result) + goto error_return; + else + tuples = PQntuples(result); + + if (tuples > 0) + { + printTableAddFooter(&cont, _("Statistics:")); + for (i = 0; i < tuples; i++) + { + printfPQExpBuffer(&buf, " "); + + /* statistics name (qualified with namespace) */ + appendPQExpBuffer(&buf, "\"%s.%s\" ", + PQgetvalue(result, i, 1), + PQgetvalue(result, i, 2)); + + /* options */ + if (!strcmp(PQgetvalue(result, i, 4), "t")) + appendPQExpBuffer(&buf, "(dependencies)"); + + appendPQExpBuffer(&buf, " ON (%s)", + PQgetvalue(result, i, 6)); + + printTableAddFooter(&cont, buf.data); + } + } + PQclear(result); + } + /* print rules */ if (tableinfo.hasrules && tableinfo.relkind != 'm') { diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index 10759c7..86acca4 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -164,10 +164,11 @@ typedef enum ObjectClass OCLASS_PUBLICATION, /* pg_publication */ OCLASS_PUBLICATION_REL, /* pg_publication_rel */ OCLASS_SUBSCRIPTION, /* pg_subscription */ - OCLASS_TRANSFORM /* pg_transform */ + OCLASS_TRANSFORM, /* pg_transform */ + OCLASS_STATISTICS /* pg_mv_statistics */ } ObjectClass; -#define LAST_OCLASS OCLASS_TRANSFORM +#define LAST_OCLASS OCLASS_STATISTICS /* flag bits for performDeletion/performMultipleDeletions: */ #define PERFORM_DELETION_INTERNAL 0x0001 /* internal action */ diff --git a/src/include/catalog/heap.h b/src/include/catalog/heap.h index 1187797..2d2a8c9 100644 --- a/src/include/catalog/heap.h +++ b/src/include/catalog/heap.h @@ -119,6 +119,7 @@ extern void RemoveAttrDefault(Oid relid, AttrNumber attnum, DropBehavior behavior, bool complain, bool internal); extern void RemoveAttrDefaultById(Oid attrdefId); extern void RemoveStatistics(Oid relid, AttrNumber attnum); +extern void RemoveMVStatistics(Oid relid, AttrNumber attnum); extern Form_pg_attribute SystemAttributeDefinition(AttrNumber attno, bool relhasoids); diff --git a/src/include/catalog/indexing.h b/src/include/catalog/indexing.h index a3635a4..e938300 100644 --- a/src/include/catalog/indexing.h +++ b/src/include/catalog/indexing.h @@ -176,6 +176,13 @@ DECLARE_UNIQUE_INDEX(pg_largeobject_loid_pn_index, 2683, on pg_largeobject using DECLARE_UNIQUE_INDEX(pg_largeobject_metadata_oid_index, 2996, on pg_largeobject_metadata using btree(oid oid_ops)); #define LargeObjectMetadataOidIndexId 2996 +DECLARE_UNIQUE_INDEX(pg_mv_statistic_oid_index, 3380, on pg_mv_statistic using btree(oid oid_ops)); +#define MvStatisticOidIndexId 3380 +DECLARE_UNIQUE_INDEX(pg_mv_statistic_name_index, 3997, on pg_mv_statistic using btree(staname name_ops, stanamespace oid_ops)); +#define MvStatisticNameIndexId 3997 +DECLARE_INDEX(pg_mv_statistic_relid_index, 3379, on pg_mv_statistic using btree(starelid oid_ops)); +#define MvStatisticRelidIndexId 3379 + DECLARE_UNIQUE_INDEX(pg_namespace_nspname_index, 2684, on pg_namespace using btree(nspname name_ops)); #define NamespaceNameIndexId 2684 DECLARE_UNIQUE_INDEX(pg_namespace_oid_index, 2685, on pg_namespace using btree(oid oid_ops)); diff --git a/src/include/catalog/namespace.h b/src/include/catalog/namespace.h index dbeb25b..35e0e2b 100644 --- a/src/include/catalog/namespace.h +++ b/src/include/catalog/namespace.h @@ -141,6 +141,8 @@ extern Oid get_collation_oid(List *collname, bool missing_ok); extern Oid get_conversion_oid(List *conname, bool missing_ok); extern Oid FindDefaultConversionProc(int32 for_encoding, int32 to_encoding); +extern Oid get_statistics_oid(List *names, bool missing_ok); + /* initialization & transaction cleanup code */ extern void InitializeSearchPath(void); extern void AtEOXact_Namespace(bool isCommit, bool parallel); diff --git a/src/include/catalog/pg_cast.h b/src/include/catalog/pg_cast.h index 80a40ab..bf39d43 100644 --- a/src/include/catalog/pg_cast.h +++ b/src/include/catalog/pg_cast.h @@ -254,6 +254,10 @@ DATA(insert ( 23 18 78 e f )); /* pg_node_tree can be coerced to, but not from, text */ DATA(insert ( 194 25 0 i b )); +/* pg_ndistinct can be coerced to, but not from, bytea and text */ +DATA(insert ( 3353 17 0 i b )); +DATA(insert ( 3353 25 0 i i )); + /* * Datetime category */ diff --git a/src/include/catalog/pg_mv_statistic.h b/src/include/catalog/pg_mv_statistic.h new file mode 100644 index 0000000..fad80a3 --- /dev/null +++ b/src/include/catalog/pg_mv_statistic.h @@ -0,0 +1,78 @@ +/*------------------------------------------------------------------------- + * + * pg_mv_statistic.h + * definition of the system "multivariate statistic" relation (pg_mv_statistic) + * along with the relation's initial contents. + * + * + * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/catalog/pg_mv_statistic.h + * + * NOTES + * the genbki.pl script reads this file and generates .bki + * information from the DATA() statements. + * + *------------------------------------------------------------------------- + */ +#ifndef PG_MV_STATISTIC_H +#define PG_MV_STATISTIC_H + +#include "catalog/genbki.h" + +/* ---------------- + * pg_mv_statistic definition. cpp turns this into + * typedef struct FormData_pg_mv_statistic + * ---------------- + */ +#define MvStatisticRelationId 3381 + +CATALOG(pg_mv_statistic,3381) +{ + /* These fields form the unique key for the entry: */ + Oid starelid; /* relation containing attributes */ + NameData staname; /* statistics name */ + Oid stanamespace; /* OID of namespace containing this statistics */ + Oid staowner; /* statistics owner */ + + /* statistics requested to build */ + bool ndist_enabled; /* build ndist coefficient? */ + + /* statistics that are available (if requested) */ + bool ndist_built; /* ndistinct coeff built */ + + /* + * variable-length fields start here, but we allow direct access to + * stakeys + */ + int2vector stakeys; /* array of column keys */ + +#ifdef CATALOG_VARLEN + pg_ndistinct standist; /* ndistinct coeff (serialized) */ +#endif + +} FormData_pg_mv_statistic; + +/* ---------------- + * Form_pg_mv_statistic corresponds to a pointer to a tuple with + * the format of pg_mv_statistic relation. + * ---------------- + */ +typedef FormData_pg_mv_statistic *Form_pg_mv_statistic; + +/* ---------------- + * compiler constants for pg_mv_statistic + * ---------------- + */ +#define Natts_pg_mv_statistic 8 +#define Anum_pg_mv_statistic_starelid 1 +#define Anum_pg_mv_statistic_staname 2 +#define Anum_pg_mv_statistic_stanamespace 3 +#define Anum_pg_mv_statistic_staowner 4 +#define Anum_pg_mv_statistic_ndist_enabled 5 +#define Anum_pg_mv_statistic_ndist_built 6 +#define Anum_pg_mv_statistic_stakeys 7 +#define Anum_pg_mv_statistic_standist 8 + +#endif /* PG_MV_STATISTIC_H */ diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 31c828a..940a991 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -2726,6 +2726,15 @@ DESCR("current user privilege on any column by rel name"); DATA(insert OID = 3029 ( has_any_column_privilege PGNSP PGUID 12 10 0 0 0 f f f f t f s s 2 0 16 "26 25" _null_ _null_ _null_ _null_ _null_ has_any_column_privilege_id _null_ _null_ _null_ )); DESCR("current user privilege on any column by rel oid"); +DATA(insert OID = 3354 ( pg_ndistinct_in PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 3353 "2275" _null_ _null_ _null_ _null_ _null_ pg_ndistinct_in _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 3355 ( pg_ndistinct_out PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 2275 "3353" _null_ _null_ _null_ _null_ _null_ pg_ndistinct_out _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 3356 ( pg_ndistinct_recv PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 3353 "2281" _null_ _null_ _null_ _null_ _null_ pg_ndistinct_recv _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 3357 ( pg_ndistinct_send PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 17 "3353" _null_ _null_ _null_ _null_ _null_ pg_ndistinct_send _null_ _null_ _null_ )); +DESCR("I/O"); + DATA(insert OID = 1928 ( pg_stat_get_numscans PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 20 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_numscans _null_ _null_ _null_ )); DESCR("statistics: number of scans done for table/index"); DATA(insert OID = 1929 ( pg_stat_get_tuples_returned PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 20 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_tuples_returned _null_ _null_ _null_ )); diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 6e4c65e..9c9caf3 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -364,6 +364,10 @@ DATA(insert OID = 194 ( pg_node_tree PGNSP PGUID -1 f b S f t \054 0 0 0 pg_node DESCR("string representing an internal node tree"); #define PGNODETREEOID 194 +DATA(insert OID = 3353 ( pg_ndistinct PGNSP PGUID -1 f b S f t \054 0 0 0 pg_ndistinct_in pg_ndistinct_out pg_ndistinct_recv pg_ndistinct_send - - - i x f 0 -1 0 100 _null_ _null_ _null_ )); +DESCR("multivariate ndistinct coefficients"); +#define PGNDISTINCTOID 3353 + DATA(insert OID = 32 ( pg_ddl_command PGNSP PGUID SIZEOF_POINTER t p P f t \054 0 0 0 pg_ddl_command_in pg_ddl_command_out pg_ddl_command_recv pg_ddl_command_send - - - ALIGNOF_POINTER p f 0 -1 0 0 _null_ _null_ _null_ )); DESCR("internal type for passing CollectedCommand"); #define PGDDLCOMMANDOID 32 diff --git a/src/include/catalog/toasting.h b/src/include/catalog/toasting.h index db7f145..37a2f7a 100644 --- a/src/include/catalog/toasting.h +++ b/src/include/catalog/toasting.h @@ -49,6 +49,7 @@ extern void BootstrapToastTable(char *relName, DECLARE_TOAST(pg_attrdef, 2830, 2831); DECLARE_TOAST(pg_constraint, 2832, 2833); DECLARE_TOAST(pg_description, 2834, 2835); +DECLARE_TOAST(pg_mv_statistic, 3439, 3440); DECLARE_TOAST(pg_proc, 2836, 2837); DECLARE_TOAST(pg_rewrite, 2838, 2839); DECLARE_TOAST(pg_seclabel, 3598, 3599); diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index 8740cee..c323e81 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -77,6 +77,10 @@ extern ObjectAddress DefineOperator(List *names, List *parameters); extern void RemoveOperatorById(Oid operOid); extern ObjectAddress AlterOperator(AlterOperatorStmt *stmt); +/* commands/statscmds.c */ +extern ObjectAddress CreateStatistics(CreateStatsStmt *stmt); +extern void RemoveStatisticsById(Oid statsOid); + /* commands/aggregatecmds.c */ extern ObjectAddress DefineAggregate(ParseState *pstate, List *name, List *args, bool oldstyle, List *parameters); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 95dd8ba..e828b43 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -272,6 +272,7 @@ typedef enum NodeTag T_PlaceHolderInfo, T_MinMaxAggInfo, T_PlannerParamItem, + T_MVStatisticInfo, /* * TAGS FOR MEMORY NODES (memnodes.h) @@ -416,6 +417,7 @@ typedef enum NodeTag T_CreateSubscriptionStmt, T_AlterSubscriptionStmt, T_DropSubscriptionStmt, + T_CreateStatsStmt, /* * TAGS FOR PARSE TREE NODES (parsenodes.h) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 07a8436..18e1dd1 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -611,6 +611,16 @@ typedef struct ColumnDef int location; /* parse location, or -1 if none/unknown */ } ColumnDef; +typedef struct CreateStatsStmt +{ + NodeTag type; + List *defnames; /* qualified name (list of Value strings) */ + RangeVar *relation; /* relation to build statistics on */ + List *keys; /* String nodes naming referenced column(s) */ + bool if_not_exists; /* do nothing if statistics already exists */ +} CreateStatsStmt; + + /* * TableLikeClause - CREATE TABLE ( ... LIKE ... ) clause */ @@ -1554,6 +1564,7 @@ typedef enum ObjectType OBJECT_SCHEMA, OBJECT_SEQUENCE, OBJECT_SUBSCRIPTION, + OBJECT_STATISTICS, OBJECT_TABCONSTRAINT, OBJECT_TABLE, OBJECT_TABLESPACE, diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 643be54..7a55151 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -525,6 +525,7 @@ typedef struct RelOptInfo List *lateral_vars; /* LATERAL Vars and PHVs referenced by rel */ Relids lateral_referencers; /* rels that reference me laterally */ List *indexlist; /* list of IndexOptInfo */ + List *mvstatlist; /* list of MVStatisticInfo */ BlockNumber pages; /* size estimates derived from pg_class */ double tuples; double allvisfrac; @@ -663,6 +664,32 @@ typedef struct ForeignKeyOptInfo List *rinfos[INDEX_MAX_KEYS]; } ForeignKeyOptInfo; +/* + * MVStatisticInfo + * Information about multivariate stats for planning/optimization + * + * This contains information about which columns are covered by the + * statistics (stakeys), which options were requested while adding the + * statistics (*_enabled), and which kinds of statistics were actually + * built and are available for the optimizer (*_built). + */ +typedef struct MVStatisticInfo +{ + NodeTag type; + + Oid mvoid; /* OID of the statistics row */ + RelOptInfo *rel; /* back-link to index's table */ + + /* enabled statistics */ + bool ndist_enabled; /* ndistinct coefficient enabled */ + + /* built/available statistics */ + bool ndist_built; /* ndistinct coefficient built */ + + /* columns in the statistics (attnums) */ + int2vector *stakeys; /* attnums of the columns covered */ + +} MVStatisticInfo; /* * EquivalenceClasses diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 686141b..4368adc 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -322,6 +322,7 @@ extern bool pg_event_trigger_ownercheck(Oid et_oid, Oid roleid); extern bool pg_extension_ownercheck(Oid ext_oid, Oid roleid); extern bool pg_publication_ownercheck(Oid pub_oid, Oid roleid); extern bool pg_subscription_ownercheck(Oid sub_oid, Oid roleid); +extern bool pg_statistics_ownercheck(Oid stat_oid, Oid roleid); extern bool has_createrole_privilege(Oid roleid); extern bool has_bypassrls_privilege(Oid roleid); diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index 5bdca82..262ee94 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -68,6 +68,12 @@ extern int float8_cmp_internal(float8 a, float8 b); extern oidvector *buildoidvector(const Oid *oids, int n); extern Oid oidparse(Node *node); +/* mvdist.c */ +extern Datum pg_ndistinct_in(PG_FUNCTION_ARGS); +extern Datum pg_ndistinct_out(PG_FUNCTION_ARGS); +extern Datum pg_ndistinct_recv(PG_FUNCTION_ARGS); +extern Datum pg_ndistinct_send(PG_FUNCTION_ARGS); + /* regexp.c */ extern char *regexp_fixed_prefix(text *text_re, bool case_insensitive, Oid collation, bool *exact); diff --git a/src/include/utils/mvstats.h b/src/include/utils/mvstats.h new file mode 100644 index 0000000..0660c59 --- /dev/null +++ b/src/include/utils/mvstats.h @@ -0,0 +1,57 @@ +/*------------------------------------------------------------------------- + * + * mvstats.h + * Multivariate statistics and selectivity estimation functions. + * + * + * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/utils/mvstats.h + * + *------------------------------------------------------------------------- + */ +#ifndef MVSTATS_H +#define MVSTATS_H + +#include "fmgr.h" +#include "commands/vacuum.h" + +#define MVSTATS_MAX_DIMENSIONS 8 /* max number of attributes */ + +#define MVSTAT_NDISTINCT_MAGIC 0xA352BFA4 /* marks serialized bytea */ +#define MVSTAT_NDISTINCT_TYPE_BASIC 1 /* basic MCV list type */ + +/* Multivariate distinct coefficients. */ +typedef struct MVNDistinctItem { + double ndistinct; + int nattrs; + int *attrs; +} MVNDistinctItem; + +typedef struct MVNDistinctData { + uint32 magic; /* magic constant marker */ + uint32 type; /* type of ndistinct (BASIC) */ + int nitems; + MVNDistinctItem items[FLEXIBLE_ARRAY_MEMBER]; +} MVNDistinctData; + +typedef MVNDistinctData *MVNDistinct; + + +MVNDistinct load_mv_ndistinct(Oid mvoid); + +bytea *serialize_mv_ndistinct(MVNDistinct ndistinct); + +/* deserialization of stats (serialization is private to analyze) */ +MVNDistinct deserialize_mv_ndistinct(bytea *data); + + +MVNDistinct build_mv_ndistinct(double totalrows, int numrows, HeapTuple *rows, + int2vector *attrs, VacAttrStats **stats); + +void build_mv_stats(Relation onerel, double totalrows, + int numrows, HeapTuple *rows, + int natts, VacAttrStats **vacattrstats); + +#endif diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index a617a7c..121d4c9 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -92,6 +92,7 @@ typedef struct RelationData bool rd_isvalid; /* relcache entry is valid */ char rd_indexvalid; /* state of rd_indexlist: 0 = not valid, 1 = * valid, 2 = temporarily forced */ + bool rd_mvstatvalid; /* state of rd_mvstatlist: true/false */ /* * rd_createSubid is the ID of the highest subtransaction the rel has @@ -136,6 +137,9 @@ typedef struct RelationData Oid rd_pkindex; /* OID of primary key, if any */ Oid rd_replidindex; /* OID of replica identity index, if any */ + /* data managed by RelationGetMVStatList: */ + List *rd_mvstatlist; /* list of OIDs of multivariate stats */ + /* data managed by RelationGetIndexAttrBitmap: */ Bitmapset *rd_indexattr; /* identifies columns used in indexes */ Bitmapset *rd_keyattr; /* cols that can be ref'd by foreign keys */ diff --git a/src/include/utils/relcache.h b/src/include/utils/relcache.h index da36b67..bee390e 100644 --- a/src/include/utils/relcache.h +++ b/src/include/utils/relcache.h @@ -39,6 +39,7 @@ extern void RelationClose(Relation relation); */ extern List *RelationGetFKeyList(Relation relation); extern List *RelationGetIndexList(Relation relation); +extern List *RelationGetMVStatList(Relation relation); extern Oid RelationGetOidIndex(Relation relation); extern Oid RelationGetPrimaryKeyIndex(Relation relation); extern Oid RelationGetReplicaIndex(Relation relation); diff --git a/src/include/utils/syscache.h b/src/include/utils/syscache.h index 66f60d2..d4ebbf7 100644 --- a/src/include/utils/syscache.h +++ b/src/include/utils/syscache.h @@ -66,6 +66,8 @@ enum SysCacheIdentifier INDEXRELID, LANGNAME, LANGOID, + MVSTATNAMENSP, + MVSTATOID, NAMESPACENAME, NAMESPACEOID, OPERNAMENSP, diff --git a/src/test/regress/expected/mv_ndistinct.out b/src/test/regress/expected/mv_ndistinct.out new file mode 100644 index 0000000..5f55091 --- /dev/null +++ b/src/test/regress/expected/mv_ndistinct.out @@ -0,0 +1,117 @@ +-- data type passed by value +CREATE TABLE ndistinct ( + a INT, + b INT, + c INT, + d INT +); +-- unknown column +CREATE STATISTICS s10 ON (unknown_column) FROM ndistinct; +ERROR: column "unknown_column" referenced in statistics does not exist +-- single column +CREATE STATISTICS s10 ON (a) FROM ndistinct; +ERROR: statistics require at least 2 columns +-- single column, duplicated +CREATE STATISTICS s10 ON (a,a) FROM ndistinct; +ERROR: duplicate column name in statistics definition +-- two columns, one duplicated +CREATE STATISTICS s10 ON (a, a, b) FROM ndistinct; +ERROR: duplicate column name in statistics definition +-- correct command +CREATE STATISTICS s10 ON (a, b, c) FROM ndistinct; +-- perfectly correlated groups +INSERT INTO ndistinct + SELECT i/100, i/100, i/100 FROM generate_series(1,10000) s(i); +ANALYZE ndistinct; +SELECT ndist_enabled, ndist_built, standist + FROM pg_mv_statistic WHERE starelid = 'ndistinct'::regclass; + ndist_enabled | ndist_built | standist +---------------+-------------+------------------------------------------------------------------------------------- + t | t | [{0, 1, 101.000000}, {0, 2, 101.000000}, {1, 2, 101.000000}, {0, 1, 2, 101.000000}] +(1 row) + +EXPLAIN (COSTS off) + SELECT COUNT(*) FROM ndistinct GROUP BY a, b; + QUERY PLAN +----------------------------- + HashAggregate + Group Key: a, b + -> Seq Scan on ndistinct +(3 rows) + +EXPLAIN (COSTS off) + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c; + QUERY PLAN +----------------------------- + HashAggregate + Group Key: a, b, c + -> Seq Scan on ndistinct +(3 rows) + +EXPLAIN (COSTS off) + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c, d; + QUERY PLAN +----------------------------- + HashAggregate + Group Key: a, b, c, d + -> Seq Scan on ndistinct +(3 rows) + +TRUNCATE TABLE ndistinct; +-- partially correlated groups +INSERT INTO ndistinct + SELECT i/50, i/100, i/200 FROM generate_series(1,10000) s(i); +ANALYZE ndistinct; +SELECT ndist_enabled, ndist_built, standist + FROM pg_mv_statistic WHERE starelid = 'ndistinct'::regclass; + ndist_enabled | ndist_built | standist +---------------+-------------+------------------------------------------------------------------------------------- + t | t | [{0, 1, 201.000000}, {0, 2, 201.000000}, {1, 2, 101.000000}, {0, 1, 2, 201.000000}] +(1 row) + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, b; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate (cost=230.00..232.01 rows=201 width=16) + Group Key: a, b + -> Seq Scan on ndistinct (cost=0.00..155.00 rows=10000 width=8) +(3 rows) + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c; + QUERY PLAN +---------------------------------------------------------------------- + HashAggregate (cost=255.00..257.01 rows=201 width=20) + Group Key: a, b, c + -> Seq Scan on ndistinct (cost=0.00..155.00 rows=10000 width=12) +(3 rows) + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c, d; + QUERY PLAN +---------------------------------------------------------------------- + HashAggregate (cost=280.00..290.00 rows=1000 width=24) + Group Key: a, b, c, d + -> Seq Scan on ndistinct (cost=0.00..155.00 rows=10000 width=16) +(3 rows) + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY b, c, d; + QUERY PLAN +---------------------------------------------------------------------- + HashAggregate (cost=255.00..265.00 rows=1000 width=20) + Group Key: b, c, d + -> Seq Scan on ndistinct (cost=0.00..155.00 rows=10000 width=12) +(3 rows) + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, d; + QUERY PLAN +--------------------------------------------------------------------- + HashAggregate (cost=230.00..240.00 rows=1000 width=16) + Group Key: a, d + -> Seq Scan on ndistinct (cost=0.00..155.00 rows=10000 width=8) +(3 rows) + +DROP TABLE ndistinct; diff --git a/src/test/regress/expected/object_address.out b/src/test/regress/expected/object_address.out index ec5ada9..2b5c022 100644 --- a/src/test/regress/expected/object_address.out +++ b/src/test/regress/expected/object_address.out @@ -38,6 +38,7 @@ CREATE TRANSFORM FOR int LANGUAGE SQL ( TO SQL WITH FUNCTION int4recv(internal)); CREATE PUBLICATION addr_pub FOR TABLE addr_nsp.gentable; CREATE SUBSCRIPTION addr_sub CONNECTION '' PUBLICATION bar WITH (DISABLED, NOCREATE SLOT); +CREATE STATISTICS addr_nsp.gentable_stat ON (a,b) FROM addr_nsp.gentable; -- test some error cases SELECT pg_get_object_address('stone', '{}', '{}'); ERROR: unrecognized object type "stone" @@ -399,7 +400,8 @@ WITH objects (type, name, args) AS (VALUES ('access method', '{btree}', '{}'), ('publication', '{addr_pub}', '{}'), ('publication relation', '{addr_nsp, gentable}', '{addr_pub}'), - ('subscription', '{addr_sub}', '{}') + ('subscription', '{addr_sub}', '{}'), + ('statistics', '{addr_nsp, gentable_stat}', '{}') ) SELECT (pg_identify_object(addr1.classid, addr1.objid, addr1.subobjid)).*, -- test roundtrip through pg_identify_object_as_address @@ -447,6 +449,7 @@ SELECT (pg_identify_object(addr1.classid, addr1.objid, addr1.subobjid)).*, trigger | | | t on addr_nsp.gentable | t operator family | pg_catalog | integer_ops | pg_catalog.integer_ops USING btree | t policy | | | genpol on addr_nsp.gentable | t + statistics | addr_nsp | gentable_stat | addr_nsp.gentable_stat | t collation | pg_catalog | "default" | pg_catalog."default" | t transform | | | for integer on language sql | t text search dictionary | addr_nsp | addr_ts_dict | addr_nsp.addr_ts_dict | t @@ -456,7 +459,7 @@ SELECT (pg_identify_object(addr1.classid, addr1.objid, addr1.subobjid)).*, subscription | | addr_sub | addr_sub | t publication | | addr_pub | addr_pub | t publication relation | | | gentable in publication addr_pub | t -(45 rows) +(46 rows) --- --- Cleanup resources diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 0bcec13..9a26205 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -817,11 +817,12 @@ WHERE c.castmethod = 'b' AND text | character | 0 | i character varying | character | 0 | i pg_node_tree | text | 0 | i + pg_ndistinct | bytea | 0 | i cidr | inet | 0 | i xml | text | 0 | a xml | character varying | 0 | a xml | character | 0 | a -(7 rows) +(8 rows) -- **************** pg_conversion **************** -- Look for illegal values in pg_conversion fields. diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 60abcad..2c54779 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1376,6 +1376,14 @@ pg_matviews| SELECT n.nspname AS schemaname, LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN pg_tablespace t ON ((t.oid = c.reltablespace))) WHERE (c.relkind = 'm'::"char"); +pg_mv_stats| SELECT n.nspname AS schemaname, + c.relname AS tablename, + s.staname, + s.stakeys AS attnums, + length((s.standist)::text) AS ndistbytes + FROM ((pg_mv_statistic s + JOIN pg_class c ON ((c.oid = s.starelid))) + LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))); pg_policies| SELECT n.nspname AS schemaname, c.relname AS tablename, pol.polname AS policyname, diff --git a/src/test/regress/expected/sanity_check.out b/src/test/regress/expected/sanity_check.out index 0af013f..9d6bd18 100644 --- a/src/test/regress/expected/sanity_check.out +++ b/src/test/regress/expected/sanity_check.out @@ -116,6 +116,7 @@ pg_init_privs|t pg_language|t pg_largeobject|t pg_largeobject_metadata|t +pg_mv_statistic|t pg_namespace|t pg_opclass|t pg_operator|t diff --git a/src/test/regress/expected/type_sanity.out b/src/test/regress/expected/type_sanity.out index 312d290..6281cef 100644 --- a/src/test/regress/expected/type_sanity.out +++ b/src/test/regress/expected/type_sanity.out @@ -67,11 +67,12 @@ WHERE p1.typtype not in ('c','d','p') AND p1.typname NOT LIKE E'\\_%' (SELECT 1 FROM pg_type as p2 WHERE p2.typname = ('_' || p1.typname)::name AND p2.typelem = p1.oid and p1.typarray = p2.oid); - oid | typname ------+-------------- - 194 | pg_node_tree - 210 | smgr -(2 rows) + oid | typname +------+-------------- + 194 | pg_node_tree + 3353 | pg_ndistinct + 210 | smgr +(3 rows) -- Make sure typarray points to a varlena array type of our own base SELECT p1.oid, p1.typname as basetype, p2.typname as arraytype, diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index e9b2bad..0273ea6 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -116,3 +116,6 @@ test: event_trigger # run stats by itself because its delay may be insufficient under heavy load test: stats + +# run tests of multivariate stats +test: mv_ndistinct diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index 7cdc0f6..f7f3a14 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -171,3 +171,4 @@ test: with test: xml test: event_trigger test: stats +test: mv_ndistinct diff --git a/src/test/regress/sql/mv_ndistinct.sql b/src/test/regress/sql/mv_ndistinct.sql new file mode 100644 index 0000000..5cef254 --- /dev/null +++ b/src/test/regress/sql/mv_ndistinct.sql @@ -0,0 +1,68 @@ +-- data type passed by value +CREATE TABLE ndistinct ( + a INT, + b INT, + c INT, + d INT +); + +-- unknown column +CREATE STATISTICS s10 ON (unknown_column) FROM ndistinct; + +-- single column +CREATE STATISTICS s10 ON (a) FROM ndistinct; + +-- single column, duplicated +CREATE STATISTICS s10 ON (a,a) FROM ndistinct; + +-- two columns, one duplicated +CREATE STATISTICS s10 ON (a, a, b) FROM ndistinct; + +-- correct command +CREATE STATISTICS s10 ON (a, b, c) FROM ndistinct; + +-- perfectly correlated groups +INSERT INTO ndistinct + SELECT i/100, i/100, i/100 FROM generate_series(1,10000) s(i); + +ANALYZE ndistinct; + +SELECT ndist_enabled, ndist_built, standist + FROM pg_mv_statistic WHERE starelid = 'ndistinct'::regclass; + +EXPLAIN (COSTS off) + SELECT COUNT(*) FROM ndistinct GROUP BY a, b; + +EXPLAIN (COSTS off) + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c; + +EXPLAIN (COSTS off) + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c, d; + +TRUNCATE TABLE ndistinct; + +-- partially correlated groups +INSERT INTO ndistinct + SELECT i/50, i/100, i/200 FROM generate_series(1,10000) s(i); + +ANALYZE ndistinct; + +SELECT ndist_enabled, ndist_built, standist + FROM pg_mv_statistic WHERE starelid = 'ndistinct'::regclass; + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, b; + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c; + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, b, c, d; + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY b, c, d; + +EXPLAIN + SELECT COUNT(*) FROM ndistinct GROUP BY a, d; + +DROP TABLE ndistinct; diff --git a/src/test/regress/sql/object_address.sql b/src/test/regress/sql/object_address.sql index e658ea3..791b942 100644 --- a/src/test/regress/sql/object_address.sql +++ b/src/test/regress/sql/object_address.sql @@ -41,6 +41,7 @@ CREATE TRANSFORM FOR int LANGUAGE SQL ( TO SQL WITH FUNCTION int4recv(internal)); CREATE PUBLICATION addr_pub FOR TABLE addr_nsp.gentable; CREATE SUBSCRIPTION addr_sub CONNECTION '' PUBLICATION bar WITH (DISABLED, NOCREATE SLOT); +CREATE STATISTICS addr_nsp.gentable_stat ON (a,b) FROM addr_nsp.gentable; -- test some error cases SELECT pg_get_object_address('stone', '{}', '{}'); @@ -179,7 +180,8 @@ WITH objects (type, name, args) AS (VALUES ('access method', '{btree}', '{}'), ('publication', '{addr_pub}', '{}'), ('publication relation', '{addr_nsp, gentable}', '{addr_pub}'), - ('subscription', '{addr_sub}', '{}') + ('subscription', '{addr_sub}', '{}'), + ('statistics', '{addr_nsp, gentable_stat}', '{}') ) SELECT (pg_identify_object(addr1.classid, addr1.objid, addr1.subobjid)).*, -- test roundtrip through pg_identify_object_as_address -- 2.5.5