From 7db4ca23cb08c1771a022ecebd38caedd9764cba Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Fri, 23 Oct 2020 14:25:30 +1300 Subject: [PATCH v32 3/3] Track collation versions for indexes. Record the current version of dependent collations in pg_depend when creating or rebuilding an index. The version is checked against the current version whenever we call get_relation_info for an index or open the parent table during non-full VACUUM or ANALYZE. Warn that the index may be corrupted if the versions don't match (but only once per session). Thanks to Doug Doole, Peter Eisentraut, Christoph Berg, Laurenz Albe, Michael Paquier, Robert Haas, Tom Lane and others for design discussion. Author: Thomas Munro Author: Julien Rouhaud Reviewed-by: Peter Eisentraut Reviewed-by: Laurenz Albe Discussion: https://postgr.es/m/CAEepm%3D0uEQCpfq_%2BLYFBdArCe4Ot98t1aR4eYiYTe%3DyavQygiQ%40mail.gmail.com --- doc/src/sgml/catalogs.sgml | 3 +- doc/src/sgml/charset.sgml | 35 +++ doc/src/sgml/func.sgml | 3 +- doc/src/sgml/ref/alter_index.sgml | 17 ++ doc/src/sgml/ref/pgupgrade.sgml | 18 ++ doc/src/sgml/ref/reindex.sgml | 9 + src/backend/catalog/dependency.c | 210 +++++++++++++-- src/backend/catalog/heap.c | 7 +- src/backend/catalog/index.c | 228 +++++++++++++++- src/backend/catalog/pg_constraint.c | 2 +- src/backend/catalog/pg_depend.c | 46 +++- src/backend/catalog/pg_type.c | 60 +++++ src/backend/commands/collationcmds.c | 22 +- src/backend/commands/tablecmds.c | 32 +++ src/backend/commands/vacuum.c | 32 +++ src/backend/nodes/copyfuncs.c | 1 + src/backend/optimizer/util/plancat.c | 9 + src/backend/parser/gram.y | 8 + src/backend/utils/adt/pg_locale.c | 51 +++- src/backend/utils/adt/pg_upgrade_support.c | 25 ++ src/backend/utils/cache/relcache.c | 2 + src/bin/pg_dump/Makefile | 2 + src/bin/pg_dump/pg_backup.h | 1 + src/bin/pg_dump/pg_dump.c | 186 ++++++++++++- src/bin/pg_dump/pg_dump.h | 3 + src/bin/pg_dump/t/002_pg_dump.pl | 248 +++++++++++++----- src/bin/pg_upgrade/dump.c | 4 +- src/bin/pg_upgrade/option.c | 7 + src/bin/pg_upgrade/pg_upgrade.h | 2 + src/bin/psql/tab-complete.c | 27 +- src/include/catalog/dependency.h | 24 +- src/include/catalog/index.h | 6 + src/include/catalog/pg_proc.dat | 4 + src/include/catalog/pg_type.h | 2 + src/include/nodes/parsenodes.h | 4 +- src/include/utils/pg_locale.h | 2 +- src/include/utils/rel.h | 1 + src/test/Makefile | 3 +- src/test/locale/.gitignore | 1 + src/test/locale/Makefile | 7 + src/test/locale/t/001_index.pl | 76 ++++++ src/test/perl/PostgresNode.pm | 6 +- .../regress/expected/collate.icu.utf8.out | 211 +++++++++++++++ src/test/regress/expected/create_index.out | 8 +- src/test/regress/sql/collate.icu.utf8.sql | 137 ++++++++++ src/tools/pgindent/typedefs.list | 2 + 46 files changed, 1640 insertions(+), 154 deletions(-) create mode 100644 src/test/locale/t/001_index.pl diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index ddf4974a9a..d5e5dd887e 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -3308,7 +3308,8 @@ SCRAM-SHA-256$<iteration count>:&l refobjversion text - An optional version for the referenced object. + An optional version for the referenced object. Currently used for + collations (see ). diff --git a/doc/src/sgml/charset.sgml b/doc/src/sgml/charset.sgml index 2745b44417..785d2afe9e 100644 --- a/doc/src/sgml/charset.sgml +++ b/doc/src/sgml/charset.sgml @@ -948,6 +948,41 @@ CREATE COLLATION ignore_accents (provider = icu, locale = 'und-u-ks-level1-kc-tr + + + Collation Versions + + + The ordering defined by a collation is not necessarily fixed over time. + When a collation changes because of an operating system upgrade, + persistent data structures such as B-trees that depend on key order might + be corrupted. + PostgreSQL defends against this by recording the + current version of each referenced collation for any index that depends on + it in the + pg_depend + catalog, if the collation provider makes that information available. If the + provider later begins to report a different version, a warning will be + issued when the index is accessed, until the + or command is used to update the version. + + + Version information is available for collations from the + icu provider on all operating systems. For the + libc provider, versions are currently only available + on systems using the GNU C library (most Linux systems) and Windows. + + + + + When using the GNU C library for collations, the C library's version + is used as a proxy for the collation version. Many Linux distributions + change collation definitions only when upgrading the C library, but this + approach is imperfect as maintainers are free to back-port newer + collation definitions to older C library releases. + + + diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 3b3d7d1c0e..be90c92e5f 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -25481,7 +25481,8 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup()); Returns the actual version of the collation object as it is currently - installed in the operating system. + installed in the operating system. An empty string is returned if the + version is unknown. diff --git a/doc/src/sgml/ref/alter_index.sgml b/doc/src/sgml/ref/alter_index.sgml index 793119d2fc..8e55b681d5 100644 --- a/doc/src/sgml/ref/alter_index.sgml +++ b/doc/src/sgml/ref/alter_index.sgml @@ -25,6 +25,7 @@ ALTER INDEX [ IF EXISTS ] name RENA ALTER INDEX [ IF EXISTS ] name SET TABLESPACE tablespace_name ALTER INDEX name ATTACH PARTITION index_name ALTER INDEX name DEPENDS ON EXTENSION extension_name +ALTER INDEX name ALTER COLLATION collation_name REFRESH VERSION ALTER INDEX [ IF EXISTS ] name SET ( storage_parameter [= value] [, ... ] ) ALTER INDEX [ IF EXISTS ] name RESET ( storage_parameter [, ... ] ) ALTER INDEX [ IF EXISTS ] name ALTER [ COLUMN ] column_number @@ -112,6 +113,22 @@ ALTER INDEX ALL IN TABLESPACE name + + ALTER COLLATION collation_name REFRESH VERSION + + + This command declares that the index is compatible with the currently + installed version of a collation that determines its order. It is used + to silence warnings caused by collation version incompatibilities and + should be issued only if the collation ordering is known not to have + changed since the index was last built. Be aware that incorrect use of + this command can hide index corruption. If you don't know whether a + collation's definition has changed, using + is a safe alternative. + + + + SET ( storage_parameter [= value] [, ... ] ) diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index b59c5697a3..527c847e27 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -215,6 +215,24 @@ PostgreSQL documentation + + + + + After upgrading from a release of PostgreSQL + before 14, warnings may be reported when collation-dependent + indexes are first accessed. This is because + pg_upgrade records the collation versions + as unknown, so PostgreSQL considers the + indexes to be potentially corrupted. If you're certain that the + collation definitions used by your operating system or ICU haven't + changed since all indexes were created, you can use this flag + to record that the indexes match the currently installed collations. + Otherwise, see . + + + + diff --git a/doc/src/sgml/ref/reindex.sgml b/doc/src/sgml/ref/reindex.sgml index fa43e3a972..7ad5e34894 100644 --- a/doc/src/sgml/ref/reindex.sgml +++ b/doc/src/sgml/ref/reindex.sgml @@ -38,6 +38,15 @@ REINDEX [ ( option [, ...] ) ] { IN several scenarios in which to use REINDEX: + + + The index depends on the sort order of a collation, and the definition + of the collation has changed. This can cause an index to fail to + find a key that is present in the index. + See for more information. + + + An index has become corrupted, and no longer contains valid diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index 1a927377e7..28278b21be 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -76,6 +76,7 @@ #include "rewrite/rewriteRemove.h" #include "storage/lmgr.h" #include "utils/acl.h" +#include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/guc.h" #include "utils/lsyscache.h" @@ -136,6 +137,7 @@ typedef struct { ObjectAddresses *addrs; /* addresses being accumulated */ List *rtables; /* list of rangetables to resolve Vars */ + NodeTag type; /* nodetag of the current node */ } find_expr_references_context; /* @@ -434,6 +436,83 @@ performMultipleDeletions(const ObjectAddresses *objects, table_close(depRel, RowExclusiveLock); } +/* + * Call a function for all objects that 'object' depend on. If the function + * returns a non-NULL pointer to a new version string, use it to update + * refobjversion. + */ +void +visitDependenciesOf(const ObjectAddress *object, + VisitDependenciesOfCB callback, + void *userdata) +{ + Relation depRel; + ScanKeyData key[3]; + SysScanDesc scan; + HeapTuple tup; + ObjectAddress otherObject; + + ScanKeyInit(&key[0], + Anum_pg_depend_classid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(object->classId)); + ScanKeyInit(&key[1], + Anum_pg_depend_objid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(object->objectId)); + ScanKeyInit(&key[2], + Anum_pg_depend_objsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum(object->objectSubId)); + + depRel = table_open(DependRelationId, RowExclusiveLock); + scan = systable_beginscan(depRel, DependDependerIndexId, true, + NULL, 3, key); + + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_depend foundDep = (Form_pg_depend) GETSTRUCT(tup); + char *new_version; + Datum depversion; + bool isnull; + + otherObject.classId = foundDep->refclassid; + otherObject.objectId = foundDep->refobjid; + otherObject.objectSubId = foundDep->refobjsubid; + + depversion = heap_getattr(tup, Anum_pg_depend_refobjversion, + RelationGetDescr(depRel), &isnull); + + new_version = callback(&otherObject, + isnull ? NULL : TextDatumGetCString(depversion), + userdata); + + /* Does the callback want to update the version? */ + if (new_version) + { + Datum values[Natts_pg_depend]; + bool nulls[Natts_pg_depend]; + bool replaces[Natts_pg_depend]; + + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + memset(replaces, false, sizeof(replaces)); + + values[Anum_pg_depend_refobjversion - 1] = + CStringGetTextDatum(new_version); + replaces[Anum_pg_depend_refobjversion - 1] = true; + + tup = heap_modify_tuple(tup, RelationGetDescr(depRel), values, + nulls, replaces); + CatalogTupleUpdate(depRel, &tup->t_self, tup); + + heap_freetuple(tup); + } + } + systable_endscan(scan); + table_close(depRel, RowExclusiveLock); +} + /* * findDependentObjects - find all objects that depend on 'object' * @@ -1566,6 +1645,38 @@ ReleaseDeletionLock(const ObjectAddress *object) AccessExclusiveLock); } +/* + * Record dependencies on a list of collations, optionally with their current + * version. + */ +void +recordDependencyOnCollations(ObjectAddress *myself, + List *collations, + bool record_version) +{ + ObjectAddresses *addrs; + ListCell *lc; + + if (list_length(collations) == 0) + return; + + addrs = new_object_addresses(); + foreach(lc, collations) + { + ObjectAddress referenced; + + ObjectAddressSet(referenced, CollationRelationId, lfirst_oid(lc)); + + add_exact_object_address(&referenced, addrs); + } + + eliminate_duplicate_dependencies(addrs); + recordMultipleDependencies(myself, addrs->refs, addrs->numrefs, + DEPENDENCY_NORMAL, record_version); + + free_object_addresses(addrs); +} + /* * recordDependencyOnExpr - find expression dependencies * @@ -1588,6 +1699,10 @@ recordDependencyOnExpr(const ObjectAddress *depender, find_expr_references_context context; context.addrs = new_object_addresses(); + if (expr) + context.type = expr->type; + else + context.type = T_Invalid; /* Set up interpretation for Vars at varlevelsup = 0 */ context.rtables = list_make1(rtable); @@ -1602,8 +1717,8 @@ recordDependencyOnExpr(const ObjectAddress *depender, recordMultipleDependencies(depender, context.addrs->refs, context.addrs->numrefs, - NULL, - behavior); + behavior, + false); free_object_addresses(context.addrs); } @@ -1630,12 +1745,17 @@ recordDependencyOnSingleRelExpr(const ObjectAddress *depender, Node *expr, Oid relId, DependencyType behavior, DependencyType self_behavior, - bool reverse_self) + bool reverse_self, + bool track_version) { find_expr_references_context context; RangeTblEntry rte; context.addrs = new_object_addresses(); + if (expr) + context.type = expr->type; + else + context.type = T_Invalid; /* We gin up a rather bogus rangetable list to handle Vars */ MemSet(&rte, 0, sizeof(rte)); @@ -1691,8 +1811,8 @@ recordDependencyOnSingleRelExpr(const ObjectAddress *depender, recordMultipleDependencies(depender, self_addrs->refs, self_addrs->numrefs, - NULL, - self_behavior); + self_behavior, + track_version); else { /* Can't use recordMultipleDependencies, so do it the hard way */ @@ -1713,8 +1833,8 @@ recordDependencyOnSingleRelExpr(const ObjectAddress *depender, recordMultipleDependencies(depender, context.addrs->refs, context.addrs->numrefs, - NULL, - behavior); + behavior, + track_version); free_object_addresses(context.addrs); } @@ -1736,8 +1856,13 @@ static bool find_expr_references_walker(Node *node, find_expr_references_context *context) { + NodeTag parent_type = context->type; + if (node == NULL) return false; + + context->type = node->type; + if (IsA(node, Var)) { Var *var = (Var *) node; @@ -1770,6 +1895,44 @@ find_expr_references_walker(Node *node, /* If it's a plain relation, reference this column */ add_object_address(OCLASS_CLASS, rte->relid, var->varattno, context->addrs); + + /* + * Record collations from the type itself, or underlying in case + * of complex type. Note that if the direct parent is a + * CollateExpr node, there's no need to record the type underlying + * collation if any. A dependency already exists for the owning + * relation, and a change in the collation sort order wouldn't + * cause any harm as the collation isn't used at all in such case. + */ + if (parent_type != T_CollateExpr) + { + /* type's collation if valid */ + if (OidIsValid(var->varcollid)) + add_object_address(OCLASS_COLLATION, var->varcollid, 0, + context->addrs); + + /* + * Otherwise, it may be a composite type having underlying + * collations. + */ + else if (var->vartype >= FirstNormalObjectId) + { + List *collations; + ListCell *lc; + + collations = GetTypeCollations(var->vartype); + + foreach(lc, collations) + { + Oid coll = lfirst_oid(lc); + + if (OidIsValid(coll)) + add_object_address(OCLASS_COLLATION, + lfirst_oid(lc), 0, + context->addrs); + } + } + } } /* @@ -1794,11 +1957,9 @@ find_expr_references_walker(Node *node, /* * We must also depend on the constant's collation: it could be * different from the datatype's, if a CollateExpr was const-folded to - * a simple constant. However we can save work in the most common - * case where the collation is "default", since we know that's pinned. + * a simple constant. */ - if (OidIsValid(con->constcollid) && - con->constcollid != DEFAULT_COLLATION_OID) + if (OidIsValid(con->constcollid)) add_object_address(OCLASS_COLLATION, con->constcollid, 0, context->addrs); @@ -1887,8 +2048,7 @@ find_expr_references_walker(Node *node, add_object_address(OCLASS_TYPE, param->paramtype, 0, context->addrs); /* and its collation, just as for Consts */ - if (OidIsValid(param->paramcollid) && - param->paramcollid != DEFAULT_COLLATION_OID) + if (OidIsValid(param->paramcollid)) add_object_address(OCLASS_COLLATION, param->paramcollid, 0, context->addrs); } @@ -1975,8 +2135,7 @@ find_expr_references_walker(Node *node, add_object_address(OCLASS_TYPE, fselect->resulttype, 0, context->addrs); /* the collation might not be referenced anywhere else, either */ - if (OidIsValid(fselect->resultcollid) && - fselect->resultcollid != DEFAULT_COLLATION_OID) + if (OidIsValid(fselect->resultcollid)) add_object_address(OCLASS_COLLATION, fselect->resultcollid, 0, context->addrs); } @@ -2006,8 +2165,7 @@ find_expr_references_walker(Node *node, add_object_address(OCLASS_TYPE, relab->resulttype, 0, context->addrs); /* the collation might not be referenced anywhere else, either */ - if (OidIsValid(relab->resultcollid) && - relab->resultcollid != DEFAULT_COLLATION_OID) + if (OidIsValid(relab->resultcollid)) add_object_address(OCLASS_COLLATION, relab->resultcollid, 0, context->addrs); } @@ -2019,8 +2177,7 @@ find_expr_references_walker(Node *node, add_object_address(OCLASS_TYPE, iocoerce->resulttype, 0, context->addrs); /* the collation might not be referenced anywhere else, either */ - if (OidIsValid(iocoerce->resultcollid) && - iocoerce->resultcollid != DEFAULT_COLLATION_OID) + if (OidIsValid(iocoerce->resultcollid)) add_object_address(OCLASS_COLLATION, iocoerce->resultcollid, 0, context->addrs); } @@ -2032,8 +2189,7 @@ find_expr_references_walker(Node *node, add_object_address(OCLASS_TYPE, acoerce->resulttype, 0, context->addrs); /* the collation might not be referenced anywhere else, either */ - if (OidIsValid(acoerce->resultcollid) && - acoerce->resultcollid != DEFAULT_COLLATION_OID) + if (OidIsValid(acoerce->resultcollid)) add_object_address(OCLASS_COLLATION, acoerce->resultcollid, 0, context->addrs); /* fall through to examine arguments */ @@ -2121,8 +2277,7 @@ find_expr_references_walker(Node *node, if (OidIsValid(wc->endInRangeFunc)) add_object_address(OCLASS_PROC, wc->endInRangeFunc, 0, context->addrs); - if (OidIsValid(wc->inRangeColl) && - wc->inRangeColl != DEFAULT_COLLATION_OID) + if (OidIsValid(wc->inRangeColl)) add_object_address(OCLASS_COLLATION, wc->inRangeColl, 0, context->addrs); /* fall through to examine substructure */ @@ -2267,7 +2422,7 @@ find_expr_references_walker(Node *node, { Oid collid = lfirst_oid(ct); - if (OidIsValid(collid) && collid != DEFAULT_COLLATION_OID) + if (OidIsValid(collid)) add_object_address(OCLASS_COLLATION, collid, 0, context->addrs); } @@ -2289,7 +2444,7 @@ find_expr_references_walker(Node *node, { Oid collid = lfirst_oid(ct); - if (OidIsValid(collid) && collid != DEFAULT_COLLATION_OID) + if (OidIsValid(collid)) add_object_address(OCLASS_COLLATION, collid, 0, context->addrs); } @@ -2685,8 +2840,9 @@ record_object_address_dependencies(const ObjectAddress *depender, { eliminate_duplicate_dependencies(referenced); recordMultipleDependencies(depender, - referenced->refs, referenced->numrefs, NULL, - behavior); + referenced->refs, referenced->numrefs, + behavior, + false); } /* diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 67144aa3c9..fc5140daa6 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -2336,7 +2336,7 @@ StoreAttrDefault(Relation rel, AttrNumber attnum, */ recordDependencyOnSingleRelExpr(&colobject, expr, RelationGetRelid(rel), DEPENDENCY_AUTO, - DEPENDENCY_AUTO, false); + DEPENDENCY_AUTO, false, false); } else { @@ -2346,7 +2346,7 @@ StoreAttrDefault(Relation rel, AttrNumber attnum, */ recordDependencyOnSingleRelExpr(&defobject, expr, RelationGetRelid(rel), DEPENDENCY_NORMAL, - DEPENDENCY_NORMAL, false); + DEPENDENCY_NORMAL, false, false); } /* @@ -3665,7 +3665,8 @@ StorePartitionKey(Relation rel, RelationGetRelid(rel), DEPENDENCY_NORMAL, DEPENDENCY_INTERNAL, - true /* reverse the self-deps */ ); + true /* reverse the self-deps */ , + false /* don't track versions */ ); /* * We must invalidate the relcache so that the next diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 0974f3e23a..4b2fc1d297 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -53,6 +53,7 @@ #include "catalog/pg_trigger.h" #include "catalog/pg_type.h" #include "catalog/storage.h" +#include "commands/defrem.h" #include "commands/event_trigger.h" #include "commands/progress.h" #include "commands/tablecmds.h" @@ -75,6 +76,7 @@ #include "utils/guc.h" #include "utils/inval.h" #include "utils/lsyscache.h" +#include "utils/pg_locale.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" #include "utils/rel.h" @@ -1020,6 +1022,8 @@ index_create(Relation heapRelation, ObjectAddress myself, referenced; ObjectAddresses *addrs; + List *colls = NIL, + *colls_no_version = NIL; ObjectAddressSet(myself, RelationRelationId, indexRelationId); @@ -1103,30 +1107,65 @@ index_create(Relation heapRelation, recordDependencyOn(&myself, &referenced, DEPENDENCY_PARTITION_SEC); } - /* placeholder for normal dependencies */ - addrs = new_object_addresses(); - - /* Store dependency on collations */ - - /* The default collation is pinned, so don't bother recording it */ + /* Get dependencies on collations for all index keys. */ for (i = 0; i < indexInfo->ii_NumIndexKeyAttrs; i++) { - if (OidIsValid(collationObjectId[i]) && - collationObjectId[i] != DEFAULT_COLLATION_OID) + Oid colloid = collationObjectId[i]; + + if (OidIsValid(colloid)) { - ObjectAddressSet(referenced, CollationRelationId, - collationObjectId[i]); - add_exact_object_address(&referenced, addrs); + Oid opclass = classObjectId[i]; + + /* + * The *_pattern_ops opclasses are special: they explicitly do + * not depend on collation order so we can save some effort. + * + * XXX With more analysis, we could also skip version tracking + * for some cases like hash indexes with deterministic + * collations, because they will never need to order strings. + */ + if (opclass == TEXT_BTREE_PATTERN_OPS_OID || + opclass == VARCHAR_BTREE_PATTERN_OPS_OID || + opclass == BPCHAR_BTREE_PATTERN_OPS_OID) + colls_no_version = lappend_oid(colls_no_version, colloid); + else + colls = lappend_oid(colls, colloid); + } + else + { + Form_pg_attribute att = TupleDescAttr(indexTupDesc, i); + + Assert(i < indexTupDesc->natts); + + /* + * Even though there is no top-level collation, there may be + * collations affecting ordering inside types, so look there + * too. + */ + colls = list_concat(colls, GetTypeCollations(att->atttypid)); } } + /* + * If we have anything in both lists, keep just the versioned one to + * save some duplication. + */ + if (colls_no_version != NIL && colls != NIL) + colls_no_version = list_difference_oid(colls_no_version, colls); + + /* Store the versioned and unversioned collation dependencies. */ + if (colls_no_version != NIL) + recordDependencyOnCollations(&myself, colls_no_version, false); + if (colls != NIL) + recordDependencyOnCollations(&myself, colls, true); + /* Store dependency on operator classes */ + addrs = new_object_addresses(); for (i = 0; i < indexInfo->ii_NumIndexKeyAttrs; i++) { ObjectAddressSet(referenced, OperatorClassRelationId, classObjectId[i]); add_exact_object_address(&referenced, addrs); } - record_object_address_dependencies(&myself, addrs, DEPENDENCY_NORMAL); free_object_addresses(addrs); @@ -1137,7 +1176,7 @@ index_create(Relation heapRelation, (Node *) indexInfo->ii_Expressions, heapRelationId, DEPENDENCY_NORMAL, - DEPENDENCY_AUTO, false); + DEPENDENCY_AUTO, false, true); } /* Store dependencies on anything mentioned in predicate */ @@ -1147,7 +1186,7 @@ index_create(Relation heapRelation, (Node *) indexInfo->ii_Predicate, heapRelationId, DEPENDENCY_NORMAL, - DEPENDENCY_AUTO, false); + DEPENDENCY_AUTO, false, true); } } else @@ -1226,6 +1265,160 @@ index_create(Relation heapRelation, return indexRelationId; } +typedef struct do_collation_version_check_context +{ + Oid relid; + List *checked_colls; +} do_collation_version_check_context; + +/* + * Raise a warning if the recorded and current collation version don't match. + * This is a callback for visitDependenciesOf(). + */ +static char * +do_collation_version_check(const ObjectAddress *otherObject, + const char *version, + void *data) +{ + do_collation_version_check_context *context = data; + char *current_version; + + /* We only care about dependencies on collations. */ + if (otherObject->classId != CollationRelationId) + return NULL; + + /* + * If we've already checked this collation, skip it. We don't expect too + * many duplicates, but it's possible, and we don't want to generate + * duplicate warnings. + */ + if (list_member_oid(context->checked_colls, otherObject->objectId)) + return NULL; + + /* Compare with the current version. */ + current_version = get_collation_version_for_oid(otherObject->objectId); + + /* XXX should we warn about "disappearing" versions? */ + if (current_version) + { + if (!version || + (strcmp(version, "") == 0 && strcmp(current_version, "") != 0)) + { + /* + * The collation provider has learned how to report versions, or + * the previous version was unknown (pg_upgrade from a release + * that didn't record them) and now it is known. + */ + ereport(WARNING, + (errmsg("index \"%s\" depends on collation \"%s\" with an unknown version, and the current version is \"%s\"", + get_rel_name(context->relid), + get_collation_name(otherObject->objectId), + current_version), + errdetail("The index may be corrupted due to changes in sort order."), + errhint("REINDEX to avoid the risk of corruption."))); + } + else if (strcmp(current_version, version) != 0) + { + /* + * The version has changed, probably due to an OS/library upgrade + * or streaming replication between different OS/library versions. + */ + ereport(WARNING, + (errmsg("index \"%s\" depends on collation \"%s\" version \"%s\", but the current version is \"%s\"", + get_rel_name(context->relid), + get_collation_name(otherObject->objectId), + version, + current_version), + errdetail("The index may be corrupted due to changes in sort order."), + errhint("REINDEX to avoid the risk of corruption."))); + } + } + + /* Remember not to complain about this collation again. */ + context->checked_colls = lappend_oid(context->checked_colls, + otherObject->objectId); + + return NULL; +} + +/* index_check_collation_versions + * Check the collation version for all dependencies on the given index. + */ +void +index_check_collation_versions(Oid relid) +{ + do_collation_version_check_context context; + ObjectAddress object; + + /* + * The callback needs the relid for error messages, and some scratch space + * to avoid duplicate warnings. + */ + context.relid = relid; + context.checked_colls = NIL; + + object.classId = RelationRelationId; + object.objectId = relid; + object.objectSubId = 0; + + visitDependenciesOf(&object, &do_collation_version_check, &context); + + list_free(context.checked_colls); +} + +typedef struct do_collation_version_update_context +{ + Oid coll; + char *import_version; +} do_collation_version_update_context; + +/* + * Update the version for collations. A callback for visitDependenciesOf(). + */ +static char * +do_collation_version_update(const ObjectAddress *otherObject, + const char *version, + void *data) +{ + do_collation_version_update_context *context = data; + + /* We only care about dependencies on collations. */ + if (otherObject->classId != CollationRelationId) + return NULL; + + /* If we're looking for a specific collation, skip non-matches. */ + if (OidIsValid(context->coll) && otherObject->objectId != context->coll) + return NULL; + + /* During pg_upgrade, this is used to import the old cluster's version. */ + if (context->import_version) + return context->import_version; + + return get_collation_version_for_oid(otherObject->objectId); +} + +/* + * Record the current versions of one or all collations that an index depends + * on. InvalidOid means all collations, otherwise only a specific collation's + * version dependency is updated. In the latter case, the version can be + * passed in, instead of asking the collation provider (for use during + * pg_upgrade, to import the version from an old cluster). + */ +void +index_update_collation_versions(Oid relid, Oid coll, char *import_version) +{ + do_collation_version_update_context context; + ObjectAddress object; + + context.coll = coll; + context.import_version = import_version; + + object.classId = RelationRelationId; + object.objectId = relid; + object.objectSubId = 0; + visitDependenciesOf(&object, &do_collation_version_update, &context); +} + /* * index_concurrently_create_copy * @@ -1686,6 +1879,10 @@ index_concurrently_swap(Oid newIndexId, Oid oldIndexId, const char *oldName) changeDependenciesOf(RelationRelationId, oldIndexId, newIndexId); changeDependenciesOn(RelationRelationId, oldIndexId, newIndexId); + /* Now we have the old index's collation versions, so fix that. */ + CommandCounterIncrement(); + index_update_collation_versions(newIndexId, InvalidOid, NULL); + /* * Copy over statistics from old to new index */ @@ -3635,6 +3832,9 @@ reindex_index(Oid indexId, bool skip_constraint_checks, char persistence, /* Close rels, but keep locks */ index_close(iRel, NoLock); table_close(heapRelation, NoLock); + + /* Record the current versions of all depended-on collations. */ + index_update_collation_versions(indexId, InvalidOid, NULL); } /* diff --git a/src/backend/catalog/pg_constraint.c b/src/backend/catalog/pg_constraint.c index 0d70cb0c3c..93774c9d21 100644 --- a/src/backend/catalog/pg_constraint.c +++ b/src/backend/catalog/pg_constraint.c @@ -362,7 +362,7 @@ CreateConstraintEntry(const char *constraintName, */ recordDependencyOnSingleRelExpr(&conobject, conExpr, relId, DEPENDENCY_NORMAL, - DEPENDENCY_NORMAL, false); + DEPENDENCY_NORMAL, false, true); } /* Post creation hook for new constraint */ diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c index 09c30b13e8..01bd17599a 100644 --- a/src/backend/catalog/pg_depend.c +++ b/src/backend/catalog/pg_depend.c @@ -19,6 +19,7 @@ #include "access/table.h" #include "catalog/dependency.h" #include "catalog/indexing.h" +#include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" #include "catalog/pg_extension.h" @@ -27,6 +28,7 @@ #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" +#include "utils/pg_locale.h" #include "utils/rel.h" @@ -45,19 +47,24 @@ recordDependencyOn(const ObjectAddress *depender, const ObjectAddress *referenced, DependencyType behavior) { - recordMultipleDependencies(depender, referenced, 1, NULL, behavior); + recordMultipleDependencies(depender, referenced, 1, behavior, false); } /* * Record multiple dependencies (of the same kind) for a single dependent * object. This has a little less overhead than recording each separately. + * + * If track_version is true, then a record is added even if the referenced + * object is pinned, and the dependency version will be retrieved according to + * the referenced object kind. For now, only collation version is + * supported. */ void recordMultipleDependencies(const ObjectAddress *depender, const ObjectAddress *referenced, int nreferenced, - const char *version, - DependencyType behavior) + DependencyType behavior, + bool track_version) { Relation dependDesc; CatalogIndexState indstate; @@ -66,6 +73,7 @@ recordMultipleDependencies(const ObjectAddress *depender, max_slots, slot_init_count, slot_stored_count; + char *version = NULL; if (nreferenced <= 0) return; /* nothing to do */ @@ -96,12 +104,38 @@ recordMultipleDependencies(const ObjectAddress *depender, slot_init_count = 0; for (i = 0; i < nreferenced; i++, referenced++) { + bool ignore_systempin = false; + + if (track_version) + { + /* For now we only know how to deal with collations. */ + if (referenced->classId == CollationRelationId) + { + /* C and POSIX don't need version tracking. */ + if (referenced->objectId == C_COLLATION_OID || + referenced->objectId == POSIX_COLLATION_OID) + continue; + + version = get_collation_version_for_oid(referenced->objectId); + + /* + * Default collation is pinned, so we need to force recording + * the dependency to store the version. + */ + if (referenced->objectId == DEFAULT_COLLATION_OID) + ignore_systempin = true; + } + } + else + Assert(!version); + /* * If the referenced object is pinned by the system, there's no real - * need to record dependencies on it. This saves lots of space in - * pg_depend, so it's worth the time taken to check. + * need to record dependencies on it, unless we need to record a + * version. This saves lots of space in pg_depend, so it's worth the + * time taken to check. */ - if (isObjectPinned(referenced, dependDesc)) + if (!ignore_systempin && isObjectPinned(referenced, dependDesc)) continue; if (slot_init_count < max_slots) diff --git a/src/backend/catalog/pg_type.c b/src/backend/catalog/pg_type.c index 0b04dff773..44eed1a0b3 100644 --- a/src/backend/catalog/pg_type.c +++ b/src/backend/catalog/pg_type.c @@ -15,6 +15,7 @@ #include "postgres.h" #include "access/htup_details.h" +#include "access/relation.h" #include "access/table.h" #include "access/xact.h" #include "catalog/binary_upgrade.h" @@ -512,6 +513,65 @@ TypeCreate(Oid newTypeOid, return address; } +/* + * Get a list of all distinct collations that the given type depends on. + */ +List * +GetTypeCollations(Oid typeoid) +{ + List *result = NIL; + HeapTuple tuple; + Form_pg_type typeTup; + + tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typeoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for type %u", typeoid); + typeTup = (Form_pg_type) GETSTRUCT(tuple); + + if (OidIsValid(typeTup->typcollation)) + result = list_append_unique_oid(result, typeTup->typcollation); + else if (typeTup->typtype == TYPTYPE_COMPOSITE) + { + Relation rel = relation_open(typeTup->typrelid, AccessShareLock); + TupleDesc desc = RelationGetDescr(rel); + + for (int i = 0; i < RelationGetNumberOfAttributes(rel); i++) + { + Form_pg_attribute att = TupleDescAttr(desc, i); + + if (OidIsValid(att->attcollation)) + result = list_append_unique_oid(result, att->attcollation); + else + result = list_concat_unique_oid(result, + GetTypeCollations(att->atttypid)); + } + + relation_close(rel, NoLock); + } + else if (typeTup->typtype == TYPTYPE_DOMAIN) + { + Assert(OidIsValid(typeTup->typbasetype)); + + result = list_concat_unique_oid(result, + GetTypeCollations(typeTup->typbasetype)); + } + else if (typeTup->typtype == TYPTYPE_RANGE) + { + Oid rangeid = get_range_subtype(typeTup->oid); + + Assert(OidIsValid(rangeid)); + + result = list_concat_unique_oid(result, GetTypeCollations(rangeid)); + } + else if (OidIsValid(typeTup->typelem)) + result = list_concat_unique_oid(result, + GetTypeCollations(typeTup->typelem)); + + ReleaseSysCache(tuple); + + return result; +} + /* * GenerateTypeDependencies: build the dependencies needed for a type * diff --git a/src/backend/commands/collationcmds.c b/src/backend/commands/collationcmds.c index 5ad8886e60..519f7a7df3 100644 --- a/src/backend/commands/collationcmds.c +++ b/src/backend/commands/collationcmds.c @@ -270,28 +270,12 @@ Datum pg_collation_actual_version(PG_FUNCTION_ARGS) { Oid collid = PG_GETARG_OID(0); - HeapTuple tp; - char *collcollate; - char collprovider; char *version; - tp = SearchSysCache1(COLLOID, ObjectIdGetDatum(collid)); - if (!HeapTupleIsValid(tp)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("collation with OID %u does not exist", collid))); - - collcollate = pstrdup(NameStr(((Form_pg_collation) GETSTRUCT(tp))->collcollate)); - collprovider = ((Form_pg_collation) GETSTRUCT(tp))->collprovider; - - ReleaseSysCache(tp); - - version = get_collation_actual_version(collprovider, collcollate); + version = get_collation_version_for_oid(collid); + Assert(version); - if (version) - PG_RETURN_TEXT_P(cstring_to_text(version)); - else - PG_RETURN_NULL(); + PG_RETURN_TEXT_P(cstring_to_text(version)); } diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index a29c14bf1c..1aa3ead9e0 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -93,6 +93,7 @@ #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/partcache.h" +#include "utils/pg_locale.h" #include "utils/relcache.h" #include "utils/ruleutils.h" #include "utils/snapmgr.h" @@ -559,6 +560,7 @@ static void refuseDupeIndexAttach(Relation parentIdx, Relation partIdx, Relation partitionTbl); static List *GetParentedForeignKeyRefs(Relation partition); static void ATDetachCheckNoForeignKeyRefs(Relation partition); +static void ATExecAlterCollationRefreshVersion(Relation rel, List *coll); /* ---------------------------------------------------------------- @@ -3986,6 +3988,10 @@ AlterTableGetLockLevel(List *cmds) cmd_lockmode = AccessShareLock; break; + case AT_AlterCollationRefreshVersion: + cmd_lockmode = AccessExclusiveLock; + break; + default: /* oops */ elog(ERROR, "unrecognized alter table type: %d", (int) cmd->subtype); @@ -4160,6 +4166,12 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, /* This command never recurses */ pass = AT_PASS_MISC; break; + case AT_AlterCollationRefreshVersion: /* ALTER COLLATION ... REFRESH + * VERSION */ + ATSimplePermissions(rel, ATT_INDEX); + /* This command never recurses */ + pass = AT_PASS_MISC; + break; case AT_SetStorage: /* ALTER COLUMN SET STORAGE */ ATSimplePermissions(rel, ATT_TABLE | ATT_MATVIEW | ATT_FOREIGN_TABLE); ATSimpleRecursion(wqueue, rel, cmd, recurse, lockmode, context); @@ -4738,6 +4750,11 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, Assert(rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); ATExecDetachPartition(rel, ((PartitionCmd *) cmd->def)->name); break; + case AT_AlterCollationRefreshVersion: + /* ATPrepCmd ensured it must be an index */ + Assert(rel->rd_rel->relkind == RELKIND_INDEX); + ATExecAlterCollationRefreshVersion(rel, cmd->object); + break; default: /* oops */ elog(ERROR, "unrecognized alter table type: %d", (int) cmd->subtype); @@ -17582,3 +17599,18 @@ ATDetachCheckNoForeignKeyRefs(Relation partition) table_close(rel, NoLock); } } + +/* + * ALTER INDEX ... ALTER COLLATION ... REFRESH VERSION + * + * This overrides an existing dependency on a specific collation for a specific + * index to depend on the current collation version. + */ +static void +ATExecAlterCollationRefreshVersion(Relation rel, List *coll) +{ + index_update_collation_versions(rel->rd_id, + get_collation_oid(coll, false), + NULL); + CacheInvalidateRelcache(rel); +} diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index ddeec870d8..e1016609dc 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -31,6 +31,8 @@ #include "access/tableam.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/catalog.h" +#include "catalog/index.h" #include "catalog/namespace.h" #include "catalog/pg_database.h" #include "catalog/pg_inherits.h" @@ -632,6 +634,36 @@ vacuum_open_relation(Oid relid, RangeVar *relation, int options, rel_lock = false; } + /* + * Perform version sanity checks on the relation underlying indexes if + * it's not a VACUUM FULL, as VACUUM FULL will recreate the index and + * update the recorded collation version. + */ + if (!(options & VACOPT_FULL) && onerel && !IsSystemRelation(onerel) && + onerel->rd_rel->relhasindex) + { + List *indexoidlist; + ListCell *l; + + indexoidlist = RelationGetIndexList(onerel); + foreach(l, indexoidlist) + { + Oid indexoid = lfirst_oid(l); + Relation indexRelation; + + indexRelation = index_open(indexoid, AccessShareLock); + + /* Warn if any dependent collations' versions have moved. */ + if (!indexRelation->rd_version_checked) + { + index_check_collation_versions(indexoid); + indexRelation->rd_version_checked = true; + } + + index_close(indexRelation, NoLock); + } + } + /* if relation is opened, leave */ if (onerel) return onerel; diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index ac8b57109c..530aac68a7 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -3215,6 +3215,7 @@ _copyAlterTableCmd(const AlterTableCmd *from) COPY_SCALAR_FIELD(subtype); COPY_STRING_FIELD(name); + COPY_NODE_FIELD(object); COPY_SCALAR_FIELD(num); COPY_NODE_FIELD(newowner); COPY_NODE_FIELD(def); diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index f9d0d67aa7..3a20dcdee5 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -28,6 +28,7 @@ #include "catalog/catalog.h" #include "catalog/dependency.h" #include "catalog/heap.h" +#include "catalog/index.h" #include "catalog/pg_am.h" #include "catalog/pg_proc.h" #include "catalog/pg_statistic_ext.h" @@ -198,6 +199,14 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, indexRelation = index_open(indexoid, lmode); index = indexRelation->rd_index; + /* Warn if any dependent collations' versions have moved. */ + if (!IsSystemRelation(relation) && + !indexRelation->rd_version_checked) + { + index_check_collation_versions(indexoid); + indexRelation->rd_version_checked = true; + } + /* * Ignore invalid indexes, since they can't safely be used for * queries. Note that this is OK because the data structure we diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 60cf7242a3..357ab93fb6 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -2591,6 +2591,14 @@ alter_table_cmd: n->subtype = AT_NoForceRowSecurity; $$ = (Node *)n; } + /* ALTER INDEX ALTER COLLATION ... REFRESH VERSION */ + | ALTER COLLATION any_name REFRESH VERSION_P + { + AlterTableCmd *n = makeNode(AlterTableCmd); + n->subtype = AT_AlterCollationRefreshVersion; + n->object = $3; + $$ = (Node *)n; + } | alter_generic_options { AlterTableCmd *n = makeNode(AlterTableCmd); diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index 514e0fa0af..faa9c7f533 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -57,7 +57,9 @@ #include "access/htup_details.h" #include "catalog/pg_collation.h" #include "catalog/pg_control.h" +#include "catalog/pg_database.h" #include "mb/pg_wchar.h" +#include "miscadmin.h" #include "utils/builtins.h" #include "utils/formatting.h" #include "utils/hsearch.h" @@ -139,6 +141,9 @@ static char *IsoLocaleName(const char *); /* MSVC specific */ static void icu_set_collation_attributes(UCollator *collator, const char *loc); #endif +static char *get_collation_actual_version(char collprovider, + const char *collcollate); + /* * pg_perm_setlocale * @@ -1630,7 +1635,7 @@ pg_newlocale_from_collation(Oid collid) * Get provider-specific collation version string for the given collation from * the operating system/library. */ -char * +static char * get_collation_actual_version(char collprovider, const char *collcollate) { char *collversion = NULL; @@ -1712,6 +1717,50 @@ get_collation_actual_version(char collprovider, const char *collcollate) return collversion; } +/* + * Get provider-specific collation version string for a given collation OID. + * + * An empty string is returned when the version is unknown. Note that NULL + * means something different ("untracked"), when stored in refobjversion. + */ +char * +get_collation_version_for_oid(Oid oid) +{ + HeapTuple tp; + char *version = NULL; + + Assert(oid != C_COLLATION_OID && oid != POSIX_COLLATION_OID); + + if (oid == DEFAULT_COLLATION_OID) + { + Form_pg_database dbform; + + tp = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(MyDatabaseId)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for database %u", MyDatabaseId); + dbform = (Form_pg_database) GETSTRUCT(tp); + version = get_collation_actual_version(COLLPROVIDER_LIBC, + NameStr(dbform->datcollate)); + } + else + { + Form_pg_collation collform; + + tp = SearchSysCache1(COLLOID, ObjectIdGetDatum(oid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for collation %u", oid); + collform = (Form_pg_collation) GETSTRUCT(tp); + version = get_collation_actual_version(collform->collprovider, + NameStr(collform->collcollate)); + } + + ReleaseSysCache(tp); + + if (!version) + return ""; + else + return version; +} #ifdef USE_ICU /* diff --git a/src/backend/utils/adt/pg_upgrade_support.c b/src/backend/utils/adt/pg_upgrade_support.c index 14d9eb2b5b..0950e3b4fc 100644 --- a/src/backend/utils/adt/pg_upgrade_support.c +++ b/src/backend/utils/adt/pg_upgrade_support.c @@ -13,6 +13,7 @@ #include "catalog/binary_upgrade.h" #include "catalog/heap.h" +#include "catalog/index.h" #include "catalog/namespace.h" #include "catalog/pg_type.h" #include "commands/extension.h" @@ -197,3 +198,27 @@ binary_upgrade_set_missing_value(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } + +Datum +binary_upgrade_set_index_coll_version(PG_FUNCTION_ARGS) +{ + Oid relid; + Oid coll; + char *version; + + CHECK_IS_BINARY_UPGRADE; + + relid = PG_GETARG_OID(0); + + /* Detect if a collation is specified */ + if (PG_ARGISNULL(1)) + coll = InvalidOid; + else + coll = PG_GETARG_OID(1); + + version = TextDatumGetCString(PG_GETARG_TEXT_PP(2)); + + index_update_collation_versions(relid, coll, version); + + PG_RETURN_VOID(); +} diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 9061af81a3..153adc157d 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -42,6 +42,7 @@ #include "access/xact.h" #include "access/xlog.h" #include "catalog/catalog.h" +#include "catalog/index.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/partition.h" @@ -5934,6 +5935,7 @@ load_relcache_init_file(bool shared) rel->rd_idattr = NULL; rel->rd_pubactions = NULL; rel->rd_statvalid = false; + rel->rd_version_checked = false; rel->rd_statlist = NIL; rel->rd_fkeyvalid = false; rel->rd_fkeylist = NIL; diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile index 2532d9183a..c58ebaa681 100644 --- a/src/bin/pg_dump/Makefile +++ b/src/bin/pg_dump/Makefile @@ -16,6 +16,8 @@ subdir = src/bin/pg_dump top_builddir = ../../.. include $(top_builddir)/src/Makefile.global +export with_icu + override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index a6a8e6f2fd..a1151e3b31 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -179,6 +179,7 @@ typedef struct _dumpOptions int sequence_data; /* dump sequence data even in schema-only mode */ int do_nothing; + int unknown_coll_compat; } DumpOptions; /* diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1d09acb64c..b82c16b151 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -46,6 +46,7 @@ #include "catalog/pg_attribute_d.h" #include "catalog/pg_cast_d.h" #include "catalog/pg_class_d.h" +#include "catalog/pg_collation_d.h" #include "catalog/pg_default_acl_d.h" #include "catalog/pg_largeobject_d.h" #include "catalog/pg_largeobject_metadata_d.h" @@ -285,6 +286,8 @@ static void binary_upgrade_extension_member(PQExpBuffer upgrade_buffer, static const char *getAttrName(int attrnum, TableInfo *tblInfo); static const char *fmtCopyColumnList(const TableInfo *ti, PQExpBuffer buffer); static bool nonemptyReloptions(const char *reloptions); +static void appendIndexCollationVersion(PQExpBuffer buffer, IndxInfo *indxinfo, + int enc, int unknown_coll_compat); static void appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions, const char *prefix, Archive *fout); static char *get_synchronized_snapshot(Archive *fout); @@ -385,6 +388,7 @@ main(int argc, char **argv) {"on-conflict-do-nothing", no_argument, &dopt.do_nothing, 1}, {"rows-per-insert", required_argument, NULL, 10}, {"include-foreign-data", required_argument, NULL, 11}, + {"unknown-collations-binary-compatible", no_argument, &dopt.unknown_coll_compat, 1}, {NULL, 0, NULL, 0} }; @@ -712,6 +716,10 @@ main(int argc, char **argv) if (archiveFormat != archDirectory && numWorkers > 1) fatal("parallel backup only supported by the directory format"); + /* Unknown collation versions can only be ignored in binary upgrade mode */ + if (dopt.unknown_coll_compat && !dopt.binary_upgrade) + fatal("option --unknown-collations-binary-compatible only works in binary upgrade mode"); + /* Open the output file */ fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, archiveMode, setupDumpWorker); @@ -7028,7 +7036,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) i_tablespace, i_indreloptions, i_indstatcols, - i_indstatvals; + i_indstatvals, + i_inddependoids, + i_inddependversions; int ntups; for (i = 0; i < numTables; i++) @@ -7064,7 +7074,62 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) * is not. */ resetPQExpBuffer(query); - if (fout->remoteVersion >= 110000) + if (fout->remoteVersion >= 140000) + { + appendPQExpBuffer(query, + "SELECT t.tableoid, t.oid, " + "t.relname AS indexname, " + "inh.inhparent AS parentidx, " + "pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, " + "i.indnkeyatts AS indnkeyatts, " + "i.indnatts AS indnatts, " + "i.indkey, i.indisclustered, " + "i.indisreplident, " + "c.contype, c.conname, " + "c.condeferrable, c.condeferred, " + "c.tableoid AS contableoid, " + "c.oid AS conoid, " + "pg_catalog.pg_get_constraintdef(c.oid, false) AS condef, " + "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " + "t.reloptions AS indreloptions, " + "(SELECT pg_catalog.array_agg(attnum ORDER BY attnum) " + " FROM pg_catalog.pg_attribute " + " WHERE attrelid = i.indexrelid AND " + " attstattarget >= 0) AS indstatcols," + "(SELECT pg_catalog.array_agg(attstattarget ORDER BY attnum) " + " FROM pg_catalog.pg_attribute " + " WHERE attrelid = i.indexrelid AND " + " attstattarget >= 0) AS indstatvals, " + "(SELECT pg_catalog.array_agg(refobjid ORDER BY refobjid) " + " FROM pg_catalog.pg_depend " + " WHERE classid = " CppAsString2(RelationRelationId) " AND " + " objid = i.indexrelid AND " + " objsubid = 0 AND " + " refclassid = " CppAsString2(CollationRelationId) " AND " + " refobjversion IS NOT NULL) AS inddependoids, " + "(SELECT pg_catalog.array_agg(quote_literal(refobjversion) ORDER BY refobjid) " + " FROM pg_catalog.pg_depend " + " WHERE classid = " CppAsString2(RelationRelationId) " AND " + " objid = i.indexrelid AND " + " objsubid = 0 AND " + " refclassid = " CppAsString2(CollationRelationId) " AND " + " refobjversion IS NOT NULL) AS inddependversions " + "FROM pg_catalog.pg_index i " + "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " + "JOIN pg_catalog.pg_class t2 ON (t2.oid = i.indrelid) " + "LEFT JOIN pg_catalog.pg_constraint c " + "ON (i.indrelid = c.conrelid AND " + "i.indexrelid = c.conindid AND " + "c.contype IN ('p','u','x')) " + "LEFT JOIN pg_catalog.pg_inherits inh " + "ON (inh.inhrelid = indexrelid) " + "WHERE i.indrelid = '%u'::pg_catalog.oid " + "AND (i.indisvalid OR t2.relkind = 'p') " + "AND i.indisready " + "ORDER BY indexname", + tbinfo->dobj.catId.oid); + } + else if (fout->remoteVersion >= 110000) { appendPQExpBuffer(query, "SELECT t.tableoid, t.oid, " @@ -7089,7 +7154,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "(SELECT pg_catalog.array_agg(attstattarget ORDER BY attnum) " " FROM pg_catalog.pg_attribute " " WHERE attrelid = i.indexrelid AND " - " attstattarget >= 0) AS indstatvals " + " attstattarget >= 0) AS indstatvals, " + "' ' AS inddependoids, " + "' ' AS inddependversions " "FROM pg_catalog.pg_index i " "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " "JOIN pg_catalog.pg_class t2 ON (t2.oid = i.indrelid) " @@ -7128,7 +7195,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " "t.reloptions AS indreloptions, " "'' AS indstatcols, " - "'' AS indstatvals " + "'' AS indstatvals, " + "' ' AS inddependoids, " + "' ' AS inddependversions " "FROM pg_catalog.pg_index i " "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " "LEFT JOIN pg_catalog.pg_constraint c " @@ -7163,7 +7232,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " "t.reloptions AS indreloptions, " "'' AS indstatcols, " - "'' AS indstatvals " + "'' AS indstatvals, " + "' ' AS inddependoids, " + "' ' AS inddependversions " "FROM pg_catalog.pg_index i " "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " "LEFT JOIN pg_catalog.pg_constraint c " @@ -7194,7 +7265,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " "t.reloptions AS indreloptions, " "'' AS indstatcols, " - "'' AS indstatvals " + "'' AS indstatvals, " + "' ' AS inddependoids, " + "' ' AS inddependversions " "FROM pg_catalog.pg_index i " "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " "LEFT JOIN pg_catalog.pg_depend d " @@ -7228,7 +7301,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) "(SELECT spcname FROM pg_catalog.pg_tablespace s WHERE s.oid = t.reltablespace) AS tablespace, " "null AS indreloptions, " "'' AS indstatcols, " - "'' AS indstatvals " + "'' AS indstatvals, " + "' ' AS inddependoids, " + "' ' AS inddependversions " "FROM pg_catalog.pg_index i " "JOIN pg_catalog.pg_class t ON (t.oid = i.indexrelid) " "LEFT JOIN pg_catalog.pg_depend d " @@ -7268,6 +7343,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) i_indreloptions = PQfnumber(res, "indreloptions"); i_indstatcols = PQfnumber(res, "indstatcols"); i_indstatvals = PQfnumber(res, "indstatvals"); + i_inddependoids = PQfnumber(res, "inddependoids"); + i_inddependversions = PQfnumber(res, "inddependversions"); tbinfo->indexes = indxinfo = (IndxInfo *) pg_malloc(ntups * sizeof(IndxInfo)); @@ -7293,6 +7370,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables) indxinfo[j].indreloptions = pg_strdup(PQgetvalue(res, j, i_indreloptions)); indxinfo[j].indstatcols = pg_strdup(PQgetvalue(res, j, i_indstatcols)); indxinfo[j].indstatvals = pg_strdup(PQgetvalue(res, j, i_indstatvals)); + indxinfo[j].inddependoids = pg_strdup(PQgetvalue(res, j, i_inddependoids)); + indxinfo[j].inddependversions = pg_strdup(PQgetvalue(res, j, i_inddependversions)); indxinfo[j].indkeys = (Oid *) pg_malloc(indxinfo[j].indnattrs * sizeof(Oid)); parseOidArray(PQgetvalue(res, j, i_indkey), indxinfo[j].indkeys, indxinfo[j].indnattrs); @@ -16359,7 +16438,8 @@ dumpIndex(Archive *fout, IndxInfo *indxinfo) /* * If there's an associated constraint, don't dump the index per se, but - * do dump any comment for it. (This is safe because dependency ordering + * do dump any comment, or in binary upgrade mode dependency on a + * collation version for it. (This is safe because dependency ordering * will have ensured the constraint is emitted first.) Note that the * emitted comment has to be shown as depending on the constraint, not the * index, in such cases. @@ -16426,6 +16506,10 @@ dumpIndex(Archive *fout, IndxInfo *indxinfo) "pg_catalog.pg_class", "INDEX", qqindxname); + if (dopt->binary_upgrade) + appendIndexCollationVersion(q, indxinfo, fout->encoding, + dopt->unknown_coll_compat); + /* If the index defines identity, we need to record that. */ if (indxinfo->indisreplident) { @@ -16454,6 +16538,21 @@ dumpIndex(Archive *fout, IndxInfo *indxinfo) if (indstatvalsarray) free(indstatvalsarray); } + else if (dopt->binary_upgrade) + { + appendIndexCollationVersion(q, indxinfo, fout->encoding, + dopt->unknown_coll_compat); + + if (indxinfo->dobj.dump & DUMP_COMPONENT_DEFINITION) + ArchiveEntry(fout, indxinfo->dobj.catId, indxinfo->dobj.dumpId, + ARCHIVE_OPTS(.tag = indxinfo->dobj.name, + .namespace = tbinfo->dobj.namespace->dobj.name, + .tablespace = indxinfo->tablespace, + .owner = tbinfo->rolname, + .description = "INDEX", + .section = SECTION_POST_DATA, + .createStmt = q->data)); + } /* Dump Index Comments */ if (indxinfo->dobj.dump & DUMP_COMPONENT_COMMENT) @@ -18438,6 +18537,77 @@ nonemptyReloptions(const char *reloptions) return (reloptions != NULL && strlen(reloptions) > 2); } +/* + * Format inddependoids and inddependversions arrays and append it to the given + * buffer in the form of binary_upgrade_set_index_coll_version() calls. + */ +static void +appendIndexCollationVersion(PQExpBuffer buffer, IndxInfo *indxinfo, int enc, + int unknown_coll_compat) +{ + char *inddependoids = indxinfo->inddependoids; + char *inddependversions = indxinfo->inddependversions; + char **inddependoidsarray = NULL; + char **inddependversionsarray = NULL; + int ninddependoids; + int ninddependversions; + int i; + + /* + * for older versions that don't record the collation depndency, issue a + * statement to mark the collation version as unknown + */ + if (strcmp(inddependoids, " ") == 0) + { + /* + * do not issue UNKNOWN VERSION if caller specified that those are + * compatible + */ + if (unknown_coll_compat) + return; + + Assert(strcmp(inddependversions, " ") == 0); + + appendPQExpBufferStr(buffer, "\n-- For binary upgrade, restore dependent collation version.\n"); + appendPQExpBuffer(buffer, "SELECT " + "pg_catalog.binary_upgrade_set_index_coll_version(%d, NULL, '');", + indxinfo->dobj.catId.oid); + return; + } + + parsePGArray(inddependoids, &inddependoidsarray, &ninddependoids); + parsePGArray(inddependversions, &inddependversionsarray, &ninddependversions); + + Assert(ninddependoids == ninddependversions); + + for (i = 0; i < ninddependoids; i++) + { + /* + * If there was an unknown version dependency recorded for this + * collation and the caller asked to mark those as depending on the + * current version, don't emit a binary_upgrade_set_index_coll_version + * function call. + */ + if ((strcmp(inddependversionsarray[i], "''")) == 0 + && unknown_coll_compat) + { + continue; + } + + appendPQExpBufferStr(buffer, "\n-- For binary upgrade, restore dependent collation version.\n"); + appendPQExpBuffer(buffer, "SELECT " + "pg_catalog.binary_upgrade_set_index_coll_version(%d, %s, %s);", + indxinfo->dobj.catId.oid, + inddependoidsarray[i], + inddependversionsarray[i]); + } + + if (inddependoidsarray) + free(inddependoidsarray); + if (inddependversionsarray) + free(inddependversionsarray); +} + /* * Format a reloptions array and append it to the given buffer. * diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index e0b42e8391..623814d1c5 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -366,6 +366,9 @@ typedef struct _indxInfo int indnattrs; /* total number of index attributes */ Oid *indkeys; /* In spite of the name 'indkeys' this field * contains both key and nonkey attributes */ + char *inddependoids; /* oids of collation this index depends on */ + char *inddependversions; /* version of collation this index depends + * on */ bool indisclustered; bool indisreplident; Oid parentidx; /* if a partition, parent index OID */ diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index ec63662060..5f323efb1f 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -53,6 +53,24 @@ my %pgdump_runs = ( "$tempdir/binary_upgrade.dump", ], }, + binary_coll_compatible => { + dump_cmd => [ + 'pg_dump', + '--no-sync', + '--format=custom', + "--file=$tempdir/binary_coll_compatible.dump", + '-w', + '--schema-only', + '--binary-upgrade', + '--unknown-collations-binary-compatible', + '-d', 'postgres', # alternative way to specify database + ], + restore_cmd => [ + 'pg_restore', '-Fc', '--verbose', + "--file=$tempdir/binary_coll_compatible.sql", + "$tempdir/binary_coll_compatible.dump", + ], + }, clean => { dump_cmd => [ 'pg_dump', @@ -387,6 +405,7 @@ my %dump_test_schema_runs = ( # are flags used to exclude specific items (ACLs, blobs, etc). my %full_runs = ( binary_upgrade => 1, + binary_coll_compatible => 1, clean => 1, clean_if_exists => 1, createdb => 1, @@ -920,9 +939,10 @@ my %tests = ( test_schema_plus_blobs => 1, }, unlike => { - binary_upgrade => 1, - no_blobs => 1, - schema_only => 1, + binary_upgrade => 1, + binary_coll_compatible => 1, + no_blobs => 1, + schema_only => 1, }, }, @@ -1184,6 +1204,7 @@ my %tests = ( }, unlike => { binary_upgrade => 1, + binary_coll_compatible => 1, exclude_dump_test_schema => 1, exclude_test_table => 1, exclude_test_table_data => 1, @@ -1209,6 +1230,7 @@ my %tests = ( }, unlike => { binary_upgrade => 1, + binary_coll_compatible => 1, exclude_dump_test_schema => 1, schema_only => 1, }, @@ -1244,6 +1266,7 @@ my %tests = ( }, unlike => { binary_upgrade => 1, + binary_coll_compatible => 1, exclude_dump_test_schema => 1, schema_only => 1, }, @@ -1266,6 +1289,7 @@ my %tests = ( }, unlike => { binary_upgrade => 1, + binary_coll_compatible => 1, exclude_dump_test_schema => 1, schema_only => 1, }, @@ -1287,6 +1311,7 @@ my %tests = ( }, unlike => { binary_upgrade => 1, + binary_coll_compatible => 1, exclude_dump_test_schema => 1, schema_only => 1, }, @@ -1308,6 +1333,7 @@ my %tests = ( }, unlike => { binary_upgrade => 1, + binary_coll_compatible => 1, exclude_dump_test_schema => 1, schema_only => 1, }, @@ -1674,6 +1700,7 @@ my %tests = ( { %full_runs, %dump_test_schema_runs, section_pre_data => 1, }, unlike => { binary_upgrade => 1, + binary_coll_compatible => 1, exclude_dump_test_schema => 1, }, }, @@ -1688,7 +1715,7 @@ my %tests = ( \n.*^ \QALTER TYPE dump_test.planets ADD VALUE 'mars';\E \n/xms, - like => { binary_upgrade => 1, }, + like => { binary_upgrade => 1, binary_coll_compatible => 1, }, }, 'CREATE TYPE dump_test.textrange AS RANGE' => { @@ -2356,6 +2383,7 @@ my %tests = ( { %full_runs, %dump_test_schema_runs, section_pre_data => 1, }, unlike => { binary_upgrade => 1, + binary_coll_compatible => 1, exclude_dump_test_schema => 1, }, }, @@ -2549,6 +2577,7 @@ my %tests = ( }, unlike => { binary_upgrade => 1, + binary_coll_compatible => 1, exclude_dump_test_schema => 1, }, }, @@ -2617,6 +2646,7 @@ my %tests = ( /xm, like => { binary_upgrade => 1, + binary_coll_compatible => 1, clean => 1, clean_if_exists => 1, createdb => 1, @@ -2688,6 +2718,7 @@ my %tests = ( /xm, like => { binary_upgrade => 1, + binary_coll_compatible => 1, clean => 1, clean_if_exists => 1, createdb => 1, @@ -3155,6 +3186,7 @@ my %tests = ( { %full_runs, %dump_test_schema_runs, section_post_data => 1, }, unlike => { binary_upgrade => 1, + binary_coll_compatible => 1, exclude_dump_test_schema => 1, schema_only => 1, }, @@ -3170,6 +3202,7 @@ my %tests = ( { %full_runs, %dump_test_schema_runs, section_post_data => 1, }, unlike => { binary_upgrade => 1, + binary_coll_compatible => 1, exclude_dump_test_schema => 1, schema_only => 1, }, @@ -3302,16 +3335,53 @@ my %tests = ( %full_runs, %dump_test_schema_runs, section_pre_data => 1, }, unlike => { exclude_dump_test_schema => 1 }, + }, + + "binary_upgrade_set_index_coll_version(oid, oid, 'not_a_version')" => { + create_order => 101, + create_sql => ' + CREATE TABLE dump_test.regress_table_coll(id integer, val text); + CREATE INDEX regress_coll_idx1 ON dump_test.regress_table_coll(val COLLATE "fr-x-icu"); + UPDATE pg_depend SET refobjversion = \'not_a_version\' WHERE refobjversion IS NOT NULL AND objid::regclass::text = \'dump_test.regress_coll_idx1\';', + regexp => qr/^ + \QCREATE INDEX regress_coll_idx1 ON dump_test.regress_table_coll USING btree (val COLLATE "fr-x-icu");\E\n + \n + \Q-- For binary upgrade, restore dependent collation version.\E\n + \QSELECT pg_catalog.binary_upgrade_set_index_coll_version\E \(\d+,\ \d+,\ 'not_a_version'\);/xm, + like => { binary_upgrade => 1, binary_coll_compatible => 1, }, + icu => 1, + }, + "binary_upgrade_set_index_coll_version(?, ?, '')" => { + create_order => 102, + create_sql => ' + CREATE TABLE dump_test.regress_table_coll_no_ver(id integer, val text); + CREATE INDEX regress_coll_no_ver_idx1 ON dump_test.regress_table_coll_no_ver(val COLLATE "fr-x-icu"); + UPDATE pg_depend SET refobjversion = \'\' WHERE refobjversion IS NOT NULL AND objid::regclass::text = \'dump_test.regress_coll_no_ver_idx1\';', + regexp => qr/SELECT pg_catalog.binary_upgrade_set_index_coll_version\(\d+, \d+, ''\)/, + like => { binary_upgrade => 1}, + # should not appear in binary_coll_compatible case! + unlike => { binary_coll_compatible => 1}, + icu => 1, }); ######################################### # Create a PG instance to test actually dumping from -my $node = get_new_node('main'); -$node->init; -$node->start; +my $main_node = get_new_node('main'); +$main_node->init; +$main_node->start; + +my $port = $main_node->port; + +# And another instance to validate the binary dump +my $bin_node = get_new_node('binary'); +$bin_node->init; +$bin_node->start; -my $port = $node->port; +my $bin_port = $bin_node->port; + +# and add a $node variable pointing to main_node for now +my $node = $main_node; # We need to see if this system supports CREATE COLLATION or not # If it doesn't then we will skip all the COLLATION-related tests. @@ -3335,6 +3405,10 @@ $node->psql('postgres', 'create database regress_pg_dump_test;'); # command_fails_like is actually 2 tests) my $num_tests = 12; +# 4 more tests for restoring globals and binary_upgrade dump, dumping it again +# and regenerating the sql file +$num_tests+= 4; + foreach my $run (sort keys %pgdump_runs) { my $test_key = $run; @@ -3385,16 +3459,29 @@ foreach my $run (sort keys %pgdump_runs) next; } + # Skip any icu-related commands if there is no icu support + if ($ENV{with_icu} ne 'yes' && defined($tests{$test}->{icu})) + { + next; + } + # If there is a like entry, but no unlike entry, then we will test the like case if ($tests{$test}->{like}->{$test_key} && !defined($tests{$test}->{unlike}->{$test_key})) { $num_tests++; + + # binary_upgrade tests are also run after being restored and + # re-dumped. + $num_tests++ if ($test_key eq 'binary_upgrade'); } else { # We will test everything that isn't a 'like' $num_tests++; + # binary_upgrade tests are also run after being restored and + # re-dumped. + $num_tests++ if ($test_key eq 'binary_upgrade'); } } } @@ -3442,6 +3529,12 @@ foreach my $test ( next; } + # Skip any icu-related commands if there is no icu support + if ($ENV{with_icu} ne 'yes' && defined($tests{$test}->{icu})) + { + next; + } + # Add terminating semicolon $create_sql{$test_db} .= $tests{$test}->{create_sql} . ";"; } @@ -3495,79 +3588,116 @@ command_fails_like( ######################################### # Run all runs -foreach my $run (sort keys %pgdump_runs) +foreach my $pass (1, 2) { - my $test_key = $run; - my $run_db = 'postgres'; - - $node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} }, - "$run: pg_dump runs"); - - if ($pgdump_runs{$run}->{restore_cmd}) + foreach my $run (sort keys %pgdump_runs) { - $node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} }, - "$run: pg_restore runs"); - } - - if ($pgdump_runs{$run}->{test_key}) - { - $test_key = $pgdump_runs{$run}->{test_key}; - } - - my $output_file = slurp_file("$tempdir/${run}.sql"); + my $test_key = $run; + my $run_db = 'postgres'; - ######################################### - # Run all tests where this run is included - # as either a 'like' or 'unlike' test. + # we only test binary upgrade on the 2nd pass + next if ($pass == 2 and $test_key ne 'binary_upgrade'); - foreach my $test (sort keys %tests) - { - my $test_db = 'postgres'; + $node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} }, + "$run: pg_dump runs"); - if (defined($pgdump_runs{$run}->{database})) + if ($pgdump_runs{$run}->{restore_cmd}) { - $run_db = $pgdump_runs{$run}->{database}; + $node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} }, + "$run: pg_restore runs"); } - if (defined($tests{$test}->{database})) + if ($pgdump_runs{$run}->{test_key}) { - $test_db = $tests{$test}->{database}; + $test_key = $pgdump_runs{$run}->{test_key}; } - # Skip any collation-related commands if there is no collation support - if (!$collation_support && defined($tests{$test}->{collation})) - { - next; - } + my $output_file = slurp_file("$tempdir/${run}.sql"); - if ($run_db ne $test_db) - { - next; - } + ######################################### + # Run all tests where this run is included + # as either a 'like' or 'unlike' test. - # Run the test listed as a like, unless it is specifically noted - # as an unlike (generally due to an explicit exclusion or similar). - if ($tests{$test}->{like}->{$test_key} - && !defined($tests{$test}->{unlike}->{$test_key})) + foreach my $test (sort keys %tests) { - if (!ok($output_file =~ $tests{$test}->{regexp}, - "$run: should dump $test")) + my $test_db = 'postgres'; + + if (defined($pgdump_runs{$run}->{database})) { - diag("Review $run results in $tempdir"); + $run_db = $pgdump_runs{$run}->{database}; } - } - else - { - if (!ok($output_file !~ $tests{$test}->{regexp}, - "$run: should not dump $test")) + + if (defined($tests{$test}->{database})) + { + $test_db = $tests{$test}->{database}; + } + + # Skip any collation-related commands if there is no collation support + if (!$collation_support && defined($tests{$test}->{collation})) + { + next; + } + + # Skip any icu-related commands if there is no icu support + if ($ENV{with_icu} ne 'yes' && defined($tests{$test}->{icu})) + { + next; + } + + if ($run_db ne $test_db) + { + next; + } + + # Run the test listed as a like, unless it is specifically noted + # as an unlike (generally due to an explicit exclusion or similar). + if ($tests{$test}->{like}->{$test_key} + && !defined($tests{$test}->{unlike}->{$test_key})) { - diag("Review $run results in $tempdir"); + if (!ok($output_file =~ $tests{$test}->{regexp}, + "$run: should dump $test")) + { + diag("Review $run results in $tempdir"); + } + } + else + { + if (!ok($output_file !~ $tests{$test}->{regexp}, + "$run: should not dump $test")) + { + diag("Review $run results in $tempdir"); + } } } } + + # After all dump have been generated, restore the binary_upgrade dump with + # the required global objects on a suitable node, and continue with the 2nd + # pass. + if ($pass == 1) + { + # Stop the original database instance as we don't need it anymore. + $node->stop('fast'); + + $bin_node->command_ok(\@{['psql', + "-d", "postgres", "-f", "$tempdir/pg_dumpall_globals.sql"]}, + "Restore globals"); + + $bin_node->stop('fast'); + $bin_node->start(binary_start => 1); + $bin_node->command_ok(\@{['pg_restore', '-p', $bin_port, + '-d', 'postgres', + "$tempdir/binary_upgrade.dump"]}, + "Restore the binary_upgrade dump"); + $bin_node->stop('fast'); + $bin_node->start; + + # And change $node to point to the freshly restored node. + $node = $bin_node; + } } ######################################### # Stop the database instance, which will be removed at the end of the tests. -$node->stop('fast'); +$bin_node->stop('fast'); diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c index 4d730adfe2..672ecda169 100644 --- a/src/bin/pg_upgrade/dump.c +++ b/src/bin/pg_upgrade/dump.c @@ -52,9 +52,11 @@ generate_old_dump(void) parallel_exec_prog(log_file_name, NULL, "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers " - "--binary-upgrade --format=custom %s --file=\"%s\" %s", + "--binary-upgrade --format=custom %s %s --file=\"%s\" %s", new_cluster.bindir, cluster_conn_opts(&old_cluster), log_opts.verbose ? "--verbose" : "", + user_opts.coll_compat ? + "--unknown-collations-binary-compatible" : "", sql_file_name, escaped_connstr.data); termPQExpBuffer(&escaped_connstr); diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c index aca1ee8b48..c7e291f7e7 100644 --- a/src/bin/pg_upgrade/option.c +++ b/src/bin/pg_upgrade/option.c @@ -56,6 +56,7 @@ parseCommandLine(int argc, char *argv[]) {"socketdir", required_argument, NULL, 's'}, {"verbose", no_argument, NULL, 'v'}, {"clone", no_argument, NULL, 1}, + {"collation-binary-compatible", no_argument, NULL, 2}, {NULL, 0, NULL, 0} }; @@ -203,6 +204,10 @@ parseCommandLine(int argc, char *argv[]) user_opts.transfer_mode = TRANSFER_MODE_CLONE; break; + case 2: + user_opts.coll_compat = true; + break; + default: fprintf(stderr, _("Try \"%s --help\" for more information.\n"), os_info.progname); @@ -307,6 +312,8 @@ usage(void) printf(_(" -v, --verbose enable verbose internal logging\n")); printf(_(" -V, --version display version information, then exit\n")); printf(_(" --clone clone instead of copying files to new cluster\n")); + printf(_(" --collation-binary-compatible mark collations as depending on current collation\n" + " versions rather than unknown if they're unknown\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\n" "Before running pg_upgrade you must:\n" diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 8b90cefbe0..68e637ce8a 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -292,6 +292,8 @@ typedef struct transferMode transfer_mode; /* copy files or link them? */ int jobs; /* number of processes/threads to use */ char *socketdir; /* directory to use for Unix sockets */ + bool coll_compat; /* should we skip marking index collations as + * unknown version */ } UserOpts; typedef struct diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index b2b4f1fd4d..912eb7409f 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -45,6 +45,7 @@ #include "catalog/pg_am_d.h" #include "catalog/pg_class_d.h" +#include "catalog/pg_collation_d.h" #include "common.h" #include "libpq-fe.h" #include "pqexpbuffer.h" @@ -820,6 +821,20 @@ static const SchemaQuery Query_for_list_of_statistics = { " (SELECT tgrelid FROM pg_catalog.pg_trigger "\ " WHERE pg_catalog.quote_ident(tgname)='%s')" +/* the silly-looking length condition is just to eat up the current word */ +#define Query_for_list_of_colls_for_one_index \ +" SELECT DISTINCT pg_catalog.quote_ident(coll.collname) " \ +" FROM pg_catalog.pg_depend d, pg_catalog.pg_collation coll, " \ +" pg_catalog.pg_class c" \ +" WHERE (%d = pg_catalog.length('%s'))" \ +" AND d.refclassid = " CppAsString2(CollationRelationId) \ +" AND d.refobjid = coll.oid " \ +" AND d.classid = " CppAsString2(RelationRelationId) \ +" AND d.objid = c.oid " \ +" AND c.relkind = " CppAsString2(RELKIND_INDEX) \ +" AND pg_catalog.pg_table_is_visible(c.oid) " \ +" AND c.relname = '%s'" + #define Query_for_list_of_ts_configurations \ "SELECT pg_catalog.quote_ident(cfgname) FROM pg_catalog.pg_ts_config "\ " WHERE substring(pg_catalog.quote_ident(cfgname),1,%d)='%s'" @@ -1715,7 +1730,8 @@ psql_completion(const char *text, int start, int end) /* ALTER INDEX */ else if (Matches("ALTER", "INDEX", MatchAny)) COMPLETE_WITH("ALTER COLUMN", "OWNER TO", "RENAME TO", "SET", - "RESET", "ATTACH PARTITION", "DEPENDS", "NO DEPENDS"); + "RESET", "ATTACH PARTITION", "DEPENDS", "NO DEPENDS", + "ALTER COLLATION"); else if (Matches("ALTER", "INDEX", MatchAny, "ATTACH")) COMPLETE_WITH("PARTITION"); else if (Matches("ALTER", "INDEX", MatchAny, "ATTACH", "PARTITION")) @@ -1765,6 +1781,15 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("ON EXTENSION"); else if (Matches("ALTER", "INDEX", MatchAny, "DEPENDS")) COMPLETE_WITH("ON EXTENSION"); + /* ALTER INDEX ALTER COLLATION */ + else if (Matches("ALTER", "INDEX", MatchAny, "ALTER", "COLLATION")) + { + completion_info_charp = prev4_wd; + COMPLETE_WITH_QUERY(Query_for_list_of_colls_for_one_index); + } + /* ALTER INDEX ALTER COLLATION */ + else if (Matches("ALTER", "INDEX", MatchAny, "ALTER", "COLLATION", MatchAny)) + COMPLETE_WITH("REFRESH VERSION"); /* ALTER LANGUAGE */ else if (Matches("ALTER", "LANGUAGE", MatchAny)) diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index 3baa5e498a..08834194d5 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -160,7 +160,8 @@ extern void recordDependencyOnSingleRelExpr(const ObjectAddress *depender, Node *expr, Oid relId, DependencyType behavior, DependencyType self_behavior, - bool reverse_self); + bool reverse_self, + bool track_version); extern ObjectClass getObjectClass(const ObjectAddress *object); @@ -180,17 +181,29 @@ extern void sort_object_addresses(ObjectAddresses *addrs); extern void free_object_addresses(ObjectAddresses *addrs); +typedef char *(*VisitDependenciesOfCB) (const ObjectAddress *otherObject, + const char *version, + void *data); + +extern void visitDependenciesOf(const ObjectAddress *object, + VisitDependenciesOfCB callback, + void *data); + /* in pg_depend.c */ extern void recordDependencyOn(const ObjectAddress *depender, const ObjectAddress *referenced, DependencyType behavior); +extern void recordDependencyOnCollations(ObjectAddress *myself, + List *collations, + bool record_version); + extern void recordMultipleDependencies(const ObjectAddress *depender, const ObjectAddress *referenced, int nreferenced, - const char *version, - DependencyType behavior); + DependencyType behavior, + bool track_version); extern void recordDependencyOnCurrentExtension(const ObjectAddress *object, bool isReplace); @@ -209,10 +222,9 @@ extern long changeDependencyFor(Oid classId, Oid objectId, Oid refClassId, Oid oldRefObjectId, Oid newRefObjectId); -extern long changeDependenciesOf(Oid classId, Oid oldObjectId, +long changeDependenciesOf(Oid classId, Oid oldObjectId, Oid newObjectId); - -extern long changeDependenciesOn(Oid refClassId, Oid oldRefObjectId, +long changeDependenciesOn(Oid refClassId, Oid oldRefObjectId, Oid newRefObjectId); extern Oid getExtensionOfObject(Oid classId, Oid objectId); diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h index f58e8675f3..3e205ab068 100644 --- a/src/include/catalog/index.h +++ b/src/include/catalog/index.h @@ -121,6 +121,8 @@ extern void FormIndexDatum(IndexInfo *indexInfo, Datum *values, bool *isnull); +extern void index_check_collation_versions(Oid relid); + extern void index_build(Relation heapRelation, Relation indexRelation, IndexInfo *indexInfo, @@ -131,6 +133,10 @@ extern void validate_index(Oid heapId, Oid indexId, Snapshot snapshot); extern void index_set_state_flags(Oid indexId, IndexStateFlagsAction action); +extern void index_update_collation_versions(Oid relid, + Oid coll, + char *import_version); + extern Oid IndexGetRelation(Oid indexId, bool missing_ok); extern void reindex_index(Oid indexId, bool skip_constraint_checks, diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index bbcac69d48..7abfa4d65d 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -10387,6 +10387,10 @@ proname => 'binary_upgrade_set_missing_value', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => 'oid text text', prosrc => 'binary_upgrade_set_missing_value' }, +{ oid => '8178', descr => 'for use by pg_upgrade', + proname => 'binary_upgrade_set_index_coll_version', provolatile => 'v', + proparallel => 'u', prorettype => 'void', proargtypes => 'oid oid text', + prosrc => 'binary_upgrade_set_index_coll_version' }, # conversion functions { oid => '4302', diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 7b37562648..f3061bca57 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -361,6 +361,8 @@ extern void GenerateTypeDependencies(HeapTuple typeTuple, bool isDependentType, bool rebuild); +extern List *GetTypeCollations(Oid typeObjectid); + extern void RenameTypeInternal(Oid typeOid, const char *newTypeName, Oid typeNamespace); diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 32dc7cd5ef..266519096e 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1849,7 +1849,8 @@ typedef enum AlterTableType AT_DetachPartition, /* DETACH PARTITION */ AT_AddIdentity, /* ADD IDENTITY */ AT_SetIdentity, /* SET identity column options */ - AT_DropIdentity /* DROP IDENTITY */ + AT_DropIdentity, /* DROP IDENTITY */ + AT_AlterCollationRefreshVersion /* ALTER COLLATION ... REFRESH VERSION */ } AlterTableType; typedef struct ReplicaIdentityStmt @@ -1865,6 +1866,7 @@ typedef struct AlterTableCmd /* one subcommand of an ALTER TABLE */ AlterTableType subtype; /* Type of table alteration to apply */ char *name; /* column, constraint, or trigger to act on, * or tablespace */ + List *object; /* collation to act on if it's a collation */ int16 num; /* attribute number for columns referenced by * number */ RoleSpec *newowner; diff --git a/src/include/utils/pg_locale.h b/src/include/utils/pg_locale.h index 9cb7d91ddf..96da132c03 100644 --- a/src/include/utils/pg_locale.h +++ b/src/include/utils/pg_locale.h @@ -103,7 +103,7 @@ typedef struct pg_locale_struct *pg_locale_t; extern pg_locale_t pg_newlocale_from_collation(Oid collid); -extern char *get_collation_actual_version(char collprovider, const char *collcollate); +extern char *get_collation_version_for_oid(Oid collid); #ifdef USE_ICU extern int32_t icu_to_uchar(UChar **buff_uchar, const char *buff, size_t nbytes); diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index 0b5957ba02..c5ffea40f2 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -63,6 +63,7 @@ typedef struct RelationData bool rd_indexvalid; /* is rd_indexlist valid? (also rd_pkindex and * rd_replidindex) */ bool rd_statvalid; /* is rd_statlist valid? */ + bool rd_version_checked; /* has version check been done yet? */ /*---------- * rd_createSubid is the ID of the highest subtransaction the rel has diff --git a/src/test/Makefile b/src/test/Makefile index 9774f534d9..14cde4f5ba 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -12,7 +12,8 @@ subdir = src/test top_builddir = ../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = perl regress isolation modules authentication recovery subscription +SUBDIRS = perl regress isolation modules authentication recovery subscription \ + locale # Test suites that are not safe by default but can be run if selected # by the user via the whitespace-separated list in variable diff --git a/src/test/locale/.gitignore b/src/test/locale/.gitignore index 620d3df425..64e1bf2a80 100644 --- a/src/test/locale/.gitignore +++ b/src/test/locale/.gitignore @@ -1 +1,2 @@ /test-ctype +/tmp_check/ diff --git a/src/test/locale/Makefile b/src/test/locale/Makefile index 22a45b65f2..73495cf16b 100644 --- a/src/test/locale/Makefile +++ b/src/test/locale/Makefile @@ -4,6 +4,7 @@ subdir = src/test/locale top_builddir = ../../.. include $(top_builddir)/src/Makefile.global +export with_icu PROGS = test-ctype DIRS = de_DE.ISO8859-1 gr_GR.ISO8859-7 koi8-r koi8-to-win1251 @@ -19,3 +20,9 @@ clean distclean maintainer-clean: # These behave like installcheck targets. check-%: all @$(MAKE) -C `echo $@ | sed 's/^check-//'` test + +check: + $(prove_check) + +installcheck: + $(prove_installcheck) diff --git a/src/test/locale/t/001_index.pl b/src/test/locale/t/001_index.pl new file mode 100644 index 0000000000..468fbb63b6 --- /dev/null +++ b/src/test/locale/t/001_index.pl @@ -0,0 +1,76 @@ +use strict; +use warnings; + +use Config; +use PostgresNode; +use TestLib; +use Test::More; + +if ($ENV{with_icu} eq 'yes') +{ + plan tests => 12; +} +else +{ + plan skip_all => 'ICU not supported by this build'; +} + +#### Set up the server + +note "setting up data directory"; +my $node = get_new_node('main'); +$node->init; + +$ENV{PGHOST} = $node->host; +$ENV{PGPORT} = $node->port; +$node->start; + +sub test_index +{ + my ($err_like, $err_comm) = @_; + + my ($ret, $out, $err) = $node->psql('postgres', + 'SET enable_seqscan = 0;' + . "EXPLAIN SELECT val FROM icu1 WHERE val = '0'"); + + is($ret, 0, 'EXPLAIN should succeed.'); + like($out, qr/icu1_fr/, 'Index icu1_fr should be used.'); + like($err, $err_like, $err_comm); +} + +$node->safe_psql('postgres', + 'CREATE TABLE icu1(val text);' + . 'INSERT INTO icu1 SELECT i::text FROM generate_series(1, 10000) i;' + . 'CREATE INDEX icu1_fr ON icu1 (val COLLATE "fr-x-icu");'); +$node->safe_psql('postgres', 'VACUUM ANALYZE icu1;'); + +test_index(qr/^$/, 'No warning should be raised'); + +# Simulate different collation version +$node->safe_psql('postgres', + "UPDATE pg_depend SET refobjversion = 'not_a_version'" + . " WHERE refobjversion IS NOT NULL" + . " AND objid::regclass::text = 'icu1_fr';"); + +test_index(qr/index "icu1_fr" depends on collation "fr-x-icu" version "not_a_version", but the current version is/, + 'Different collation version warning should be raised.'); + +# Simulate unknown collation version +$node->safe_psql('postgres', + "UPDATE pg_depend SET refobjversion = ''" + . " WHERE refobjversion IS NOT NULL" + . " AND objid::regclass::text = 'icu1_fr';"); + +test_index(qr/index "icu1_fr" depends on collation "fr-x-icu" with an unknown version, and the current version is/, + 'Unknown collation version warning should be raised.'); + +# Simulate previously unhandled collation versioning +$node->safe_psql('postgres', + "UPDATE pg_depend SET refobjversion = NULL" + . " WHERE refobjversion IS NOT NULL" + . " AND objid::regclass::text = 'icu1_fr';"); + +test_index(qr/index "icu1_fr" depends on collation "fr-x-icu" with an unknown version, and the current version is/, + 'Unknown collation version warning should be raised.'); + +$node->stop; diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index ebcaeb44fe..c1b5d4aabd 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -769,10 +769,14 @@ sub start local %ENV = %ENV; delete $ENV{PGAPPNAME}; + my $options = "--cluster-name=$name"; + + $options .= ' -b' if ($params{binary_start}); + # Note: We set the cluster_name here, not in postgresql.conf (in # sub init) so that it does not get copied to standbys. $ret = TestLib::system_log('pg_ctl', '-D', $self->data_dir, '-l', - $self->logfile, '-o', "--cluster-name=$name", 'start'); + $self->logfile, '-o', $options, 'start'); } if ($ret != 0) diff --git a/src/test/regress/expected/collate.icu.utf8.out b/src/test/regress/expected/collate.icu.utf8.out index 60d9263a2f..da5a03663d 100644 --- a/src/test/regress/expected/collate.icu.utf8.out +++ b/src/test/regress/expected/collate.icu.utf8.out @@ -1897,6 +1897,217 @@ SELECT (SELECT count(*) FROM test33_0) <> (SELECT count(*) FROM test33_1); t (1 row) +-- collation versioning support +CREATE TYPE t_en_fr AS (fr text COLLATE "fr-x-icu", en text COLLATE "en-x-icu"); +CREATE DOMAIN d_en_fr AS t_en_fr; +CREATE DOMAIN d_es AS text COLLATE "es-x-icu"; +CREATE TYPE t_en_fr_ga AS (en_fr t_en_fr, ga text COLLATE "ga-x-icu"); +CREATE DOMAIN d_en_fr_ga AS t_en_fr_ga; +CREATE TYPE t_custom AS (meh text, meh2 text); +CREATE DOMAIN d_custom AS t_custom; +CREATE COLLATION custom ( + LOCALE = 'fr-x-icu', PROVIDER = 'icu' +); +CREATE TYPE myrange AS range (subtype = text); +CREATE TYPE myrange_en_fr_ga AS range(subtype = t_en_fr_ga); +CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); +CREATE TABLE collate_test ( + id integer, + val text COLLATE "fr-x-icu", + t_en_fr t_en_fr, + d_en_fr d_en_fr, + d_es d_es, + t_en_fr_ga t_en_fr_ga, + d_en_fr_ga d_en_fr_ga, + d_en_fr_ga_arr d_en_fr_ga[], + myrange myrange, + myrange_en_fr_ga myrange_en_fr_ga, + mood mood +); +CREATE INDEX icuidx00_val ON collate_test(val); +-- shouldn't get duplicated dependencies +CREATE INDEX icuidx00_val_val ON collate_test(val, val); +-- shouldn't track version +CREATE INDEX icuidx00_val_pattern ON collate_test(val text_pattern_ops); +-- should have single dependency, no version tracked +CREATE INDEX icuidx00_val_pattern_val_pattern ON collate_test(val text_pattern_ops, val text_pattern_ops); +-- should have single dependency, with version tracked +CREATE INDEX icuidx00_val_pattern_val ON collate_test(val text_pattern_ops, val); +-- should have single dependency, with version tracked +CREATE INDEX icuidx00_val_val_pattern ON collate_test(val, val text_pattern_ops); +-- should have single dependency, with version tracked +CREATE INDEX icuidx00_val_pattern_where ON collate_test(val text_pattern_ops) WHERE val >= val; +-- should have single dependency, with version tracked +CREATE INDEX icuidx00_val_where ON collate_test(val) WHERE val >= val; +-- should ideally have single dependency, no version tracked, but expression walker will find a dependency on the collation and will ask to track the version +CREATE INDEX icuidx00_val_pattern_expr_pattern ON collate_test(val varchar_pattern_ops, (val || val) text_pattern_ops); +-- should have single dependency, with version tracked +CREATE INDEX icuidx00_val_pattern_expr ON collate_test(val varchar_pattern_ops, (val || val)); +CREATE INDEX icuidx01_t_en_fr__d_es ON collate_test (t_en_fr, d_es); +CREATE INDEX icuidx02_d_en_fr ON collate_test (d_en_fr); +CREATE INDEX icuidx03_t_en_fr_ga ON collate_test (t_en_fr_ga); +CREATE INDEX icuidx04_d_en_fr_ga ON collate_test (d_en_fr_ga); +CREATE INDEX icuidx05_d_en_fr_ga_arr ON collate_test (d_en_fr_ga_arr); +CREATE INDEX icuidx06_d_en_fr_ga ON collate_test(id) WHERE (d_en_fr_ga).en_fr.fr = 'foo'; +CREATE INDEX icuidx07_d_en_fr_ga ON collate_test(id) WHERE (d_en_fr_ga).ga = 'foo'; +CREATE INDEX icuidx08_d_en_fr_ga ON collate_test(id) WHERE (t_en_fr_ga) = ('foo', 'bar', 'baz'); +CREATE INDEX icuidx09_d_en_fr_ga ON collate_test(id) WHERE (d_en_fr_ga) = ('foo', 'bar', 'baz'); +CREATE INDEX icuidx10_d_en_fr_ga_es ON collate_test(id) WHERE (d_en_fr_ga) = ('foo', 'bar', 'baz' COLLATE "es-x-icu"); +CREATE INDEX icuidx11_d_es ON collate_test(id) WHERE (d_es) = ('foo'); +CREATE INDEX icuidx12_custom ON collate_test(id) WHERE ('foo', 'bar')::d_custom = ('foo', 'bar' COLLATE custom)::d_custom; +CREATE INDEX icuidx13_custom ON collate_test(id) WHERE ('foo' COLLATE custom, 'bar')::d_custom = ('foo', 'bar')::d_custom; +CREATE INDEX icuidx14_myrange ON collate_test(myrange); +CREATE INDEX icuidx15_myrange_en_fr_ga ON collate_test USING gist (myrange_en_fr_ga); +CREATE INDEX icuidx16_mood ON collate_test(id) WHERE mood > 'ok' COLLATE "fr-x-icu"; +CREATE TABLE collate_part(id integer, val text COLLATE "en-x-icu") PARTITION BY range(id); +CREATE TABLE collate_part_0 PARTITION OF collate_part FOR VALUES FROM (0) TO (1); +CREATE TABLE collate_part_1 PARTITION OF collate_part FOR VALUES FROM (1) TO (1000000); +CREATE INDEX icuidx17_part ON collate_part_1 (val); +-- for key columns, hash indexes should record dependency on the collation but +-- not the version +CREATE INDEX icuidx18_hash_d_es ON collate_test USING hash (d_es); +CREATE INDEX icuidx19_hash_id_d_es_eq ON collate_test USING hash (id) WHERE (d_es) = 'foo'; +CREATE INDEX icuidx20_hash_id_d_es_lt ON collate_test USING hash (id) WHERE (d_es) < 'foo'; +SELECT objid::regclass, refobjid::regcollation, +CASE +WHEN refobjversion IS NULL THEN 'version not tracked' +ELSE CASE + WHEN refobjversion = pg_collation_actual_version(refobjid) THEN 'up to date' + ELSE 'out of date' + END +END AS version +FROM pg_depend d +LEFT JOIN pg_class c ON c.oid = d.objid +WHERE refclassid = 'pg_collation'::regclass +AND coalesce(relkind, 'i') = 'i' +AND relname LIKE 'icuidx%' +ORDER BY 1, 2; + objid | refobjid | version +-----------------------------------+------------+--------------------- + icuidx00_val | "fr-x-icu" | up to date + icuidx00_val_val | "fr-x-icu" | up to date + icuidx00_val_pattern | "fr-x-icu" | version not tracked + icuidx00_val_pattern_val_pattern | "fr-x-icu" | version not tracked + icuidx00_val_pattern_val | "fr-x-icu" | up to date + icuidx00_val_val_pattern | "fr-x-icu" | up to date + icuidx00_val_pattern_where | "fr-x-icu" | version not tracked + icuidx00_val_pattern_where | "fr-x-icu" | up to date + icuidx00_val_where | "fr-x-icu" | up to date + icuidx00_val_where | "fr-x-icu" | up to date + icuidx00_val_pattern_expr_pattern | "fr-x-icu" | up to date + icuidx00_val_pattern_expr_pattern | "fr-x-icu" | version not tracked + icuidx00_val_pattern_expr | "fr-x-icu" | up to date + icuidx00_val_pattern_expr | "fr-x-icu" | up to date + icuidx01_t_en_fr__d_es | "en-x-icu" | up to date + icuidx01_t_en_fr__d_es | "es-x-icu" | up to date + icuidx01_t_en_fr__d_es | "fr-x-icu" | up to date + icuidx02_d_en_fr | "en-x-icu" | up to date + icuidx02_d_en_fr | "fr-x-icu" | up to date + icuidx03_t_en_fr_ga | "en-x-icu" | up to date + icuidx03_t_en_fr_ga | "fr-x-icu" | up to date + icuidx03_t_en_fr_ga | "ga-x-icu" | up to date + icuidx04_d_en_fr_ga | "en-x-icu" | up to date + icuidx04_d_en_fr_ga | "fr-x-icu" | up to date + icuidx04_d_en_fr_ga | "ga-x-icu" | up to date + icuidx05_d_en_fr_ga_arr | "en-x-icu" | up to date + icuidx05_d_en_fr_ga_arr | "fr-x-icu" | up to date + icuidx05_d_en_fr_ga_arr | "ga-x-icu" | up to date + icuidx06_d_en_fr_ga | "default" | up to date + icuidx06_d_en_fr_ga | "en-x-icu" | up to date + icuidx06_d_en_fr_ga | "fr-x-icu" | up to date + icuidx06_d_en_fr_ga | "ga-x-icu" | up to date + icuidx07_d_en_fr_ga | "default" | up to date + icuidx07_d_en_fr_ga | "en-x-icu" | up to date + icuidx07_d_en_fr_ga | "fr-x-icu" | up to date + icuidx07_d_en_fr_ga | "ga-x-icu" | up to date + icuidx08_d_en_fr_ga | "en-x-icu" | up to date + icuidx08_d_en_fr_ga | "fr-x-icu" | up to date + icuidx08_d_en_fr_ga | "ga-x-icu" | up to date + icuidx09_d_en_fr_ga | "en-x-icu" | up to date + icuidx09_d_en_fr_ga | "fr-x-icu" | up to date + icuidx09_d_en_fr_ga | "ga-x-icu" | up to date + icuidx10_d_en_fr_ga_es | "en-x-icu" | up to date + icuidx10_d_en_fr_ga_es | "es-x-icu" | up to date + icuidx10_d_en_fr_ga_es | "fr-x-icu" | up to date + icuidx10_d_en_fr_ga_es | "ga-x-icu" | up to date + icuidx11_d_es | "default" | up to date + icuidx11_d_es | "es-x-icu" | up to date + icuidx12_custom | "default" | up to date + icuidx12_custom | custom | up to date + icuidx13_custom | "default" | up to date + icuidx13_custom | custom | up to date + icuidx14_myrange | "default" | up to date + icuidx15_myrange_en_fr_ga | "en-x-icu" | up to date + icuidx15_myrange_en_fr_ga | "fr-x-icu" | up to date + icuidx15_myrange_en_fr_ga | "ga-x-icu" | up to date + icuidx16_mood | "fr-x-icu" | up to date + icuidx17_part | "en-x-icu" | up to date + icuidx18_hash_d_es | "es-x-icu" | up to date + icuidx19_hash_id_d_es_eq | "default" | up to date + icuidx19_hash_id_d_es_eq | "es-x-icu" | up to date + icuidx20_hash_id_d_es_lt | "default" | up to date + icuidx20_hash_id_d_es_lt | "es-x-icu" | up to date +(63 rows) + +-- Validate that REINDEX will update the stored version. +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text LIKE 'icuidx%' +AND refobjversion IS NOT NULL; +REINDEX TABLE collate_test; +REINDEX TABLE collate_part_0; +REINDEX TABLE collate_part_1; +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + objid +------- +(0 rows) + +-- Validate that REINDEX CONCURRENTLY will update the stored version. +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text LIKE 'icuidx%' +AND refobjversion IS NOT NULL; +REINDEX TABLE CONCURRENTLY collate_test; +REINDEX TABLE CONCURRENTLY collate_part_0; +REINDEX INDEX CONCURRENTLY icuidx17_part; +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + objid +------- +(0 rows) + +-- Validate that VACUUM FULL will update the stored version. +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text LIKE 'icuidx%' +AND refobjversion IS NOT NULL; +VACUUM FULL collate_test; +VACUUM FULL collate_part_0; +VACUUM FULL collate_part_1; +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + objid +------- +(0 rows) + +-- Test ALTER INDEX name ALTER COLLATION name REFRESH VERSION +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text = 'icuidx17_part' +AND refobjversion IS NOT NULL; +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + objid +--------------- + icuidx17_part +(1 row) + +ALTER INDEX icuidx17_part ALTER COLLATION "en-x-icu" REFRESH VERSION; +SELECT objid::regclass, refobjversion = 'not a version' AS ver FROM pg_depend +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text = 'icuidx17_part'; + objid | ver +---------------+----- + icuidx17_part | f +(1 row) + -- cleanup RESET search_path; SET client_min_messages TO warning; diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index 6ace7662ee..6cb7786e13 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2065,14 +2065,16 @@ WHERE classid = 'pg_class'::regclass AND obj | objref | deptype ------------------------------------------+------------------------------------------------------------+--------- index concur_reindex_ind1 | constraint concur_reindex_ind1 on table concur_reindex_tab | i + index concur_reindex_ind2 | collation "default" | n index concur_reindex_ind2 | column c2 of table concur_reindex_tab | a index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a index concur_reindex_ind3 | table concur_reindex_tab | a + index concur_reindex_ind4 | collation "default" | n index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a index concur_reindex_ind4 | column c2 of table concur_reindex_tab | a materialized view concur_reindex_matview | schema public | n table concur_reindex_tab | schema public | n -(8 rows) +(10 rows) REINDEX INDEX CONCURRENTLY concur_reindex_ind1; REINDEX TABLE CONCURRENTLY concur_reindex_tab; @@ -2092,14 +2094,16 @@ WHERE classid = 'pg_class'::regclass AND obj | objref | deptype ------------------------------------------+------------------------------------------------------------+--------- index concur_reindex_ind1 | constraint concur_reindex_ind1 on table concur_reindex_tab | i + index concur_reindex_ind2 | collation "default" | n index concur_reindex_ind2 | column c2 of table concur_reindex_tab | a index concur_reindex_ind3 | column c1 of table concur_reindex_tab | a index concur_reindex_ind3 | table concur_reindex_tab | a + index concur_reindex_ind4 | collation "default" | n index concur_reindex_ind4 | column c1 of table concur_reindex_tab | a index concur_reindex_ind4 | column c2 of table concur_reindex_tab | a materialized view concur_reindex_matview | schema public | n table concur_reindex_tab | schema public | n -(8 rows) +(10 rows) -- Check that comments are preserved CREATE TABLE testcomment (i int); diff --git a/src/test/regress/sql/collate.icu.utf8.sql b/src/test/regress/sql/collate.icu.utf8.sql index 35acf91fbf..c9afb88f7d 100644 --- a/src/test/regress/sql/collate.icu.utf8.sql +++ b/src/test/regress/sql/collate.icu.utf8.sql @@ -716,6 +716,143 @@ INSERT INTO test33 VALUES (2, 'DEF'); -- they end up in the same partition (but it's platform-dependent which one) SELECT (SELECT count(*) FROM test33_0) <> (SELECT count(*) FROM test33_1); +-- collation versioning support +CREATE TYPE t_en_fr AS (fr text COLLATE "fr-x-icu", en text COLLATE "en-x-icu"); +CREATE DOMAIN d_en_fr AS t_en_fr; +CREATE DOMAIN d_es AS text COLLATE "es-x-icu"; +CREATE TYPE t_en_fr_ga AS (en_fr t_en_fr, ga text COLLATE "ga-x-icu"); +CREATE DOMAIN d_en_fr_ga AS t_en_fr_ga; +CREATE TYPE t_custom AS (meh text, meh2 text); +CREATE DOMAIN d_custom AS t_custom; + +CREATE COLLATION custom ( + LOCALE = 'fr-x-icu', PROVIDER = 'icu' +); + +CREATE TYPE myrange AS range (subtype = text); +CREATE TYPE myrange_en_fr_ga AS range(subtype = t_en_fr_ga); + +CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); + +CREATE TABLE collate_test ( + id integer, + val text COLLATE "fr-x-icu", + t_en_fr t_en_fr, + d_en_fr d_en_fr, + d_es d_es, + t_en_fr_ga t_en_fr_ga, + d_en_fr_ga d_en_fr_ga, + d_en_fr_ga_arr d_en_fr_ga[], + myrange myrange, + myrange_en_fr_ga myrange_en_fr_ga, + mood mood +); + +CREATE INDEX icuidx00_val ON collate_test(val); +-- shouldn't get duplicated dependencies +CREATE INDEX icuidx00_val_val ON collate_test(val, val); +-- shouldn't track version +CREATE INDEX icuidx00_val_pattern ON collate_test(val text_pattern_ops); +-- should have single dependency, no version tracked +CREATE INDEX icuidx00_val_pattern_val_pattern ON collate_test(val text_pattern_ops, val text_pattern_ops); +-- should have single dependency, with version tracked +CREATE INDEX icuidx00_val_pattern_val ON collate_test(val text_pattern_ops, val); +-- should have single dependency, with version tracked +CREATE INDEX icuidx00_val_val_pattern ON collate_test(val, val text_pattern_ops); +-- should have single dependency, with version tracked +CREATE INDEX icuidx00_val_pattern_where ON collate_test(val text_pattern_ops) WHERE val >= val; +-- should have single dependency, with version tracked +CREATE INDEX icuidx00_val_where ON collate_test(val) WHERE val >= val; +-- should ideally have single dependency, no version tracked, but expression walker will find a dependency on the collation and will ask to track the version +CREATE INDEX icuidx00_val_pattern_expr_pattern ON collate_test(val varchar_pattern_ops, (val || val) text_pattern_ops); +-- should have single dependency, with version tracked +CREATE INDEX icuidx00_val_pattern_expr ON collate_test(val varchar_pattern_ops, (val || val)); +CREATE INDEX icuidx01_t_en_fr__d_es ON collate_test (t_en_fr, d_es); +CREATE INDEX icuidx02_d_en_fr ON collate_test (d_en_fr); +CREATE INDEX icuidx03_t_en_fr_ga ON collate_test (t_en_fr_ga); +CREATE INDEX icuidx04_d_en_fr_ga ON collate_test (d_en_fr_ga); +CREATE INDEX icuidx05_d_en_fr_ga_arr ON collate_test (d_en_fr_ga_arr); +CREATE INDEX icuidx06_d_en_fr_ga ON collate_test(id) WHERE (d_en_fr_ga).en_fr.fr = 'foo'; +CREATE INDEX icuidx07_d_en_fr_ga ON collate_test(id) WHERE (d_en_fr_ga).ga = 'foo'; +CREATE INDEX icuidx08_d_en_fr_ga ON collate_test(id) WHERE (t_en_fr_ga) = ('foo', 'bar', 'baz'); +CREATE INDEX icuidx09_d_en_fr_ga ON collate_test(id) WHERE (d_en_fr_ga) = ('foo', 'bar', 'baz'); +CREATE INDEX icuidx10_d_en_fr_ga_es ON collate_test(id) WHERE (d_en_fr_ga) = ('foo', 'bar', 'baz' COLLATE "es-x-icu"); +CREATE INDEX icuidx11_d_es ON collate_test(id) WHERE (d_es) = ('foo'); +CREATE INDEX icuidx12_custom ON collate_test(id) WHERE ('foo', 'bar')::d_custom = ('foo', 'bar' COLLATE custom)::d_custom; +CREATE INDEX icuidx13_custom ON collate_test(id) WHERE ('foo' COLLATE custom, 'bar')::d_custom = ('foo', 'bar')::d_custom; +CREATE INDEX icuidx14_myrange ON collate_test(myrange); +CREATE INDEX icuidx15_myrange_en_fr_ga ON collate_test USING gist (myrange_en_fr_ga); +CREATE INDEX icuidx16_mood ON collate_test(id) WHERE mood > 'ok' COLLATE "fr-x-icu"; + +CREATE TABLE collate_part(id integer, val text COLLATE "en-x-icu") PARTITION BY range(id); +CREATE TABLE collate_part_0 PARTITION OF collate_part FOR VALUES FROM (0) TO (1); +CREATE TABLE collate_part_1 PARTITION OF collate_part FOR VALUES FROM (1) TO (1000000); +CREATE INDEX icuidx17_part ON collate_part_1 (val); +-- for key columns, hash indexes should record dependency on the collation but +-- not the version +CREATE INDEX icuidx18_hash_d_es ON collate_test USING hash (d_es); +CREATE INDEX icuidx19_hash_id_d_es_eq ON collate_test USING hash (id) WHERE (d_es) = 'foo'; +CREATE INDEX icuidx20_hash_id_d_es_lt ON collate_test USING hash (id) WHERE (d_es) < 'foo'; + +SELECT objid::regclass, refobjid::regcollation, +CASE +WHEN refobjversion IS NULL THEN 'version not tracked' +ELSE CASE + WHEN refobjversion = pg_collation_actual_version(refobjid) THEN 'up to date' + ELSE 'out of date' + END +END AS version +FROM pg_depend d +LEFT JOIN pg_class c ON c.oid = d.objid +WHERE refclassid = 'pg_collation'::regclass +AND coalesce(relkind, 'i') = 'i' +AND relname LIKE 'icuidx%' +ORDER BY 1, 2; + +-- Validate that REINDEX will update the stored version. +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text LIKE 'icuidx%' +AND refobjversion IS NOT NULL; + +REINDEX TABLE collate_test; +REINDEX TABLE collate_part_0; +REINDEX TABLE collate_part_1; + +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + +-- Validate that REINDEX CONCURRENTLY will update the stored version. +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text LIKE 'icuidx%' +AND refobjversion IS NOT NULL; +REINDEX TABLE CONCURRENTLY collate_test; +REINDEX TABLE CONCURRENTLY collate_part_0; +REINDEX INDEX CONCURRENTLY icuidx17_part; + +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + +-- Validate that VACUUM FULL will update the stored version. +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text LIKE 'icuidx%' +AND refobjversion IS NOT NULL; +VACUUM FULL collate_test; +VACUUM FULL collate_part_0; +VACUUM FULL collate_part_1; + +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; + +-- Test ALTER INDEX name ALTER COLLATION name REFRESH VERSION +UPDATE pg_depend SET refobjversion = 'not a version' +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text = 'icuidx17_part' +AND refobjversion IS NOT NULL; +SELECT objid::regclass FROM pg_depend WHERE refobjversion = 'not a version'; +ALTER INDEX icuidx17_part ALTER COLLATION "en-x-icu" REFRESH VERSION; +SELECT objid::regclass, refobjversion = 'not a version' AS ver FROM pg_depend +WHERE refclassid = 'pg_collation'::regclass +AND objid::regclass::text = 'icuidx17_part'; -- cleanup RESET search_path; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index ff853634bc..da3e5f73d0 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2913,6 +2913,8 @@ dlist_head dlist_iter dlist_mutable_iter dlist_node +do_collation_version_check_context +do_collation_version_update_context ds_state dsa_area dsa_area_control -- 2.20.1