From 9619edd924c13337843f3fde221096b389701012 Mon Sep 17 00:00:00 2001 From: Kyotaro Horiguchi Date: Thu, 16 Nov 2017 16:18:54 +0900 Subject: [PATCH 2/4] Add vacuum_required to pg_stat_all_tables If vacuum of a table has been failed for a long time for some reasons, it is hard for uses to distinguish between that the server judged vacuuming of the table is not required and that rquired but failed. This offers convenient way to check that as the first step of trouble shooting. --- doc/src/sgml/config.sgml | 5 +- doc/src/sgml/maintenance.sgml | 4 +- doc/src/sgml/monitoring.sgml | 5 ++ src/backend/catalog/system_views.sql | 1 + src/backend/commands/cluster.c | 2 +- src/backend/commands/vacuum.c | 69 ++++++++++++++++++--- src/backend/commands/vacuumlazy.c | 14 +---- src/backend/postmaster/autovacuum.c | 115 +++++++++++++++++++++++++++++++++++ src/backend/utils/adt/pgstatfuncs.c | 9 +++ src/include/catalog/pg_proc.h | 2 + src/include/commands/vacuum.h | 3 +- src/include/postmaster/autovacuum.h | 1 + src/test/regress/expected/rules.out | 3 + 13 files changed, 210 insertions(+), 23 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 41f0858..7262ffb 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6579,7 +6579,10 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; VACUUM performs an aggressive scan if the table's pg_class.relfrozenxid field has reached - the age specified by this setting. An aggressive scan differs from + the age specified by this setting. It is indicated + as aggressive in vacuum_required + of . An aggressive scan + differs from a regular VACUUM in that it visits every page that might contain unfrozen XIDs or MXIDs, not just those that might contain dead tuples. The default is 150 million transactions. Although users can diff --git a/doc/src/sgml/maintenance.sgml b/doc/src/sgml/maintenance.sgml index 1a37905..d045b09 100644 --- a/doc/src/sgml/maintenance.sgml +++ b/doc/src/sgml/maintenance.sgml @@ -514,7 +514,9 @@ autovacuum_freeze_max_age wouldn't make sense because an anti-wraparound autovacuum would be triggered at that point anyway, and the 0.95 multiplier leaves some breathing room to run a manual - VACUUM before that happens. As a rule of thumb, + VACUUM before that happens. It is indicated + as close to freeze-limit xid in vacuum_required + of . As a rule of thumb, vacuum_freeze_table_age should be set to a value somewhat below autovacuum_freeze_max_age, leaving enough gap so that a regularly scheduled VACUUM or an autovacuum triggered by diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 71823c5..e8a8f77 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -2547,6 +2547,11 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i Estimated number of rows modified since this table was last analyzed + vacuum_required + text + Vacuum requirement status. "partial", "aggressive", "required", "not requried" or "close to freeze-limit xid". + + last_vacuum timestamp with time zone Last time at which this table was manually vacuumed diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index cf6621d..97bafb8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -523,6 +523,7 @@ CREATE VIEW pg_stat_all_tables AS pg_stat_get_live_tuples(C.oid) AS n_live_tup, pg_stat_get_dead_tuples(C.oid) AS n_dead_tup, pg_stat_get_mod_since_analyze(C.oid) AS n_mod_since_analyze, + pg_stat_get_vacuum_necessity(C.oid) AS vacuum_required, pg_stat_get_last_vacuum_time(C.oid) as last_vacuum, pg_stat_get_last_autovacuum_time(C.oid) as last_autovacuum, pg_stat_get_last_analyze_time(C.oid) as last_analyze, diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 48f1e6e..403b76d 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -850,7 +850,7 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, */ vacuum_set_xid_limits(OldHeap, 0, 0, 0, 0, &OldestXmin, &FreezeXid, NULL, &MultiXactCutoff, - NULL); + NULL, NULL, NULL); /* * FreezeXid will become the table's new relfrozenxid, and that mustn't go diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index cbd6e9b..f51dcdb 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -585,6 +585,10 @@ get_all_vacuum_rels(void) * Xmax. * - mxactFullScanLimit is a value against which a table's relminmxid value is * compared to produce a full-table vacuum, as with xidFullScanLimit. + * - aggressive is set if it is not NULL and set true if the table needs + * aggressive scan. + * - close_to_wrap_around_limit is set if it is not NULL and set true if it is + * in anti-anti-wraparound window. * * xidFullScanLimit and mxactFullScanLimit can be passed as NULL if caller is * not interested. @@ -599,9 +603,11 @@ vacuum_set_xid_limits(Relation rel, TransactionId *freezeLimit, TransactionId *xidFullScanLimit, MultiXactId *multiXactCutoff, - MultiXactId *mxactFullScanLimit) + MultiXactId *mxactFullScanLimit, + bool *aggressive, bool *close_to_wrap_around_limit) { int freezemin; + int freezemax; int mxid_freezemin; int effective_multixact_freeze_max_age; TransactionId limit; @@ -701,11 +707,13 @@ vacuum_set_xid_limits(Relation rel, *multiXactCutoff = mxactLimit; - if (xidFullScanLimit != NULL) + if (xidFullScanLimit != NULL || aggressive != NULL) { int freezetable; + bool maybe_anti_wrapround = false; - Assert(mxactFullScanLimit != NULL); + /* these two output should be requested together */ + Assert(xidFullScanLimit == NULL || mxactFullScanLimit != NULL); /* * Determine the table freeze age to use: as specified by the caller, @@ -717,7 +725,14 @@ vacuum_set_xid_limits(Relation rel, freezetable = freeze_table_age; if (freezetable < 0) freezetable = vacuum_freeze_table_age; - freezetable = Min(freezetable, autovacuum_freeze_max_age * 0.95); + + freezemax = autovacuum_freeze_max_age * 0.95; + if (freezemax < freezetable) + { + /* We may be in anti-anti-warparound window */ + freezetable = freezemax; + maybe_anti_wrapround = true; + } Assert(freezetable >= 0); /* @@ -728,7 +743,8 @@ vacuum_set_xid_limits(Relation rel, if (!TransactionIdIsNormal(limit)) limit = FirstNormalTransactionId; - *xidFullScanLimit = limit; + if (xidFullScanLimit) + *xidFullScanLimit = limit; /* * Similar to the above, determine the table freeze age to use for @@ -741,10 +757,20 @@ vacuum_set_xid_limits(Relation rel, freezetable = multixact_freeze_table_age; if (freezetable < 0) freezetable = vacuum_multixact_freeze_table_age; - freezetable = Min(freezetable, - effective_multixact_freeze_max_age * 0.95); + + freezemax = effective_multixact_freeze_max_age * 0.95; + if (freezemax < freezetable) + { + /* We may be in anti-anti-warparound window */ + freezetable = freezemax; + maybe_anti_wrapround = true; + } Assert(freezetable >= 0); + /* We may be in anti-anti-warparound window */ + if (effective_multixact_freeze_max_age * 0.95 < freezetable) + maybe_anti_wrapround = true; + /* * Compute MultiXact limit causing a full-table vacuum, being careful * to generate a valid MultiXact value. @@ -753,11 +779,38 @@ vacuum_set_xid_limits(Relation rel, if (mxactLimit < FirstMultiXactId) mxactLimit = FirstMultiXactId; - *mxactFullScanLimit = mxactLimit; + if (mxactFullScanLimit) + *mxactFullScanLimit = mxactLimit; + + /* + * We request an aggressive scan if the table's frozen Xid is now + * older than or equal to the requested Xid full-table scan limit; or + * if the table's minimum MultiXactId is older than or equal to the + * requested mxid full-table scan limit. + */ + if (aggressive) + { + *aggressive = + TransactionIdPrecedesOrEquals(rel->rd_rel->relfrozenxid, + limit); + *aggressive |= + MultiXactIdPrecedesOrEquals(rel->rd_rel->relminmxid, + mxactLimit); + + /* set close_to_wrap_around_limit if requested */ + if (close_to_wrap_around_limit) + *close_to_wrap_around_limit = + (*aggressive && maybe_anti_wrapround); + } + else + { + Assert (!close_to_wrap_around_limit); + } } else { Assert(mxactFullScanLimit == NULL); + Assert(aggressive == NULL); } } diff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.c index c482c8e..4274043 100644 --- a/src/backend/commands/vacuumlazy.c +++ b/src/backend/commands/vacuumlazy.c @@ -227,18 +227,10 @@ lazy_vacuum_rel(Relation onerel, int options, VacuumParams *params, params->multixact_freeze_min_age, params->multixact_freeze_table_age, &OldestXmin, &FreezeLimit, &xidFullScanLimit, - &MultiXactCutoff, &mxactFullScanLimit); + &MultiXactCutoff, &mxactFullScanLimit, + &aggressive, NULL); - /* - * We request an aggressive scan if the table's frozen Xid is now older - * than or equal to the requested Xid full-table scan limit; or if the - * table's minimum MultiXactId is older than or equal to the requested - * mxid full-table scan limit; or if DISABLE_PAGE_SKIPPING was specified. - */ - aggressive = TransactionIdPrecedesOrEquals(onerel->rd_rel->relfrozenxid, - xidFullScanLimit); - aggressive |= MultiXactIdPrecedesOrEquals(onerel->rd_rel->relminmxid, - mxactFullScanLimit); + /* force aggressive scan if DISABLE_PAGE_SKIPPING was specified */ if (options & VACOPT_DISABLE_PAGE_SKIPPING) aggressive = true; diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 48765bb..abbf660 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -831,6 +831,121 @@ shutdown: } /* + * Returns status string of auto vacuum on the relation + */ +char * +AutoVacuumRequirement(Oid reloid) +{ + Relation classRel; + Relation rel; + TupleDesc pg_class_desc; + HeapTuple tuple; + Form_pg_class classForm; + AutoVacOpts *relopts; + PgStat_StatTabEntry *tabentry; + PgStat_StatDBEntry *shared; + PgStat_StatDBEntry *dbentry; + int effective_multixact_freeze_max_age; + bool dovacuum; + bool doanalyze; + bool wraparound; + bool aggressive; + bool xid_calculated = false; + bool in_anti_wa_window = false; + char *ret = "not requried"; + + /* Compute the multixact age for which freezing is urgent. */ + effective_multixact_freeze_max_age = MultiXactMemberFreezeThreshold(); + + /* Fetch the pgclass entry for this relation */ + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(reloid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for relation %u", reloid); + classForm = (Form_pg_class) GETSTRUCT(tuple); + + /* extract relopts for autovacuum */ + classRel = heap_open(RelationRelationId, AccessShareLock); + pg_class_desc = RelationGetDescr(classRel); + relopts = extract_autovac_opts(tuple, pg_class_desc); + heap_close(classRel, AccessShareLock); + + /* Fetch the pgstat shared entry and entry for this database */ + shared = pgstat_fetch_stat_dbentry(InvalidOid); + dbentry = pgstat_fetch_stat_dbentry(MyDatabaseId); + + /* Fetch the pgstat entry for this table */ + tabentry = get_pgstat_tabentry_relid(reloid, classForm->relisshared, + shared, dbentry); + + /* + * Check if the relation needs vacuum. This function is intended to + * suggest aggresive vacuum for the last 5% window in + * autovacuum_freeze_max_age so the variable wraparound is ignored + * here. See vacuum_set_xid_limits for details. + */ + relation_needs_vacanalyze(reloid, relopts, classForm, tabentry, + effective_multixact_freeze_max_age, + &dovacuum, &doanalyze, &wraparound); + ReleaseSysCache(tuple); + + /* get further information if needed */ + rel = NULL; + + /* don't get stuck with lock */ + if (ConditionalLockRelationOid(reloid, AccessShareLock)) + rel = try_relation_open(reloid, NoLock); + + if (rel) + { + TransactionId OldestXmin, FreezeLimit; + MultiXactId MultiXactCutoff; + + vacuum_set_xid_limits(rel, + vacuum_freeze_min_age, + vacuum_freeze_table_age, + vacuum_multixact_freeze_min_age, + vacuum_multixact_freeze_table_age, + &OldestXmin, &FreezeLimit, NULL, + &MultiXactCutoff, NULL, + &aggressive, &in_anti_wa_window); + + xid_calculated = true; + relation_close(rel, AccessShareLock); + } + + /* choose the proper message according to the calculation above */ + if (xid_calculated) + { + if (dovacuum) + { + /* we don't care anti-wraparound if autovacuum is on */ + if (aggressive) + ret = "aggressive"; + else + ret = "partial"; + } + else if (in_anti_wa_window) + ret = "close to freeze-limit xid"; + /* otherwise just "not requried" */ + } + else + { + /* + * failed to compute xid limits. show less-grained messages. We can + * use just "required" in the autovacuum case is enough to distinguish + * from full-grained messages, but we require additional words in the + * case where autovacuum is turned off. + */ + if (dovacuum) + ret = "required"; + else + ret = "not required (lock not acquired)"; + } + + return ret; +} + +/* * Determine the time to sleep, based on the database list. * * The "canlaunch" parameter indicates whether we can start a worker right now, diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 2956356..ab80794 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -23,6 +23,7 @@ #include "pgstat.h" #include "postmaster/bgworker_internals.h" #include "postmaster/postmaster.h" +#include "postmaster/autovacuum.h" #include "storage/proc.h" #include "storage/procarray.h" #include "utils/acl.h" @@ -195,6 +196,14 @@ pg_stat_get_mod_since_analyze(PG_FUNCTION_ARGS) } Datum +pg_stat_get_vacuum_necessity(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + + PG_RETURN_TEXT_P(cstring_to_text(AutoVacuumRequirement(relid))); +} + +Datum pg_stat_get_last_vacuum_index_scans(PG_FUNCTION_ARGS) { Oid relid = PG_GETARG_OID(0); diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index f3b606b..6b84c9a 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -2887,6 +2887,8 @@ DATA(insert OID = 3317 ( pg_stat_get_wal_receiver PGNSP PGUID 12 1 0 0 0 f f f DESCR("statistics: information about WAL receiver"); DATA(insert OID = 6118 ( pg_stat_get_subscription PGNSP PGUID 12 1 0 0 0 f f f f f f s r 1 0 2249 "26" "{26,26,26,23,3220,1184,1184,3220,1184}" "{i,o,o,o,o,o,o,o,o}" "{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}" _null_ _null_ pg_stat_get_subscription _null_ _null_ _null_ )); DESCR("statistics: information about subscription"); +DATA(insert OID = 2579 ( pg_stat_get_vacuum_necessity PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 25 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_vacuum_necessity _null_ _null_ _null_ )); +DESCR("statistics: true if needs vacuum"); DATA(insert OID = 3281 ( pg_stat_get_last_vacuum_index_scans PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 23 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_last_vacuum_index_scans _null_ _null_ _null_ )); DESCR("statistics: number of index scans in the last vacuum"); DATA(insert OID = 2026 ( pg_backend_pid PGNSP PGUID 12 1 0 0 0 f f f f t f s r 0 0 23 "" _null_ _null_ _null_ _null_ _null_ pg_backend_pid _null_ _null_ _null_ )); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 60586b2..84bec74 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -182,7 +182,8 @@ extern void vacuum_set_xid_limits(Relation rel, TransactionId *freezeLimit, TransactionId *xidFullScanLimit, MultiXactId *multiXactCutoff, - MultiXactId *mxactFullScanLimit); + MultiXactId *mxactFullScanLimit, + bool *aggressive, bool *in_wa_window); extern void vac_update_datfrozenxid(void); extern void vacuum_delay_point(void); diff --git a/src/include/postmaster/autovacuum.h b/src/include/postmaster/autovacuum.h index 3469915..848a322 100644 --- a/src/include/postmaster/autovacuum.h +++ b/src/include/postmaster/autovacuum.h @@ -49,6 +49,7 @@ extern int Log_autovacuum_min_duration; extern bool AutoVacuumingActive(void); extern bool IsAutoVacuumLauncherProcess(void); extern bool IsAutoVacuumWorkerProcess(void); +extern char *AutoVacuumRequirement(Oid reloid); #define IsAnyAutoVacuumProcess() \ (IsAutoVacuumLauncherProcess() || IsAutoVacuumWorkerProcess()) diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index c334d20..2144269 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1759,6 +1759,7 @@ pg_stat_all_tables| SELECT c.oid AS relid, pg_stat_get_live_tuples(c.oid) AS n_live_tup, pg_stat_get_dead_tuples(c.oid) AS n_dead_tup, pg_stat_get_mod_since_analyze(c.oid) AS n_mod_since_analyze, + pg_stat_get_vacuum_necessity(c.oid) AS vacuum_required, pg_stat_get_last_vacuum_time(c.oid) AS last_vacuum, pg_stat_get_last_autovacuum_time(c.oid) AS last_autovacuum, pg_stat_get_last_analyze_time(c.oid) AS last_analyze, @@ -1907,6 +1908,7 @@ pg_stat_sys_tables| SELECT pg_stat_all_tables.relid, pg_stat_all_tables.n_live_tup, pg_stat_all_tables.n_dead_tup, pg_stat_all_tables.n_mod_since_analyze, + pg_stat_all_tables.vacuum_required, pg_stat_all_tables.last_vacuum, pg_stat_all_tables.last_autovacuum, pg_stat_all_tables.last_analyze, @@ -1951,6 +1953,7 @@ pg_stat_user_tables| SELECT pg_stat_all_tables.relid, pg_stat_all_tables.n_live_tup, pg_stat_all_tables.n_dead_tup, pg_stat_all_tables.n_mod_since_analyze, + pg_stat_all_tables.vacuum_required, pg_stat_all_tables.last_vacuum, pg_stat_all_tables.last_autovacuum, pg_stat_all_tables.last_analyze, -- 2.9.2