From 7decfda337bbc422fece4c736a719b6fcfdc5cf3 Mon Sep 17 00:00:00 2001 From: Atsushi Torikoshi Date: Mon, 31 Aug 2020 18:20:34 +0900 Subject: [PATCH] Enabled pg_get_backend_memory_contexts() to collect arbitrary backend process's memory contexts. Previously, pg_get_backend_memory_contexts() could only get the memory contexts of the process which kicked it. This patch enables to get memory contexts of the arbitrary process which PID is specified by the argument. --- doc/src/sgml/func.sgml | 13 + src/backend/catalog/system_views.sql | 4 +- src/backend/replication/basebackup.c | 3 + src/backend/storage/ipc/procsignal.c | 4 + src/backend/tcop/postgres.c | 5 + src/backend/utils/adt/mcxtfuncs.c | 315 ++++++++++++++++++- src/backend/utils/init/globals.c | 1 + src/bin/initdb/initdb.c | 3 +- src/bin/pg_basebackup/t/010_pg_basebackup.pl | 2 +- src/bin/pg_rewind/filemap.c | 3 + src/include/catalog/pg_proc.dat | 7 +- src/include/miscadmin.h | 1 + src/include/storage/procsignal.h | 1 + src/include/utils/mcxtfuncs.h | 21 ++ src/test/regress/expected/rules.out | 2 +- 15 files changed, 364 insertions(+), 21 deletions(-) create mode 100644 src/include/utils/mcxtfuncs.h diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index b9f591296a..cc9a458334 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21062,6 +21062,19 @@ SELECT * FROM pg_ls_dir('.') WITH ORDINALITY AS t(ls,n); + + + + pg_get_backend_memory_contexts + + pg_get_backend_memory_contexts ( integer ) + setof records + + + Returns all the memory contexts of the specified process ID. + + + diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index a2d61302f9..88fb837ecd 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -555,10 +555,10 @@ REVOKE ALL ON pg_shmem_allocations FROM PUBLIC; REVOKE EXECUTE ON FUNCTION pg_get_shmem_allocations() FROM PUBLIC; CREATE VIEW pg_backend_memory_contexts AS - SELECT * FROM pg_get_backend_memory_contexts(); + SELECT * FROM pg_get_backend_memory_contexts(-1); REVOKE ALL ON pg_backend_memory_contexts FROM PUBLIC; -REVOKE EXECUTE ON FUNCTION pg_get_backend_memory_contexts() FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION pg_get_backend_memory_contexts FROM PUBLIC; -- Statistics views diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 6064384e32..f69d851b6b 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -184,6 +184,9 @@ static const char *const excludeDirContents[] = /* Contents zeroed on startup, see StartupSUBTRANS(). */ "pg_subtrans", + /* Skip memory context dumped files. */ + "pg_memusage", + /* end of list */ NULL }; diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index 4fa385b0ec..5966729897 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -28,6 +28,7 @@ #include "storage/shmem.h" #include "storage/sinval.h" #include "tcop/tcopprot.h" +#include "utils/mcxtfuncs.h" /* * The SIGUSR1 signal is multiplexed to support signaling multiple event @@ -567,6 +568,9 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_BARRIER)) HandleProcSignalBarrierInterrupt(); + if (CheckProcSignal(PROCSIG_DUMP_MEMORY)) + HandleProcSignalDumpMemory(); + if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_DATABASE)) RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_DATABASE); diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index c9424f167c..d30426cf0f 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -76,6 +76,7 @@ #include "tcop/utility.h" #include "utils/lsyscache.h" #include "utils/memutils.h" +#include "utils/mcxtfuncs.h" #include "utils/ps_status.h" #include "utils/snapmgr.h" #include "utils/timeout.h" @@ -539,6 +540,10 @@ ProcessClientReadInterrupt(bool blocked) /* Process notify interrupts, if any */ if (notifyInterruptPending) ProcessNotifyInterrupt(); + + /* Process memory contexts dump interrupts, if any */ + if (ProcSignalDumpMemoryPending) + ProcessDumpMemoryInterrupt(); } else if (ProcDiePending) { diff --git a/src/backend/utils/adt/mcxtfuncs.c b/src/backend/utils/adt/mcxtfuncs.c index 50e1b07ff0..12f7ecc116 100644 --- a/src/backend/utils/adt/mcxtfuncs.c +++ b/src/backend/utils/adt/mcxtfuncs.c @@ -15,34 +15,44 @@ #include "postgres.h" +#include + #include "funcapi.h" #include "miscadmin.h" #include "mb/pg_wchar.h" +#include "storage/latch.h" +#include "storage/procsignal.h" #include "utils/builtins.h" +#include "utils/mcxtfuncs.h" /* ---------- * The max bytes for showing identifiers of MemoryContext. * ---------- */ -#define MEMORY_CONTEXT_IDENT_DISPLAY_SIZE 1024 +#define MEMORY_CONTEXT_DISPLAY_SIZE 1024 + +#define PG_GET_BACKEND_MEMORY_CONTEXTS_COLS 9 /* * PutMemoryContextsStatsTupleStore * One recursion level for pg_get_backend_memory_contexts. + * + * Note: When fpout is not NULL, ferror() check must be done + * by the caller. */ static void PutMemoryContextsStatsTupleStore(Tuplestorestate *tupstore, TupleDesc tupdesc, MemoryContext context, - const char *parent, int level) + const char *parent, int level, FILE *fpout) { -#define PG_GET_BACKEND_MEMORY_CONTEXTS_COLS 9 - Datum values[PG_GET_BACKEND_MEMORY_CONTEXTS_COLS]; bool nulls[PG_GET_BACKEND_MEMORY_CONTEXTS_COLS]; + char clipped_ident[MEMORY_CONTEXT_DISPLAY_SIZE]; MemoryContextCounters stat; MemoryContext child; const char *name; const char *ident; + int idlen; AssertArg(MemoryContextIsValid(context)); @@ -73,22 +83,23 @@ PutMemoryContextsStatsTupleStore(Tuplestorestate *tupstore, if (ident) { - int idlen = strlen(ident); - char clipped_ident[MEMORY_CONTEXT_IDENT_DISPLAY_SIZE]; - + idlen = strlen(ident); /* * Some identifiers such as SQL query string can be very long, * truncate oversize identifiers. */ - if (idlen >= MEMORY_CONTEXT_IDENT_DISPLAY_SIZE) - idlen = pg_mbcliplen(ident, idlen, MEMORY_CONTEXT_IDENT_DISPLAY_SIZE - 1); + if (idlen >= MEMORY_CONTEXT_DISPLAY_SIZE) + idlen = pg_mbcliplen(ident, idlen, MEMORY_CONTEXT_DISPLAY_SIZE - 1); memcpy(clipped_ident, ident, idlen); clipped_ident[idlen] = '\0'; values[1] = CStringGetTextDatum(clipped_ident); } else + { + idlen = 0; nulls[1] = true; + } if (parent) values[2] = CStringGetTextDatum(parent); @@ -101,12 +112,41 @@ PutMemoryContextsStatsTupleStore(Tuplestorestate *tupstore, values[6] = Int64GetDatum(stat.freespace); values[7] = Int64GetDatum(stat.freechunks); values[8] = Int64GetDatum(stat.totalspace - stat.freespace); - tuplestore_putvalues(tupstore, tupdesc, values, nulls); + + if(fpout == NULL) + /* pg_get_backend_memory_contexts() is called from local process */ + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + + else + { + int rc; + int parent_len = strlen(parent); + int name_len = strlen(name); + + /* + * write out the current memory context information. + * Since some elements of values are reusable, we write it out. + */ + fputc('D', fpout); + rc = fwrite(values, sizeof(values), 1, fpout); + rc = fwrite(nulls, sizeof(nulls), 1, fpout); + + /* write out information which is not resuable from serialized values */ + rc = fwrite(&name_len, sizeof(int), 1, fpout); + rc = fwrite(name, name_len, 1, fpout); + rc = fwrite(&idlen, sizeof(int), 1, fpout); + rc = fwrite(clipped_ident, idlen, 1, fpout); + rc = fwrite(&level, sizeof(int), 1, fpout); + rc = fwrite(&parent_len, sizeof(int), 1, fpout); + rc = fwrite(parent, parent_len, 1, fpout); + (void) rc; /* we'll check for error with ferror */ + + } for (child = context->firstchild; child != NULL; child = child->nextchild) { PutMemoryContextsStatsTupleStore(tupstore, tupdesc, - child, name, level + 1); + child, name, level + 1, fpout); } } @@ -117,6 +157,8 @@ PutMemoryContextsStatsTupleStore(Tuplestorestate *tupstore, Datum pg_get_backend_memory_contexts(PG_FUNCTION_ARGS) { + int pid = PG_GETARG_INT32(0); + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; TupleDesc tupdesc; Tuplestorestate *tupstore; @@ -147,11 +189,258 @@ pg_get_backend_memory_contexts(PG_FUNCTION_ARGS) MemoryContextSwitchTo(oldcontext); - PutMemoryContextsStatsTupleStore(tupstore, tupdesc, - TopMemoryContext, NULL, 0); + if (pid == -1) + { + /* + * Since pid -1 indicates target is the local process, simply + * traverse memory contexts. + */ + PutMemoryContextsStatsTupleStore(tupstore, tupdesc, + TopMemoryContext, "", 0, NULL); + } + else + { + /* + * Send signal for dumping memory contexts to the target process, + * and read the dumped file. + */ + FILE *fpin; + char dumpfile[MAXPGPATH]; + + SendProcSignal(pid, PROCSIG_DUMP_MEMORY, InvalidBackendId); + + snprintf(dumpfile, sizeof(dumpfile), "pg_memusage/%d", pid); + + while (true) + { + CHECK_FOR_INTERRUPTS(); + + pg_usleep(10000L); + + if ((fpin = AllocateFile(dumpfile, PG_BINARY_R)) == NULL) + { + if (errno != ENOENT) + ereport(LOG, (errcode_for_file_access(), + errmsg("could not open temporary memory dump file \"%s\": %m", + dumpfile))); + } + else + { + elog(DEBUG1, "Succeeded opening temporary memory dump file \"%s\": %m", + dumpfile); + break; + } + } + + /* read dumped files */ + while (true) + { + Datum values[PG_GET_BACKEND_MEMORY_CONTEXTS_COLS]; + bool nulls[PG_GET_BACKEND_MEMORY_CONTEXTS_COLS]; + char name[MEMORY_CONTEXT_DISPLAY_SIZE]; + char parent[MEMORY_CONTEXT_DISPLAY_SIZE]; + char clipped_ident[MEMORY_CONTEXT_DISPLAY_SIZE]; + int level; + int name_len; + int parent_len; + int idlen; + + memset(values, 0, sizeof(values)); + memset(nulls, 0, sizeof(nulls)); + + switch (fgetc(fpin)) + { + /* 'D' A memory context information follows. */ + case 'D': + if (fread(values, 1, sizeof(values), fpin) != sizeof(values)) + { + ereport(WARNING, + (errmsg("corrupted memory dump file \"%s\"", + dumpfile))); + goto done; + } + if (fread(nulls, 1, sizeof(nulls), fpin) != sizeof(nulls)) + { + ereport(WARNING, + (errmsg("corrupted memory dump file \"%s\"", + dumpfile))); + goto done; + } + if (fread(&name_len, 1, sizeof(int), fpin) != sizeof(int)) + { + ereport(WARNING, + (errmsg("corrupted memory dump file \"%s\"", + dumpfile))); + goto done; + } + if (fread(name, 1, name_len, fpin) != name_len) + { + ereport(WARNING, + (errmsg("corrupted memory dump file \"%s\"", + dumpfile))); + goto done; + } + name[name_len] = '\0'; + if (fread(&idlen, 1, sizeof(int), fpin) != sizeof(int)) + { + ereport(WARNING, + (errmsg("corrupted memory dump file \"%s\"", + dumpfile))); + goto done; + } + if (fread(clipped_ident, 1, idlen, fpin) != idlen) + { + ereport(WARNING, + (errmsg("corrupted memory dump file \"%s\"", + dumpfile))); + goto done; + } + clipped_ident[idlen] = '\0'; + if (fread(&level, 1, sizeof(int), fpin) != sizeof(int)) + { + ereport(WARNING, + (errmsg("corrupted memory dump file \"%s\"", + dumpfile))); + goto done; + } + if (fread(&parent_len, 1, sizeof(int), fpin) != sizeof(int)) + { + ereport(WARNING, + (errmsg("corrupted memory dump file \"%s\"", + dumpfile))); + goto done; + } + if (fread(parent, 1, parent_len, fpin) != parent_len) + { + ereport(WARNING, + (errmsg("corrupted memory dump file \"%s\"", + dumpfile))); + goto done; + } + parent[parent_len] = '\0'; + + /* + * Overwrite some elements of values which are not available + * from serialized values. + */ + if (!nulls[0]) + values[0] = CStringGetTextDatum(name); + + if (!nulls[1]) + values[1] = CStringGetTextDatum(clipped_ident); + + if (!nulls[2]) + values[2] = CStringGetTextDatum(parent); + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + break; + + case 'E': + goto done; + + default: + ereport(WARNING, + (errmsg("corrupted memory dump file \"%s\"", + dumpfile))); + goto done; + } + } +done: + FreeFile(fpin); + unlink(dumpfile); + } /* clean up and return the tuplestore */ tuplestore_donestoring(tupstore); return (Datum) 0; } + +/* + * dump_memory_contexts + * Dumping local memory contexts to a file. + * This function does not delete the file as it is intended to be read by + * another process. + */ +static void +dump_memory_contexts(void) +{ + FILE *fpout; + char tmpfile[MAXPGPATH]; + char dumpfile[MAXPGPATH]; + + snprintf(tmpfile, sizeof(tmpfile), "pg_memusage/%d.tmp", MyProcPid); + snprintf(dumpfile, sizeof(dumpfile), "pg_memusage/%d", MyProcPid); + + /* + * Open a temp file to dump the current memory context. + */ + fpout = AllocateFile(tmpfile, PG_BINARY_W); + if (fpout == NULL) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not write temporary memory context file \"%s\": %m", + tmpfile))); + return; + } + + PutMemoryContextsStatsTupleStore(NULL, NULL, + TopMemoryContext, "", 0, fpout); + + /* No more output to be done. */ + fputc('E', fpout); + + if (ferror(fpout)) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not write temporary memory context dump file \"%s\": %m", + tmpfile))); + FreeFile(fpout); + unlink(tmpfile); + } + else if (FreeFile(fpout) < 0) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not close temporary memory context file \"%s\": %m", + tmpfile))); + unlink(tmpfile); + } + else if (rename(tmpfile, dumpfile) < 0) + { + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not rename dump file \"%s\" to \"%s\": %m", + tmpfile, dumpfile))); + unlink(tmpfile); + } +} + +/* + * ProcessDumpMemoryInterrupt + * + * The portion of memory dump interrupt handling that runs + * outside of the signal handler. + */ +void +ProcessDumpMemoryInterrupt(void) +{ + ProcSignalDumpMemoryPending = false; + dump_memory_contexts(); +} + +/* + * Handle receipt of an interrupt indicating a dump memory context. + * + * Signal handler portion of interrupt handling. + */ +void +HandleProcSignalDumpMemory(void) +{ + ProcSignalDumpMemoryPending = true; + + /* make sure the event is processed in due course */ + SetLatch(MyLatch); +} diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index 6ab8216839..463337f661 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -33,6 +33,7 @@ volatile sig_atomic_t ProcDiePending = false; volatile sig_atomic_t ClientConnectionLost = false; volatile sig_atomic_t IdleInTransactionSessionTimeoutPending = false; volatile sig_atomic_t ProcSignalBarrierPending = false; +volatile sig_atomic_t ProcSignalDumpMemoryPending = false; volatile uint32 InterruptHoldoffCount = 0; volatile uint32 QueryCancelHoldoffCount = 0; volatile uint32 CritSectionCount = 0; diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 786672b1b6..c3130dc6f9 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -220,7 +220,8 @@ static const char *const subdirs[] = { "pg_xact", "pg_logical", "pg_logical/snapshots", - "pg_logical/mappings" + "pg_logical/mappings", + "pg_memusage" }; diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index f674a7c94e..683c2d5408 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -124,7 +124,7 @@ is_deeply( # Contents of these directories should not be copied. foreach my $dirname ( - qw(pg_dynshmem pg_notify pg_replslot pg_serial pg_snapshots pg_stat_tmp pg_subtrans) + qw(pg_dynshmem pg_notify pg_replslot pg_serial pg_snapshots pg_stat_tmp pg_subtrans pg_memusage) ) { is_deeply( diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index 1879257b66..23471007d8 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -85,6 +85,9 @@ static const char *excludeDirContents[] = /* Contents zeroed on startup, see StartupSUBTRANS(). */ "pg_subtrans", + /* Skip memory context dumped files. */ + "pg_memusage", + /* end of list */ NULL }; diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 1dd325e0e6..b6859b23e8 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -7812,9 +7812,10 @@ { oid => '2282', descr => 'information about all memory contexts of local backend', proname => 'pg_get_backend_memory_contexts', prorows => '100', proretset => 't', provolatile => 'v', proparallel => 'r', prorettype => 'record', proargtypes => '', - proallargtypes => '{text,text,text,int4,int8,int8,int8,int8,int8}', - proargmodes => '{o,o,o,o,o,o,o,o,o}', - proargnames => '{name, ident, parent, level, total_bytes, total_nblocks, free_bytes, free_chunks, used_bytes}', + provolatile => 'v', proparallel => 'r', prorettype => 'record', proargtypes => 'int4', + proallargtypes => '{int4,text,text,text,int4,int8,int8,int8,int8,int8}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o}', + proargnames => '{pid, name, ident, parent, level, total_bytes, total_nblocks, free_bytes, free_chunks, used_bytes}', prosrc => 'pg_get_backend_memory_contexts' }, # non-persistent series generator diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 72e3352398..812032bb15 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -83,6 +83,7 @@ extern PGDLLIMPORT volatile sig_atomic_t QueryCancelPending; extern PGDLLIMPORT volatile sig_atomic_t ProcDiePending; extern PGDLLIMPORT volatile sig_atomic_t IdleInTransactionSessionTimeoutPending; extern PGDLLIMPORT volatile sig_atomic_t ProcSignalBarrierPending; +extern PGDLLIMPORT volatile sig_atomic_t ProcSignalDumpMemoryPending; extern PGDLLIMPORT volatile sig_atomic_t ClientConnectionLost; diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index 5cb39697f3..5db92a9a52 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -34,6 +34,7 @@ typedef enum PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ PROCSIG_WALSND_INIT_STOPPING, /* ask walsenders to prepare for shutdown */ PROCSIG_BARRIER, /* global barrier interrupt */ + PROCSIG_DUMP_MEMORY, /* request dumping memory context interrupt */ /* Recovery conflict reasons */ PROCSIG_RECOVERY_CONFLICT_DATABASE, diff --git a/src/include/utils/mcxtfuncs.h b/src/include/utils/mcxtfuncs.h new file mode 100644 index 0000000000..cd74bb6700 --- /dev/null +++ b/src/include/utils/mcxtfuncs.h @@ -0,0 +1,21 @@ +/*------------------------------------------------------------------------- + * + * mcxtfuncs.h + * Declarations for showing backend memory context. + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/utils/adt/mcxtfuncs.h + * + *------------------------------------------------------------------------- + */ +#ifndef MCXT_H +#define MCXT_H + +extern void ProcessDumpMemoryInterrupt(void); +extern void HandleProcSignalDumpMemory(void); + +#endif /* MCXT_H */ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 2a18dc423e..990b54d777 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1333,7 +1333,7 @@ pg_backend_memory_contexts| SELECT pg_get_backend_memory_contexts.name, pg_get_backend_memory_contexts.free_bytes, pg_get_backend_memory_contexts.free_chunks, pg_get_backend_memory_contexts.used_bytes - FROM pg_get_backend_memory_contexts() pg_get_backend_memory_contexts(name, ident, parent, level, total_bytes, total_nblocks, free_bytes, free_chunks, used_bytes); + FROM pg_get_backend_memory_contexts('-1'::integer) pg_get_backend_memory_contexts(name, ident, parent, level, total_bytes, total_nblocks, free_bytes, free_chunks, used_bytes); pg_config| SELECT pg_config.name, pg_config.setting FROM pg_config() pg_config(name, setting); -- 2.18.1