From 6f6821f2db77b70502a3b7c2688e003d144fbf2b Mon Sep 17 00:00:00 2001 From: Beena Emerson Date: Sun, 10 Sep 2017 22:28:58 +0530 Subject: [PATCH 2/2] Make wal segment size configurable at initdb time. Author: Beena Emerson Reviewed-By: Andres Freund, David Steele, Kuntal Ghosh, Michael Paquier, Peter Eisentraut, Robert Hass, Tushar Ahuja, Discussion: https://postgr.es/m/CAOG9ApEAcQ--1ieKbhFzXSQPw_YLmepaa4hNdnY5+ZULpt81Mw@mail.gmail.com --- configure | 54 ----- configure.in | 31 --- contrib/pg_standby/pg_standby.c | 93 ++++++-- doc/src/sgml/backup.sgml | 2 +- doc/src/sgml/installation.sgml | 14 -- doc/src/sgml/ref/initdb.sgml | 15 ++ doc/src/sgml/wal.sgml | 13 +- src/backend/access/transam/twophase.c | 3 +- src/backend/access/transam/xlog.c | 285 +++++++++++++++--------- src/backend/access/transam/xlogarchive.c | 14 +- src/backend/access/transam/xlogfuncs.c | 10 +- src/backend/access/transam/xlogreader.c | 32 +-- src/backend/access/transam/xlogutils.c | 36 +-- src/backend/bootstrap/bootstrap.c | 10 +- src/backend/postmaster/checkpointer.c | 5 +- src/backend/postmaster/postmaster.c | 4 + src/backend/replication/basebackup.c | 34 +-- src/backend/replication/logical/logical.c | 2 +- src/backend/replication/logical/reorderbuffer.c | 19 +- src/backend/replication/slot.c | 2 +- src/backend/replication/walreceiver.c | 14 +- src/backend/replication/walreceiverfuncs.c | 4 +- src/backend/replication/walsender.c | 16 +- src/backend/utils/misc/guc.c | 20 +- src/backend/utils/misc/pg_controldata.c | 5 +- src/backend/utils/misc/postgresql.conf.sample | 2 +- src/bin/initdb/initdb.c | 61 ++++- src/bin/pg_basebackup/pg_basebackup.c | 7 +- src/bin/pg_basebackup/pg_receivewal.c | 16 +- src/bin/pg_basebackup/receivelog.c | 36 +-- src/bin/pg_basebackup/streamutil.c | 76 +++++++ src/bin/pg_basebackup/streamutil.h | 2 + src/bin/pg_controldata/pg_controldata.c | 15 +- src/bin/pg_resetwal/pg_resetwal.c | 55 +++-- src/bin/pg_rewind/parsexlog.c | 26 ++- src/bin/pg_rewind/pg_rewind.c | 12 +- src/bin/pg_rewind/pg_rewind.h | 1 + src/bin/pg_test_fsync/pg_test_fsync.c | 7 +- src/bin/pg_waldump/pg_waldump.c | 248 +++++++++++++++------ src/include/access/xlog.h | 3 + src/include/access/xlog_internal.h | 76 ++++--- src/include/access/xlogreader.h | 8 +- src/include/catalog/pg_control.h | 2 +- src/include/pg_config.h.in | 5 - src/include/pg_config_manual.h | 6 + src/tools/msvc/Solution.pm | 2 - 46 files changed, 900 insertions(+), 503 deletions(-) diff --git a/configure b/configure index 0d76e5e..5c38149 100755 --- a/configure +++ b/configure @@ -821,7 +821,6 @@ enable_tap_tests with_blocksize with_segsize with_wal_blocksize -with_wal_segsize with_CC enable_depend enable_cassert @@ -1518,8 +1517,6 @@ Optional Packages: --with-segsize=SEGSIZE set table segment size in GB [1] --with-wal-blocksize=BLOCKSIZE set WAL block size in kB [8] - --with-wal-segsize=SEGSIZE - set WAL segment size in MB [16] --with-CC=CMD set compiler (deprecated) --with-icu build with ICU support --with-tcl build Tcl modules (PL/Tcl) @@ -3734,57 +3731,6 @@ _ACEOF # -# WAL segment size -# -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for WAL segment size" >&5 -$as_echo_n "checking for WAL segment size... " >&6; } - - - -# Check whether --with-wal-segsize was given. -if test "${with_wal_segsize+set}" = set; then : - withval=$with_wal_segsize; - case $withval in - yes) - as_fn_error $? "argument required for --with-wal-segsize option" "$LINENO" 5 - ;; - no) - as_fn_error $? "argument required for --with-wal-segsize option" "$LINENO" 5 - ;; - *) - wal_segsize=$withval - ;; - esac - -else - wal_segsize=16 -fi - - -case ${wal_segsize} in - 1) ;; - 2) ;; - 4) ;; - 8) ;; - 16) ;; - 32) ;; - 64) ;; - 128) ;; - 256) ;; - 512) ;; - 1024) ;; - *) as_fn_error $? "Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64,128,256,512,1024." "$LINENO" 5 -esac -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: ${wal_segsize}MB" >&5 -$as_echo "${wal_segsize}MB" >&6; } - - -cat >>confdefs.h <<_ACEOF -#define XLOG_SEG_SIZE (${wal_segsize} * 1024 * 1024) -_ACEOF - - -# # C compiler # diff --git a/configure.in b/configure.in index bdc41b0..176b29a 100644 --- a/configure.in +++ b/configure.in @@ -344,37 +344,6 @@ AC_DEFINE_UNQUOTED([XLOG_BLCKSZ], ${XLOG_BLCKSZ}, [ ]) # -# WAL segment size -# -AC_MSG_CHECKING([for WAL segment size]) -PGAC_ARG_REQ(with, wal-segsize, [SEGSIZE], [set WAL segment size in MB [16]], - [wal_segsize=$withval], - [wal_segsize=16]) -case ${wal_segsize} in - 1) ;; - 2) ;; - 4) ;; - 8) ;; - 16) ;; - 32) ;; - 64) ;; - 128) ;; - 256) ;; - 512) ;; - 1024) ;; - *) AC_MSG_ERROR([Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64,128,256,512,1024.]) -esac -AC_MSG_RESULT([${wal_segsize}MB]) - -AC_DEFINE_UNQUOTED([XLOG_SEG_SIZE], [(${wal_segsize} * 1024 * 1024)], [ - XLOG_SEG_SIZE is the size of a single WAL file. This must be a power of 2 - and larger than XLOG_BLCKSZ (preferably, a great deal larger than - XLOG_BLCKSZ). - - Changing XLOG_SEG_SIZE requires an initdb. -]) - -# # C compiler # diff --git a/contrib/pg_standby/pg_standby.c b/contrib/pg_standby/pg_standby.c index d7fa2a8..9ab4cbd 100644 --- a/contrib/pg_standby/pg_standby.c +++ b/contrib/pg_standby/pg_standby.c @@ -36,6 +36,8 @@ const char *progname; +int WalSegsz; + /* Options and defaults */ int sleeptime = 5; /* amount of time to sleep between file checks */ int waittime = -1; /* how long we have been waiting, -1 no wait @@ -100,6 +102,75 @@ int nextWALFileType; struct stat stat_buf; +static bool SetWALFileNameForCleanup(void); + +/* + * Try to set the wal segment size from the WAL file specified by WALFilePath. + * If successful, set the cleanup WAL filename and need_cleanup. + */ +static bool +SetWALSegSizeAndCleanupValues(void) +{ + bool ret_val = false; + int fd; + char *buf = (char *) malloc(XLOG_BLCKSZ); + + /* Already set a valid WalSegsz? */ + if (IsValidWalSegSize(WalSegsz)) + return true; + + if ((fd = open(WALFilePath, O_RDWR, 0)) < 0) + { + fprintf(stderr, "%s: couldn't open WAL file \"%s\"\n", + progname, WALFilePath); + return false; + } + if (read(fd, buf, XLOG_BLCKSZ) == XLOG_BLCKSZ) + { + XLogPageHeader hdr = (XLogPageHeader) buf; + XLogLongPageHeader longhdr = (XLogLongPageHeader) hdr; + + WalSegsz = longhdr->xlp_seg_size; + + if (IsValidWalSegSize(WalSegsz)) + { + /* successfully retrieved WAL segment size */ + ret_val = true; + + /* check if clean up is necessary */ + need_cleanup = SetWALFileNameForCleanup(); + if (debug) + { + fprintf(stderr, + _("WAL segment size: %d \n"), WalSegsz); + fprintf(stderr, "Keep archive history: "); + + if (need_cleanup) + fprintf(stderr, "%s and later\n", + exclusiveCleanupFileName); + else + fprintf(stderr, "no cleanup required\n"); + } + } + else + fprintf(stderr, + "%s: WAL segment size must be a power of two between 1MB and 1GB, but the WAL file header specifies %d bytes\n", + progname, WalSegsz); + close(fd); + } + else + { + if (errno != 0) + fprintf(stderr, "could not read file \"%s\": %s", + WALFilePath, strerror(errno)); + else + fprintf(stderr, "not enough data in file \"%s\"", WALFilePath); + } + + fflush(stderr); + return ret_val; +} + /* ===================================================================== * * Customizable section @@ -184,7 +255,9 @@ CustomizableNextWALFileReady(void) nextWALFileType = XLOG_BACKUP_LABEL; return true; } - else if (stat_buf.st_size == XLOG_SEG_SIZE) + else if (!SetWALSegSizeAndCleanupValues()) + return false; + else if (stat_buf.st_size == WalSegsz) { #ifdef WIN32 @@ -204,7 +277,7 @@ CustomizableNextWALFileReady(void) /* * If still too small, wait until it is the correct size */ - if (stat_buf.st_size > XLOG_SEG_SIZE) + if (stat_buf.st_size > WalSegsz) { if (debug) { @@ -218,8 +291,6 @@ CustomizableNextWALFileReady(void) return false; } -#define MaxSegmentsPerLogFile ( 0xFFFFFFFF / XLOG_SEG_SIZE ) - static void CustomizableCleanupPriorWALFiles(void) { @@ -315,6 +386,7 @@ SetWALFileNameForCleanup(void) uint32 log_diff = 0, seg_diff = 0; bool cleanup = false; + int max_segments_per_logfile = (0xFFFFFFFF / WalSegsz); if (restartWALFileName) { @@ -336,12 +408,12 @@ SetWALFileNameForCleanup(void) sscanf(nextWALFileName, "%08X%08X%08X", &tli, &log, &seg); if (tli > 0 && seg > 0) { - log_diff = keepfiles / MaxSegmentsPerLogFile; - seg_diff = keepfiles % MaxSegmentsPerLogFile; + log_diff = keepfiles / max_segments_per_logfile; + seg_diff = keepfiles % max_segments_per_logfile; if (seg_diff > seg) { log_diff++; - seg = MaxSegmentsPerLogFile - (seg_diff - seg); + seg = max_segments_per_logfile - (seg_diff - seg); } else seg -= seg_diff; @@ -708,8 +780,6 @@ main(int argc, char **argv) CustomizableInitialize(); - need_cleanup = SetWALFileNameForCleanup(); - if (debug) { fprintf(stderr, "Trigger file: %s\n", triggerPath ? triggerPath : ""); @@ -721,11 +791,6 @@ main(int argc, char **argv) fprintf(stderr, "Max wait interval: %d %s\n", maxwaittime, (maxwaittime > 0 ? "seconds" : "forever")); fprintf(stderr, "Command for restore: %s\n", restoreCommand); - fprintf(stderr, "Keep archive history: "); - if (need_cleanup) - fprintf(stderr, "%s and later\n", exclusiveCleanupFileName); - else - fprintf(stderr, "no cleanup required\n"); fflush(stderr); } diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index 95aeb35..bd55e8b 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -562,7 +562,7 @@ tar -cf backup.tar /usr/local/pgsql/data produces an indefinitely long sequence of WAL records. The system physically divides this sequence into WAL segment files, which are normally 16MB apiece (although the segment size - can be altered when building PostgreSQL). The segment + can be altered during initdb). The segment files are given numeric names that reflect their position in the abstract WAL sequence. When not using WAL archiving, the system normally creates just a few segment files and then diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 12866b4..0f512ba 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -1059,20 +1059,6 @@ su - postgres - - - - Set the WAL segment size, in megabytes. This is - the size of each individual file in the WAL log. It may be useful - to adjust this size to control the granularity of WAL log shipping. - The default size is 16 megabytes. - The value must be a power of 2 between 1 and 1024 (megabytes). - Note that changing this value requires an initdb. - - - - - diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml index 6efb2e4..732feca 100644 --- a/doc/src/sgml/ref/initdb.sgml +++ b/doc/src/sgml/ref/initdb.sgml @@ -316,6 +316,21 @@ PostgreSQL documentation + + + + Set the WAL segment size, in megabytes. This is + the size of each individual file in the WAL log. It may be useful + to adjust this size to control the granularity of WAL log shipping. + This option can only be set during initialization, and cannot be + changed later. + The default size is 16 megabytes. + The value must be a power of 2 between 1 and 1024 (megabytes). + + + + + diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index 940c37b..ddcef5f 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -752,13 +752,12 @@ WAL logs are stored in the directory pg_wal under the data directory, as a set of segment files, normally each 16 MB in size (but the size can be changed - by altering the