From 5e0dec3961f338f621d55bd3f666f69b5063fd90 Mon Sep 17 00:00:00 2001 From: Mark Dilger Date: Sat, 23 Jan 2021 16:24:32 -0800 Subject: [PATCH v33 3/8] Preparing for move of parallel slot infrastructure Preparing to move src/bin/scripts/scripts_parallel.[ch] parallel slot code into fe_utils. ParallelSlotSetup() uses struct ConnParams, functions connectDatabase() and disconnectDatabase(), and enum trivalue. Moving those from src/bin/scripts/common.h into new file src/include/fe_utils/connect_utils.h. Since connectDatabase() uses executeQuery(), also defined in src/bin/scripts/common.h, executeQuery() also needs to be moved to fe_utils. These moves are somewhat messy, because connectDatabase() and disconnectDatabase() are defined next to connectMaintenanceDatabase, the latter not directly implicated in the anticipated move of parallel slot code to fe_utils. But connectMaintenanceDatabase will ultimately need to be moved for use by pg_amcheck, so moving these three connection management functions to fe_utils/connect_utils.[ch] now seems fine. Also messy is that executeQuery() is defined next to executeCommand() and executeMaintenanceCommand(), but these other functions are also not directly implicated in the anticipated move of parallel slot code to fe_utils. We will ultimately need executeCommand() in pg_amcheck, but not executeMaintenanceCommand(). It seems wrong to split these up, though, so moving all three query functions to fe_utils/query_utils.[ch] now seems acceptable. The functions consumeQueryResult() and processQueryResult() in src/bin/scripts/common.[ch] might seem to belong in fe_utils/query_utils.[ch] for the same reasons that executeCommand() and executeMaintenanceCommand(), but unlike the other functions we are moving in this commit, they are used exclusively by the parallel slot infrastructure and will therefore be moved/refactored at the same time as it is. --- src/bin/scripts/clusterdb.c | 1 + src/bin/scripts/common.c | 241 +-------------------------- src/bin/scripts/common.h | 39 +---- src/bin/scripts/reindexdb.c | 1 + src/bin/scripts/vacuumdb.c | 1 + src/fe_utils/Makefile | 2 + src/fe_utils/connect_utils.c | 170 +++++++++++++++++++ src/fe_utils/query_utils.c | 92 ++++++++++ src/include/fe_utils/connect_utils.h | 48 ++++++ src/include/fe_utils/query_utils.h | 26 +++ 10 files changed, 343 insertions(+), 278 deletions(-) create mode 100644 src/fe_utils/connect_utils.c create mode 100644 src/fe_utils/query_utils.c create mode 100644 src/include/fe_utils/connect_utils.h create mode 100644 src/include/fe_utils/query_utils.h diff --git a/src/bin/scripts/clusterdb.c b/src/bin/scripts/clusterdb.c index 7d25bb31d4..24a5a549b4 100644 --- a/src/bin/scripts/clusterdb.c +++ b/src/bin/scripts/clusterdb.c @@ -13,6 +13,7 @@ #include "common.h" #include "common/logging.h" #include "fe_utils/cancel.h" +#include "fe_utils/query_utils.h" #include "fe_utils/simple_list.h" #include "fe_utils/string_utils.h" diff --git a/src/bin/scripts/common.c b/src/bin/scripts/common.c index 21ef297e6e..62645fd276 100644 --- a/src/bin/scripts/common.c +++ b/src/bin/scripts/common.c @@ -22,6 +22,7 @@ #include "common/logging.h" #include "common/string.h" #include "fe_utils/cancel.h" +#include "fe_utils/query_utils.h" #include "fe_utils/string_utils.h" #define ERRCODE_UNDEFINED_TABLE "42P01" @@ -49,246 +50,6 @@ handle_help_version_opts(int argc, char *argv[], } } - -/* - * Make a database connection with the given parameters. - * - * An interactive password prompt is automatically issued if needed and - * allowed by cparams->prompt_password. - * - * If allow_password_reuse is true, we will try to re-use any password - * given during previous calls to this routine. (Callers should not pass - * allow_password_reuse=true unless reconnecting to the same database+user - * as before, else we might create password exposure hazards.) - */ -PGconn * -connectDatabase(const ConnParams *cparams, const char *progname, - bool echo, bool fail_ok, bool allow_password_reuse) -{ - PGconn *conn; - bool new_pass; - static char *password = NULL; - - /* Callers must supply at least dbname; other params can be NULL */ - Assert(cparams->dbname); - - if (!allow_password_reuse && password) - { - free(password); - password = NULL; - } - - if (cparams->prompt_password == TRI_YES && password == NULL) - password = simple_prompt("Password: ", false); - - /* - * Start the connection. Loop until we have a password if requested by - * backend. - */ - do - { - const char *keywords[8]; - const char *values[8]; - int i = 0; - - /* - * If dbname is a connstring, its entries can override the other - * values obtained from cparams; but in turn, override_dbname can - * override the dbname component of it. - */ - keywords[i] = "host"; - values[i++] = cparams->pghost; - keywords[i] = "port"; - values[i++] = cparams->pgport; - keywords[i] = "user"; - values[i++] = cparams->pguser; - keywords[i] = "password"; - values[i++] = password; - keywords[i] = "dbname"; - values[i++] = cparams->dbname; - if (cparams->override_dbname) - { - keywords[i] = "dbname"; - values[i++] = cparams->override_dbname; - } - keywords[i] = "fallback_application_name"; - values[i++] = progname; - keywords[i] = NULL; - values[i++] = NULL; - Assert(i <= lengthof(keywords)); - - new_pass = false; - conn = PQconnectdbParams(keywords, values, true); - - if (!conn) - { - pg_log_error("could not connect to database %s: out of memory", - cparams->dbname); - exit(1); - } - - /* - * No luck? Trying asking (again) for a password. - */ - if (PQstatus(conn) == CONNECTION_BAD && - PQconnectionNeedsPassword(conn) && - cparams->prompt_password != TRI_NO) - { - PQfinish(conn); - if (password) - free(password); - password = simple_prompt("Password: ", false); - new_pass = true; - } - } while (new_pass); - - /* check to see that the backend connection was successfully made */ - if (PQstatus(conn) == CONNECTION_BAD) - { - if (fail_ok) - { - PQfinish(conn); - return NULL; - } - pg_log_error("%s", PQerrorMessage(conn)); - exit(1); - } - - /* Start strict; callers may override this. */ - PQclear(executeQuery(conn, ALWAYS_SECURE_SEARCH_PATH_SQL, echo)); - - return conn; -} - -/* - * Try to connect to the appropriate maintenance database. - * - * This differs from connectDatabase only in that it has a rule for - * inserting a default "dbname" if none was given (which is why cparams - * is not const). Note that cparams->dbname should typically come from - * a --maintenance-db command line parameter. - */ -PGconn * -connectMaintenanceDatabase(ConnParams *cparams, - const char *progname, bool echo) -{ - PGconn *conn; - - /* If a maintenance database name was specified, just connect to it. */ - if (cparams->dbname) - return connectDatabase(cparams, progname, echo, false, false); - - /* Otherwise, try postgres first and then template1. */ - cparams->dbname = "postgres"; - conn = connectDatabase(cparams, progname, echo, true, false); - if (!conn) - { - cparams->dbname = "template1"; - conn = connectDatabase(cparams, progname, echo, false, false); - } - return conn; -} - -/* - * Disconnect the given connection, canceling any statement if one is active. - */ -void -disconnectDatabase(PGconn *conn) -{ - char errbuf[256]; - - Assert(conn != NULL); - - if (PQtransactionStatus(conn) == PQTRANS_ACTIVE) - { - PGcancel *cancel; - - if ((cancel = PQgetCancel(conn))) - { - (void) PQcancel(cancel, errbuf, sizeof(errbuf)); - PQfreeCancel(cancel); - } - } - - PQfinish(conn); -} - -/* - * Run a query, return the results, exit program on failure. - */ -PGresult * -executeQuery(PGconn *conn, const char *query, bool echo) -{ - PGresult *res; - - if (echo) - printf("%s\n", query); - - res = PQexec(conn, query); - if (!res || - PQresultStatus(res) != PGRES_TUPLES_OK) - { - pg_log_error("query failed: %s", PQerrorMessage(conn)); - pg_log_info("query was: %s", query); - PQfinish(conn); - exit(1); - } - - return res; -} - - -/* - * As above for a SQL command (which returns nothing). - */ -void -executeCommand(PGconn *conn, const char *query, bool echo) -{ - PGresult *res; - - if (echo) - printf("%s\n", query); - - res = PQexec(conn, query); - if (!res || - PQresultStatus(res) != PGRES_COMMAND_OK) - { - pg_log_error("query failed: %s", PQerrorMessage(conn)); - pg_log_info("query was: %s", query); - PQfinish(conn); - exit(1); - } - - PQclear(res); -} - - -/* - * As above for a SQL maintenance command (returns command success). - * Command is executed with a cancel handler set, so Ctrl-C can - * interrupt it. - */ -bool -executeMaintenanceCommand(PGconn *conn, const char *query, bool echo) -{ - PGresult *res; - bool r; - - if (echo) - printf("%s\n", query); - - SetCancelConn(conn); - res = PQexec(conn, query); - ResetCancelConn(); - - r = (res && PQresultStatus(res) == PGRES_COMMAND_OK); - - if (res) - PQclear(res); - - return r; -} - /* * Consume all the results generated for the given connection until * nothing remains. If at least one error is encountered, return false. diff --git a/src/bin/scripts/common.h b/src/bin/scripts/common.h index 5630975712..ae19c420df 100644 --- a/src/bin/scripts/common.h +++ b/src/bin/scripts/common.h @@ -10,54 +10,17 @@ #define COMMON_H #include "common/username.h" +#include "fe_utils/connect_utils.h" #include "getopt_long.h" /* pgrminclude ignore */ #include "libpq-fe.h" #include "pqexpbuffer.h" /* pgrminclude ignore */ -enum trivalue -{ - TRI_DEFAULT, - TRI_NO, - TRI_YES -}; - -/* Parameters needed by connectDatabase/connectMaintenanceDatabase */ -typedef struct _connParams -{ - /* These fields record the actual command line parameters */ - const char *dbname; /* this may be a connstring! */ - const char *pghost; - const char *pgport; - const char *pguser; - enum trivalue prompt_password; - /* If not NULL, this overrides the dbname obtained from command line */ - /* (but *only* the DB name, not anything else in the connstring) */ - const char *override_dbname; -} ConnParams; - typedef void (*help_handler) (const char *progname); extern void handle_help_version_opts(int argc, char *argv[], const char *fixed_progname, help_handler hlp); -extern PGconn *connectDatabase(const ConnParams *cparams, - const char *progname, - bool echo, bool fail_ok, - bool allow_password_reuse); - -extern PGconn *connectMaintenanceDatabase(ConnParams *cparams, - const char *progname, bool echo); - -extern void disconnectDatabase(PGconn *conn); - -extern PGresult *executeQuery(PGconn *conn, const char *query, bool echo); - -extern void executeCommand(PGconn *conn, const char *query, bool echo); - -extern bool executeMaintenanceCommand(PGconn *conn, const char *query, - bool echo); - extern bool consumeQueryResult(PGconn *conn); extern bool processQueryResult(PGconn *conn, PGresult *result); diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c index dece8200fa..c9289ae78d 100644 --- a/src/bin/scripts/reindexdb.c +++ b/src/bin/scripts/reindexdb.c @@ -16,6 +16,7 @@ #include "common/connect.h" #include "common/logging.h" #include "fe_utils/cancel.h" +#include "fe_utils/query_utils.h" #include "fe_utils/simple_list.h" #include "fe_utils/string_utils.h" #include "scripts_parallel.h" diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c index 8246327770..55c974ff6d 100644 --- a/src/bin/scripts/vacuumdb.c +++ b/src/bin/scripts/vacuumdb.c @@ -18,6 +18,7 @@ #include "common/connect.h" #include "common/logging.h" #include "fe_utils/cancel.h" +#include "fe_utils/query_utils.h" #include "fe_utils/simple_list.h" #include "fe_utils/string_utils.h" #include "scripts_parallel.h" diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile index bc77d42dbe..9ddf324584 100644 --- a/src/fe_utils/Makefile +++ b/src/fe_utils/Makefile @@ -23,11 +23,13 @@ OBJS = \ archive.o \ cancel.o \ conditional.o \ + connect_utils.o \ exit_utils.o \ mbprint.o \ pgreshandler.o \ print.o \ psqlscan.o \ + query_utils.o \ recovery_gen.o \ simple_list.o \ string_utils.o diff --git a/src/fe_utils/connect_utils.c b/src/fe_utils/connect_utils.c new file mode 100644 index 0000000000..7475e2f366 --- /dev/null +++ b/src/fe_utils/connect_utils.c @@ -0,0 +1,170 @@ +#include "postgres_fe.h" + +#include "common/connect.h" +#include "common/logging.h" +#include "common/string.h" +#include "fe_utils/connect_utils.h" +#include "fe_utils/query_utils.h" + +/* + * Make a database connection with the given parameters. + * + * An interactive password prompt is automatically issued if needed and + * allowed by cparams->prompt_password. + * + * If allow_password_reuse is true, we will try to re-use any password + * given during previous calls to this routine. (Callers should not pass + * allow_password_reuse=true unless reconnecting to the same database+user + * as before, else we might create password exposure hazards.) + */ +PGconn * +connectDatabase(const ConnParams *cparams, const char *progname, + bool echo, bool fail_ok, bool allow_password_reuse) +{ + PGconn *conn; + bool new_pass; + static char *password = NULL; + + /* Callers must supply at least dbname; other params can be NULL */ + Assert(cparams->dbname); + + if (!allow_password_reuse && password) + { + free(password); + password = NULL; + } + + if (cparams->prompt_password == TRI_YES && password == NULL) + password = simple_prompt("Password: ", false); + + /* + * Start the connection. Loop until we have a password if requested by + * backend. + */ + do + { + const char *keywords[8]; + const char *values[8]; + int i = 0; + + /* + * If dbname is a connstring, its entries can override the other + * values obtained from cparams; but in turn, override_dbname can + * override the dbname component of it. + */ + keywords[i] = "host"; + values[i++] = cparams->pghost; + keywords[i] = "port"; + values[i++] = cparams->pgport; + keywords[i] = "user"; + values[i++] = cparams->pguser; + keywords[i] = "password"; + values[i++] = password; + keywords[i] = "dbname"; + values[i++] = cparams->dbname; + if (cparams->override_dbname) + { + keywords[i] = "dbname"; + values[i++] = cparams->override_dbname; + } + keywords[i] = "fallback_application_name"; + values[i++] = progname; + keywords[i] = NULL; + values[i++] = NULL; + Assert(i <= lengthof(keywords)); + + new_pass = false; + conn = PQconnectdbParams(keywords, values, true); + + if (!conn) + { + pg_log_error("could not connect to database %s: out of memory", + cparams->dbname); + exit(1); + } + + /* + * No luck? Trying asking (again) for a password. + */ + if (PQstatus(conn) == CONNECTION_BAD && + PQconnectionNeedsPassword(conn) && + cparams->prompt_password != TRI_NO) + { + PQfinish(conn); + if (password) + free(password); + password = simple_prompt("Password: ", false); + new_pass = true; + } + } while (new_pass); + + /* check to see that the backend connection was successfully made */ + if (PQstatus(conn) == CONNECTION_BAD) + { + if (fail_ok) + { + PQfinish(conn); + return NULL; + } + pg_log_error("%s", PQerrorMessage(conn)); + exit(1); + } + + /* Start strict; callers may override this. */ + PQclear(executeQuery(conn, ALWAYS_SECURE_SEARCH_PATH_SQL, echo)); + + return conn; +} + +/* + * Try to connect to the appropriate maintenance database. + * + * This differs from connectDatabase only in that it has a rule for + * inserting a default "dbname" if none was given (which is why cparams + * is not const). Note that cparams->dbname should typically come from + * a --maintenance-db command line parameter. + */ +PGconn * +connectMaintenanceDatabase(ConnParams *cparams, + const char *progname, bool echo) +{ + PGconn *conn; + + /* If a maintenance database name was specified, just connect to it. */ + if (cparams->dbname) + return connectDatabase(cparams, progname, echo, false, false); + + /* Otherwise, try postgres first and then template1. */ + cparams->dbname = "postgres"; + conn = connectDatabase(cparams, progname, echo, true, false); + if (!conn) + { + cparams->dbname = "template1"; + conn = connectDatabase(cparams, progname, echo, false, false); + } + return conn; +} + +/* + * Disconnect the given connection, canceling any statement if one is active. + */ +void +disconnectDatabase(PGconn *conn) +{ + char errbuf[256]; + + Assert(conn != NULL); + + if (PQtransactionStatus(conn) == PQTRANS_ACTIVE) + { + PGcancel *cancel; + + if ((cancel = PQgetCancel(conn))) + { + (void) PQcancel(cancel, errbuf, sizeof(errbuf)); + PQfreeCancel(cancel); + } + } + + PQfinish(conn); +} diff --git a/src/fe_utils/query_utils.c b/src/fe_utils/query_utils.c new file mode 100644 index 0000000000..a70ae3c082 --- /dev/null +++ b/src/fe_utils/query_utils.c @@ -0,0 +1,92 @@ +/*------------------------------------------------------------------------- + * + * Facilities for frontend code to query a databases. + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/fe_utils/query_utils.c + * + *------------------------------------------------------------------------- + */ +#include "postgres_fe.h" + +#include "common/logging.h" +#include "fe_utils/cancel.h" +#include "fe_utils/query_utils.h" + +/* + * Run a query, return the results, exit program on failure. + */ +PGresult * +executeQuery(PGconn *conn, const char *query, bool echo) +{ + PGresult *res; + + if (echo) + printf("%s\n", query); + + res = PQexec(conn, query); + if (!res || + PQresultStatus(res) != PGRES_TUPLES_OK) + { + pg_log_error("query failed: %s", PQerrorMessage(conn)); + pg_log_info("query was: %s", query); + PQfinish(conn); + exit(1); + } + + return res; +} + + +/* + * As above for a SQL command (which returns nothing). + */ +void +executeCommand(PGconn *conn, const char *query, bool echo) +{ + PGresult *res; + + if (echo) + printf("%s\n", query); + + res = PQexec(conn, query); + if (!res || + PQresultStatus(res) != PGRES_COMMAND_OK) + { + pg_log_error("query failed: %s", PQerrorMessage(conn)); + pg_log_info("query was: %s", query); + PQfinish(conn); + exit(1); + } + + PQclear(res); +} + + +/* + * As above for a SQL maintenance command (returns command success). + * Command is executed with a cancel handler set, so Ctrl-C can + * interrupt it. + */ +bool +executeMaintenanceCommand(PGconn *conn, const char *query, bool echo) +{ + PGresult *res; + bool r; + + if (echo) + printf("%s\n", query); + + SetCancelConn(conn); + res = PQexec(conn, query); + ResetCancelConn(); + + r = (res && PQresultStatus(res) == PGRES_COMMAND_OK); + + if (res) + PQclear(res); + + return r; +} diff --git a/src/include/fe_utils/connect_utils.h b/src/include/fe_utils/connect_utils.h new file mode 100644 index 0000000000..8fde0ea2a0 --- /dev/null +++ b/src/include/fe_utils/connect_utils.h @@ -0,0 +1,48 @@ +/*------------------------------------------------------------------------- + * + * Facilities for frontend code to connect to and disconnect from databases. + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/fe_utils/connect_utils.h + * + *------------------------------------------------------------------------- + */ +#ifndef CONNECT_UTILS_H +#define CONNECT_UTILS_H + +#include "libpq-fe.h" + +enum trivalue +{ + TRI_DEFAULT, + TRI_NO, + TRI_YES +}; + +/* Parameters needed by connectDatabase/connectMaintenanceDatabase */ +typedef struct _connParams +{ + /* These fields record the actual command line parameters */ + const char *dbname; /* this may be a connstring! */ + const char *pghost; + const char *pgport; + const char *pguser; + enum trivalue prompt_password; + /* If not NULL, this overrides the dbname obtained from command line */ + /* (but *only* the DB name, not anything else in the connstring) */ + const char *override_dbname; +} ConnParams; + +extern PGconn *connectDatabase(const ConnParams *cparams, + const char *progname, + bool echo, bool fail_ok, + bool allow_password_reuse); + +extern PGconn *connectMaintenanceDatabase(ConnParams *cparams, + const char *progname, bool echo); + +extern void disconnectDatabase(PGconn *conn); + +#endif /* CONNECT_UTILS_H */ diff --git a/src/include/fe_utils/query_utils.h b/src/include/fe_utils/query_utils.h new file mode 100644 index 0000000000..1f5812bbf6 --- /dev/null +++ b/src/include/fe_utils/query_utils.h @@ -0,0 +1,26 @@ +/*------------------------------------------------------------------------- + * + * Facilities for frontend code to query a databases. + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/fe_utils/query_utils.h + * + *------------------------------------------------------------------------- + */ +#ifndef QUERY_UTILS_H +#define QUERY_UTILS_H + +#include "postgres_fe.h" + +#include "libpq-fe.h" + +extern PGresult *executeQuery(PGconn *conn, const char *query, bool echo); + +extern void executeCommand(PGconn *conn, const char *query, bool echo); + +extern bool executeMaintenanceCommand(PGconn *conn, const char *query, + bool echo); + +#endif /* QUERY_UTILS_H */ -- 2.21.1 (Apple Git-122.3)