From 1ee11643fa63c59b032aca475b1144c8f1f6063b Mon Sep 17 00:00:00 2001 From: dilipkumar Date: Sat, 24 Oct 2020 20:16:48 +0530 Subject: [PATCH v10 6/6] Support compression methods options --- contrib/cmzlib/zlib.c | 74 ++++++++++++++-- doc/src/sgml/ref/alter_table.sgml | 18 ++-- doc/src/sgml/ref/create_table.sgml | 8 +- src/backend/access/common/indextuple.c | 2 +- src/backend/access/common/reloptions.c | 42 ++++++++++ src/backend/access/common/toast_internals.c | 9 +- src/backend/access/compression/lz4.c | 4 +- src/backend/access/compression/pglz.c | 93 ++++++++++++++++++++- src/backend/access/table/toast_helper.c | 25 +++++- src/backend/bootstrap/bootparse.y | 1 + src/backend/catalog/heap.c | 15 +++- src/backend/catalog/index.c | 3 +- src/backend/catalog/toasting.c | 1 + src/backend/commands/cluster.c | 1 + src/backend/commands/compressioncmds.c | 18 +++- src/backend/commands/foreigncmds.c | 44 ---------- src/backend/commands/tablecmds.c | 34 ++++++-- src/backend/nodes/copyfuncs.c | 1 + src/backend/nodes/equalfuncs.c | 1 + src/backend/nodes/outfuncs.c | 1 + src/backend/parser/gram.y | 16 +++- src/include/access/compressionapi.h | 14 +++- src/include/access/toast_helper.h | 1 + src/include/access/toast_internals.h | 2 +- src/include/catalog/heap.h | 2 + src/include/catalog/pg_attribute.h | 3 + src/include/commands/defrem.h | 3 +- src/include/nodes/parsenodes.h | 1 + src/test/regress/expected/create_cm.out | 14 ++++ src/test/regress/expected/misc_sanity.out | 3 +- src/test/regress/sql/create_cm.sql | 8 ++ 31 files changed, 369 insertions(+), 93 deletions(-) diff --git a/contrib/cmzlib/zlib.c b/contrib/cmzlib/zlib.c index f24fc1c936..7d0a256f6f 100644 --- a/contrib/cmzlib/zlib.c +++ b/contrib/cmzlib/zlib.c @@ -14,6 +14,7 @@ #include "postgres.h" #include "access/compressionapi.h" #include "access/toast_internals.h" +#include "commands/defrem.h" #include "fmgr.h" #include "utils/builtins.h" @@ -45,6 +46,61 @@ typedef struct unsigned int dictlen; } zlib_state; +/* + * Check options if specified. All validation is located here so + * we don't need do it again in cminitstate function. + */ +static void +zlib_cmcheck(List *options) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, "level") == 0) + { + int8 level = pg_atoi(defGetString(def), sizeof(int8), 0); + if (level < 0 || level > 9) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unexpected value for zlib compression level: \"%s\"", + defGetString(def)), + errhint("expected value between 0 and 9") + )); + } + else + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unexpected parameter for zlib: \"%s\"", def->defname))); + } +} + +static void * +zlib_cminitstate(List *options) +{ + zlib_state *state = NULL; + + state = palloc0(sizeof(zlib_state)); + state->level = Z_DEFAULT_COMPRESSION; + + if (list_length(options) > 0) + { + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, "level") == 0) + state->level = pg_atoi(defGetString(def), sizeof(int), 0); + } + } + + return state; +} + /* * zlib_cmcompress - compression routine for zlib compression method * @@ -52,23 +108,21 @@ typedef struct * Returns the compressed varlena, or NULL if compression fails. */ static struct varlena * -zlib_cmcompress(const struct varlena *value, int32 header_size) +zlib_cmcompress(const struct varlena *value, int32 header_size, void *options) { - int32 valsize, - len; + int32 valsize, + len; struct varlena *tmp = NULL; - z_streamp zp; - int res; - zlib_state state; - - state.level = Z_DEFAULT_COMPRESSION; + z_streamp zp; + int res; + zlib_state *state = (zlib_state *) options; zp = (z_streamp) palloc(sizeof(z_stream)); zp->zalloc = Z_NULL; zp->zfree = Z_NULL; zp->opaque = Z_NULL; - if (deflateInit(zp, state.level) != Z_OK) + if (deflateInit(zp, state->level) != Z_OK) elog(ERROR, "could not initialize compression library: %s", zp->msg); valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); @@ -150,6 +204,8 @@ zlibhandler(PG_FUNCTION_ARGS) { CompressionRoutine *routine = makeNode(CompressionRoutine); + routine->cmcheck = zlib_cmcheck; + routine->cminitstate = zlib_cminitstate; routine->cmcompress = zlib_cmcompress; routine->cmdecompress = zlib_cmdecompress; routine->cmdecompress_slice = NULL; diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 733ee0cf20..58782153a6 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -54,7 +54,7 @@ ALTER TABLE [ IF EXISTS ] name ALTER [ COLUMN ] column_name SET ( attribute_option = value [, ... ] ) ALTER [ COLUMN ] column_name RESET ( attribute_option [, ... ] ) ALTER [ COLUMN ] column_name SET STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN } - ALTER [ COLUMN ] column_name SET COMPRESSION compression_method [ PRESERVE (compression_preserve_list) | PRESERVE ALL ] + ALTER [ COLUMN ] column_name SET COMPRESSION compression_method [ WITH (compression_method_options) ] [ PRESERVE (compression_preserve_list) | PRESERVE ALL ] ADD table_constraint [ NOT VALID ] ADD table_constraint_using_index ALTER CONSTRAINT constraint_name [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] @@ -386,17 +386,19 @@ WITH ( MODULUS numeric_literal, REM - SET COMPRESSION compression_method [ PRESERVE (compression_preserve_list) | PRESERVE ALL ] + SET COMPRESSION compression_method [ WITH (compression_method_options) ] [ PRESERVE (compression_preserve_list) | PRESERVE ALL ] This clause adds compression to a column. Compression method can be - set from available built-in compression methods. The PRESERVE list - contains list of compression methods used on the column and determines - which of them should be kept on the column. Without PRESERVE or if all - the previous compression methods are not preserved then the table will - be rewritten. If PRESERVE ALL is specified then all the previous methods - will be preserved and the table will not be rewritten. + set from available built-in compression methods. If compression + method has options they could be specified with WITH + parameter. The PRESERVE list contains list of compression methods used + on the column and determines which of them should be kept on the column. + Without PRESERVE or if all the previous compression methods are not + preserved then the table will be rewritten. If PRESERVE ALL is specified + then all the previous methods will be preserved and the table will not be + rewritten. diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index f083602f7f..c32a7ae835 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -69,7 +69,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( sequence_options ) ] | UNIQUE index_parameters | PRIMARY KEY index_parameters | - COMPRESSION compression_method | + COMPRESSION compression_method [ WITH (compression_options) ] | REFERENCES reftable [ ( refcolumn ) ] [ MATCH FULL | MATCH PARTIAL | MATCH SIMPLE ] [ ON DELETE referential_action ] [ ON UPDATE referential_action ] } [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] @@ -968,13 +968,15 @@ WITH ( MODULUS numeric_literal, REM - COMPRESSION compression_method + COMPRESSION compression_method [ WITH (compression_am_options) ] This clause adds the compression method to a column. Compression method can be set from the available built-in compression methods. The available options are pglz and lz4. The - default compression method is pglz. + default compression method is pglz. If compression + method has options they could be specified by WITH + parameter. diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c index f81fc8ea66..616f31ba47 100644 --- a/src/backend/access/common/indextuple.c +++ b/src/backend/access/common/indextuple.c @@ -105,7 +105,7 @@ index_form_tuple(TupleDesc tupleDescriptor, att->attstorage == TYPSTORAGE_MAIN)) { Datum cvalue = toast_compress_datum(untoasted_values[i], - att->attcompression); + att->attcompression, NULL); if (DatumGetPointer(cvalue) != NULL) { diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index 8ccc228a8c..3185082034 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -2112,3 +2112,45 @@ AlterTableGetRelOptionsLockLevel(List *defList) return lockmode; } + +/* + * Convert a DefElem list to the text array format that is used in + * pg_foreign_data_wrapper, pg_foreign_server, pg_user_mapping, and + * pg_foreign_table. + * + * Returns the array in the form of a Datum, or PointerGetDatum(NULL) + * if the list is empty. + * + * Note: The array is usually stored to database without further + * processing, hence any validation should be done before this + * conversion. + */ +Datum +optionListToArray(List *options) +{ + ArrayBuildState *astate = NULL; + ListCell *cell; + + foreach(cell, options) + { + DefElem *def = lfirst(cell); + const char *value; + Size len; + text *t; + + value = defGetString(def); + len = VARHDRSZ + strlen(def->defname) + 1 + strlen(value); + t = palloc(len + 1); + SET_VARSIZE(t, len); + sprintf(VARDATA(t), "%s=%s", def->defname, value); + + astate = accumArrayResult(astate, PointerGetDatum(t), + false, TEXTOID, + CurrentMemoryContext); + } + + if (astate) + return makeArrayResult(astate, CurrentMemoryContext); + + return PointerGetDatum(NULL); +} diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index d69dd907c9..34343695ea 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -62,10 +62,11 @@ toast_set_compressed_datum_info(struct varlena *val, CompressionId cmid, * ---------- */ Datum -toast_compress_datum(Datum value, Oid cmoid) +toast_compress_datum(Datum value, Oid cmoid, List *cmoptions) { struct varlena *tmp = NULL; int32 valsize; + void *options = NULL; CompressionId cmid; CompressionRoutine *cmroutine; @@ -80,10 +81,14 @@ toast_compress_datum(Datum value, Oid cmoid) cmroutine = GetCompressionRoutine(cmoid); cmid = GetCompressionId(cmoid); + if (cmroutine->cminitstate) + options = cmroutine->cminitstate(cmoptions); + /* Call the actual compression function */ tmp = cmroutine->cmcompress((const struct varlena *) value, IsCustomCompression(cmid) ? - TOAST_CUSTOM_COMPRESS_HDRSZ : TOAST_COMPRESS_HDRSZ); + TOAST_CUSTOM_COMPRESS_HDRSZ : TOAST_COMPRESS_HDRSZ, + options); if (!tmp) return PointerGetDatum(NULL); diff --git a/src/backend/access/compression/lz4.c b/src/backend/access/compression/lz4.c index 52c167a35d..fc5bcaa351 100644 --- a/src/backend/access/compression/lz4.c +++ b/src/backend/access/compression/lz4.c @@ -29,7 +29,7 @@ * compressed varlena, or NULL if compression fails. */ static struct varlena * -lz4_cmcompress(const struct varlena *value, int32 header_size) +lz4_cmcompress(const struct varlena *value, int32 header_size, void *options) { int32 valsize; int32 len; @@ -120,6 +120,8 @@ lz4handler(PG_FUNCTION_ARGS) #else CompressionRoutine *routine = makeNode(CompressionRoutine); + routine->cmcheck = NULL; + routine->cminitstate = NULL; routine->cmcompress = lz4_cmcompress; routine->cmdecompress = lz4_cmdecompress; routine->cmdecompress_slice = lz4_cmdecompress_slice; diff --git a/src/backend/access/compression/pglz.c b/src/backend/access/compression/pglz.c index 931394f779..607849a73d 100644 --- a/src/backend/access/compression/pglz.c +++ b/src/backend/access/compression/pglz.c @@ -15,11 +15,92 @@ #include "access/compressionapi.h" #include "access/toast_internals.h" +#include "commands/defrem.h" #include "common/pg_lzcompress.h" #include "fmgr.h" #include "utils/builtins.h" +#define PGLZ_OPTIONS_COUNT 6 + +static char *PGLZ_options[PGLZ_OPTIONS_COUNT] = { + "min_input_size", + "max_input_size", + "min_comp_rate", + "first_success_by", + "match_size_good", + "match_size_drop" +}; + +/* + * Convert value from reloptions to int32, and report if it is not correct. + * Also checks parameter names + */ +static int32 +parse_option(char *name, char *value) +{ + int i; + + for (i = 0; i < PGLZ_OPTIONS_COUNT; i++) + { + if (strcmp(PGLZ_options[i], name) == 0) + return pg_atoi(value, 4, 0); + } + + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unexpected parameter for pglz: \"%s\"", name))); +} + +/* + * Check PGLZ options if specified + */ +static void +pglz_cmcheck(List *options) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + parse_option(def->defname, defGetString(def)); + } +} + +/* + * Configure PGLZ_Strategy struct for compression function + */ +static void * +pglz_cminitstate(List *options) +{ + ListCell *lc; + PGLZ_Strategy *strategy = palloc(sizeof(PGLZ_Strategy)); + + /* initialize with default strategy values */ + memcpy(strategy, PGLZ_strategy_default, sizeof(PGLZ_Strategy)); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + int32 val = parse_option(def->defname, defGetString(def)); + + /* fill the strategy */ + if (strcmp(def->defname, "min_input_size") == 0) + strategy->min_input_size = val; + else if (strcmp(def->defname, "max_input_size") == 0) + strategy->max_input_size = val; + else if (strcmp(def->defname, "min_comp_rate") == 0) + strategy->min_comp_rate = val; + else if (strcmp(def->defname, "first_success_by") == 0) + strategy->first_success_by = val; + else if (strcmp(def->defname, "match_size_good") == 0) + strategy->match_size_good = val; + else if (strcmp(def->defname, "match_size_drop") == 0) + strategy->match_size_drop = val; + } + return (void *) strategy; +} + /* * pglz_cmcompress - compression routine for pglz compression method * @@ -27,20 +108,22 @@ * compressed varlena, or NULL if compression fails. */ static struct varlena * -pglz_cmcompress(const struct varlena *value, int32 header_size) +pglz_cmcompress(const struct varlena *value, int32 header_size, void *options) { int32 valsize, len; struct varlena *tmp = NULL; + PGLZ_Strategy *strategy; valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + strategy = (PGLZ_Strategy *) options; /* * No point in wasting a palloc cycle if value size is out of the allowed * range for compression */ - if (valsize < PGLZ_strategy_default->min_input_size || - valsize > PGLZ_strategy_default->max_input_size) + if (valsize < strategy->min_input_size || + valsize > strategy->max_input_size) return NULL; tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + @@ -49,7 +132,7 @@ pglz_cmcompress(const struct varlena *value, int32 header_size) len = pglz_compress(VARDATA_ANY(DatumGetPointer(value)), valsize, (char *) tmp + header_size, - NULL); + strategy); if (len >= 0) { @@ -122,6 +205,8 @@ pglzhandler(PG_FUNCTION_ARGS) { CompressionRoutine *routine = makeNode(CompressionRoutine); + routine->cmcheck = pglz_cmcheck; + routine->cminitstate = pglz_cminitstate; routine->cmcompress = pglz_cmcompress; routine->cmdecompress = pglz_cmdecompress; routine->cmdecompress_slice = pglz_cmdecompress_slice; diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c index b33c925a4b..1f842abfaf 100644 --- a/src/backend/access/table/toast_helper.c +++ b/src/backend/access/table/toast_helper.c @@ -15,11 +15,12 @@ #include "postgres.h" #include "access/detoast.h" +#include "access/reloptions.h" #include "access/table.h" #include "access/toast_helper.h" #include "access/toast_internals.h" #include "catalog/pg_type_d.h" - +#include "utils/syscache.h" /* * Prepare to TOAST a tuple. @@ -43,6 +44,7 @@ toast_tuple_init(ToastTupleContext *ttc) TupleDesc tupleDesc = ttc->ttc_rel->rd_att; int numAttrs = tupleDesc->natts; int i; + Datum attcmoptions; ttc->ttc_flags = 0; @@ -51,10 +53,28 @@ toast_tuple_init(ToastTupleContext *ttc) Form_pg_attribute att = TupleDescAttr(tupleDesc, i); struct varlena *old_value; struct varlena *new_value; + HeapTuple attr_tuple; + bool isNull; ttc->ttc_attr[i].tai_colflags = 0; ttc->ttc_attr[i].tai_oldexternal = NULL; ttc->ttc_attr[i].tai_compression = att->attcompression; + attr_tuple = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(att->attrelid), + Int16GetDatum(att->attnum)); + if (!HeapTupleIsValid(attr_tuple)) + elog(ERROR, "cache lookup failed for attribute %d of relation %u", + att->attnum, att->attrelid); + + attcmoptions = SysCacheGetAttr(ATTNUM, attr_tuple, + Anum_pg_attribute_attcmoptions, + &isNull); + if (isNull) + ttc->ttc_attr[i].tai_cmoptions = NULL; + else + ttc->ttc_attr[i].tai_cmoptions = untransformRelOptions(attcmoptions); + + ReleaseSysCache(attr_tuple); if (ttc->ttc_oldvalues != NULL) { @@ -230,7 +250,8 @@ toast_tuple_try_compression(ToastTupleContext *ttc, int attribute) Datum new_value; ToastAttrInfo *attr = &ttc->ttc_attr[attribute]; - new_value = toast_compress_datum(*value, attr->tai_compression); + new_value = toast_compress_datum(*value, attr->tai_compression, + attr->tai_cmoptions); if (DatumGetPointer(new_value) != NULL) { diff --git a/src/backend/bootstrap/bootparse.y b/src/backend/bootstrap/bootparse.y index 6bb0c6ed1e..bbc849b541 100644 --- a/src/backend/bootstrap/bootparse.y +++ b/src/backend/bootstrap/bootparse.y @@ -239,6 +239,7 @@ Boot_CreateStmt: true, false, InvalidOid, + NULL, NULL); elog(DEBUG4, "relation created with OID %u", id); } diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 5f77b61a9b..d7b201fe5a 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -732,6 +732,7 @@ InsertPgAttributeTuples(Relation pg_attribute_rel, TupleDesc tupdesc, Oid new_rel_oid, Datum *attoptions, + Datum *attcmoptions, CatalogIndexState indstate) { TupleTableSlot **slot; @@ -787,6 +788,11 @@ InsertPgAttributeTuples(Relation pg_attribute_rel, else slot[slotCount]->tts_isnull[Anum_pg_attribute_attoptions - 1] = true; + if (attcmoptions && attcmoptions[natts] != (Datum) 0) + slot[slotCount]->tts_values[Anum_pg_attribute_attcmoptions - 1] = attcmoptions[natts]; + else + slot[slotCount]->tts_isnull[Anum_pg_attribute_attcmoptions - 1] = true; + /* start out with empty permissions and empty options */ slot[slotCount]->tts_isnull[Anum_pg_attribute_attacl - 1] = true; slot[slotCount]->tts_isnull[Anum_pg_attribute_attfdwoptions - 1] = true; @@ -834,6 +840,7 @@ InsertPgAttributeTuples(Relation pg_attribute_rel, static void AddNewAttributeTuples(Oid new_rel_oid, TupleDesc tupdesc, + Datum *acoption, char relkind) { Relation rel; @@ -852,7 +859,8 @@ AddNewAttributeTuples(Oid new_rel_oid, /* set stats detail level to a sane default */ for (int i = 0; i < natts; i++) tupdesc->attrs[i].attstattarget = -1; - InsertPgAttributeTuples(rel, tupdesc, new_rel_oid, NULL, indstate); + InsertPgAttributeTuples(rel, tupdesc, new_rel_oid, + NULL, acoption, indstate); /* add dependencies on their datatypes and collations */ for (int i = 0; i < natts; i++) @@ -884,7 +892,7 @@ AddNewAttributeTuples(Oid new_rel_oid, td = CreateTupleDesc(lengthof(SysAtt), (FormData_pg_attribute **) &SysAtt); - InsertPgAttributeTuples(rel, td, new_rel_oid, NULL, indstate); + InsertPgAttributeTuples(rel, td, new_rel_oid, NULL, NULL, indstate); FreeTupleDesc(td); } @@ -1150,6 +1158,7 @@ heap_create_with_catalog(const char *relname, bool allow_system_table_mods, bool is_internal, Oid relrewrite, + Datum *acoptions, ObjectAddress *typaddress) { Relation pg_class_desc; @@ -1407,7 +1416,7 @@ heap_create_with_catalog(const char *relname, /* * now add tuples to pg_attribute for the attributes in our new relation. */ - AddNewAttributeTuples(relid, new_rel_desc->rd_att, relkind); + AddNewAttributeTuples(relid, new_rel_desc->rd_att, acoptions, relkind); /* * Make a dependency link to force the relation to be deleted if its diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 2dcad80e69..4f898bb65e 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -505,7 +505,8 @@ AppendAttributeTuples(Relation indexRelation, Datum *attopts) */ indexTupDesc = RelationGetDescr(indexRelation); - InsertPgAttributeTuples(pg_attribute, indexTupDesc, InvalidOid, attopts, indstate); + InsertPgAttributeTuples(pg_attribute, indexTupDesc, InvalidOid, + attopts, NULL, indstate); CatalogCloseIndexes(indstate); diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index c40d25b301..6f8dd45d23 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -255,6 +255,7 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, true, true, InvalidOid, + NULL, NULL); Assert(toast_relid != InvalidOid); diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 0d647e912c..5c77a36699 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -680,6 +680,7 @@ make_new_heap(Oid OIDOldHeap, Oid NewTableSpace, char relpersistence, true, true, OIDOldHeap, + NULL, NULL); Assert(OIDNewHeap != InvalidOid); diff --git a/src/backend/commands/compressioncmds.c b/src/backend/commands/compressioncmds.c index d47d55b119..21afea729a 100644 --- a/src/backend/commands/compressioncmds.c +++ b/src/backend/commands/compressioncmds.c @@ -228,7 +228,7 @@ IsCompressionSupported(Form_pg_attribute att, Oid cmoid) */ Oid GetAttributeCompression(Form_pg_attribute att, ColumnCompression *compression, - bool *need_rewrite) + Datum *acoptions, bool *need_rewrite) { Oid cmoid; ListCell *cell; @@ -247,6 +247,22 @@ GetAttributeCompression(Form_pg_attribute att, ColumnCompression *compression, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("compression type \"%s\" not recognized", compression->cmname))); + /* if compression options are given then check them */ + if (compression->options) + { + CompressionRoutine *routine = GetCompressionRoutine(cmoid); + + /* we need routine only to call cmcheck function */ + if (routine->cmcheck != NULL) + routine->cmcheck(compression->options); + + pfree(routine); + + *acoptions = optionListToArray(compression->options); + } + else + *acoptions = PointerGetDatum(NULL); + /* * Determine if the column needs rewrite or not. Rewrite conditions: SET * COMPRESSION without PRESERVE - SET COMPRESSION with PRESERVE but not diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index de31ddd1f3..95bdcb1874 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -49,50 +49,6 @@ typedef struct /* Internal functions */ static void import_error_callback(void *arg); - -/* - * Convert a DefElem list to the text array format that is used in - * pg_foreign_data_wrapper, pg_foreign_server, pg_user_mapping, and - * pg_foreign_table. - * - * Returns the array in the form of a Datum, or PointerGetDatum(NULL) - * if the list is empty. - * - * Note: The array is usually stored to database without further - * processing, hence any validation should be done before this - * conversion. - */ -static Datum -optionListToArray(List *options) -{ - ArrayBuildState *astate = NULL; - ListCell *cell; - - foreach(cell, options) - { - DefElem *def = lfirst(cell); - const char *value; - Size len; - text *t; - - value = defGetString(def); - len = VARHDRSZ + strlen(def->defname) + 1 + strlen(value); - t = palloc(len + 1); - SET_VARSIZE(t, len); - sprintf(VARDATA(t), "%s=%s", def->defname, value); - - astate = accumArrayResult(astate, PointerGetDatum(t), - false, TEXTOID, - CurrentMemoryContext); - } - - if (astate) - return makeArrayResult(astate, CurrentMemoryContext); - - return PointerGetDatum(NULL); -} - - /* * Transform a list of DefElem into text array format. This is substantially * the same thing as optionListToArray(), except we recognize SET/ADD/DROP diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 80bdb79530..358c56ec03 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -610,6 +610,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, LOCKMODE parentLockmode; const char *accessMethod = NULL; Oid accessMethodId = InvalidOid; + Datum *acoptions; /* * Truncate relname to appropriate length (probably a waste of time, as @@ -814,6 +815,8 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, cookedDefaults = NIL; attnum = 0; + acoptions = (Datum *) palloc0(sizeof(Datum) * descriptor->natts); + foreach(listptr, stmt->tableElts) { ColumnDef *colDef = lfirst(listptr); @@ -867,8 +870,9 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, if (!IsBinaryUpgrade && (relkind == RELKIND_RELATION || relkind == RELKIND_PARTITIONED_TABLE)) - attr->attcompression = - GetAttributeCompression(attr, colDef->compression, NULL); + attr->attcompression = GetAttributeCompression(attr, + colDef->compression, + &acoptions[attnum - 1], NULL); else attr->attcompression = InvalidOid; } @@ -922,8 +926,11 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, allowSystemTableMods, false, InvalidOid, + acoptions, typaddress); + pfree(acoptions); + /* * We must bump the command counter to make the newly-created relation * tuple visible for opening. @@ -6136,6 +6143,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, AclResult aclresult; ObjectAddress address; TupleDesc tupdesc; + Datum acoptions = PointerGetDatum(NULL); FormData_pg_attribute *aattr[] = {&attribute}; /* At top level, permission check was done in ATPrepCmd, else do it */ @@ -6298,7 +6306,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, if (rel->rd_rel->relkind == RELKIND_RELATION || rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) attribute.attcompression = GetAttributeCompression(&attribute, - colDef->compression, NULL); + colDef->compression, &acoptions, NULL); else attribute.attcompression = InvalidOid; @@ -6308,7 +6316,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, tupdesc = CreateTupleDesc(lengthof(aattr), (FormData_pg_attribute **) &aattr); - InsertPgAttributeTuples(attrdesc, tupdesc, myrelid, NULL, NULL); + InsertPgAttributeTuples(attrdesc, tupdesc, myrelid, NULL, &acoptions, NULL); table_close(attrdesc, RowExclusiveLock); @@ -15046,9 +15054,11 @@ ATExecSetCompression(AlteredTableInfo *tab, AttrNumber attnum; Oid cmoid; bool need_rewrite; + HeapTuple newtuple; Datum values[Natts_pg_attribute]; bool nulls[Natts_pg_attribute]; bool replace[Natts_pg_attribute]; + Datum acoptions; ObjectAddress address; ListCell *lc; @@ -15082,7 +15092,8 @@ ATExecSetCompression(AlteredTableInfo *tab, memset(replace, false, sizeof(replace)); /* Get the attribute compression method. */ - cmoid = GetAttributeCompression(atttableform, compression, &need_rewrite); + cmoid = GetAttributeCompression(atttableform, compression, + &acoptions, &need_rewrite); if (atttableform->attcompression != cmoid) add_column_compression_dependency(atttableform->attrelid, @@ -15091,7 +15102,18 @@ ATExecSetCompression(AlteredTableInfo *tab, tab->rewrite |= AT_REWRITE_ALTER_COMPRESSION; atttableform->attcompression = cmoid; - CatalogTupleUpdate(attrel, &atttuple->t_self, atttuple); + + /* update existing entry */ + if (acoptions) + { + values[Anum_pg_attribute_attcmoptions - 1] = PointerGetDatum(acoptions); + replace[Anum_pg_attribute_attcmoptions - 1] = true; + newtuple = heap_modify_tuple(atttuple, RelationGetDescr(attrel), + values, nulls, replace); + CatalogTupleUpdate(attrel, &newtuple->t_self, newtuple); + } + else + CatalogTupleUpdate(attrel, &atttuple->t_self, atttuple); InvokeObjectPostAlterHook(RelationRelationId, RelationGetRelid(rel), diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 53c8ad462e..30833e32a4 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2955,6 +2955,7 @@ _copyColumnCompression(const ColumnCompression *from) COPY_STRING_FIELD(cmname); COPY_SCALAR_FIELD(preserve_all); + COPY_NODE_FIELD(options); COPY_NODE_FIELD(preserve); return newnode; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index f3023ae03f..919b0745f8 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -2622,6 +2622,7 @@ _equalColumnCompression(const ColumnCompression *a, const ColumnCompression *b) { COMPARE_STRING_FIELD(cmname); COMPARE_SCALAR_FIELD(preserve_all); + COMPARE_NODE_FIELD(options); COMPARE_NODE_FIELD(preserve); return true; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 2ef640b3b7..76473a9749 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2857,6 +2857,7 @@ _outColumnCompression(StringInfo str, const ColumnCompression *node) WRITE_STRING_FIELD(cmname); WRITE_BOOL_FIELD(preserve_all); + WRITE_NODE_FIELD(options); WRITE_NODE_FIELD(preserve); } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 35c69744e2..9bac7f1932 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -415,6 +415,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); transform_element_list transform_type_list TriggerTransitions TriggerReferencing publication_name_list + optCompressionParameters vacuum_relation_list opt_vacuum_relation_list drop_option_list @@ -3454,11 +3455,17 @@ compressionClause: COMPRESSION name { $$ = pstrdup($2); } ; +optCompressionParameters: + WITH '(' generic_option_list ')' { $$ = $3; } + | /*EMPTY*/ { $$ = NULL; } + ; + optColumnCompression: - compressionClause + compressionClause optCompressionParameters { ColumnCompression *n = makeNode(ColumnCompression); n->cmname = $1; + n->options = (List *) $2; n->preserve = NIL; $$ = (Node *) n; } @@ -3466,14 +3473,15 @@ optColumnCompression: ; alterColumnCompression: - compressionClause optCompressionPreserve + compressionClause optCompressionParameters optCompressionPreserve { ColumnCompression *n = makeNode(ColumnCompression); n->cmname = $1; - n->preserve = (List *) $2; + n->options = (List *) $2; + n->preserve = (List *) $3; $$ = (Node *) n; } - | compressionClause PRESERVE ALL + | compressionClause optCompressionParameters PRESERVE ALL { ColumnCompression *n = makeNode(ColumnCompression); n->cmname = $1; diff --git a/src/include/access/compressionapi.h b/src/include/access/compressionapi.h index 4d4326ff57..fb8e44754e 100644 --- a/src/include/access/compressionapi.h +++ b/src/include/access/compressionapi.h @@ -17,6 +17,7 @@ #include "catalog/pg_compression_d.h" #include "nodes/nodes.h" +#include "nodes/pg_list.h" /* * Built-in compression method-id. The toast compression header will store @@ -42,7 +43,12 @@ typedef enum CompressionId typedef struct CompressionRoutine CompressionRoutine; /* compresion handler routines */ +typedef void (*cmcheck_function) (List *options); +typedef void *(*cminitstate_function) (List *options); typedef struct varlena *(*cmcompress_function)(const struct varlena *value, + int32 toast_header_size, + void *options); +typedef struct varlena *(*cmdecompress_function)(const struct varlena *value, int32 toast_header_size); typedef struct varlena *(*cmdecompress_slice_function)( const struct varlena *value, @@ -61,11 +67,17 @@ struct CompressionRoutine /* name of the compression method */ char cmname[64]; + /* compression option check, can be NULL */ + cmcheck_function cmcheck; + + /* compression option intialize, can be NULL */ + cminitstate_function cminitstate; + /* compression routine for the compression method */ cmcompress_function cmcompress; /* decompression routine for the compression method */ - cmcompress_function cmdecompress; + cmdecompress_function cmdecompress; /* slice decompression routine for the compression method */ cmdecompress_slice_function cmdecompress_slice; diff --git a/src/include/access/toast_helper.h b/src/include/access/toast_helper.h index a88e3daa6a..9071e80714 100644 --- a/src/include/access/toast_helper.h +++ b/src/include/access/toast_helper.h @@ -33,6 +33,7 @@ typedef struct int32 tai_size; uint8 tai_colflags; Oid tai_compression; + List *tai_cmoptions; } ToastAttrInfo; /* diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h index 48ca172eae..53becd2057 100644 --- a/src/include/access/toast_internals.h +++ b/src/include/access/toast_internals.h @@ -67,7 +67,7 @@ do { \ #define TOAST_COMPRESS_SET_CMID(ptr, oid) \ (((toast_compress_header_custom *) (ptr))->cmoid = (oid)) -extern Datum toast_compress_datum(Datum value, Oid cmoid); +extern Datum toast_compress_datum(Datum value, Oid cmoid, List *cmoptions); extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock); extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative); diff --git a/src/include/catalog/heap.h b/src/include/catalog/heap.h index d31141c1a2..fd25321f1a 100644 --- a/src/include/catalog/heap.h +++ b/src/include/catalog/heap.h @@ -81,6 +81,7 @@ extern Oid heap_create_with_catalog(const char *relname, bool allow_system_table_mods, bool is_internal, Oid relrewrite, + Datum *acoptions, ObjectAddress *typaddress); extern void heap_drop_with_catalog(Oid relid); @@ -97,6 +98,7 @@ extern void InsertPgAttributeTuples(Relation pg_attribute_rel, TupleDesc tupdesc, Oid new_rel_oid, Datum *attoptions, + Datum *attcmoptions, CatalogIndexState indstate); extern void InsertPgClassTuple(Relation pg_class_desc, diff --git a/src/include/catalog/pg_attribute.h b/src/include/catalog/pg_attribute.h index 7b27df7464..69132d4b1d 100644 --- a/src/include/catalog/pg_attribute.h +++ b/src/include/catalog/pg_attribute.h @@ -171,6 +171,9 @@ CATALOG(pg_attribute,1249,AttributeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(75, /* Column-level FDW options */ text attfdwoptions[1] BKI_DEFAULT(_null_); + /* current compression options */ + text attcmoptions[1] BKI_DEFAULT(_null_); + /* * Missing value for added columns. This is a one element array which lets * us store a value of the attribute type here. diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index 8952b2b70d..fcbdbfa401 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -137,6 +137,7 @@ extern Datum transformGenericOptions(Oid catalogId, Datum oldOptions, List *options, Oid fdwvalidator); +extern Datum optionListToArray(List *options); /* commands/amcmds.c */ extern ObjectAddress CreateAccessMethod(CreateAmStmt *stmt); @@ -150,7 +151,7 @@ extern ObjectAddress CreateCompressionMethod(CreateCmStmt *stmt); extern Oid get_cm_oid(const char *cmname, bool missing_ok); extern Oid GetAttributeCompression(Form_pg_attribute att, ColumnCompression *compression, - bool *need_rewrite); + Datum *acoptions, bool *need_rewrite); extern ColumnCompression *MakeColumnCompression(Oid atttcompression); extern bool IsCompressionSupported(Form_pg_attribute att, Oid cmoid); diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index f7870c469b..3bd758b7d8 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -635,6 +635,7 @@ typedef struct ColumnCompression NodeTag type; char *cmname; bool preserve_all; + List *options; List *preserve; } ColumnCompression; diff --git a/src/test/regress/expected/create_cm.out b/src/test/regress/expected/create_cm.out index 897a61689f..8a989e0ddd 100644 --- a/src/test/regress/expected/create_cm.out +++ b/src/test/regress/expected/create_cm.out @@ -121,4 +121,18 @@ SELECT length(f1) FROM cmmove2; --------+------+-----------+----------+---------+----------+-------------+--------------+------------- f1 | text | | | | extended | pglz2 | | +-- compression options +DROP TABLE cmmove1; +CREATE TABLE cmmove1(f1 TEXT COMPRESSION pglz WITH (min_input_size '100')); +INSERT INTO cmmove1 VALUES (repeat('1234567890',1004)); +ALTER TABLE cmmove1 ALTER COLUMN f1 SET COMPRESSION lz4 WITH (level '9'); +ALTER TABLE cmmove1 ALTER COLUMN f1 SET COMPRESSION pglz WITH (min_input_size '200') PRESERVE (lz4); +SELECT length(f1) FROM cmmove2; + length +-------- + 10000 + 10040 + 10040 +(3 rows) + DROP TABLE cmmove1, cmmove2, cmmove3, lz4test; diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out index 8d4f9e5ea0..7935170622 100644 --- a/src/test/regress/expected/misc_sanity.out +++ b/src/test/regress/expected/misc_sanity.out @@ -98,6 +98,7 @@ ORDER BY 1, 2; relname | attname | atttypid -------------------------+---------------+-------------- pg_attribute | attacl | aclitem[] + pg_attribute | attcmoptions | text[] pg_attribute | attfdwoptions | text[] pg_attribute | attmissingval | anyarray pg_attribute | attoptions | text[] @@ -108,5 +109,5 @@ ORDER BY 1, 2; pg_index | indpred | pg_node_tree pg_largeobject | data | bytea pg_largeobject_metadata | lomacl | aclitem[] -(11 rows) +(12 rows) diff --git a/src/test/regress/sql/create_cm.sql b/src/test/regress/sql/create_cm.sql index a0a0aa13f0..0df4d3b60c 100644 --- a/src/test/regress/sql/create_cm.sql +++ b/src/test/regress/sql/create_cm.sql @@ -58,4 +58,12 @@ INSERT INTO cmmove2 VALUES (repeat('1234567890',1004)); SELECT length(f1) FROM cmmove2; \d+ cmmove2 +-- compression options +DROP TABLE cmmove1; +CREATE TABLE cmmove1(f1 TEXT COMPRESSION pglz WITH (min_input_size '100')); +INSERT INTO cmmove1 VALUES (repeat('1234567890',1004)); +ALTER TABLE cmmove1 ALTER COLUMN f1 SET COMPRESSION lz4 WITH (level '9'); +ALTER TABLE cmmove1 ALTER COLUMN f1 SET COMPRESSION pglz WITH (min_input_size '200') PRESERVE (lz4); +SELECT length(f1) FROM cmmove2; + DROP TABLE cmmove1, cmmove2, cmmove3, lz4test; -- 2.23.0