V1 pg11 compat (#190)

* Make cstore compile under PG 11 * Modify truncate tests for output differences in PG11 Add an alternate output file for pre PG11 * Change the tupleDescriptor access method PG11 changed TupleDesc copying to not copy defaults and constraints. We changed the way to access TupleDesc data and kept access to default values for columns. * Address pykello's feedback.
citusdata · Dec 14, 2018 · c44da98 · c44da98
1 parent a15864a
commit c44da98
Show file tree

Hide file tree

Showing 10 changed files with 380 additions and 52 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -15,8 +15,9 @@ env:
     - PGVERSION=9.5
     - PGVERSION=9.6
     - PGVERSION=10
+    - PGVERSION=11
 before_install:
-  - git clone -b v0.6.3 --depth 1 https://github.com/citusdata/tools.git
+  - git clone -b v0.7.9 --depth 1 https://github.com/citusdata/tools.git
   - sudo make -C tools install
   - setup_apt
   - nuke_pg

diff --git a/Makefile b/Makefile
@@ -44,8 +44,8 @@ ifndef MAJORVERSION
     MAJORVERSION := $(basename $(VERSION))
 endif
 
-ifeq (,$(findstring $(MAJORVERSION), 9.3 9.4 9.5 9.6 10))
-    $(error PostgreSQL 9.3 or 9.4 or 9.5 or 9.6 or 10 is required to compile this extension)
+ifeq (,$(findstring $(MAJORVERSION), 9.3 9.4 9.5 9.6 10 11))
+    $(error PostgreSQL 9.3 or 9.4 or 9.5 or 9.6 or 10 or 11 is required to compile this extension)
 endif
 
 cstore.pb-c.c: cstore.proto

diff --git a/cstore_fdw.c b/cstore_fdw.c
@@ -16,6 +16,7 @@
 
 #include "postgres.h"
 #include "cstore_fdw.h"
+#include "cstore_version_compat.h"
 
 #include <sys/stat.h>
 #include <unistd.h>
@@ -65,20 +66,6 @@
 #include "utils/tqual.h"
 
 
-#define PREVIOUS_UTILITY (PreviousProcessUtilityHook != NULL \
-						  ? PreviousProcessUtilityHook : standard_ProcessUtility)
-#if PG_VERSION_NUM >= 100000
-#define CALL_PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, \
-							  destReceiver, completionTag) \
-	PREVIOUS_UTILITY(plannedStatement, queryString, context, paramListInfo, \
-					 queryEnvironment, destReceiver, completionTag)
-#else
-#define CALL_PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, \
-							  destReceiver, completionTag) \
-	PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, destReceiver, \
-					 completionTag)
-#endif
-
 /* local functions forward declarations */
 static void RegisterCStoreTable(Oid relationId, Oid fileNodeOid);
 static void UnregisterCStoreTable(Oid relationId);
@@ -676,9 +663,7 @@ CopyIntoCStoreTable(const CopyStmt *copyStatement, const char *queryString)
 	 */
 	tupleContext = AllocSetContextCreate(CurrentMemoryContext,
 										 "CStore COPY Row Memory Context",
-										 ALLOCSET_DEFAULT_MINSIZE,
-										 ALLOCSET_DEFAULT_INITSIZE,
-										 ALLOCSET_DEFAULT_MAXSIZE);
+										 ALLOCSET_DEFAULT_SIZES);
 
 	/* init state to read from COPY data source */
 #if (PG_VERSION_NUM >= 100000)
@@ -964,7 +949,7 @@ OpenRelationsForTruncate(List *cstoreTableList)
 											   ACL_TRUNCATE);
 		if (aclresult != ACLCHECK_OK)
 		{
-			aclcheck_error(aclresult, ACL_KIND_CLASS, get_rel_name(relationId));
+			aclcheck_error(aclresult, ACLCHECK_OBJECT_TABLE, get_rel_name(relationId));
 		}
 
 		/* check if this relation is repeated */
@@ -1753,7 +1738,6 @@ ColumnList(RelOptInfo *baserel, Oid foreignTableId)
 	const AttrNumber wholeRow = 0;
 	Relation relation = heap_open(foreignTableId, AccessShareLock);
 	TupleDesc tupleDescriptor = RelationGetDescr(relation);
-	Form_pg_attribute *attributeFormArray = tupleDescriptor->attrs;
 
 	/* first add the columns used in joins and projections */
 	foreach(targetColumnCell, targetColumnList)
@@ -1812,7 +1796,7 @@ ColumnList(RelOptInfo *baserel, Oid foreignTableId)
 			}
 			else if (neededColumn->varattno == wholeRow)
 			{
-				Form_pg_attribute attributeForm = attributeFormArray[columnIndex - 1];
+				Form_pg_attribute attributeForm =  TupleDescAttr(tupleDescriptor, columnIndex - 1);
 				Index tableId = neededColumn->varno;
 
 				column = makeVar(tableId, columnIndex, attributeForm->atttypid,
@@ -1858,13 +1842,12 @@ static void
 CStoreBeginForeignScan(ForeignScanState *scanState, int executorFlags)
 {
 	TableReadState *readState = NULL;
-	TupleTableSlot *tupleSlot = scanState->ss.ss_ScanTupleSlot;
-	TupleDesc tupleDescriptor = tupleSlot->tts_tupleDescriptor;
+	Relation currentRelation = scanState->ss.ss_currentRelation;
+	TupleDesc tupleDescriptor = RelationGetDescr(currentRelation);
 	List *columnList = NIL;
 	ForeignScan *foreignScan = NULL;
 	List *foreignPrivateList = NIL;
 	List *whereClauseList = NIL;
-	Relation currentRelation = scanState->ss.ss_currentRelation;
 
 	/* if Explain with no Analyze, do nothing */
 	if (executorFlags & EXEC_FLAG_EXPLAIN_ONLY)
@@ -1990,13 +1973,13 @@ CStoreAcquireSampleRows(Relation relation, int logLevel,
 
 	TupleDesc tupleDescriptor = RelationGetDescr(relation);
 	uint32 columnCount = tupleDescriptor->natts;
-	Form_pg_attribute *attributeFormArray = tupleDescriptor->attrs;
+
 
 	/* create list of columns of the relation */
 	uint32 columnIndex = 0;
 	for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
 	{
-		Form_pg_attribute attributeForm = attributeFormArray[columnIndex];
+		Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
 		const Index tableId = 1;
 
 		if (!attributeForm->attisdropped)
@@ -2015,7 +1998,11 @@ CStoreAcquireSampleRows(Relation relation, int logLevel,
 	/* set up tuple slot */
 	columnValues = palloc0(columnCount * sizeof(Datum));
 	columnNulls = palloc0(columnCount * sizeof(bool));
+#if PG_VERSION_NUM >= 110000
+	scanTupleSlot = MakeTupleTableSlot(NULL);
+#else
 	scanTupleSlot = MakeTupleTableSlot();
+#endif
 	scanTupleSlot->tts_tupleDescriptor = tupleDescriptor;
 	scanTupleSlot->tts_values = columnValues;
 	scanTupleSlot->tts_isnull = columnNulls;
@@ -2032,9 +2019,7 @@ CStoreAcquireSampleRows(Relation relation, int logLevel,
 	 */
 	tupleContext = AllocSetContextCreate(CurrentMemoryContext,
 										 "cstore_fdw temporary context",
-										 ALLOCSET_DEFAULT_MINSIZE,
-										 ALLOCSET_DEFAULT_INITSIZE,
-										 ALLOCSET_DEFAULT_MAXSIZE);
+										 ALLOCSET_DEFAULT_SIZES);
 
 	CStoreBeginForeignScan(scanState, executorFlags);
 

diff --git a/cstore_fdw.h b/cstore_fdw.h
@@ -322,6 +322,7 @@ extern Datum cstore_ddl_event_end_trigger(PG_FUNCTION_ARGS);
 
 /* Function declarations for utility UDFs */
 extern Datum cstore_table_size(PG_FUNCTION_ARGS);
+extern Datum cstore_clean_table_resources(PG_FUNCTION_ARGS);
 
 /* Function declarations for foreign data wrapper */
 extern Datum cstore_fdw_handler(PG_FUNCTION_ARGS);

diff --git a/cstore_reader.c b/cstore_reader.c
@@ -17,6 +17,7 @@
 #include "postgres.h"
 #include "cstore_fdw.h"
 #include "cstore_metadata_serialization.h"
+#include "cstore_version_compat.h"
 
 #include "access/nbtree.h"
 #include "access/skey.h"
@@ -56,7 +57,7 @@ static StripeSkipList * LoadStripeSkipList(Relation relation,
 										   StripeFooter *stripeFooter,
 										   uint32 columnCount,
 										   bool *projectedColumnMask,
-										   Form_pg_attribute *attributeFormArray);
+										   TupleDesc tupleDescriptor);
 static bool * SelectedBlockMask(StripeSkipList *stripeSkipList,
 								List *projectedColumnList, List *whereClauseList);
 static List * BuildRestrictInfoList(List *whereClauseList);
@@ -76,7 +77,6 @@ static void DeserializeDatumArray(StringInfo datumBuffer, bool *existsArray,
 								  int datumTypeLength, char datumTypeAlign,
 								  Datum *datumArray);
 static void DeserializeBlockData(StripeBuffers *stripeBuffers, uint64 blockIndex,
-								 Form_pg_attribute *attributeFormArray,
 								 uint32 rowCount, ColumnBlockData **blockDataArray,
 								 TupleDesc tupleDescriptor);
 static Datum ColumnDefaultValue(TupleConstr *tupleConstraints,
@@ -117,9 +117,7 @@ CStoreBeginRead(Relation relation, TupleDesc tupleDescriptor,
 	 */
 	stripeReadContext = AllocSetContextCreate(CurrentMemoryContext,
 											  "Stripe Read Memory Context",
-											  ALLOCSET_DEFAULT_MINSIZE,
-											  ALLOCSET_DEFAULT_INITSIZE,
-											  ALLOCSET_DEFAULT_MAXSIZE);
+											  ALLOCSET_DEFAULT_SIZES);
 
 	columnCount = tupleDescriptor->natts;
 	projectedColumnMask = ProjectedColumnMask(columnCount, projectedColumnList);
@@ -311,7 +309,6 @@ CStoreReadNextRow(TableReadState *readState, Datum *columnValues, bool *columnNu
 	uint32 blockIndex = 0;
 	uint32 blockRowIndex = 0;
 	TableFooter *tableFooter = readState->tableFooter;
-	Form_pg_attribute *attributeFormArray = readState->tupleDescriptor->attrs;
 	MemoryContext oldContext = NULL;
 
 	/*
@@ -379,7 +376,7 @@ CStoreReadNextRow(TableReadState *readState, Datum *columnValues, bool *columnNu
 
 		oldContext = MemoryContextSwitchTo(readState->stripeReadContext);
 
-		DeserializeBlockData(readState->stripeBuffers, blockIndex, attributeFormArray,
+		DeserializeBlockData(readState->stripeBuffers, blockIndex,
 							 blockRowCount, readState->blockDataArray,
 							 readState->tupleDescriptor);
 
@@ -540,7 +537,6 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
 	ColumnBuffers **columnBuffersArray = NULL;
 	uint64 currentColumnFileOffset = 0;
 	uint32 columnIndex = 0;
-	Form_pg_attribute *attributeFormArray = tupleDescriptor->attrs;
 	uint32 columnCount = tupleDescriptor->natts;
 
 	bool *projectedColumnMask = ProjectedColumnMask(columnCount, projectedColumnList);
@@ -551,7 +547,7 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
 	StripeSkipList *stripeSkipList = LoadStripeSkipList(relation, stripeMetadata,
 														stripeFooter, columnCount,
 														projectedColumnMask,
-														attributeFormArray);
+														tupleDescriptor);
 
 	bool *selectedBlockMask = SelectedBlockMask(stripeSkipList, projectedColumnList,
 												whereClauseList);
@@ -575,7 +571,7 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
 		{
 			ColumnBlockSkipNode *blockSkipNode =
 				selectedBlockSkipList->blockSkipNodeArray[columnIndex];
-			Form_pg_attribute attributeForm = attributeFormArray[columnIndex];
+			Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
 			uint32 blockCount = selectedBlockSkipList->blockCount;
 
 			ColumnBuffers *columnBuffers = LoadColumnBuffers(relation, blockSkipNode,
@@ -716,7 +712,7 @@ static StripeSkipList *
 LoadStripeSkipList(Relation relation, StripeMetadata *stripeMetadata,
 				   StripeFooter *stripeFooter, uint32 columnCount,
 				   bool *projectedColumnMask,
-				   Form_pg_attribute *attributeFormArray)
+				   TupleDesc tupleDescriptor)
 {
 	StripeSkipList *stripeSkipList = NULL;
 	ColumnBlockSkipNode **blockSkipNodeArray = NULL;
@@ -748,7 +744,7 @@ LoadStripeSkipList(Relation relation, StripeMetadata *stripeMetadata,
 		 */
 		if (projectedColumnMask[columnIndex] || firstColumn)
 		{
-			Form_pg_attribute attributeForm = attributeFormArray[columnIndex];
+			Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
 
 			StringInfo columnSkipListBuffer =
 				ReadFromFile(relation, currentColumnSkipListFileOffset, columnSkipListSize);
@@ -1242,14 +1238,14 @@ DeserializeDatumArray(StringInfo datumBuffer, bool *existsArray, uint32 datumCou
  */
 static void
 DeserializeBlockData(StripeBuffers *stripeBuffers, uint64 blockIndex,
-					 Form_pg_attribute *attributeFormArray, uint32 rowCount,
+					 uint32 rowCount,
 					 ColumnBlockData **blockDataArray, TupleDesc tupleDescriptor)
 {
 	int columnIndex = 0;
 	for (columnIndex = 0; columnIndex < stripeBuffers->columnCount; columnIndex++)
 	{
 		ColumnBlockData *blockData = blockDataArray[columnIndex];
-		Form_pg_attribute attributeForm = attributeFormArray[columnIndex];
+		Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
 		ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex];
 		bool columnAdded = false;
 

diff --git a/cstore_version_compat.h b/cstore_version_compat.h
@@ -0,0 +1,50 @@
+/*-------------------------------------------------------------------------
+ *
+ * cstore_version_compat.h
+ *
+ * 	Compatibility macros for writing code agnostic to PostgreSQL versions
+ *
+ * Copyright (c) 2018, Citus Data, Inc.
+ *
+ * $Id$
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef CSTORE_COMPAT_H
+#define CSTORE_COMPAT_H
+
+#if PG_VERSION_NUM < 100000
+
+/* Accessor for the i'th attribute of tupdesc. */
+#define TupleDescAttr(tupdesc, i) ((tupdesc)->attrs[(i)])
+
+#endif
+
+#if PG_VERSION_NUM < 110000
+#define ALLOCSET_DEFAULT_SIZES ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE
+#define ACLCHECK_OBJECT_TABLE ACL_KIND_CLASS
+#else
+#define ACLCHECK_OBJECT_TABLE OBJECT_TABLE
+
+#define ExplainPropertyLong(qlabel, value, es) \
+	ExplainPropertyInteger(qlabel, NULL, value, es)
+#endif
+
+#define PREVIOUS_UTILITY (PreviousProcessUtilityHook != NULL \
+						  ? PreviousProcessUtilityHook : standard_ProcessUtility)
+#if PG_VERSION_NUM >= 100000
+#define CALL_PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, \
+							  destReceiver, completionTag) \
+	PREVIOUS_UTILITY(plannedStatement, queryString, context, paramListInfo, \
+					 queryEnvironment, destReceiver, completionTag)
+#else
+#define CALL_PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, \
+							  destReceiver, completionTag) \
+	PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, destReceiver, \
+					 completionTag)
+#endif
+
+
+
+#endif /* CSTORE_COMPAT_H */
diff --git a/cstore_writer.c b/cstore_writer.c
@@ -17,6 +17,7 @@
 #include "postgres.h"
 #include "cstore_fdw.h"
 #include "cstore_metadata_serialization.h"
+#include "cstore_version_compat.h"
 
 #include <sys/stat.h>
 #include "access/heapam.h"
@@ -130,7 +131,7 @@ CStoreBeginWrite(Relation relation, CompressionType compressionType,
 	for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
 	{
 		FmgrInfo *comparisonFunction = NULL;
-		FormData_pg_attribute *attributeForm = tupleDescriptor->attrs[columnIndex];
+		FormData_pg_attribute *attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
 
 		if (!attributeForm->attisdropped)
 		{
@@ -149,9 +150,7 @@ CStoreBeginWrite(Relation relation, CompressionType compressionType,
 	 */
 	stripeWriteContext = AllocSetContextCreate(CurrentMemoryContext,
 											   "Stripe Write Memory Context",
-											   ALLOCSET_DEFAULT_MINSIZE,
-											   ALLOCSET_DEFAULT_INITSIZE,
-											   ALLOCSET_DEFAULT_MAXSIZE);
+											   ALLOCSET_DEFAULT_SIZES);
 
 	columnMaskArray = palloc(columnCount * sizeof(bool));
 	memset(columnMaskArray, true, columnCount);
@@ -267,7 +266,7 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul
 			FmgrInfo *comparisonFunction =
 				writeState->comparisonFunctionArray[columnIndex];
 			Form_pg_attribute attributeForm =
-				writeState->tupleDescriptor->attrs[columnIndex];
+				TupleDescAttr(writeState->tupleDescriptor, columnIndex);
 			bool columnTypeByValue = attributeForm->attbyval;
 			int columnTypeLength = attributeForm->attlen;
 			Oid columnCollation = attributeForm->attcollation;
@@ -773,7 +772,7 @@ CreateSkipListBufferArray(StripeSkipList *stripeSkipList, TupleDesc tupleDescrip
 		StringInfo skipListBuffer = NULL;
 		ColumnBlockSkipNode *blockSkipNodeArray =
 			stripeSkipList->blockSkipNodeArray[columnIndex];
-		Form_pg_attribute attributeForm = tupleDescriptor->attrs[columnIndex];
+		Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
 
 		skipListBuffer = SerializeColumnSkipList(blockSkipNodeArray,
 												 stripeSkipList->blockCount,

diff --git a/expected/truncate.out b/expected/truncate.out
@@ -1,6 +1,14 @@
 --
 -- Test the TRUNCATE TABLE command for cstore_fdw tables.
 --
+-- print whether we're using version > 10 to make version-specific tests clear
+SHOW server_version \gset
+SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten;
+ version_above_ten 
+-------------------
+ t
+(1 row)
+
 -- CREATE a cstore_fdw table, fill with some data --
 CREATE FOREIGN TABLE cstore_truncate_test (a int, b int) SERVER cstore_server;
 CREATE FOREIGN TABLE cstore_truncate_test_second (a int, b int) SERVER cstore_server;
@@ -222,7 +230,7 @@ SELECT count(*) FROM truncate_schema.truncate_tbl;
 (1 row)
 
 TRUNCATE TABLE truncate_schema.truncate_tbl;
-ERROR:  permission denied for relation truncate_tbl
+ERROR:  permission denied for table truncate_tbl
 SELECT count(*) FROM truncate_schema.truncate_tbl;
  count 
 -------