- MDEV-11295: developing handling files contained in ZIP file.

  Enable using multiple zip files
  modified:   storage/connect/filamzip.cpp
  modified:   storage/connect/ha_connect.cc
  modified:   storage/connect/ha_connect.h
  modified:   storage/connect/mycat.h
  modified:   storage/connect/tabdos.cpp
  modified:   storage/connect/tabdos.h
  modified:   storage/connect/tabfmt.cpp
  modified:   storage/connect/tabjson.cpp

Author: Buggynours

storage/connect/filamzip.cpp

@@ -48,11 +48,11 @@
 ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp)
 {
 	zipfile = NULL;
-	zfn = tdp->Zipfn;
-	target = tdp->Fn;
+//zfn = tdp->Fn;
+	target = tdp->Entry;
 //*fn = 0;
 	entryopen = false;
-	multiple = tdp->Multiple;
+	multiple = (target && !(strchr(target, '*') || strchr(target, '?'))) ? 0 : 1;
 
 	// Init the case mapping table.
 #if defined(__WIN__)
@@ -65,7 +65,7 @@ ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp)
 ZIPFAM::ZIPFAM(PZIPFAM txfp) : MAPFAM(txfp)
 {
 	zipfile = txfp->zipfile;
-	zfn = txfp->zfn;
+//zfn = txfp->zfn;
 	target = txfp->target;
 //strcpy(fn, txfp->fn);
 	finfo = txfp->finfo;
@@ -129,7 +129,7 @@ int ZIPFAM::GetFileLength(PGLOBAL g)
 bool ZIPFAM::open(PGLOBAL g, const char *filename)
 {
 	if (!zipfile && !(zipfile = unzOpen64(filename)))
-		sprintf(g->Message, "Zipfile open error");
+		sprintf(g->Message, "Zipfile open error on %s", filename);
 
 	return (zipfile == NULL);
 }	// end of open
@@ -205,7 +205,7 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g)
 	/*********************************************************************/
 	if (mode == MODE_READ) {
 		//  We used the file name relative to recorded datapath
-		PlugSetPath(filename, zfn, Tdbp->GetPath());
+		PlugSetPath(filename, To_File, Tdbp->GetPath());
 
 		bool b = open(g, filename);
 
@@ -218,7 +218,7 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g)
 
 					if (rc == UNZ_END_OF_LIST_OF_FILE) {
 						sprintf(g->Message, "Target file %s not in %s", target, filename);
-						return true;
+						return false;
 					} else if (rc != UNZ_OK) {
 						sprintf(g->Message, "unzLocateFile rc=%d", rc);
 						return true;
@@ -229,7 +229,7 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g)
 						return true;
 					else if (rc == RC_NF) {
 						sprintf(g->Message, "No match of %s in %s", target, filename);
-						return true;
+						return false;
 					} // endif rc
 
 				} // endif multiple
@@ -258,7 +258,8 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g)
 			} // endif fp
 
 			To_Fb = fp;                               // Useful when closing
-		}	// endif b
+		} else
+			return true;
 
 	} else {
 		strcpy(g->Message, "Only READ mode supported for ZIP files");

storage/connect/filamzip.h

@@ -51,7 +51,7 @@ class DllExport ZIPFAM : public MAPFAM {
 
 	// Members
 	unzFile         zipfile;							   // The ZIP container file
-	PSZ             zfn;									   // The ZIP file name
+//PSZ             zfn;									   // The ZIP file name
 	PSZ             target;									 // The target file name
 	unz_file_info   finfo;									 // The current file info
 //char            fn[FILENAME_MAX];     	 // The current file name

storage/connect/ha_connect.cc

@@ -171,9 +171,9 @@
 #define JSONMAX      10             // JSON Default max grp size
 
 extern "C" {
-       char version[]= "Version 1.04.0009 December 09, 2016";
+       char version[]= "Version 1.05.0001 December 13, 2016";
 #if defined(__WIN__)
-       char compver[]= "Version 1.04.0009 " __DATE__ " "  __TIME__;
+       char compver[]= "Version 1.05.0001 " __DATE__ " "  __TIME__;
        char slash= '\\';
 #else   // !__WIN__
        char slash= '/';
@@ -512,13 +512,13 @@ ha_create_table_option connect_table_option_list[]=
   HA_TOPTION_NUMBER("QUOTED", quoted, (ulonglong) -1, 0, 3, 1),
   HA_TOPTION_NUMBER("ENDING", ending, (ulonglong) -1, 0, INT_MAX32, 1),
   HA_TOPTION_NUMBER("COMPRESS", compressed, 0, 0, 2, 1),
-//HA_TOPTION_BOOL("COMPRESS", compressed, 0),
   HA_TOPTION_BOOL("MAPPED", mapped, 0),
   HA_TOPTION_BOOL("HUGE", huge, 0),
   HA_TOPTION_BOOL("SPLIT", split, 0),
   HA_TOPTION_BOOL("READONLY", readonly, 0),
   HA_TOPTION_BOOL("SEPINDEX", sepindex, 0),
-  HA_TOPTION_END
+	HA_TOPTION_BOOL("ZIPPED", zipped, 0),
+	HA_TOPTION_END
 };
 
 
@@ -532,7 +532,6 @@ ha_create_table_option connect_field_option_list[]=
 {
   HA_FOPTION_NUMBER("FLAG", offset, (ulonglong) -1, 0, INT_MAX32, 1),
   HA_FOPTION_NUMBER("MAX_DIST", freq, 0, 0, INT_MAX32, 1), // BLK_INDX
-//HA_FOPTION_NUMBER("DISTRIB", opt, 0, 0, 2, 1),  // used for BLK_INDX
   HA_FOPTION_NUMBER("FIELD_LENGTH", fldlen, 0, 0, INT_MAX32, 1),
   HA_FOPTION_STRING("DATE_FORMAT", dateformat),
   HA_FOPTION_STRING("FIELD_FORMAT", fieldformat),
@@ -678,7 +677,6 @@ static int connect_init_func(void *p)
   connect_hton= (handlerton *)p;
   connect_hton->state= SHOW_OPTION_YES;
   connect_hton->create= connect_create_handler;
-//connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED | HTON_NO_PARTITION;
   connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED;
   connect_hton->table_options= connect_table_option_list;
   connect_hton->field_options= connect_field_option_list;
@@ -1135,7 +1133,9 @@ bool GetBooleanTableOption(PGLOBAL g, PTOS options, char *opname, bool bdef)
     opval= options->sepindex;
   else if (!stricmp(opname, "Header"))
     opval= (options->header != 0);   // Is Boolean for some table types
-  else if (options->oplist)
+	else if (!stricmp(opname, "Zipped"))
+		opval = options->zipped;
+	else if (options->oplist)
     if ((pv= GetListOption(g, opname, options->oplist)))
       opval= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0);
 

storage/connect/ha_connect.h

@@ -83,42 +83,9 @@ extern handlerton *connect_hton;
 
   These can be specified in the CREATE TABLE:
   CREATE TABLE ( ... ) {...here...}
-*/
-#if 0  // moved to mycat.h
-typedef struct ha_table_option_struct TOS, *PTOS;
-
-struct ha_table_option_struct {
-  const char *type;
-  const char *filename;
-  const char *optname;
-  const char *tabname;
-  const char *tablist;
-  const char *dbname;
-  const char *separator;
-//const char *connect;
-  const char *qchar;
-  const char *module;
-  const char *subtype;
-  const char *catfunc;
-  const char *srcdef;
-  const char *colist;
-  const char *oplist;
-  const char *data_charset;
-  ulonglong lrecl;
-  ulonglong elements;
-//ulonglong estimate;
-  ulonglong multiple;
-  ulonglong header;
-  ulonglong quoted;
-  ulonglong ending;
-  ulonglong compressed;
-  bool mapped;
-  bool huge;
-  bool split;
-  bool readonly;
-  bool sepindex;
-  };
-#endif // 0
+
+	------ Was moved to mycat.h ------
+	*/
 
 /**
   structure for CREATE TABLE options (field options)

storage/connect/mycat.h

@@ -62,6 +62,7 @@ struct ha_table_option_struct {
   bool split;
   bool readonly;
   bool sepindex;
+	bool zipped;
   };
 
 // Possible value for catalog functions

storage/connect/tabdos.cpp

@@ -96,11 +96,12 @@ DOSDEF::DOSDEF(void)
   Pseudo = 3;
   Fn = NULL;
   Ofn = NULL;
-	Zipfn = NULL;
+	Entry = NULL;
   To_Indx = NULL;
   Recfm = RECFM_VAR;
   Mapped = false;
-  Padded = false;
+	Zipped = false;
+	Padded = false;
   Huge = false;
   Accept = false;
   Eof = false;
@@ -131,20 +132,11 @@ bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int)
              : (am && !stricmp(am, "DBF"))        ? "D" : "V";
 
 	if (*dfm != 'D')
-		Zipfn = GetStringCatInfo(g, "Zipfile", NULL);
-
-	if (Zipfn && Multiple) {
-		// Prevent Fn to default to table name
-		Desc = GetStringCatInfo(g, "Filename", NULL);
-		Fn = GetStringCatInfo(g, "Filename", "<%>");
-
-		if (!strcmp(Fn, "<%>"))
-			Fn = NULL;
-
-	} else
-	  Desc = Fn = GetStringCatInfo(g, "Filename", NULL);
+		Zipped = GetBoolCatInfo("Zipped", false);
 
+  Desc = Fn = GetStringCatInfo(g, "Filename", NULL);
   Ofn = GetStringCatInfo(g, "Optname", Fn);
+	Entry = GetStringCatInfo(g, "Entry", NULL);
   GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf));
   Recfm = (toupper(*buf) == 'F') ? RECFM_FIX :
           (toupper(*buf) == 'B') ? RECFM_BIN :
@@ -350,15 +342,14 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode)
   /*  Allocate table and file processing class of the proper type.     */
   /*  Column blocks will be allocated only when needed.                */
   /*********************************************************************/
-	if (Zipfn) {
+	if (Zipped) {
 #if defined(ZIP_SUPPORT)
 		if (Recfm == RECFM_VAR)
 		  txfp = new(g) ZIPFAM(this);
 		else
 			txfp = new(g) ZPXFAM(this);
 
 		tdbp = new(g) TDBDOS(this, txfp);
-		return tdbp;
 #else   // !ZIP_SUPPORT
 		strcpy(g->Message, "ZIP not supported");
 		return NULL;

storage/connect/tabdos.h

@@ -59,7 +59,7 @@ class DllExport DOSDEF : public TABDEF {  /* Logical table description */
 
   // Methods
   virtual int  Indexable(void)
-          {return (!Multiple && !Zipfn && Compressed != 1) ? 1 : 0;}
+          {return (!Multiple && !Zipped && Compressed != 1) ? 1 : 0;}
   virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf);
   virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff);
   virtual PTDB GetTable(PGLOBAL g, MODE mode);
@@ -73,11 +73,12 @@ class DllExport DOSDEF : public TABDEF {  /* Logical table description */
   // Members
   PSZ     Fn;                 /* Path/Name of corresponding file       */
   PSZ     Ofn;                /* Base Path/Name of matching index files*/
-	PSZ     Zipfn;						  /* Zip container name										 */
+	PSZ     Entry;						  /* Zip entry name or pattern						 */
 	PIXDEF  To_Indx;            /* To index definitions blocks           */
   RECFM   Recfm;              /* 0:VAR, 1:FIX, 2:BIN, 3:VCT, 6:DBF     */
   bool    Mapped;             /* 0: disk file, 1: memory mapped file   */
-  bool    Padded;             /* true for padded table file            */
+	bool    Zipped;             /* true for zipped table file            */
+	bool    Padded;             /* true for padded table file            */
   bool    Huge;               /* true for files larger than 2GB        */
   bool    Accept;             /* true if wrong lines are accepted      */
   bool    Eof;                /* true if an EOF (0xA) character exists */

storage/connect/tabfmt.cpp

@@ -108,6 +108,11 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info)
     goto skipit;
     } // endif info
 
+	if (GetIntegerTableOption(g, topt, "Multiple", 0)) {
+		strcpy(g->Message, "Cannot find column definition for multiple table");
+		return NULL;
+	}	// endif Multiple
+
 //      num_max = atoi(p+1);             // Max num of record to test
   imax = hmax = nerr = 0;
 
@@ -123,18 +128,16 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info)
   /*********************************************************************/
 	tdp = new(g) CSVDEF;
 #if defined(ZIP_SUPPORT)
-	tdp->Zipfn = GetStringTableOption(g, topt, "Zipfile", NULL);
-	tdp->Multiple = GetIntegerTableOption(g, topt, "Multiple", 0);
+	tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL);
+	tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false);
 #endif   // ZIP_SUPPORT
-	tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL);
+	fn = tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL);
 
-	if (!tdp->Fn && !tdp->Zipfn && !tdp->Multiple) {
+	if (!tdp->Fn) {
 		strcpy(g->Message, MSG(MISSING_FNAME));
 		return NULL;
 	} // endif Fn
 
-	fn = (tdp->Fn) ? tdp->Fn : "unnamed";
-
 	if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0)))
 		tdp->Lrecl = 4096;
 
@@ -174,7 +177,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info)
 		htrc("File %s Sep=%c Qot=%c Header=%d maxerr=%d\n",
 		SVP(tdp->Fn), tdp->Sep, tdp->Qot, tdp->Header, tdp->Maxerr);
 
-	if (tdp->Zipfn)
+	if (tdp->Zipped)
 		tdbp = new(g) TDBCSV(tdp, new(g) ZIPFAM(tdp));
 	else
 		tdbp = new(g) TDBCSV(tdp, new(g) DOSFAM(tdp));
@@ -493,16 +496,9 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode)
     /*******************************************************************/
     /*  Allocate a file processing class of the proper type.           */
     /*******************************************************************/
-		if (Zipfn) {
+		if (Zipped) {
 #if defined(ZIP_SUPPORT)
 			txfp = new(g) ZIPFAM(this);
-
-			if (!Fmtd)
-				tdbp = new(g) TDBCSV(this, txfp);
-			else
-				tdbp = new(g) TDBFMT(this, txfp);
-
-			return tdbp;
 #else   // !ZIP_SUPPORT
 			strcpy(g->Message, "ZIP not supported");
 			return NULL;