This commit includes changes done in a previous (deleted) branch plus new ones.

From the previous branch:
commit eda4928ff122a0845baf5ade83b4aa29244a3a89
Author: Olivier Bertrand <bertrandop@gmail.com>
Date:   Mon Mar 9 22:34:56 2015 +0100

- Add discovery to JSON tables
  When columns are not defined, CONNECT analyses the json file to find column definitions.
  This wors only on table that are an array of objects. Pair keys are used to generate the
  column names and pair values are used for its definition. When the LEVEL option is defined
  as a not null integer, the eventual JPATH is scanned up to the LEVEL value.

From the current one:
- Fix MDEV-7521 when column names are utf8 encoded (not a general multi-charset fix)

- Adds more to JSON discovery processing and UDF's

- Use PlugDup everywhere it replaces PlugSubAlloc + strcpy.

Author: Buggynours

storage/connect/filamap.cpp

@@ -5,7 +5,7 @@
 /*                                                                     */
 /* COPYRIGHT:                                                          */
 /* ----------                                                          */
-/*  (C) Copyright to the author Olivier BERTRAND          2005-2014    */
+/*  (C) Copyright to the author Olivier BERTRAND          2005-2015    */
 /*                                                                     */
 /* WHAT THIS PROGRAM DOES:                                             */
 /* -----------------------                                             */
@@ -207,8 +207,7 @@ bool MAPFAM::OpenTableFile(PGLOBAL g)
     /*******************************************************************/
     fp = (PFBLOCK)PlugSubAlloc(g, NULL, sizeof(FBLOCK));
     fp->Type = TYPE_FB_MAP;
-    fp->Fname = (char*)PlugSubAlloc(g, NULL, strlen(filename) + 1);
-    strcpy((char*)fp->Fname, filename);
+    fp->Fname = PlugDup(g, filename);
     fp->Next = dbuserp->Openlist;
     dbuserp->Openlist = fp;
     fp->Count = 1;

storage/connect/filamtxt.cpp

@@ -5,7 +5,7 @@
 /*                                                                     */
 /* COPYRIGHT:                                                          */
 /* ----------                                                          */
-/*  (C) Copyright to the author Olivier BERTRAND          2005-2014    */
+/*  (C) Copyright to the author Olivier BERTRAND          2005-2015    */
 /*                                                                     */
 /* WHAT THIS PROGRAM DOES:                                             */
 /* -----------------------                                             */
@@ -289,8 +289,7 @@ bool TXTFAM::AddListValue(PGLOBAL g, int type, void *val, PPARM *top)
       pp->Intval = *(int*)val;
       break;
 //  case TYPE_STRING:
-//    pp->Value = PlugSubAlloc(g, NULL, strlen((char*)val) + 1);
-//    strcpy((char*)pp->Value, (char*)val);
+//    pp->Value = PlugDup(g, (char*)val);
 //    break;
     case TYPE_PCHAR:
       pp->Value = val;
@@ -325,8 +324,7 @@ int TXTFAM::StoreValues(PGLOBAL g, bool upd)
     if (Tdbp->PrepareWriting(g))
       return RC_FX;
 
-    buf = (char*)PlugSubAlloc(g, NULL, strlen(Tdbp->GetLine()) + 1);
-    strcpy(buf, Tdbp->GetLine());
+    buf = PlugDup(g, Tdbp->GetLine());
     rc = AddListValue(g, TYPE_PCHAR, buf, &To_Upd);
     } // endif upd
 

storage/connect/filamvct.cpp

@@ -5,7 +5,7 @@
 /*                                                                     */
 /* COPYRIGHT:                                                          */
 /* ----------                                                          */
-/*  (C) Copyright to the author Olivier BERTRAND          2005-2014    */
+/*  (C) Copyright to the author Olivier BERTRAND          2005-2015    */
 /*                                                                     */
 /* WHAT THIS PROGRAM DOES:                                             */
 /* -----------------------                                             */
@@ -1451,8 +1451,7 @@ bool VCMFAM::OpenTableFile(PGLOBAL g)
     /*******************************************************************/
     fp = (PFBLOCK)PlugSubAlloc(g, NULL, sizeof(FBLOCK));
     fp->Type = TYPE_FB_MAP;
-    fp->Fname = (char*)PlugSubAlloc(g, NULL, strlen(filename) + 1);
-    strcpy((char*)fp->Fname, filename);
+    fp->Fname = PlugDup(g, filename);
     fp->Next = dbuserp->Openlist;
     dbuserp->Openlist = fp;
     fp->Count = 1;
@@ -2844,8 +2843,7 @@ bool VMPFAM::MapColumnFile(PGLOBAL g, MODE mode, int i)
     /*******************************************************************/
     fp = (PFBLOCK)PlugSubAlloc(g, NULL, sizeof(FBLOCK));
     fp->Type = TYPE_FB_MAP;
-    fp->Fname = (char*)PlugSubAlloc(g, NULL, strlen(filename) + 1);
-    strcpy((char*)fp->Fname, filename);
+    fp->Fname = PlugDup(g, filename);
     fp->Next = dup->Openlist;
     dup->Openlist = fp;
     fp->Count = 1;

storage/connect/ha_connect.cc

@@ -171,7 +171,7 @@
 #define JSONMAX      10             // JSON Default max grp size
 
 extern "C" {
-       char  version[]= "Version 1.03.0006 February 06, 2015";
+       char  version[]= "Version 1.03.0006 March 16, 2015";
 
 #if defined(WIN32)
        char  compver[]= "Version 1.03.0006 " __DATE__ " "  __TIME__;
@@ -211,6 +211,8 @@ extern "C" {
 /***********************************************************************/
 PQRYRES OEMColumns(PGLOBAL g, PTOS topt, char *tab, char *db, bool info);
 PQRYRES VirColumns(PGLOBAL g, char *tab, char *db, bool info);
+PQRYRES JSONColumns(PGLOBAL g, char *dp, const char *fn, char *objn,
+                    int pretty, int lvl, int mxr, bool info);
 void    PushWarning(PGLOBAL g, THD *thd, int level);
 bool    CheckSelf(PGLOBAL g, TABLE_SHARE *s, const char *host,
                   const char *db, char *tab, const char *src, int port);
@@ -1020,8 +1022,7 @@ char *GetListOption(PGLOBAL g, const char *opname,
     } // endif pv
 
     if (!stricmp(opname, key)) {
-      opval= (char*)PlugSubAlloc(g, NULL, strlen(val) + 1);
-      strcpy(opval, val);
+      opval= PlugDup(g, val);
       break;
     } else if (!pn)
       break;
@@ -1506,8 +1507,7 @@ PIXDEF ha_connect::GetIndexInfo(TABLE_SHARE *s)
 
     // Now get index information
     pn= (char*)s->keynames.type_names[n];
-    name= (char*)PlugSubAlloc(g, NULL, strlen(pn) + 1);
-    strcpy(name, pn);    // This is probably unuseful
+    name= PlugDup(g, pn);
     unique= (kp.flags & 1) != 0;
     pkp= NULL;
 
@@ -1517,8 +1517,7 @@ PIXDEF ha_connect::GetIndexInfo(TABLE_SHARE *s)
     // Get the the key parts info
     for (int k= 0; (unsigned)k < kp.user_defined_key_parts; k++) {
       pn= (char*)kp.key_part[k].field->field_name;
-      name= (char*)PlugSubAlloc(g, NULL, strlen(pn) + 1);
-      strcpy(name, pn);    // This is probably unuseful
+      name= PlugDup(g, pn);
 
       // Allocate the key part description block
       kpp= new(g) KPARTDEF(name, k + 1);
@@ -4738,9 +4737,9 @@ static char *encode(PGLOBAL g, const char *cnm)
   @return
     Return 0 if ok
 */
-static bool add_field(String *sql, const char *field_name, int typ,
-                      int len, int dec, char *key, uint tm, const char *rem,
-                      char *dft, char *xtra, int flag, bool dbf, char v)
+static bool add_field(String *sql, const char *field_name, int typ, int len,
+                      int dec, char *key, uint tm, const char *rem, char *dft,
+                      char *xtra, char *fmt, int flag, bool dbf, char v)
 {
   char var = (len > 255) ? 'V' : v;
   bool q, error= false;
@@ -4809,6 +4808,12 @@ static bool add_field(String *sql, const char *field_name, int typ,
     error|= sql->append("'");
     } // endif rem
 
+  if (fmt && *fmt) {
+    error|= sql->append(" FIELD_FORMAT='");
+    error|= sql->append_for_single_quote(fmt, strlen(fmt));
+    error|= sql->append("'");
+    } // endif flag
+
   if (flag) {
     error|= sql->append(" FLAG=");
     error|= sql->append_ulonglong(flag);
@@ -4952,12 +4957,12 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
   const char *fncn= "?";
   const char *user, *fn, *db, *host, *pwd, *sep, *tbl, *src;
   const char *col, *ocl, *rnk, *pic, *fcl, *skc;
-  char       *tab, *dsn, *shm, *dpath; 
+  char       *tab, *dsn, *shm, *dpath, *objn; 
 #if defined(WIN32)
   char       *nsp= NULL, *cls= NULL;
 #endif   // WIN32
   int         port= 0, hdr= 0, mxr __attribute__((unused))= 0, mxe= 0, rc= 0;
-  int         cop __attribute__((unused)) = 0;
+  int         cop __attribute__((unused))= 0, pty= 2, lrecl= 0, lvl= 0;
 #if defined(ODBC_SUPPORT)
   POPARM      sop = NULL;
   char       *ucnc = NULL;
@@ -4987,7 +4992,7 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
   if (!g)
     return HA_ERR_INTERNAL_ERROR;
 
-  user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= dsn= NULL;
+  user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= dsn= objn= NULL;
 
   // Get the useful create options
   ttp= GetTypeID(topt->type);
@@ -5003,6 +5008,7 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
   hdr= (int)topt->header;
   tbl= topt->tablist;
   col= topt->colist;
+  lrecl= (int)topt->lrecl;
 
   if (topt->oplist) {
     host= GetListOption(g, "host", topt->oplist, "localhost");
@@ -5017,6 +5023,7 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
     skc= GetListOption(g, "skipcol", topt->oplist, NULL);
     rnk= GetListOption(g, "rankcol", topt->oplist, NULL);
     pwd= GetListOption(g, "password", topt->oplist);
+    objn= GetListOption(g, "Object", topt->oplist, NULL);
 #if defined(WIN32)
     nsp= GetListOption(g, "namespace", topt->oplist);
     cls= GetListOption(g, "class", topt->oplist);
@@ -5034,6 +5041,8 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
 #if defined(PROMPT_OK)
     cop= atoi(GetListOption(g, "checkdsn", topt->oplist, "0"));
 #endif   // PROMPT_OK
+    pty= atoi(GetListOption(g,"Pretty", topt->oplist, "2"));
+    lvl= atoi(GetListOption(g,"Level", topt->oplist, "0"));
   } else {
     host= "localhost";
     user= (ttp == TAB_ODBC ? NULL : "root");
@@ -5077,8 +5086,7 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
         goto err;
         } // endif tbl
 
-      tab= (char*)PlugSubAlloc(g, NULL, strlen(tbl) + 1);
-      strcpy(tab, tbl);
+      tab= PlugDup(g, tbl);
 
       if ((p= strchr(tab, ',')))
         *p= 0;
@@ -5204,6 +5212,13 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
       else
         strcpy(g->Message, "Missing OEM module or subtype");
 
+      break;
+    case TAB_JSON:
+      if (!fn)
+        sprintf(g->Message, "Missing %s file name", topt->type);
+      else
+        ok= true;
+
       break;
     case TAB_VIR:
       ok= true;
@@ -5226,7 +5241,7 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
     } // endif src
 
   if (ok) {
-    char   *cnm, *rem, *dft, *xtra, *key;
+    char   *cnm, *rem, *dft, *xtra, *key, *fmt;
     int     i, len, prec, dec, typ, flg;
 
 //  if (cat)
@@ -5315,6 +5330,9 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
       case TAB_VIR:
         qrp= VirColumns(g, tab, (char*)db, fnc == FNC_COL);
         break;
+      case TAB_JSON:
+        qrp= JSONColumns(g, (char*)db, fn, objn, pty, lrecl, lvl, fnc == FNC_COL);
+        break;
       case TAB_OEM:
         qrp= OEMColumns(g, topt, tab, (char*)db, fnc == FNC_COL);
         break;
@@ -5331,7 +5349,7 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
     if (fnc != FNC_NO || src || ttp == TAB_PIVOT) {
       // Catalog like table
       for (crp= qrp->Colresp; !rc && crp; crp= crp->Next) {
-        cnm= encode(g, crp->Name);
+        cnm= (ttp == TAB_PIVOT) ? crp->Name : encode(g, crp->Name);
         typ= crp->Type;
         len= crp->Length;
         dec= crp->Prec;
@@ -5347,7 +5365,7 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
                        NOT_NULL_FLAG, "", flg, dbf, v);
 #else   // !NEW_WAY
         if (add_field(&sql, cnm, typ, len, dec, NULL, NOT_NULL_FLAG,
-                      NULL, NULL, NULL, flg, dbf, v))
+                      NULL, NULL, NULL, NULL, flg, dbf, v))
           rc= HA_ERR_OUT_OF_MEM;
 #endif  // !NEW_WAY
       } // endfor crp
@@ -5368,7 +5386,7 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
         typ= len= prec= dec= 0;
         tm= NOT_NULL_FLAG;
         cnm= (char*)"noname";
-        dft= xtra= key= NULL;
+        dft= xtra= key= fmt= NULL;
         v= ' ';
 #if defined(NEW_WAY)
         rem= "";
@@ -5407,6 +5425,9 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
               if (crp->Kdata->GetIntValue(i))
                 tm= 0;               // Nullable
 
+              break;
+            case FLD_FORMAT:
+              fmt= (crp->Kdata) ? crp->Kdata->GetCharValue(i) : NULL;
               break;
             case FLD_REM:
               rem= crp->Kdata->GetCharValue(i);
@@ -5485,7 +5506,7 @@ static int connect_assisted_discovery(handlerton *hton, THD* thd,
                        tm, rem, 0, dbf, v);
 #else   // !NEW_WAY
         if (add_field(&sql, cnm, typ, prec, dec, key, tm, rem, dft, xtra,
-                      0, dbf, v))
+                      fmt, 0, dbf, v))
           rc= HA_ERR_OUT_OF_MEM;
 #endif  // !NEW_WAY
         } // endfor i
@@ -6545,7 +6566,7 @@ maria_declare_plugin(connect)
   0x0103,                                       /* version number (1.03) */
   NULL,                                         /* status variables */
   connect_system_variables,                     /* system variables */
-  "1.03.0005",                                  /* string version */
+  "1.03.0006",                                  /* string version */
   MariaDB_PLUGIN_MATURITY_BETA                  /* maturity */
 }
 maria_declare_plugin_end;

storage/connect/ha_connect.h

@@ -1,4 +1,4 @@
-/* Copyright (C) Olivier Bertrand 2004 - 2014
+/* Copyright (C) Olivier Bertrand 2004 - 2015
 
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -56,11 +56,7 @@ class XCHK : public BLOCK {
               oldopn= newopn= NULL;
               oldpix= newpix= NULL;}
 
-  inline char *SetName(PGLOBAL g, char *name) {
-    char *nm= NULL;
-    if (name) {nm= (char*)PlugSubAlloc(g, NULL, strlen(name) + 1);
-               strcpy(nm, name);}
-    return nm;}
+  inline char *SetName(PGLOBAL g, char *name) {return PlugDup(g, name);}
 
   bool         oldsep;              // Sepindex before create/alter
   bool         newsep;              // Sepindex after create/alter

storage/connect/json.cpp

@@ -1,5 +1,5 @@
 /*************** json CPP Declares Source Code File (.H) ***************/
-/*  Name: json.cpp   Version 1.0                                       */
+/*  Name: json.cpp   Version 1.1                                       */
 /*                                                                     */
 /*  (C) Copyright to the author Olivier BERTRAND          2014 - 2015  */
 /*                                                                     */
@@ -1157,6 +1157,6 @@ void JVALUE::SetString(PGLOBAL g, PSZ s)
 /***********************************************************************/
 bool JVALUE::IsNull(void)
 {
-  return (Jsp) ? Jsp->IsNull() : (Value) ? Value->IsNull() : true;
+  return (Jsp) ? Jsp->IsNull() : (Value) ? Value->IsZero() : true;
 } // end of IsNull