feat(db) add options.nulls to request explicit nulls from the DAO

Minimal changes to add a { nulls = true } options argument to the new
DAO, so that by default it returns Lua `nil` on null values, but can
also return explicit `ngx.null` upon request. Adjusts the Admin API
endpoint generator to pass this option, and also the router (so that
code changes to the router at this point are minimal).

Fix #3609
Fix #3617

kong/api/endpoints.lua

@@ -24,6 +24,9 @@ local ERRORS_HTTP_CODES = {
 }
 
 
+local op_nulls = { nulls = true }
+
+
 local function handle_error(err_t)
   local status = ERRORS_HTTP_CODES[err_t.code]
   if not status or status == 500 then
@@ -44,16 +47,17 @@ local function query_entity(context, self, db, schema)
 
   local id = unescape_uri(self.params[schema.name])
   if utils.is_valid_uuid(id) then
-    return dao[context](dao, { id = id }, args)
+    return dao[context](dao, { id = id }, args, op_nulls)
   end
 
   if schema.endpoint_key then
     local field = schema.fields[schema.endpoint_key]
     local inferred_value = arguments.infer_value(id, field)
-    return dao[context .. "_by_" .. schema.endpoint_key](dao, inferred_value, args)
+    return dao[context .. "_by_" .. schema.endpoint_key](dao, inferred_value,
+                                                         args, op_nulls)
   end
 
-  return dao[context](dao, { id = id })
+  return dao[context](dao, { id = id }, op_nulls)
 end
 
 
@@ -90,7 +94,8 @@ end
 local function get_collection_endpoint(schema, foreign_schema, foreign_field_name)
   return not foreign_schema and function(self, db, helpers)
     local data, _, err_t, offset = db[schema.name]:page(self.args.size,
-                                                        self.args.offset)
+                                                        self.args.offset,
+                                                        op_nulls)
     if err_t then
       return handle_error(err_t)
     end
@@ -115,7 +120,8 @@ local function get_collection_endpoint(schema, foreign_schema, foreign_field_nam
 
     local dao = db[schema.name]
     local data, _, err_t, offset = dao["for_" .. foreign_field_name](dao, { id = foreign_entity.id },
-                                                                     self.args.size, self.args.offset)
+                                                                     self.args.size, self.args.offset,
+                                                                     op_nulls)
     if err_t then
       return handle_error(err_t)
     end
@@ -163,7 +169,7 @@ local function post_collection_endpoint(schema, foreign_schema, foreign_field_na
       self.args.post[foreign_field_name] = { id = foreign_entity.id }
     end
 
-    local entity, _, err_t = db[schema.name]:insert(self.args.post)
+    local entity, _, err_t = db[schema.name]:insert(self.args.post, op_nulls)
     if err_t then
       return handle_error(err_t)
     end
@@ -200,7 +206,7 @@ local function get_entity_endpoint(schema, foreign_schema, foreign_field_name)
         return helpers.responses.send_HTTP_NOT_FOUND()
       end
 
-      entity, _, err_t = db[foreign_schema.name]:select(id)
+      entity, _, err_t = db[foreign_schema.name]:select(id, op_nulls)
       if err_t then
         return handle_error(err_t)
       end
@@ -253,7 +259,7 @@ local function put_entity_endpoint(schema, foreign_schema, foreign_field_name)
       return helpers.responses.send_HTTP_NOT_FOUND()
     end
 
-    entity, _, err_t = db[foreign_schema.name]:upsert(id, self.args.post)
+    entity, _, err_t = db[foreign_schema.name]:upsert(id, self.args.post, op_nulls)
     if err_t then
       return handle_error(err_t)
     end
@@ -305,7 +311,7 @@ local function patch_entity_endpoint(schema, foreign_schema, foreign_field_name)
       return helpers.responses.send_HTTP_NOT_FOUND()
     end
 
-    entity, _, err_t = db[foreign_schema.name]:update(id, self.args.post)
+    entity, _, err_t = db[foreign_schema.name]:update(id, self.args.post, op_nulls)
     if err_t then
       return handle_error(err_t)
     end

kong/api/routes/certificates.lua

@@ -53,14 +53,14 @@ return {
 
       -- cert was found via id or sni inside `before` section
       if utils.is_valid_uuid(id) then
-        cert, _, err_t = db.certificates:upsert({ id = id }, args)
+        cert, _, err_t = db.certificates:upsert({ id = id }, args, { nulls = true })
 
       else -- create a new cert. Add extra sni if provided on url
         if self.new_put_sni then
           args.snis = Set.values(Set(args.snis or {}) + self.new_put_sni)
           self.new_put_sni = nil
         end
-        cert, _, err_t = db.certificates:insert(args)
+        cert, _, err_t = db.certificates:insert(args, { nulls = true })
       end
 
       if err_t then

kong/api/routes/consumers.lua

@@ -23,7 +23,8 @@ return {
       end
 
       local data, _, err_t, offset = db.consumers:page(self.args.size,
-                                                       self.args.offset)
+                                                       self.args.offset,
+                                                       { nulls = true })
       if err_t then
         return Endpoints.handle_error(err_t)
       end

kong/db/dao/init.lua

@@ -93,9 +93,13 @@ local function generate_foreign_key_methods(schema)
         end
       end
 
-      methods["select_by_" .. name] = function(self, unique_value)
+      methods["select_by_" .. name] = function(self, unique_value, options)
         validate_unique_value(unique_value)
 
+        if options ~= nil and type(options) ~= "table" then
+          error("options must be a table", 2)
+        end
+
         local row, err_t = self.strategy:select_by_field(name, unique_value)
         if err_t then
           return nil, tostring(err_t), err_t
@@ -105,12 +109,16 @@ local function generate_foreign_key_methods(schema)
           return nil
         end
 
-        return self:row_to_entity(row)
+        return self:row_to_entity(row, options)
       end
 
-      methods["update_by_" .. name] = function(self, unique_value, entity)
+      methods["update_by_" .. name] = function(self, unique_value, entity, options)
         validate_unique_value(unique_value)
 
+        if options ~= nil and type(options) ~= "table" then
+          error("options must be a table", 2)
+        end
+
         local entity_to_update, err = self.schema:process_auto_fields(entity, "update")
         if not entity_to_update then
           local err_t = self.errors:schema_violation(err)
@@ -129,7 +137,7 @@ local function generate_foreign_key_methods(schema)
           return nil, tostring(err_t), err_t
         end
 
-        row, err, err_t = self:row_to_entity(row)
+        row, err, err_t = self:row_to_entity(row, options)
         if not row then
           return nil, err, err_t
         end
@@ -139,9 +147,13 @@ local function generate_foreign_key_methods(schema)
         return row
       end
 
-      methods["upsert_by_" .. name] = function(self, unique_value, entity)
+      methods["upsert_by_" .. name] = function(self, unique_value, entity, options)
         validate_unique_value(unique_value)
 
+        if options ~= nil and type(options) ~= "table" then
+          error("options must be a table", 2)
+        end
+
         local entity_to_upsert, err = self.schema:process_auto_fields(entity, "upsert")
         if not entity_to_upsert then
           local err_t = self.errors:schema_violation(err)
@@ -162,7 +174,7 @@ local function generate_foreign_key_methods(schema)
           return nil, tostring(err_t), err_t
         end
 
-        row, err, err_t = self:row_to_entity(row)
+        row, err, err_t = self:row_to_entity(row, options)
         if not row then
           return nil, err, err_t
         end
@@ -227,11 +239,15 @@ function DAO:truncate()
 end
 
 
-function DAO:select(primary_key)
+function DAO:select(primary_key, options)
   if type(primary_key) ~= "table" then
     error("primary_key must be a table", 2)
   end
 
+  if options ~= nil and type(options) ~= "table" then
+    error("options must be a table", 2)
+  end
+
   local ok, errors = self.schema:validate_primary_key(primary_key)
   if not ok then
     local err_t = self.errors:invalid_primary_key(errors)
@@ -247,11 +263,11 @@ function DAO:select(primary_key)
     return nil
   end
 
-  return self:row_to_entity(row)
+  return self:row_to_entity(row, options)
 end
 
 
-function DAO:page(size, offset)
+function DAO:page(size, offset, options)
   size = tonumber(size == nil and 100 or size)
 
   if not size then
@@ -268,12 +284,16 @@ function DAO:page(size, offset)
     error("offset must be a string", 2)
   end
 
+  if options ~= nil and type(options) ~= "table" then
+    error("options must be a table", 2)
+  end
+
   local rows, err_t, offset = self.strategy:page(size, offset)
   if err_t then
     return nil, tostring(err_t), err_t
   end
 
-  local entities, err, err_t = self:rows_to_entities(rows)
+  local entities, err, err_t = self:rows_to_entities(rows, options)
   if not entities then
     return nil, err, err_t
   end
@@ -282,7 +302,7 @@ function DAO:page(size, offset)
 end
 
 
-function DAO:each(size)
+function DAO:each(size, options)
   size = tonumber(size == nil and 100 or size)
 
   if not size then
@@ -295,6 +315,10 @@ function DAO:each(size)
     error("size must be positive (> 0)", 2)
   end
 
+  if options ~= nil and type(options) ~= "table" then
+    error("options must be a table", 2)
+  end
+
   local next_row = self.strategy:each(size)
 
   return function()
@@ -308,7 +332,7 @@ function DAO:each(size)
     end
 
     local err
-    row, err, err_t = self:row_to_entity(row)
+    row, err, err_t = self:row_to_entity(row, options)
     if not row then
       return nil, err, err_t
     end
@@ -318,11 +342,15 @@ function DAO:each(size)
 end
 
 
-function DAO:insert(entity)
+function DAO:insert(entity, options)
   if type(entity) ~= "table" then
     error("entity must be a table", 2)
   end
 
+  if options ~= nil and type(options) ~= "table" then
+    error("options must be a table", 2)
+  end
+
   local entity_to_insert, err = self.schema:process_auto_fields(entity, "insert")
   if not entity_to_insert then
     local err_t = self.errors:schema_violation(err)
@@ -340,7 +368,7 @@ function DAO:insert(entity)
     return nil, tostring(err_t), err_t
   end
 
-  row, err, err_t = self:row_to_entity(row)
+  row, err, err_t = self:row_to_entity(row, options)
   if not row then
     return nil, err, err_t
   end
@@ -351,7 +379,7 @@ function DAO:insert(entity)
 end
 
 
-function DAO:update(primary_key, entity)
+function DAO:update(primary_key, entity, options)
   if type(primary_key) ~= "table" then
     error("primary_key must be a table", 2)
   end
@@ -360,6 +388,10 @@ function DAO:update(primary_key, entity)
     error("entity must be a table", 2)
   end
 
+  if options ~= nil and type(options) ~= "table" then
+    error("options must be a table", 2)
+  end
+
   local ok, errors = self.schema:validate_primary_key(primary_key)
   if not ok then
     local err_t = self.errors:invalid_primary_key(errors)
@@ -383,7 +415,7 @@ function DAO:update(primary_key, entity)
     return nil, tostring(err_t), err_t
   end
 
-  row, err, err_t = self:row_to_entity(row)
+  row, err, err_t = self:row_to_entity(row, options)
   if not row then
     return nil, err, err_t
   end
@@ -394,7 +426,7 @@ function DAO:update(primary_key, entity)
 end
 
 
-function DAO:upsert(primary_key, entity)
+function DAO:upsert(primary_key, entity, options)
   if type(primary_key) ~= "table" then
     error("primary_key must be a table", 2)
   end
@@ -403,6 +435,10 @@ function DAO:upsert(primary_key, entity)
     error("entity must be a table", 2)
   end
 
+  if options ~= nil and type(options) ~= "table" then
+    error("options must be a table", 2)
+  end
+
   local ok, errors = self.schema:validate_primary_key(primary_key)
   if not ok then
     local err_t = self.errors:invalid_primary_key(errors)
@@ -426,7 +462,7 @@ function DAO:upsert(primary_key, entity)
     return nil, tostring(err_t), err_t
   end
 
-  row, err, err_t = self:row_to_entity(row)
+  row, err, err_t = self:row_to_entity(row, options)
   if not row then
     return nil, err, err_t
   end
@@ -467,7 +503,7 @@ function DAO:delete(primary_key)
 end
 
 
-function DAO:rows_to_entities(rows)
+function DAO:rows_to_entities(rows, options)
   local count = #rows
   if count == 0 then
     return setmetatable(rows, cjson.empty_array_mt)
@@ -476,7 +512,7 @@ function DAO:rows_to_entities(rows)
   local entities = new_tab(count, 0)
 
   for i = 1, count do
-    local entity, err, err_t = self:row_to_entity(rows[i])
+    local entity, err, err_t = self:row_to_entity(rows[i], options)
     if not entity then
       return nil, err, err_t
     end
@@ -488,8 +524,14 @@ function DAO:rows_to_entities(rows)
 end
 
 
-function DAO:row_to_entity(row)
-  local entity, errors = self.schema:process_auto_fields(row, "select")
+function DAO:row_to_entity(row, options)
+  if options ~= nil and type(options) ~= "table" then
+    error("options must be a table", 2)
+  end
+
+  local nulls = options and options.nulls
+
+  local entity, errors = self.schema:process_auto_fields(row, "select", nulls)
   if not entity then
     local err_t = self.errors:schema_violation(errors)
     return nil, tostring(err_t), err_t