feat(cmd) implement kong config db_export command

kong/cmd/config.lua

@@ -18,6 +18,37 @@ local accepted_formats = {
 }
 
 
+local function db_export(filename, conf)
+  if pl_file.access_time(filename) then
+    error(filename .. " already exists. Will not overwrite it.")
+  end
+
+  _G.kong = kong_global.new()
+  kong_global.init_pdk(_G.kong, conf, nil) -- nil: latest PDK
+
+  local db = assert(DB.new(conf))
+  assert(db:init_connector())
+  assert(db:connect())
+  assert(db.plugins:load_plugin_schemas(conf.loaded_plugins))
+
+  _G.kong.db = db
+
+  local fd, err = io.open(filename, "w")
+  if not fd then
+    return nil, err
+  end
+
+  local ok, err = declarative.export_from_db(fd)
+  if not ok then
+    error(err)
+  end
+
+  fd:close()
+
+  os.exit(0)
+end
+
+
 local function generate_init()
   if pl_file.access_time(INIT_FILE) then
     error(INIT_FILE .. " already exists in the current directory.\n" ..
@@ -46,23 +77,29 @@ local function execute(args)
     conf = assert(conf_loader(conf.kong_env))
   end
 
-  if args.command == "db-import" then
-    args.command = "db_import"
-  end
+  args.command = args.command:gsub("%-", "_")
 
   if args.command == "db_import" and conf.database == "off" then
     error("'kong config db_import' only works with a database.\n" ..
           "When using database=off, reload your declarative configuration\n" ..
           "using the /config endpoint.")
   end
 
+  if args.command == "db_export" and conf.database == "off" then
+    error("'kong config db_export' only works with a database.")
+  end
+
   package.path = conf.lua_package_path .. ";" .. package.path
 
   local dc, err = declarative.new_config(conf)
   if not dc then
     error(err)
   end
 
+  if args.command == "db_export" then
+    return db_export(args[1] or "kong.yml", conf)
+  end
+
   if args.command == "db_import" or args.command == "parse" then
     local filename = args[1]
     if not filename then
@@ -126,6 +163,9 @@ The available commands are:
   db_import <file>                    Import a declarative config file into
                                       the Kong database.
 
+  db_export <file>                    Export the Kong database into a
+                                      declarative config file.
+
   parse <file>                        Parse a declarative config file (check
                                       its syntax) but do not load it into Kong.
 
@@ -140,6 +180,7 @@ return {
   sub_commands = {
     init = true,
     db_import = true,
+    db_export = true,
     parse = true,
   },
 }

kong/db/declarative/init.lua

@@ -10,6 +10,7 @@ local deepcopy = tablex.deepcopy
 local null = ngx.null
 local SHADOW = true
 local md5 = ngx.md5
+local REMOVE_FIRST_LINE_PATTERN = "^[^\n]+\n(.+)$"
 
 
 local declarative = {}
@@ -231,6 +232,60 @@ function declarative.load_into_db(dc_table)
 end
 
 
+function declarative.export_from_db(fd)
+  local schemas = {}
+  for _, dao in pairs(kong.db.daos) do
+    table.insert(schemas, dao.schema)
+  end
+  local sorted_schemas, err = topological_sort(schemas)
+  if not sorted_schemas then
+    return nil, err
+  end
+
+  fd:write(declarative.to_yaml_string({
+    _format_version = "1.1",
+  }))
+
+  for _, schema in ipairs(sorted_schemas) do
+    if schema.db_export == false then
+      goto continue
+    end
+
+    local name = schema.name
+    local fks = {}
+    for name, field in schema:each_field() do
+      if field.type == "foreign" then
+        table.insert(fks, name)
+      end
+    end
+
+    local first_row = true
+    for row, err in kong.db[name]:each() do
+      for _, fname in ipairs(fks) do
+        if type(row[fname]) == "table" then
+          local id = row[fname].id
+          if id ~= nil then
+            row[fname] = id
+          end
+        end
+      end
+
+      local yaml = declarative.to_yaml_string({ [name] = { row } })
+      if not first_row then
+        yaml = assert(yaml:match(REMOVE_FIRST_LINE_PATTERN))
+      end
+      first_row = false
+
+      fd:write(yaml)
+    end
+
+    ::continue::
+  end
+
+  return true
+end
+
+
 local function remove_nulls(tbl)
   for k,v in pairs(tbl) do
     if v == null then

kong/db/schema/entities/cluster_ca.lua

@@ -6,6 +6,7 @@ local openssl_x509 = require "openssl.x509"
 return {
   name = "cluster_ca",
   generate_admin_api = false,
+  db_export = false,
 
   -- Cassandra *requires* a primary key.
   -- To keep it happy we add a superfluous boolean column that is always true.

kong/db/schema/entities/tags.lua

@@ -5,6 +5,7 @@ return {
   primary_key  = { "tag" },
   endpoint_key = "tag",
   dao          = "kong.db.dao.tags",
+  db_export = false,
 
   fields = {
     { tag          = typedefs.tag, },

kong/db/schema/metaschema.lua

@@ -374,6 +374,13 @@ local MetaSchema = Schema.new({
         nilable = true,
       }
     },
+    {
+      db_export = {
+        type = "boolean",
+        nilable = true,
+        default = true,
+      }
+    },
     {
       subschema_key = {
         type = "string",

spec/02-integration/02-cmd/11-config_spec.lua

@@ -1,14 +1,18 @@
 local helpers = require "spec.helpers"
 local constants = require "kong.constants"
 local cjson = require "cjson"
+local lyaml = require "lyaml"
 
 
+local function sort_by_name(a, b)
+  return a.name < b.name
+end
+
 describe("kong config", function()
-  local db
+  local bp, db
 
   lazy_setup(function()
-    local _
-    _, db = helpers.get_db_utils(nil, {}) -- runs migrations
+    bp, db = helpers.get_db_utils(nil, {}) -- runs migrations
   end)
   after_each(function()
     helpers.kill_all()
@@ -296,4 +300,100 @@ describe("kong config", function()
 
     assert(helpers.stop_kong())
   end)
+
+  it("#db config db_export exports a yaml file", function()
+    assert(db.plugins:truncate())
+    assert(db.routes:truncate())
+    assert(db.services:truncate())
+    assert(db.consumers:truncate())
+    assert(db.acls:truncate())
+
+    local filename = os.tmpname()
+    os.remove(filename)
+    filename = filename .. ".yml"
+
+    -- starting kong just so the prefix is properly initialized
+    assert(helpers.start_kong())
+
+    local service1 = bp.services:insert({ name = "service1" })
+    local route1 = bp.routes:insert({ service = service1, methods = { "POST" }, name = "a" })
+    local plugin1 = bp.hmac_auth_plugins:insert({
+      service = service1,
+    })
+    local plugin2 = bp.key_auth_plugins:insert({
+      service = service1,
+    })
+
+    local service2 = bp.services:insert({ name = "service2" })
+    local route2 = bp.routes:insert({ service = service2, methods = { "GET" }, name = "b" })
+    local plugin3 = bp.tcp_log_plugins:insert({
+      service = service2,
+    })
+    local consumer = bp.consumers:insert()
+    local acls = bp.acls:insert({ consumer = consumer })
+
+    assert(helpers.kong_exec("config db_export " .. filename, {
+      prefix = helpers.test_conf.prefix,
+    }))
+
+    finally(function()
+      os.remove(filename)
+    end)
+
+    local f = assert(io.open(filename, "rb"))
+    local content = f:read("*all")
+    f:close()
+    local yaml = assert(lyaml.load(content))
+
+    local toplevel_keys = {}
+    for k in pairs(yaml) do
+      toplevel_keys[#toplevel_keys + 1] = k
+    end
+    table.sort(toplevel_keys)
+    assert.same({
+      "_format_version",
+      "acls",
+      "consumers",
+      "plugins",
+      "routes",
+      "services",
+    }, toplevel_keys)
+
+    assert.equals("1.1", yaml._format_version)
+
+    assert.equals(2, #yaml.services)
+    table.sort(yaml.services, sort_by_name)
+    assert.same(service1, yaml.services[1])
+    assert.same(service2, yaml.services[2])
+
+    assert.equals(2, #yaml.routes)
+    table.sort(yaml.routes, sort_by_name)
+    assert.equals(route1.id, yaml.routes[1].id)
+    assert.equals(route1.name, yaml.routes[1].name)
+    assert.equals(service1.id, yaml.routes[1].service)
+    assert.equals(route2.id, yaml.routes[2].id)
+    assert.equals(route2.name, yaml.routes[2].name)
+    assert.equals(service2.id, yaml.routes[2].service)
+
+    assert.equals(3, #yaml.plugins)
+    table.sort(yaml.plugins, sort_by_name)
+    assert.equals(plugin1.id, yaml.plugins[1].id)
+    assert.equals(plugin1.name, yaml.plugins[1].name)
+    assert.equals(service1.id, yaml.plugins[1].service)
+
+    assert.equals(plugin2.id, yaml.plugins[2].id)
+    assert.equals(plugin2.name, yaml.plugins[2].name)
+    assert.equals(service1.id, yaml.plugins[2].service)
+
+    assert.equals(plugin3.id, yaml.plugins[3].id)
+    assert.equals(plugin3.name, yaml.plugins[3].name)
+    assert.equals(service2.id, yaml.plugins[3].service)
+
+    assert.equals(1, #yaml.consumers)
+    assert.same(consumer, yaml.consumers[1])
+
+    assert.equals(1, #yaml.acls)
+    assert.equals(acls.group, yaml.acls[1].group)
+    assert.equals(consumer.id, yaml.acls[1].consumer)
+  end)
 end)