From 0cbd796aea4980ad8541e096c4cecdc899532e99 Mon Sep 17 00:00:00 2001 From: Hisham Muhammad Date: Tue, 9 Jul 2019 12:58:58 -0300 Subject: [PATCH] feat(cmd) kong config db_export command --- kong/cmd/config.lua | 47 ++++++- kong/db/declarative/init.lua | 55 ++++++++ kong/db/schema/entities/cluster_ca.lua | 1 + kong/db/schema/entities/tags.lua | 1 + kong/db/schema/metaschema.lua | 7 + spec/02-integration/02-cmd/11-config_spec.lua | 106 ++++++++++++++- .../03-db/08-declarative_spec.lua | 122 +++++++++++++++--- 7 files changed, 314 insertions(+), 25 deletions(-) diff --git a/kong/cmd/config.lua b/kong/cmd/config.lua index eb8768912f55..87b27f891118 100644 --- a/kong/cmd/config.lua +++ b/kong/cmd/config.lua @@ -18,6 +18,37 @@ local accepted_formats = { } +local function db_export(filename, conf) + if pl_file.access_time(filename) then + error(filename .. " already exists. Will not overwrite it.") + end + + _G.kong = kong_global.new() + kong_global.init_pdk(_G.kong, conf, nil) -- nil: latest PDK + + local db = assert(DB.new(conf)) + assert(db:init_connector()) + assert(db:connect()) + assert(db.plugins:load_plugin_schemas(conf.loaded_plugins)) + + _G.kong.db = db + + local fd, err = io.open(filename, "w") + if not fd then + return nil, err + end + + local ok, err = declarative.export_from_db(fd) + if not ok then + error(err) + end + + fd:close() + + os.exit(0) +end + + local function generate_init() if pl_file.access_time(INIT_FILE) then error(INIT_FILE .. " already exists in the current directory.\n" .. @@ -46,9 +77,7 @@ local function execute(args) conf = assert(conf_loader(conf.kong_env)) end - if args.command == "db-import" then - args.command = "db_import" - end + args.command = args.command:gsub("%-", "_") if args.command == "db_import" and conf.database == "off" then error("'kong config db_import' only works with a database.\n" .. @@ -56,6 +85,10 @@ local function execute(args) "using the /config endpoint.") end + if args.command == "db_export" and conf.database == "off" then + error("'kong config db_export' only works with a database.") + end + package.path = conf.lua_package_path .. ";" .. package.path local dc, err = declarative.new_config(conf) @@ -63,6 +96,10 @@ local function execute(args) error(err) end + if args.command == "db_export" then + return db_export(args[1] or "kong.yml", conf) + end + if args.command == "db_import" or args.command == "parse" then local filename = args[1] if not filename then @@ -126,6 +163,9 @@ The available commands are: db_import Import a declarative config file into the Kong database. + db_export Export the Kong database into a + declarative config file. + parse Parse a declarative config file (check its syntax) but do not load it into Kong. @@ -140,6 +180,7 @@ return { sub_commands = { init = true, db_import = true, + db_export = true, parse = true, }, } diff --git a/kong/db/declarative/init.lua b/kong/db/declarative/init.lua index 595bc4c7fc50..c490d7070c95 100644 --- a/kong/db/declarative/init.lua +++ b/kong/db/declarative/init.lua @@ -10,6 +10,7 @@ local deepcopy = tablex.deepcopy local null = ngx.null local SHADOW = true local md5 = ngx.md5 +local REMOVE_FIRST_LINE_PATTERN = "^[^\n]+\n(.+)$" local declarative = {} @@ -231,6 +232,60 @@ function declarative.load_into_db(dc_table) end +function declarative.export_from_db(fd) + local schemas = {} + for _, dao in pairs(kong.db.daos) do + table.insert(schemas, dao.schema) + end + local sorted_schemas, err = topological_sort(schemas) + if not sorted_schemas then + return nil, err + end + + fd:write(declarative.to_yaml_string({ + _format_version = "1.1", + })) + + for _, schema in ipairs(sorted_schemas) do + if schema.db_export == false then + goto continue + end + + local name = schema.name + local fks = {} + for name, field in schema:each_field() do + if field.type == "foreign" then + table.insert(fks, name) + end + end + + local first_row = true + for row, err in kong.db[name]:each() do + for _, fname in ipairs(fks) do + if type(row[fname]) == "table" then + local id = row[fname].id + if id ~= nil then + row[fname] = id + end + end + end + + local yaml = declarative.to_yaml_string({ [name] = { row } }) + if not first_row then + yaml = assert(yaml:match(REMOVE_FIRST_LINE_PATTERN)) + end + first_row = false + + fd:write(yaml) + end + + ::continue:: + end + + return true +end + + local function remove_nulls(tbl) for k,v in pairs(tbl) do if v == null then diff --git a/kong/db/schema/entities/cluster_ca.lua b/kong/db/schema/entities/cluster_ca.lua index 071d06743b3d..5b0640d6534c 100644 --- a/kong/db/schema/entities/cluster_ca.lua +++ b/kong/db/schema/entities/cluster_ca.lua @@ -6,6 +6,7 @@ local openssl_x509 = require "openssl.x509" return { name = "cluster_ca", generate_admin_api = false, + db_export = false, -- Cassandra *requires* a primary key. -- To keep it happy we add a superfluous boolean column that is always true. diff --git a/kong/db/schema/entities/tags.lua b/kong/db/schema/entities/tags.lua index e83882d65cf7..d2f7d94e3751 100644 --- a/kong/db/schema/entities/tags.lua +++ b/kong/db/schema/entities/tags.lua @@ -5,6 +5,7 @@ return { primary_key = { "tag" }, endpoint_key = "tag", dao = "kong.db.dao.tags", + db_export = false, fields = { { tag = typedefs.tag, }, diff --git a/kong/db/schema/metaschema.lua b/kong/db/schema/metaschema.lua index c2a54045531e..4fb93a7c0730 100644 --- a/kong/db/schema/metaschema.lua +++ b/kong/db/schema/metaschema.lua @@ -374,6 +374,13 @@ local MetaSchema = Schema.new({ nilable = true, } }, + { + db_export = { + type = "boolean", + nilable = true, + default = true, + } + }, { subschema_key = { type = "string", diff --git a/spec/02-integration/02-cmd/11-config_spec.lua b/spec/02-integration/02-cmd/11-config_spec.lua index 897c28b5c266..85e1f9c67824 100644 --- a/spec/02-integration/02-cmd/11-config_spec.lua +++ b/spec/02-integration/02-cmd/11-config_spec.lua @@ -1,14 +1,18 @@ local helpers = require "spec.helpers" local constants = require "kong.constants" local cjson = require "cjson" +local lyaml = require "lyaml" +local function sort_by_name(a, b) + return a.name < b.name +end + describe("kong config", function() - local db + local bp, db lazy_setup(function() - local _ - _, db = helpers.get_db_utils(nil, {}) -- runs migrations + bp, db = helpers.get_db_utils(nil, {}) -- runs migrations end) after_each(function() helpers.kill_all() @@ -296,4 +300,100 @@ describe("kong config", function() assert(helpers.stop_kong()) end) + + it("#db config db_export exports a yaml file", function() + assert(db.plugins:truncate()) + assert(db.routes:truncate()) + assert(db.services:truncate()) + assert(db.consumers:truncate()) + assert(db.acls:truncate()) + + local filename = os.tmpname() + os.remove(filename) + filename = filename .. ".yml" + + -- starting kong just so the prefix is properly initialized + assert(helpers.start_kong()) + + local service1 = bp.services:insert({ name = "service1" }) + local route1 = bp.routes:insert({ service = service1, methods = { "POST" }, name = "a" }) + local plugin1 = bp.hmac_auth_plugins:insert({ + service = service1, + }) + local plugin2 = bp.key_auth_plugins:insert({ + service = service1, + }) + + local service2 = bp.services:insert({ name = "service2" }) + local route2 = bp.routes:insert({ service = service2, methods = { "GET" }, name = "b" }) + local plugin3 = bp.tcp_log_plugins:insert({ + service = service2, + }) + local consumer = bp.consumers:insert() + local acls = bp.acls:insert({ consumer = consumer }) + + assert(helpers.kong_exec("config db_export " .. filename, { + prefix = helpers.test_conf.prefix, + })) + + finally(function() + os.remove(filename) + end) + + local f = assert(io.open(filename, "rb")) + local content = f:read("*all") + f:close() + local yaml = assert(lyaml.load(content)) + + local toplevel_keys = {} + for k in pairs(yaml) do + toplevel_keys[#toplevel_keys + 1] = k + end + table.sort(toplevel_keys) + assert.same({ + "_format_version", + "acls", + "consumers", + "plugins", + "routes", + "services", + }, toplevel_keys) + + assert.equals("1.1", yaml._format_version) + + assert.equals(2, #yaml.services) + table.sort(yaml.services, sort_by_name) + assert.same(service1, yaml.services[1]) + assert.same(service2, yaml.services[2]) + + assert.equals(2, #yaml.routes) + table.sort(yaml.routes, sort_by_name) + assert.equals(route1.id, yaml.routes[1].id) + assert.equals(route1.name, yaml.routes[1].name) + assert.equals(service1.id, yaml.routes[1].service) + assert.equals(route2.id, yaml.routes[2].id) + assert.equals(route2.name, yaml.routes[2].name) + assert.equals(service2.id, yaml.routes[2].service) + + assert.equals(3, #yaml.plugins) + table.sort(yaml.plugins, sort_by_name) + assert.equals(plugin1.id, yaml.plugins[1].id) + assert.equals(plugin1.name, yaml.plugins[1].name) + assert.equals(service1.id, yaml.plugins[1].service) + + assert.equals(plugin2.id, yaml.plugins[2].id) + assert.equals(plugin2.name, yaml.plugins[2].name) + assert.equals(service1.id, yaml.plugins[2].service) + + assert.equals(plugin3.id, yaml.plugins[3].id) + assert.equals(plugin3.name, yaml.plugins[3].name) + assert.equals(service2.id, yaml.plugins[3].service) + + assert.equals(1, #yaml.consumers) + assert.same(consumer, yaml.consumers[1]) + + assert.equals(1, #yaml.acls) + assert.equals(acls.group, yaml.acls[1].group) + assert.equals(consumer.id, yaml.acls[1].consumer) + end) end) diff --git a/spec/02-integration/03-db/08-declarative_spec.lua b/spec/02-integration/03-db/08-declarative_spec.lua index 5b3c2c75e5f6..74463e806bce 100644 --- a/spec/02-integration/03-db/08-declarative_spec.lua +++ b/spec/02-integration/03-db/08-declarative_spec.lua @@ -1,6 +1,7 @@ local declarative = require "kong.db.declarative" local ssl_fixtures = require "spec.fixtures.ssl" local helpers = require "spec.helpers" +local lyaml = require "lyaml" for _, strategy in helpers.each_strategy() do describe("declarative config #" .. strategy, function() @@ -31,7 +32,8 @@ for _, strategy in helpers.each_strategy() do read_timeout = 60000, retries = 5, updated_at = 1549025889, - write_timeout = 60000 + write_timeout = 60000, + tags = { "potato", "carrot" }, } local route_def = { @@ -97,27 +99,28 @@ for _, strategy in helpers.each_strategy() do consumer = { id = consumer_def.id }, group = "The A Team" } + before_each(function() + db.acls:truncate() + db.plugins:truncate() + db.routes:truncate() + db.services:truncate() + db.snis:truncate() + db.certificates:truncate() + db.consumers:truncate() + + assert(declarative.load_into_db({ + snis = { [sni_def.id] = sni_def }, + certificates = { [certificate_def.id] = certificate_def }, + routes = { [route_def.id] = route_def }, + services = { [service_def.id] = service_def }, + consumers = { [consumer_def.id] = consumer_def }, + plugins = { [plugin_def.id] = plugin_def }, + acls = { [acl_def.id] = acl_def }, + })) + end) describe("load_into_db", function() it("imports base and custom entities with associations", function() - db.acls:truncate() - db.plugins:truncate() - db.routes:truncate() - db.services:truncate() - db.snis:truncate() - db.certificates:truncate() - db.consumers:truncate() - - assert(declarative.load_into_db({ - snis = { [sni_def.id] = sni_def }, - certificates = { [certificate_def.id] = certificate_def }, - routes = { [route_def.id] = route_def }, - services = { [service_def.id] = service_def }, - consumers = { [consumer_def.id] = consumer_def }, - plugins = { [plugin_def.id] = plugin_def }, - acls = { [acl_def.id] = acl_def }, - })) - local sni = assert(db.snis:select_by_name("baz")) assert.equals(sni_def.id, sni.id) assert.equals(certificate_def.id, sni.certificate.id) @@ -154,6 +157,87 @@ for _, strategy in helpers.each_strategy() do assert.equals("The A Team", acl.group) end) end) + + describe("export_from_db", function() + it("exports base and custom entities with associations", function() + + local fake_file = { + buffer = {}, + write = function(self, str) + self.buffer[#self.buffer + 1] = str + end, + } + + assert(declarative.export_from_db(fake_file)) + + local exported_str = table.concat(fake_file.buffer) + local yaml = lyaml.load(exported_str) + + -- ensure tags and cluster_ca are not being exported + local toplevel_keys = {} + for k in pairs(yaml) do + toplevel_keys[#toplevel_keys + 1] = k + end + table.sort(toplevel_keys) + assert.same({ + "_format_version", + "acls", + "certificates", + "consumers", + "plugins", + "routes", + "services", + "snis" + }, toplevel_keys) + + assert.equals("1.1", yaml._format_version) + + assert.equals(1, #yaml.snis) + local sni = assert(yaml.snis[1]) + assert.equals(sni_def.id, sni.id) + assert.equals(sni_def.name, sni.name) + assert.equals(certificate_def.id, sni.certificate) + + assert.equals(1, #yaml.certificates) + local cert = assert(yaml.certificates[1]) + assert.equals(certificate_def.id, cert.id) + assert.equals(ssl_fixtures.key, cert.key) + assert.equals(ssl_fixtures.cert, cert.cert) + + assert.equals(1, #yaml.services) + local service = assert(yaml.services[1]) + assert.equals(service_def.id, service.id) + assert.equals("example.com", service.host) + assert.equals("https", service.protocol) + table.sort(service.tags) + assert.same({"carrot", "potato"}, service.tags) + + assert.equals(1, #yaml.routes) + local route = assert(yaml.routes[1]) + assert.equals(route_def.id, route.id) + assert.equals("example.com", route.hosts[1]) + assert.same({ "http", "https" }, route.protocols) + assert.equals(service_def.id, route.service) + + assert.equals(1, #yaml.consumers) + local consumer = assert(yaml.consumers[1]) + assert.equals(consumer_def.id, consumer.id) + assert.equals("andru", consumer_def.username) + assert.equals("donalds", consumer_def.custom_id) + + assert.equals(1, #yaml.plugins) + local plugin = assert(yaml.plugins[1]) + assert.equals(plugin_def.id, plugin.id) + assert.equals(service.id, plugin.service) + assert.equals("acl", plugin.name) + assert.same(plugin_def.config, plugin.config) + + assert.equals(1, #yaml.acls) + local acl = assert(yaml.acls[1]) + assert.equals(consumer_def.id, acl.consumer) + assert.equals("The A Team", acl.group) + end) + end) end) end