ci/relx changes (#4641)

backport of #4634

add name

don't override config with env

template name/cookie

don't hardcode name-arg, use vm.args template

update release vm.args with name/cookie

add templates

fix sed

set the cookie

remove templates

set cookie on command line

remove echo

short and silent

changes after cherry-pick

.gitignore

@@ -20,9 +20,6 @@
 sup.bash
 
 _rel/
-rel/relx.config
-rel/dev-vm.args
-rel/vm.args
 
 erlang.mk
 .erlang.mk

Makefile

@@ -7,7 +7,7 @@ KAZOODIRS = core/Makefile applications/Makefile
 
 .PHONY: $(KAZOODIRS) deps core apps xref xref_release dialyze dialyze-it dialyze-apps dialyze-core dialyze-kazoo clean clean-test clean-release build-release build-ci-release tar-release release read-release-cookie elvis install ci diff fmt bump-copyright apis validate-swagger sdks coverage-report fs-headers docs validate-schemas
 
-all: compile rel/dev-vm.args
+all: compile
 
 compile: ACTION = all
 compile: deps $(KAZOODIRS)
@@ -79,46 +79,22 @@ $(RELX):
 
 clean-release:
 	$(if $(wildcard _rel/), rm -r _rel/)
-	$(if $(wildcard rel/relx.config rel/vm.args rel/dev-vm.args), \
-	  rm $(wildcard rel/relx.config rel/vm.args rel/dev-vm.args)  )
 
-build-release: $(RELX) clean-release rel/relx.config rel/vm.args
+build-release: $(RELX) clean-release rel/relx.config rel/relx.config.script rel/sys.config rel/vm.args
 	$(RELX) --config rel/relx.config -V 2 release --relname 'kazoo'
-	patch _rel/'kazoo'/bin/'kazoo' -i rel/relx.patch
-build-all-release: build-release
-	for path in applications/*/; do \
-	  app=$$(echo $$path | cut -d/ -f2) ; \
-	  if [ $$app = 'skel' ]; then continue; fi ; \
-	  $(RELX) --config rel/relx.config -V 2 release --relname $$app ; \
-	  patch _rel/$$app/bin/$$app -i rel/relx.patch ; \
-	done
-build-dev-release: $(RELX) clean-release rel/relx.config-dev rel/vm.args
-	$(RELX) --dev-mode true --config rel/relx.config -V 2 release --relname 'kazoo'
-	patch _rel/kazoo/bin/kazoo -i rel/relx.patch
-build-ci-release: $(RELX) clean-release rel/relx.config rel/vm.args
-	$(RELX) --config rel/relx.config -V 2 release --relname 'kazoo' --sys_config rel/ci-sys.config
-	patch _rel/kazoo/bin/kazoo -i rel/relx.patch
-tar-release: $(RELX) rel/relx.config rel/vm.args
+build-dev-release: $(RELX) clean-release rel/dev.relx.config rel/dev.relx.config.script rel/dev.vm.args rel/dev.sys.config
+	$(RELX) --dev-mode true --config rel/dev.relx.config -V 2 release --relname 'kazoo'
+build-ci-release: $(RELX) clean-release rel/ci.relx.config rel/ci.relx.config.script rel/ci.sys.config rel/ci.vm.args
+	$(RELX) --config rel/ci.relx.config -V 2 release --relname 'kazoo'
+tar-release: $(RELX) rel/relx.config rel/relx.config.script rel/sys.config rel/vm.args
 	$(RELX) --config rel/relx.config -V 2 release tar --relname 'kazoo'
-rel/relx.config: rel/relx.config.src
-	$(ROOT)/scripts/src2any.escript $<
-rel/relx.config-dev: export KAZOO_DEV='true'
-rel/relx.config-dev: rel/relx.config.src
-	$(ROOT)/scripts/src2any.escript $<
-
-rel/dev-vm.args: rel/args  # Used by scripts/dev-start-*.sh
-	cp $^ $@
-rel/vm.args: rel/args rel/dev-vm.args
-	( echo '-setcookie $${COOKIE}'; cat $<; echo '-name $${NODE_NAME}' ) > $@
 
 ## More ACTs at //github.com/erlware/relx/priv/templates/extended_bin
 release: ACT ?= console # start | attach | stop | console | foreground
 release: REL ?= kazoo_apps # kazoo_apps | ecallmgr | …
-ifneq ($(findstring kazoo_apps,$(REL)),kazoo_apps)
-release: export KAZOO_APPS = 'ecallmgr'
-endif
+release: COOKIE ?= change_me
 release:
-	@NODE_NAME='$(REL)' COOKIE='change_me' $(ROOT)/scripts/dev/kazoo.sh $(ACT) "$$@"
+	@NODE_NAME="$(REL)" COOKIE="$(COOKIE)" $(ROOT)/scripts/dev/kazoo.sh $(ACT) "$$@"
 
 install: compile build-release
 	cp -a _rel/kazoo /opt
@@ -147,7 +123,7 @@ dialyze-kazoo: dialyze
 dialyze-apps:  TO_DIALYZE  = $(shell find $(ROOT)/applications -name ebin)
 dialyze-apps: dialyze
 dialyze-core:  TO_DIALYZE  = $(shell find $(ROOT)/core         -name ebin)
-dialyze-core: dialyze
+dialyze-core: dialyze-it
 dialyze:       TO_DIALYZE ?= $(shell find $(ROOT)/applications -name ebin)
 dialyze: dialyze-it
 
@@ -205,6 +181,9 @@ apis:
 	@$(ROOT)/scripts/format-json.sh applications/crossbar/priv/api/*.json
 	@ERL_LIBS=deps/:core/:applications/ $(ROOT)/scripts/generate-fs-headers-hrl.escript
 
+schemas:
+	@ERL_LIBS=deps/:core/:applications/ $(ROOT)/scripts/generate-schemas.escript
+
 DOCS_ROOT=$(ROOT)/doc/mkdocs
 docs: docs-validate docs-report docs-setup docs-build
 

VERSION

@@ -0,0 +1 @@
+4.1

applications/ecallmgr/src/ecallmgr_fs_config.erl

@@ -119,6 +119,16 @@ handle_info({'fetch', 'configuration', <<"configuration">>, <<"name">>, Conf, ID
     lager:debug("fetch configuration request from ~s: ~s", [Node, ID]),
     _ = kz_util:spawn(fun handle_config_req/4, [Node, ID, Conf, Data]),
     {'noreply', State};
+handle_info({'fetch', 'configuration', <<"configuration">>, <<"name">>, Conf, ID, [UUID | Data]}, #state{node=Node}=State)
+  when is_binary(UUID) ->
+    lager:debug("fetch configuration request from ~s: ~s", [Node, ID]),
+    _ = kz_util:spawn(fun handle_config_req/4, [Node, ID, Conf, Data]),
+    {'noreply', State};
+handle_info({'fetch', 'configuration', <<"configuration">>, <<"name">>, Conf, ID, Data}, #state{node=Node}=State)
+  when is_list(Data) ->
+    lager:debug("fetch configuration request from ~s: ~s", [Node, ID]),
+    _ = kz_util:spawn(fun handle_config_req/4, [Node, ID, Conf, Data]),
+    {'noreply', State};
 handle_info({_Fetch, _Section, _Something, _Key, _Value, ID, _Data}, #state{node=Node}=State) ->
     lager:debug("unhandled fetch from section ~s for ~s:~s", [_Section, _Something, _Key]),
     {'ok', Resp} = ecallmgr_fs_xml:not_found(),

circle.yml

@@ -65,3 +65,9 @@ test:
     - . ~/.kerl/$OTP_VERSION/activate && make elvis
     - . ~/.kerl/$OTP_VERSION/activate && make build-ci-release
     - . ~/.kerl/$OTP_VERSION/activate && ./scripts/check-release-startup.sh
+  post:
+    - mkdir $CIRCLE_ARTIFACTS/logs
+    - mv ./_rel/kazoo/log/* $CIRCLE_ARTIFACTS/logs/
+    - cp ./rel/ci.relx.config $CIRCLE_ARTIFACTS/
+    - find ./_rel/kazoo/releases -name kazoo.rel -exec cp {} $CIRCLE_ARTIFACTS/ \;
+

core/kazoo_apps/src/kapps_maintenance.erl

@@ -297,9 +297,11 @@ refresh(?KZ_SIP_DB) ->
     kapps_util:update_views(?KZ_SIP_DB, Views, 'true');
 refresh(?KZ_SCHEMA_DB) ->
     kz_datamgr:db_create(?KZ_SCHEMA_DB),
+    lager:notice("starting system schemas update"),
     kz_datamgr:suppress_change_notice(),
     kz_datamgr:revise_docs_from_folder(?KZ_SCHEMA_DB, 'crossbar', "schemas"),
     kz_datamgr:enable_change_notice(),
+    lager:notice("finished system schemas update"),
     'ok';
 refresh(?KZ_MEDIA_DB) ->
     kz_datamgr:db_create(?KZ_MEDIA_DB),

core/sup/priv/sup

@@ -1,9 +1,9 @@
-#!/bin/bash
+#!/bin/sh
 
-# The file to copy/symlink under /usr/bin or similar
+# look for kazoo release root directory
 DEFAULT_ROOT=${KAZOO_ROOT:-_rel/kazoo}
 if [ -d "$DEFAULT_ROOT/_rel/kazoo" ]; then
-    DEFAULT_ROOT="$DEFAULT_ROOT"
+    DEFAULT_ROOT="$DEFAULT_ROOT/_rel/kazoo"
 elif [ -d "$DEFAULT_ROOT/bin" ]; then
     DEFAULT_ROOT="$DEFAULT_ROOT"
 elif [ -d /opt/kazoo/_rel/kazoo ]; then
@@ -15,5 +15,22 @@ else
     exit -1
 fi
 
-"${DEFAULT_ROOT}"/bin/kazoo escript lib/sup-*/priv/sup.escript "$*"
+while getopts "n:c:" optname
+do
+    case "$optname" in
+        "n")
+            SUP_NODE_ARG=$OPTARG
+            ;;
+        "c")
+            SUP_COOKIE_ARG=$OPTARG
+            ;;
+    esac
+done
+
+shift $(($OPTIND - 1))
+
+KAZOO_NODE=${SUP_NODE_ARG:-${KAZOO_NODE:-kazoo_apps}}
+KAZOO_COOKIE=${SUP_COOKIE_ARG:-${KAZOO_COOKIE:-change_me}}
+
+${DEFAULT_ROOT}/bin/kazoo escript lib/sup-*/priv/sup.escript -n ${KAZOO_NODE} -c ${KAZOO_COOKIE} $*
 exit $?

make/deps.mk

@@ -75,7 +75,7 @@ dep_couchbeam = git https://github.com/2600hz/couchbeam 1.4.1b
 dep_jesse = git https://github.com/2600hz/jesse 1.5-rc7
 ##dep_jesse = git https://github.com/for-GET/jesse 1.5.0-rc2
 
-dep_lager = git https://github.com/erlang-lager/lager 3.2.1
+dep_lager = git https://github.com/erlang-lager/lager 3.5.2
 dep_trie = git https://github.com/okeuday/trie v1.5.4
 dep_fs_event = git https://github.com/jamhed/fs_event 783400da08c2b55c295dbec81df0d926960c0346
 dep_fs_sync = git https://github.com/jamhed/fs_sync 2cf85cf5861221128f020c453604d509fd37cd53

make/kz.mk

@@ -29,6 +29,8 @@ ifeq ($(PLATFORM),)
     export PLATFORM
 endif
 
+VERSION=$(shell $(ROOT)/scripts/next_version)
+
 ## pipefail enforces that the command fails even when run through a pipe
 SHELL = /bin/bash -o pipefail
 
@@ -60,7 +62,8 @@ compile: $(COMPILE_MOAR) ebin/$(PROJECT).app json
 ebin/$(PROJECT).app: $(SOURCES)
 	@mkdir -p ebin/
 	ERL_LIBS=$(ELIBS) erlc -v $(ERLC_OPTS) $(PA) -o ebin/ $?
-	@sed "s/{modules, \[\]}/{modules, \[`echo ebin/*.beam | sed 's%\.beam ebin/%, %g;s%ebin/%%;s/\.beam//'`\]}/" src/$(PROJECT).app.src > $@
+	@sed "s/{modules, \[\]}/{modules, \[`echo ebin/*.beam | sed 's%\.beam ebin/%, %g;s%ebin/%%;s/\.beam//'`\]}/" src/$(PROJECT).app.src \
+        | sed -e "s/{vsn,\([^}]*\)}/\{vsn,\"$(VERSION)\"}/g" > $@
 
 
 json: JSON = $(if $(wildcard priv/), $(shell find priv/ -name '*.json'))
@@ -143,3 +146,4 @@ perf: ERLC_OPTS += -pa $(ROOT)/deps/horse/ebin -DPERF +'{parse_transform, horse_
 perf: compile-test
 	$(gen_verbose) @ERL_LIBS=$(ELIBS) erl -noshell  -pa $(ROOT)/deps/horse/ebin -pa $(TEST_PA) \
 		-eval 'horse:app_perf($(PROJECT)), init:stop().'
+

rel/ci-config.ini → rel/ci.config.ini

@@ -5,6 +5,5 @@ uri = "amqp://guest:guest@127.0.0.1:5672"
 pool_size = 10
 
 [log]
-syslog=debug
-console=debug
+console=info
 error=error

rel/ci.relx.config

@@ -0,0 +1,20 @@
+{lib_dirs, ["applications"
+           ,"core"
+           ,"deps"
+           ]}.
+
+{include_src, false}.
+{include_erts, true}.
+{dev_mode, false}.
+
+{generate_start_script, false}.
+{extended_start_script, false}.
+
+{sys_config, "rel/ci.sys.config"}.
+{vm_args, "rel/ci.vm.args"}.
+
+{overlay, [{copy, "../core/sup/priv/sup", "{{output_dir}}/bin/"}
+          ,{copy, "nodetool", "{{output_dir}}/bin/"}
+          ,{template, "kazoo", "{{output_dir}}/bin/kazoo"}
+          ]}.
+

rel/ci.relx.config.script

@@ -0,0 +1,39 @@
+ToName = fun (App) -> list_to_atom(hd(string:tokens(App,"-"))) end,
+
+Apps = [list_to_atom(App)  || "applications/"++App = Dir <- filelib:wildcard("applications/*"), filelib:is_dir(Dir)],
+Core = [list_to_atom(App)  ||         "core/"++App = Dir <- filelib:wildcard(        "core/*"), filelib:is_dir(Dir)],
+Deps = [list_to_atom(App)  ||         "deps/"++App = Dir <- filelib:wildcard(        "deps/*"), filelib:is_dir(Dir)],
+
+ToFilterOut = [rabbitmq_codegen
+              ,'.erlang.mk'
+              ,'ci.erlang.mk'
+              ,'.settings'
+              ,skel
+              ,parse_trans
+              ,meck
+              ],
+Preparer = fun (List) -> [{E,load} || E <- List, not lists:member(E, ToFilterOut)] end,
+Base = [runtime_tools
+       ,wx
+       ,observer
+       ,debugger
+       ,sasl
+       ],
+VSN = os:cmd("echo -n $(scripts/next_version 2>/dev/null)"),
+
+BASE = [{default_release, {kazoo, VSN}}
+       ,{release, {kazoo, VSN}
+       ,Base ++ Preparer(Apps ++ Core ++ Deps)
+       }],
+
+APPS = [begin
+          {ok, [{application, App, Settings}]} = file:consult(list_to_binary(["applications/", atom_to_list(App), "/ebin/", atom_to_list(App), ".app"])),
+          AppDeps = proplists:get_value(applications, Settings, []),
+          {release, {App, VSN}, Base ++ Preparer(AppDeps)}
+        end
+        || App <- Apps,
+           not lists:member(App, ToFilterOut)
+       ],
+
+CONFIG ++ BASE ++ APPS.
+

rel/ci-sys.config → rel/ci.sys.config

@@ -1,9 +1,10 @@
 [
  {lager, [
           {handlers, [
-                      {lager_console_backend, [info,
-                                               {lager_default_formatter,
-                                                [time, color, " [", severity, "] |", {callid,<<"0000000000">>}, "|", module,":",line, " (",pid,") ", "\e[0m", message, "\n"]}]}
+                      {lager_console_backend, [{level, info}
+                                              ,{formatter, lager_default_formatter}
+                                              ,{formatter_config, [time, color, " [", severity, "] |", {callid,<<"0000000000">>}, "|", module,":",line, " (",pid,") ", "\e[0m", message, "\n"]}
+                                              ]}
                      ,{lager_file_backend, [{file, "log/error.log"}, {level, error}, {size, 10485760}, {date, "$D0"}, {count, 5}
                                            ,{formatter, lager_default_formatter}
                                            ,{formatter_config, [time," [",severity,"] ", "|", {callid, <<"0000000000">>}, "|", module, ":", line, " (",pid, ") ", message, "\n"]}
@@ -12,8 +13,13 @@
                                            ,{formatter, lager_default_formatter}
                                            ,{formatter_config, [time," [",severity,"] ", "|", {callid, <<"0000000000">>}, "|", module, ":", line, " (",pid, ") ", message, "\n"]}
                                            ]}
-                     ]}
+                     ]
+          }
+         ,{extra_sinks, [{data_lager_event,[]}
+                        ]
+          }
          ,{colored, true}
          ,{error_logger_hwm, 5000}
          ]}
 ].
+

rel/args → rel/ci.vm.args

@@ -9,22 +9,24 @@
 +K true
 
 # Start a pool of asynchronous IO threads
-+A 25
++A 100
 
 # Treat error_logger:warning_msg/2 as warnings instead of errors (default)
 +W w
 
+# utf8
+-pc unicode
+
 # Hearbeat disbled by default
 ##-heart
 
 # GC more often
-##-env ERL_FULLSWEEP_AFTER 10
+-env ERL_FULLSWEEP_AFTER 10
 
 # Increase number of concurrent ports / sockets
 ##-env ERL_MAX_PORTS 4096
 
 # When crashing, write dump with highest priority
 -env ERL_CRASH_DUMP_NICE 1
-
 -s lager
 -s kazoo_apps_app

rel/dev.relx.config

@@ -0,0 +1,19 @@
+{lib_dirs, ["applications"
+           ,"core"
+           ,"deps"
+           ]}.
+
+{include_src, false}.
+{include_erts, false}.
+{dev_mode, true}.
+
+{generate_start_script, false}.
+{extended_start_script, false}.
+
+{sys_config, "rel/dev.sys.config"}.
+{vm_args, "rel/dev.vm.args"}.
+
+{overlay, [{copy, "../core/sup/priv/sup", "{{output_dir}}/bin/"}
+          ,{copy, "nodetool", "{{output_dir}}/bin/"}
+          ,{template, "kazoo", "{{output_dir}}/bin/kazoo"}
+          ]}.

rel/dev.relx.config.script

@@ -0,0 +1,46 @@
+ToName = fun (App) -> list_to_atom(hd(string:tokens(App,"-"))) end,
+
+Apps = [list_to_atom(App)  || "applications/"++App = Dir <- filelib:wildcard("applications/*"), filelib:is_dir(Dir)],
+Core = [list_to_atom(App)  ||         "core/"++App = Dir <- filelib:wildcard(        "core/*"), filelib:is_dir(Dir)],
+Deps = [list_to_atom(App)  ||         "deps/"++App = Dir <- filelib:wildcard(        "deps/*"), filelib:is_dir(Dir)],
+
+ToFilterOut = [rabbitmq_codegen
+              ,'.erlang.mk'
+              ,'ci.erlang.mk'
+              ,'.settings'
+              ,skel
+              ,parse_trans
+              ,meck
+              ,'kazoo_slack'
+              ,'exml'
+              ,'escalus'
+              ,'ananke'
+              ,'amqp_cron'
+              ,'amqp_leader'
+              ,'amqp_dist'
+              ],
+Preparer = fun (List) -> [{E,load} || E <- List, not lists:member(E, ToFilterOut)] end,
+Base = [runtime_tools
+       ,wx
+       ,observer
+       ,debugger
+       ,sasl
+       ],
+VSN = os:cmd("echo -n $(scripts/next_version 2>/dev/null)"),
+
+BASE = [{default_release, {kazoo, VSN}}
+       ,{release, {kazoo, VSN}
+       ,Base ++ Preparer(Apps ++ Core ++ Deps)
+       }],
+
+APPS = [begin
+          {ok, [{application, App, Settings}]} = file:consult(list_to_binary(["applications/", atom_to_list(App), "/ebin/", atom_to_list(App), ".app"])),
+          AppDeps = proplists:get_value(applications, Settings, []),
+          {release, {App, VSN}, Base ++ Preparer(AppDeps)}
+        end
+        || App <- Apps,
+           not lists:member(App, ToFilterOut)
+       ],
+
+CONFIG ++ BASE ++ APPS.
+