Skip to content

Commit

Permalink
Fix capistrano deployments from master.
Browse files Browse the repository at this point in the history
Capistrano-based deploys (to overlay changes from master on top of an
existing package installation) had become broken somewhere along the way
to the v0.14 release. This fixes the deployments by refactoring the
deployments to leverage more of the normal build process, so our
existing build process for gems/lua/admin-ui can be reused.

NREL#351
  • Loading branch information
GUI committed Jun 16, 2017
1 parent 137afa6 commit e7c870c
Show file tree
Hide file tree
Showing 12 changed files with 176 additions and 191 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Expand Up @@ -33,6 +33,8 @@
/build/package/work
/build/package/verify/.rnd
/deploy/.env
/deploy/log
/deploy/tmp
/log/*
/test/reports
/test/tmp
Expand Down
134 changes: 76 additions & 58 deletions CMakeLists.txt
Expand Up @@ -3,6 +3,7 @@ project(api-umbrella)

option(ENABLE_HADOOP_ANALYTICS "Build dependencies for Hadoop analytics" off)
option(ENABLE_TEST_DEPENDENCIES "Build dependencies for running tests" off)
option(ENABLE_DEPLOY_ONLY "Only build dependencies for a deployment overlaying an existing package install" off)

# Installation prefix
if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
Expand Down Expand Up @@ -32,7 +33,12 @@ set(HADOOP_ANALYTICS_STAGE_EMBEDDED_DIR ${HADOOP_ANALYTICS_STAGE_DIR}${INSTALL_P
# Where to install app-level vendor dependencies.
set(VENDOR_DIR ${WORK_DIR}/vendor)
set(VENDOR_LUA_DIR ${VENDOR_DIR}/share/lua/5.1)
set(LUAROCKS_CMD env LUA_PATH=${STAGE_EMBEDDED_DIR}/openresty/luajit/share/lua/5.1/?.lua$<SEMICOLON>${STAGE_EMBEDDED_DIR}/openresty/luajit/share/lua/5.1/?/init.lua$<SEMICOLON>$<SEMICOLON> ${STAGE_EMBEDDED_DIR}/bin/luarocks)
if(ENABLE_DEPLOY_ONLY)
set(LUA_PREFIX ${INSTALL_PREFIX_EMBEDDED})
else()
set(LUA_PREFIX ${STAGE_EMBEDDED_DIR})
endif()
set(LUAROCKS_CMD env LUA_PATH=${LUA_PREFIX}/openresty/luajit/share/lua/5.1/?.lua$<SEMICOLON>${LUA_PREFIX}/openresty/luajit/share/lua/5.1/?/init.lua$<SEMICOLON>$<SEMICOLON> ${LUA_PREFIX}/bin/luarocks)

# Where to install development-only dependencies.
set(DEV_INSTALL_PREFIX ${WORK_DIR}/dev-env)
Expand Down Expand Up @@ -60,62 +66,74 @@ function(require_program name)
endif()
endfunction(require_program)

find_package(LibXml2 REQUIRED)
find_package(PkgConfig REQUIRED)
pkg_search_module(LIBUUID REQUIRED uuid)
pkg_search_module(LIBFFI REQUIRED libffi)
require_program(rsync)

include(${CMAKE_SOURCE_DIR}/build/cmake/versions.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/dev/nodejs.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/elasticsearch.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/libcidr.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/libgeoip.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/mongodb.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/mora.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/openresty.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/luarocks.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/perp.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/runit_svlogd.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/ruby.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/rsyslog.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/trafficserver.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/static-site.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/core.cmake)
if(ENABLE_HADOOP_ANALYTICS)
include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/flume.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/kylin.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/presto.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/processor.cmake)
endif()
if(ENABLE_DEPLOY_ONLY)
# Create stub/empty targets for things the core build process depends on. But
# for deploy-based builds, we'll assume these dependencies have already been
# installed (since we're assuming the deploys are overlaying a package
# installation).
add_custom_target(bundler)
add_custom_target(libcidr)
add_custom_target(luarocks)

include(${CMAKE_SOURCE_DIR}/build/cmake/versions.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/core.cmake)
else()
find_package(LibXml2 REQUIRED)
find_package(PkgConfig REQUIRED)
pkg_search_module(LIBUUID REQUIRED uuid)
pkg_search_module(LIBFFI REQUIRED libffi)
require_program(rsync)

include(${CMAKE_SOURCE_DIR}/build/cmake/versions.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/elasticsearch.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/libcidr.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/libgeoip.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/mongodb.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/mora.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/openresty.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/luarocks.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/perp.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/runit_svlogd.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/ruby.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/rsyslog.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/trafficserver.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/static-site.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/core.cmake)
if(ENABLE_HADOOP_ANALYTICS)
include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/flume.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/kylin.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/presto.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/processor.cmake)
endif()

#
# Testing
#
if(ENABLE_TEST_DEPENDENCIES)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/lua-deps.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/mailhog.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/mongo-orchestration.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/openldap.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/phantomjs.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/shellcheck.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/unbound.cmake)
#
# Testing
#
if(ENABLE_TEST_DEPENDENCIES)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/lua-deps.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/mailhog.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/mongo-orchestration.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/openldap.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/phantomjs.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/shellcheck.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/unbound.cmake)
endif()
include(${CMAKE_SOURCE_DIR}/build/cmake/test/bundle.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/test.cmake)

#
# Installation
#
include(${CMAKE_SOURCE_DIR}/build/cmake/install.cmake)

#
# Packaging
#
include(${CMAKE_SOURCE_DIR}/build/cmake/package.cmake)

#
# Clean Task
#
include(${CMAKE_SOURCE_DIR}/build/cmake/clean-download-archives.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/distclean.cmake)
endif()
include(${CMAKE_SOURCE_DIR}/build/cmake/test/bundle.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/test/test.cmake)

#
# Installation
#
include(${CMAKE_SOURCE_DIR}/build/cmake/install.cmake)

#
# Packaging
#
include(${CMAKE_SOURCE_DIR}/build/cmake/package.cmake)

#
# Clean Task
#
include(${CMAKE_SOURCE_DIR}/build/cmake/clean-download-archives.cmake)
include(${CMAKE_SOURCE_DIR}/build/cmake/distclean.cmake)
2 changes: 2 additions & 0 deletions build/cmake/core-admin-ui.cmake
@@ -1,3 +1,5 @@
include(${CMAKE_SOURCE_DIR}/build/cmake/dev/nodejs.cmake)

file(GLOB_RECURSE admin_ui_files
${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/app/*.hbs
${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/app/*.html
Expand Down
4 changes: 3 additions & 1 deletion build/cmake/core-lua-deps.cmake
@@ -1,7 +1,9 @@
include(${CMAKE_SOURCE_DIR}/build/cmake/luarocks_install.cmake)

# LuaRock app dependencies
luarocks_install(argparse ${LUAROCK_ARGPARSE_VERSION} ${LUAROCK_ARGPARSE_HASH})
luarocks_install(inspect ${LUAROCK_INSPECT_VERSION} ${LUAROCK_INSPECT_HASH})
luarocks_install(libcidr-ffi ${LUAROCK_LIBCIDR_VERSION} ${LUAROCK_LIBCIDR_HASH} CIDR_DIR=${STAGE_EMBEDDED_DIR} libcidr)
luarocks_install(libcidr-ffi ${LUAROCK_LIBCIDR_VERSION} ${LUAROCK_LIBCIDR_HASH} CIDR_DIR=${LUA_PREFIX} libcidr)
luarocks_install(lua-cmsgpack ${LUAROCK_CMSGPACK_VERSION} ${LUAROCK_CMSGPACK_HASH})
luarocks_install(lua-iconv ${LUAROCK_ICONV_VERSION} ${LUAROCK_ICONV_HASH})
luarocks_install(lua-resty-auto-ssl ${LUAROCK_RESTY_AUTO_SSL_VERSION} ${LUAROCK_RESTY_AUTO_SSL_HASH})
Expand Down
40 changes: 0 additions & 40 deletions build/cmake/deploy/CMakeLists.txt

This file was deleted.

23 changes: 0 additions & 23 deletions build/cmake/luarocks.cmake
Expand Up @@ -12,26 +12,3 @@ ExternalProject_Add(
COMMAND rm -rf ${VENDOR_DIR}/share/lua ${VENDOR_DIR}/lib/luarocks
COMMAND rm -rf ${TEST_VENDOR_DIR}/share/lua ${TEST_VENDOR_DIR}/lib/luarocks
)

function(_luarocks_install tree_dir package version hash)
ExternalProject_Add(
luarock_${package}
DEPENDS luarocks ${ARGV5}
URL https://luarocks.org/${package}-${version}.rockspec
URL_HASH MD5=${hash}
DOWNLOAD_NO_EXTRACT 1
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ${LUAROCKS_CMD} --tree=${tree_dir} install ${package} ${version} ${ARGV4}
COMMAND find ${tree_dir} -name *.so -exec chrpath -d {} $<SEMICOLON>
)
endfunction()


function(luarocks_install package version hash)
_luarocks_install(${VENDOR_DIR} ${package} ${version} ${hash} ${ARGV3} ${ARGV4})
endfunction()

function(test_luarocks_install package version hash)
_luarocks_install(${TEST_VENDOR_DIR} ${package} ${version} ${hash} ${ARGV3} ${ARGV4})
endfunction()
21 changes: 21 additions & 0 deletions build/cmake/luarocks_install.cmake
@@ -0,0 +1,21 @@
function(_luarocks_install tree_dir package version hash)
ExternalProject_Add(
luarock_${package}
DEPENDS luarocks ${ARGV5}
URL https://luarocks.org/${package}-${version}.rockspec
URL_HASH MD5=${hash}
DOWNLOAD_NO_EXTRACT 1
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ${LUAROCKS_CMD} --tree=${tree_dir} install ${package} ${version} ${ARGV4}
COMMAND find ${tree_dir} -name *.so -exec chrpath -d {} $<SEMICOLON>
)
endfunction()

function(luarocks_install package version hash)
_luarocks_install(${VENDOR_DIR} ${package} ${version} ${hash} ${ARGV3} ${ARGV4})
endfunction()

function(test_luarocks_install package version hash)
_luarocks_install(${TEST_VENDOR_DIR} ${package} ${version} ${hash} ${ARGV3} ${ARGV4})
endfunction()
2 changes: 1 addition & 1 deletion configure
Expand Up @@ -34,7 +34,7 @@ if [ -e "${TOP_SRCDIR}/.configure-custom.sh" ]; then
fi

PREFIX=/opt/api-umbrella
ENABLE_VARS="hadoop-analytics|on|ENABLE_HADOOP_ANALYTICS test-dependencies|on|ENABLE_TEST_DEPENDENCIES"
ENABLE_VARS="hadoop-analytics|on|ENABLE_HADOOP_ANALYTICS test-dependencies|on|ENABLE_TEST_DEPENDENCIES deploy-only|on|ENABLE_DEPLOY_ONLY"

quote() {
echo "$1" | sed -e "s|'|'\\\\''|g; 1s/^/'/; \$s/\$/'/"
Expand Down
5 changes: 3 additions & 2 deletions deploy/Gemfile
@@ -1,4 +1,5 @@
source "https://rubygems.org"

gem "capistrano", "~> 3.4.0"
gem "dotenv", "~> 2.0.2"
gem "capistrano", "~> 3.6.1"
gem "capistrano-rsync-bladrak", "~> 1.4.2"
gem "dotenv", "~> 2.2.1"
31 changes: 20 additions & 11 deletions deploy/Gemfile.lock
@@ -1,25 +1,34 @@
GEM
remote: https://rubygems.org/
specs:
capistrano (3.4.0)
airbrussh (1.2.0)
sshkit (>= 1.6.1, != 1.7.0)
capistrano (3.6.1)
airbrussh (>= 1.0.0)
capistrano-harrow
i18n
rake (>= 10.0.0)
sshkit (~> 1.3)
colorize (0.7.7)
dotenv (2.0.2)
i18n (0.7.0)
sshkit (>= 1.9.0)
capistrano-harrow (0.5.3)
capistrano-rsync-bladrak (1.4.2)
capistrano (>= 3.0.0.pre14, < 4)
dotenv (2.2.1)
i18n (0.8.4)
net-scp (1.2.1)
net-ssh (>= 2.6.5)
net-ssh (3.0.1)
rake (10.4.2)
sshkit (1.7.1)
colorize (>= 0.7.0)
net-ssh (4.1.0)
rake (12.0.0)
sshkit (1.13.1)
net-scp (>= 1.1.2)
net-ssh (>= 2.8.0)

PLATFORMS
ruby

DEPENDENCIES
capistrano (~> 3.4.0)
dotenv (~> 2.0.2)
capistrano (~> 3.6.1)
capistrano-rsync-bladrak (~> 1.4.2)
dotenv (~> 2.2.1)

BUNDLED WITH
1.15.1

0 comments on commit e7c870c

Please sign in to comment.