Skip to content

Commit

Permalink
Remove redundant script from Travis build and improve log information (…
Browse files Browse the repository at this point in the history 
…#1140)

* Improve Travis build log information

* Improve Travis build log information

* Improve Travis build log information

* Improve Travis builds

* Improve Travis builds

* improve task information for logs

* improve task information for logs

* improve task information for logs

* Add spew for export commands that get SIGINT faults

* Add spew for export commands that get SIGINT faults

* Add trace around intermittent failings on export project

* Cleanup changes and add comments and TODOs
  • Loading branch information
@mrutkows
mrutkows committed Apr 7, 2021
1 parent d653592 commit ff6238d
Show file tree
Hide file tree
Showing 13 changed files with 221 additions and 96 deletions.
3 changes: 2 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ _testmain.go
*.prof


#OS X
# OS X
**/.DS_Store

#binaries
Expand All @@ -43,6 +43,7 @@ Packages/
.gradle
.gogradle
build/
release/

#emacs
*~
Expand Down
71 changes: 59 additions & 12 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,33 +36,64 @@ notifications:
# travis2slack webhook to enable DMs on openwhisk-team.slack.com to PR authors with TravisCI results
secure: "NaDCtHwbFUEy0h0u/QrAt395o0/9FstJ5/Tv4uBSLCBdaUNU+0q6/RXVH76dKbujoxrLded2gyeJSAbI4kl4SM7FyilSHzKTikF0VsgM3G5veh65tVY5ztsvl40hDmK2E/PmWI3HkZNeWJO32WjraqPmhSWL0BZOWhA/4gJSCIsLnNRQXmpZoDqC7V8IG1BWYGwd5qcNkEjItLjYCUy1s2K8bj3QmhshYRVLTNbXxh0yJVKjM04I+bWhMXqvL8GS55qeicJ3fUwm5g1PDVCtzxLwAHCPo2jusjtNSnV+BaZgMBSahv3MD7ApxfF58e1buejggH3qZAhNGccC4bYJCahqVv/KKoA10kO6exH5iGwlHPWQTjMuF++PHmAk+FeQ1jh+JoUSBnIHExPnaD4CQIArHvUuQom+WJBnOz+L1H755VyPGzeDQ+ZUDlhOiQ6CDP/sqaRH3Wmo4IzhsqTsLaQs0dS1YeEWOS8gcawwNTVf/WtqiedYu24rQN9RiUXelRoEaXJy/pldb2KpyQRo2hbKoStDDB5fEij1xyGsj4kSXZ8uk5G10oDGnZYd+okduzJazN8wr0flrLvH32DRgDZDcG/D90I19JmqqfqbcG/SEmhxTikHZeVQd5DfWcqssLvWNrtQa/EJgAQkmZ2gaSsZ73AvPk/KyMV30+m8xTU="

# The complete sequence of phases of the lifecycle:
# ---------------------------------------------------------
# before_install
# install
# before_script
# script
# before_cache (OPTIONAL) (if and only if caching is effective)
# after_success (or after_failure)
# before_deploy (OPTIONAL) (if and only if deployment is active)
# deploy (OPTIONAL)
# after_deploy (OPTIONAL) (if and only if deployment is active)
# after_script

before_install:
- "./tools/travis/cloneutils.sh"
- sudo apt-get install tree

# Install tools we use for linting and unit/integration testing
# TODO: build.gradle (gradlew) already installs 'golint' and runs linting
# in the `goLint` task; see if we can remove here.
# TODO: The "gogradle" plugin in build.gradle (gradlew) already provides a
# task called "goTest" which runs all `unit` tests. See if we can use that
# instead and only run `integration` tests here.
install:
- export DEPLOY_BUILD_READY=false
- go get -u golang.org/x/lint/golint
- go get -u github.com/stretchr/testify

# Identify Golang files that are not properly formatted
# TODO: See if we can create custom tasks to perform this in build.gradle
# as a custom `goFmt` task
before_script:
- GO_FILES=$(find . -iname '*.go' -type f -not -path "./wski18n/i18n_resources.go")
- export BAD_GO=$(gofmt -s -l $(echo $GO_FILES))
- echo "["$BAD_GO"]"
#- test -z "$BAD_GO"
#- test -z "$(gofmt -s -l $(echo $GO_FILES))"

# Clone and build the OpenWhisk platform (i.e., apache/openwhisk)
# and then use Ansible to deploy it and prepare it to run our
# integration tests.
# TODO: Do not use Makefile for running integration tests (in script.sh);
# change to using gradle wrapper
script:
- echo $TRAVIS
- echo $TRAVIS_PULL_REQUEST
- echo $TRAVIS_SECURE_ENV_VARS
- printenv
- echo "Installing and building OpenWhisk platform using Ansible..."
- "./tools/travis/showenv.sh"
- "./tools/travis/script.sh"

after_success:
- DEPLOY_BUILD_READY=true
- if [ "$TRAVIS_EVENT_TYPE" == "cron" ] ; then
export DEPLOY_BUILD_READY=false;
fi

after_script:
- make clean
# If this is a "tagged" GitHub build, then use Gradle Wrapper
# to build binaries for all supported OS platforms and architectures.
# The result would be a "/build" directory that has a subdirectory named
# for each supported GOOS/GOARCH
before_deploy:
- export build_file_name=wskdeploy
- go get github.com/inconshreveable/mousetrap
Expand All @@ -72,10 +103,14 @@ before_deploy:
export GIT_TAG=$TRAVIS_TAG;
export TAG=true;
fi
- echo "Calling Gradle Wrapper task to build and 'releaseBinaries'..."
- "./tools/travis/showenv.sh"
- ./gradlew --console=plain releaseBinaries -PpackageVersion=$GIT_TAG -PgitCommit=$(git rev-parse HEAD) -PbuildDate=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
- "./tools/travis/build_tag_releases.sh $build_file_name $GIT_TAG"
- echo "Build binaries and release archives created:"
- tree /build
- tree /release
- export RELEASE_PKG_FILE="$(cd "$TRAVIS_BUILD_DIR/release" && ls ${zip_file_name}-*.tgz ${zip_file_name}-*.zip)"
- echo "Deploying $RELEASE_PKG_FILE to GitHub releases."
- echo "Deploying "$RELEASE_PKG_FILE" to GitHub releases..."
- if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_EVENT_TYPE" == "push" ] && [ "$TRAVIS_OS_NAME" == "linux" ] ; then
git config --global user.email "builds@travis-ci.com";
git config --global user.name "Travis CI";
Expand All @@ -84,11 +119,11 @@ before_deploy:
GIT_COMMITTER_DATE="$(git show --format=%aD | head -1)" git tag $GIT_TAG -a -m "Generated tag from Travis CI build $TRAVIS_BUILD_NUMBER";
git push -f -q https://$API_KEY@github.com/apache/openwhisk-wskdeploy $GIT_TAG;
fi
- echo "TRAVIS_EVENT_TYPE is "
- echo $TRAVIS_EVENT_TYPE
- echo "GIT_TAG is "
- echo $GIT_TAG
- echo "Completing before_deploy stage"

# Utilize two providers to create a github release and publish a docker image.
# For information on the "releases" provider,
# see https://docs.travis-ci.com/user/deployment/releases/
deploy:
- provider: releases
api_key:
Expand All @@ -112,6 +147,18 @@ deploy:
repo: apache/openwhisk-wskdeploy
tags: true
condition: "$DEPLOY_BUILD_READY = true"

# Clean up wskdeploy binary (single)
# Fundamentally, remove both /build and /release directory trees and contents
# TODO: See if this is really needed in Travis environments as the
# Makefile is not used elsewhere here (especially if we do not use th
# build_tag_release.sh script) as it is for Windows developers primarily
# and we are building using Linux.
after_script:
- make clean
- tree /build
- tree /release

env:
global:
- BLUEMIX_APIHOST=openwhisk.ng.bluemix.net
Expand Down
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ rat {
storage is in "GO" format:
OS: linux, darwin, windows
Arch: 386, amd64, s390x, ppc64le, arm
Arch: 386, amd64, s390x, ppc64le, arm, arm64
TODO - It may be appropriate to refactor into a general purpose Platform
class for all builds, then to extend with specifics needed for
Expand Down
150 changes: 88 additions & 62 deletions cmd/export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@ import (
"github.com/apache/openwhisk-wskdeploy/parsers"
"github.com/apache/openwhisk-wskdeploy/runtimes"
"github.com/apache/openwhisk-wskdeploy/utils"
"github.com/apache/openwhisk-wskdeploy/wskderrors"
"github.com/apache/openwhisk-wskdeploy/wski18n"
"github.com/apache/openwhisk-wskdeploy/wskprint"
"github.com/davecgh/go-spew/spew"
"github.com/spf13/cobra"
)

Expand Down Expand Up @@ -117,9 +119,6 @@ func ExportAction(actionName string, packageName string, maniyaml *parsers.YAML,

func exportProject(projectName string, targetManifest string) error {

whisk.SetVerbose(utils.Flags.Verbose)
whisk.SetDebug(utils.Flags.Trace)

maniyaml := &parsers.YAML{}
maniyaml.Project.Name = projectName

Expand All @@ -129,13 +128,19 @@ func exportProject(projectName string, targetManifest string) error {
return err
}

// Emit additional trace data (primarily in Travis)
if utils.Flags.Trace {
spew.Dump(packages)
}

var bindings = make(map[string]whisk.Binding)

// iterate over each package to find managed annotations
// check if "managed" annotation is attached to a package
// add to export when managed project name matches with the
// specified project name
for _, pkg := range packages {

if a := pkg.Annotations.GetValue(utils.MANAGED); a != nil {
// decode the JSON blob and retrieve __OW_PROJECT_NAME
pa := a.(map[string]interface{})
Expand Down Expand Up @@ -199,59 +204,66 @@ func exportProject(projectName string, targetManifest string) error {

// iterate over the list of triggers to determine whether any of them part of specified managed project
for _, trg := range triggers {
// trigger has attached managed annotation
if a := trg.Annotations.GetValue(utils.MANAGED); a != nil {
// decode the JSON blob and retrieve __OW_PROJECT_NAME
ta := a.(map[string]interface{})
if ta[utils.OW_PROJECT_NAME] == projectName {

for pkgName := range maniyaml.Packages {
if maniyaml.Packages[pkgName].Namespace == trg.Namespace {
if maniyaml.Packages[pkgName].Triggers == nil {
pkg := maniyaml.Packages[pkgName]
pkg.Triggers = make(map[string]parsers.Trigger)
maniyaml.Packages[pkgName] = pkg
}

// export trigger to manifest

if feedname, isFeed := utils.IsFeedAction(&trg); isFeed {
// check if feed name starts with namespace and workaround it
// the current problem is that client has user namespace and when feed specified with different namespace it will fail to invoke the feed action
// we need to transform the path from e.g.
// /api/v1/namespaces/kpavel@il.ibm.com_uspace/actions//whisk.system/alarms/interval?blocking=true
// in to
// /api/v1/namespaces/kpavel@il.ibm.com_uspace/actions/../../whisk.system/actions/alarms/interval?blocking=true
if strings.HasPrefix(feedname, "/") {
// /whisk.system/alarms/interval -> ../../whisk.system/actions/alarms/interval
prts := strings.SplitN(feedname, "/", 3)
feedname = "../../" + prts[1] + "/actions/" + prts[2]
// trigger has attached managed annotation
if trg.Annotations != nil {
if a := trg.Annotations.GetValue(utils.MANAGED); a != nil {
// decode the JSON blob and retrieve __OW_PROJECT_NAME
ta := a.(map[string]interface{})
if ta[utils.OW_PROJECT_NAME] == projectName {

for pkgName := range maniyaml.Packages {
if maniyaml.Packages[pkgName].Namespace == trg.Namespace {
if maniyaml.Packages[pkgName].Triggers == nil {
pkg := maniyaml.Packages[pkgName]
pkg.Triggers = make(map[string]parsers.Trigger)
maniyaml.Packages[pkgName] = pkg
}

// export feed input parameters
params := make(map[string]interface{})
params["authKey"] = client.Config.AuthToken
params["lifecycleEvent"] = "READ"
params["triggerName"] = "/" + client.Namespace + "/" + trg.Name
res, _, err := client.Actions.Invoke(feedname, params, true, true)
if err != nil {
return err
}
feedConfig := res["config"]
// export trigger to manifest

if feedname, isFeed := utils.IsFeedAction(&trg); isFeed {
// check if feed name starts with namespace and workaround it
// the current problem is that client has user namespace and when feed specified with different namespace it will fail to invoke the feed action
// we need to transform the path from e.g.
// /api/v1/namespaces/kpavel@il.ibm.com_uspace/actions//whisk.system/alarms/interval?blocking=true
// in to
// /api/v1/namespaces/kpavel@il.ibm.com_uspace/actions/../../whisk.system/actions/alarms/interval?blocking=true
if strings.HasPrefix(feedname, "/") {
// /whisk.system/alarms/interval -> ../../whisk.system/actions/alarms/interval
prts := strings.SplitN(feedname, "/", 3)
feedname = "../../" + prts[1] + "/actions/" + prts[2]
}

// export feed input parameters
params := make(map[string]interface{})
params["authKey"] = client.Config.AuthToken
params["lifecycleEvent"] = "READ"
params["triggerName"] = "/" + client.Namespace + "/" + trg.Name
res, _, err := client.Actions.Invoke(feedname, params, true, true)
if err != nil {
return err
}
feedConfig := res["config"]

if feedConfig != nil {
for key, val := range feedConfig.(map[string]interface{}) {
if key != "startDate" {
trg.Parameters = trg.Parameters.AddOrReplace(&whisk.KeyValue{Key: key, Value: val})
if feedConfig != nil {
for key, val := range feedConfig.(map[string]interface{}) {
if key != "startDate" {
trg.Parameters = trg.Parameters.AddOrReplace(&whisk.KeyValue{Key: key, Value: val})
}
}
}
}
}

maniyaml.Packages[pkgName].Triggers[trg.Name] = *maniyaml.ComposeParsersTrigger(trg)
maniyaml.Packages[pkgName].Triggers[trg.Name] = *maniyaml.ComposeParsersTrigger(trg)
}
}
}
}
} else {
// Emit additional trace data on common Travis failures
spew.Dump(trg)
return wskderrors.NewCommandError("Export", "Trigger missing annotations.")
}
}

Expand All @@ -263,29 +275,36 @@ func exportProject(projectName string, targetManifest string) error {

// iterate over the list of rules to determine whether any of them is part of the manage dproject
for _, rule := range rules {

// get rule from OW
wskRule, _, _ := client.Rules.Get(rule.Name)
// rule has attached managed annotation
if a := wskRule.Annotations.GetValue(utils.MANAGED); a != nil {
// decode the JSON blob and retrieve __OW_PROJECT_NAME
ta := a.(map[string]interface{})
if ta[utils.OW_PROJECT_NAME] == projectName {

for pkgName := range maniyaml.Packages {
if maniyaml.Packages[pkgName].Namespace == wskRule.Namespace {
if maniyaml.Packages[pkgName].Rules == nil {
pkg := maniyaml.Packages[pkgName]
pkg.Rules = make(map[string]parsers.Rule)
maniyaml.Packages[pkgName] = pkg
}

// export rule to manifest
maniyaml.Packages[pkgName].Rules[wskRule.Name] = *maniyaml.ComposeParsersRule(*wskRule)
if wskRule.Annotations != nil {
// rule has attached managed annotation
if a := wskRule.Annotations.GetValue(utils.MANAGED); a != nil {
// decode the JSON blob and retrieve __OW_PROJECT_NAME
ta := a.(map[string]interface{})
if ta[utils.OW_PROJECT_NAME] == projectName {

for pkgName := range maniyaml.Packages {
if maniyaml.Packages[pkgName].Namespace == wskRule.Namespace {
if maniyaml.Packages[pkgName].Rules == nil {
pkg := maniyaml.Packages[pkgName]
pkg.Rules = make(map[string]parsers.Rule)
maniyaml.Packages[pkgName] = pkg
}

// export rule to manifest
maniyaml.Packages[pkgName].Rules[wskRule.Name] = *maniyaml.ComposeParsersRule(*wskRule)
}
}
}
}
} else {
// Emit additional trace data on common Travis failures
spew.Dump(wskRule)
return wskderrors.NewCommandError("Export", "Rule missing annotations.")
}

}

// API Gateway is an optional component. Export APIs only when ApigwAccessToken is configured
Expand Down Expand Up @@ -386,7 +405,14 @@ func exportProject(projectName string, targetManifest string) error {

// find exported manifest parent directory
manifestDir := filepath.Dir(utils.Flags.ManifestPath)
os.MkdirAll(manifestDir, os.ModePerm)
errMkDir := os.MkdirAll(manifestDir, os.ModePerm)

// Exit if unable to create export dir. structure
// TODO: This sometimes fails in Travis, perhaps retry?
if errMkDir != nil {
wskprint.PrintOpenWhiskError(errMkDir.Error())
return errMkDir
}

// export manifest to file
parsers.Write(maniyaml, targetManifest)
Expand Down
4 changes: 1 addition & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,15 @@ go 1.15
require (
github.com/apache/openwhisk-client-go v0.0.0-20210311185314-87edc2364717
github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/fatih/color v1.10.0
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
github.com/google/go-querystring v1.0.0 // indirect
github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519
github.com/magiconair/properties v1.8.4 // indirect
github.com/mattn/go-colorable v0.1.8
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/nicksnyder/go-i18n v1.10.1
github.com/nxadm/tail v1.4.8 // indirect
github.com/onsi/ginkgo v1.15.0 // indirect
github.com/pelletier/go-toml v1.8.1 // indirect
github.com/spf13/afero v1.5.1 // indirect
github.com/spf13/cast v1.3.1 // indirect
Expand All @@ -26,5 +25,4 @@ require (
golang.org/x/text v0.3.5 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)
Loading

0 comments on commit ff6238d

Please sign in to comment.