Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update Docker build and add CircleCI + CodeCov support #44

Closed
wants to merge 21 commits into from
Closed

Update Docker build and add CircleCI + CodeCov support #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
0b350f9
Add nightly Dockerfile for Moveit2
ruffsl Mar 12, 2019
826b152
Add CircleCI config
ruffsl Mar 12, 2019
1118615
Create .repos file nightly Dockerfile
ruffsl Mar 12, 2019
b6f4a93
Add code coverage script
ruffsl Mar 13, 2019
ac10b22
Update .repo file path
ruffsl Mar 19, 2019
95c3036
Remove unrelated config
ruffsl Mar 19, 2019
da182b3
Add Comment to describe purpose of checksum.txt
ruffsl Mar 19, 2019
4073586
Move code coverage script
ruffsl Mar 19, 2019
e93c8b9
Update Docker Hub image repo
ruffsl Mar 19, 2019
c5d41c1
Add TODO for dally cron job checks
ruffsl Mar 19, 2019
da3173c
Remove old ros1 dockerfile
ruffsl Mar 19, 2019
7d2b7ac
Move CircleCI Dockerfile
ruffsl Mar 19, 2019
fc71c65
Fix comment typo
davetcoleman Mar 19, 2019
8a5f875
Move .repo file to project root
ruffsl Mar 19, 2019
f42ba65
Remove unrelated config
ruffsl Mar 20, 2019
bfdbc03
Enable nightly cron and update TODO
ruffsl Mar 20, 2019
4cff4af
Update comment
ruffsl Mar 20, 2019
bf95820
Move Dockerfile back to root
ruffsl Mar 21, 2019
adc87a7
Add .dockerignore
ruffsl Mar 21, 2019
6d642ca
Revert "Remove old ros1 dockerfile"
ruffsl Mar 21, 2019
5940894
Update .circleci/config.yml
ruffsl May 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
251 changes: 251 additions & 0 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
# CircleCi Configuration for MoveIt
# Historically MoveIt has used Travis for continuous integration testing, but we are starting to migrate
# to CircleCi for its greater feature set. In particular, testing coverage metrics.
#
# `checksum.txt` derives the key for the `save_cache` and `restore_cache` steps.
ruffsl marked this conversation as resolved.
Show resolved Hide resolved
# As we can't cache upon the state of the file system directly,
# we instead compute a checksum of the files we wish to cache indirectly.
# CircleCI checksums this file, a simple list of checksum values itself,
# as part of the key to restore or break the respective cache value.

version: 2.1
references:
pre_checkout: &pre_checkout
run:
name: Pre Checkout
command: |
stat -c '%y' /opt/ros/$ROS_DISTRO/setup.sh | \
sha256sum | \
awk '{print "distro "$1}' > /opt/ros/$ROS_DISTRO/checksum.txt
ruffsl marked this conversation as resolved.
Show resolved Hide resolved
cp $MOVEIT2_WS/src/moveit2/moveit2.repos $ROS_WS/
rm -rf $MOVEIT2_WS/*
post_checkout: &post_checkout
run:
name: Post Checkout
command: |
if ! cmp $MOVEIT2_WS/src/moveit2/moveit2.repos $ROS_WS/moveit2.repos >/dev/null 2>&1
then
echo "Cleaning Upstream"
rm -rf $ROS_WS/*
mkdir -p $ROS_WS/src
vcs import $ROS_WS/src < $MOVEIT2_WS/src/moveit2/moveit2.repos
fi
install_upstream: &install_upstream
run:
name: Install Upstream
command: |
cat /opt/ros/$ROS_DISTRO/checksum.txt > $ROS_WS/checksum.txt
find $ROS_WS/src -not -iwholename '*.git/*' -type f -exec sha256sum {} \; | \
sort -k 2 | \
cut -d" " -f1 | \
sha256sum | \
awk '{print "upstream source "$1}' >> $ROS_WS/checksum.txt
apt-get update
dependencies=$(
rosdep install -q -y \
--from-paths \
$ROS_WS/src \
--ignore-src \
--verbose | \
awk '$1 ~ /^resolution\:/' | \
awk -F'[][]' '{print $2}' | \
tr -d \, | xargs -n1 | sort -u | xargs)
dpkg -s $dependencies | \
sha256sum | \
awk '{print "upstream dependencies "$1}' >> $ROS_WS/checksum.txt
restore_upstream_cache: &restore_upstream_cache
restore_cache:
name: Restore Upstream Cache
key: upstream-cache-v1-{{ arch }}-{{ checksum "/opt/ros_ws/checksum.txt" }}
build_upstream: &build_upstream
run:
name: Build Upstream
command: |
cd $ROS_WS
if [ -d install ]
then
echo "Skipping Build Upstream"
else
. /opt/ros/$ROS_DISTRO/setup.sh
colcon build \
--symlink-install \
--cmake-args \
-DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE
fi
save_upstream_cache: &save_upstream_cache
save_cache:
name: Save Upstream Cache
key: upstream-cache-v1-{{ arch }}-{{ checksum "/opt/ros_ws/checksum.txt" }}
paths:
- /opt/ros_ws
install_dependencies: &install_dependencies
run:
name: Install Dependencies
command: |
cat $ROS_WS/checksum.txt > $MOVEIT2_WS/checksum.txt
find $MOVEIT2_WS/src -not -iwholename '*.git/*' -type f -exec sha256sum {} \; | \
sort -k 2 | \
cut -d" " -f1 | \
sha256sum | \
awk '{print "source "$1}' >> $MOVEIT2_WS/checksum.txt
apt-get update
dependencies=$(
rosdep install -q -y \
--from-paths \
$ROS_WS/src \
src \
--ignore-src \
--verbose | \
awk '$1 ~ /^resolution\:/' | \
awk -F'[][]' '{print $2}' | \
tr -d \, | xargs -n1 | sort -u | xargs)
dpkg -s $dependencies | \
sha256sum | \
awk '{print "dependencies "$1}' >> $MOVEIT2_WS/checksum.txt
store_build_checksum: &store_build_checksum
store_artifacts:
path: checksum.txt
restore_build_cache: &restore_build_cache
restore_cache:
name: Restore Build Cache
key: build-cache-v1-{{ arch }}-{{ .Branch }}-{{ checksum "/opt/moveit2_ws/checksum.txt" }}
source_build: &source_build
run:
name: Source Build
command: |
if [ -d install ]
then
echo "Skipping Source Build"
else
. $ROS_WS/install/setup.sh
colcon build \
--symlink-install \
--cmake-args \
-DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \
-DCOVERAGE_ENABLED=$COVERAGE_ENABLED
fi
save_build_cache: &save_build_cache
save_cache:
name: Save Build Cache
key: build-cache-v1-{{ arch }}-{{ .Branch }}-{{ checksum "/opt/moveit2_ws/checksum.txt" }}
paths:
- /opt/moveit2_ws
copy_build_logs: &copy_build_logs
run:
name: Copy Build Logs
when: always
command: cp -rH log/latest_build log/build
store_build_logs: &store_build_logs
store_artifacts:
path: log/build
test_build: &test_build
run:
name: Test Build
command: |
. install/setup.sh
src/moveit2/tools/run_test_suite.bash
copy_test_logs: &copy_test_logs
run:
name: Copy Test Logs
when: always
command: cp -rH log/latest_test log/test
store_test_logs: &store_test_logs
store_artifacts:
path: log/test
report_code_coverage: &report_code_coverage
run:
name: Report Code Coverage
command: |
if [ "$COVERAGE_ENABLED" = "True" ]
then
. install/setup.sh
src/moveit2/moveit/scripts/code_coverage_report.bash codecovio
else
echo "Skipping Code Coverage Report"
fi

commands:
checkout_source:
description: "Checkout Source"
steps:
- *pre_checkout
- checkout:
path: src/moveit2
- *post_checkout
setup_upstream:
description: "Setup Upstream"
steps:
- *install_upstream
- *restore_upstream_cache
- *build_upstream
- *save_upstream_cache
build_source:
description: "Build Source"
steps:
- *install_dependencies
- *store_build_checksum
- *restore_build_cache
- *source_build
- *save_build_cache
- *copy_build_logs
- *store_build_logs
test_source:
description: "Test Source"
steps:
- *test_build
- *copy_test_logs
- *store_test_logs
report_coverage:
description: "Report Coverage"
steps:
- *report_code_coverage

executors:
docker_exec:
docker:
- image: moveit/moveit2:master
ruffsl marked this conversation as resolved.
Show resolved Hide resolved
working_directory: /opt/moveit2_ws
environment:
MAKEFLAGS: "-j 1 -l 1"

jobs:
debug_build:
executor: docker_exec
environment:
CMAKE_BUILD_TYPE: "Debug"
COVERAGE_ENABLED: "True"
steps:
- checkout_source
- setup_upstream
- build_source
- test_source
# - report_coverage
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
# - report_coverage
# TODO: re-enable report coverage when...
# - report_coverage

when? or remove comment

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is code coverage working now? I was waiting on this until this repo uses something like nav2's cmake changes to enable codecov at build time.

release_build:
executor: docker_exec
environment:
CMAKE_BUILD_TYPE: "Release"
COVERAGE_ENABLED: "False"
steps:
- checkout_source
- setup_upstream
- build_source
- test_source

workflows:
version: 2
build-test:
jobs:
- debug_build
- release_build
# TODO: Make PR jobs use docker image for target branch, this needs: https://circleci.com/ideas/?idea=CCI-I-894
nightly:
triggers:
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- master
jobs:
- debug_build
- release_build
13 changes: 13 additions & 0 deletions .dockerignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
################################################################################
# Repo

.git/*
.dockerignore
.gitignore
Dockerfile
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
Dockerfile
Dockerfile
CONTRIBUTING.md
LICENSE.txt
MIGRATION.md
README.md

this improves caching, right?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I the .md and .txt files are inconsequential to the build, perhaps is worth blanketing them all in the root directory?

*.md
*.txt

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sounds good to me


################################################################################
# JetBrains IDEs

# User-specific stuff
**/.idea/**
81 changes: 81 additions & 0 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
# This dockerfile expects proxies to be set via --build-arg if needed
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we have a .docker folder:
https://github.com/ros-planning/moveit2/tree/master/.docker

@mlautman has been setting up docker for ROS2:
#46

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Optimizations of the CI config heavily rely upon the assumed setup of this Dockerfile, so they go hand-in-hand. Perhaps @mlautman could mention what changes should be added into this CI dockerfile?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would recommend testing with the .docker/ci/Dockerfile in the MoveIt2 Ci PR as it is very close to being merged in.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I made some comments on #46 (review) that the Dockerfile here also addresses. However, like I mentioned, the Dockerfile here and config go rather hand-in-hand due to the build caching strategy optimizations. That said, I'm not opposed to multiple dockerfiles; we could have one for the existing travis setup and one for CircleCI.

Copy link
Contributor

@mlautman mlautman Mar 20, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We certainly can have multiple (we currently have about 4) but it is important that we remove redundancies wherever possible. Looking through the Dockerfile (albeit rather briefly) it seems that the docker container created isn't very different from moveit/moveit2:crystal-source The main difference that I see is that this Dockerfile builds the workspace whiel crystal-source does not. (Although it certainly could)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@davetcoleman and myself were just discussing this and it makes sense, at least in the short term, to have multiple dockerfiles with one for the existing travis setup and one for CricleCi

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, I would also like to prune Dockerfile redundancies. I have an idea to address this later using build args in the FROM calls and overriding the Dockerhub build hooks. See bfdbc03 and https://docs.docker.com/docker-hub/builds/advanced as I linked to above with https://github.com/ruffsl/navigation2/tree/foo/.dockerhub/devel

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we move this file to our .docker folder at least?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
# This dockerfile expects proxies to be set via --build-arg if needed
# Docker specifically for CircleCi setup, not for Travis
#
# This dockerfile expects proxies to be set via --build-arg if needed

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we move this file to our .docker folder at least?

The Dockerfile is kept in the root of the project so that the build context may encompass the entire repo so that we can COPY all the separate source files into the image. It is possible to have different Dockerfile and build context locations, but from a user perspective, the expected build context location is no longer obvious. DockerHub has recently revamped the build rules setting, allowing us to specify separate paths, and we can document the use required pwd and --file for uses building locally.

https://docs.docker.com/engine/reference/commandline/build/#extended-description
https://docs.docker.com/engine/reference/commandline/build/#text-files

# It also expects to be contained in the /moveit2 root folder for file copy
# Example build command:
# export http_proxy=http://my.proxy.com:80
# export CMAKE_BUILD_TYPE=Debug
# docker build -t moveit2:master --build-arg http_proxy --build-arg CMAKE_BUILD_TYPE ./
FROM osrf/ros2:nightly

# setup keys
ARG http_proxy
RUN if [ "$http_proxy" != "" ]; \
then \
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 \
--keyserver-options http-proxy=$http_proxy \
--recv-keys 421C365BD9FF1F717815A3895523BAEEB01FA116; \
fi

# install ROS2 dependencies
RUN apt-get update && apt-get install -q -y \
build-essential \
cmake \
git \
python3-colcon-common-extensions \
python3-vcstool \
wget \
&& rm -rf /var/lib/apt/lists/*

# copy ros package repo
ENV MOVEIT2_WS /opt/moveit2_ws
RUN mkdir -p $MOVEIT2_WS/src
WORKDIR $MOVEIT2_WS/src
COPY ./ moveit2/

# clone dependency package repos
ENV ROS_WS /opt/ros_ws
RUN mkdir -p $ROS_WS/src
WORKDIR $ROS_WS
RUN vcs import src < $MOVEIT2_WS/src/moveit2/tools/ros2_dependencies.repos

# install dependency package dependencies
RUN . /opt/ros/$ROS_DISTRO/setup.sh && \
apt-get update && \
rosdep install -q -y \
--from-paths \
src \
--ignore-src \
&& rm -rf /var/lib/apt/lists/*

# build dependency package source
ARG CMAKE_BUILD_TYPE=Release

RUN . /opt/ros/$ROS_DISTRO/setup.sh && \
colcon build \
--symlink-install \
--cmake-args \
-DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE

# install moveit2 package dependencies
WORKDIR $MOVEIT2_WS
RUN . $ROS_WS/install/setup.sh && \
apt-get update && \
rosdep install -q -y \
--from-paths \
$ROS_WS/src \
src \
--ignore-src \
&& rm -rf /var/lib/apt/lists/*

# build moveit2 package source
ARG COVERAGE_ENABLED=False
RUN . $ROS_WS/install/setup.sh && \
colcon build \
--symlink-install \
--cmake-args \
-DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \
-DCOVERAGE_ENABLED=$COVERAGE_ENABLED

# source moveit2 workspace from entrypoint
RUN sed --in-place \
's|^source .*|source "$MOVEIT2_WS/install/setup.bash"|' \
/ros_entrypoint.sh
Loading