Skip to content

Commit

Permalink
Merge pull request espressif#7 from me-no-dev/master
Browse files Browse the repository at this point in the history
Build system overhaul
  • Loading branch information
me-no-dev committed Sep 27, 2019
2 parents 7dbfa98 + 5fdfb88 commit b78b168
Show file tree
Hide file tree
Showing 12 changed files with 364 additions and 52 deletions.
39 changes: 39 additions & 0 deletions .github/workflows/cron.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
name: Cron Build

on:
schedule:
# ┌───────────── minute (0 - 59)
# │ ┌───────────── hour (0 - 23)
# │ │ ┌───────────── day of the month (1 - 31)
# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
# │ │ │ │ │
# │ │ │ │ │
# │ │ │ │ │
# * * * * *
- cron: '0 */6 * * *'

jobs:
run:
name: Build with IDF ${{ matrix.idf_branch }}
runs-on: ubuntu-latest

strategy:
matrix:
idf_branch: [release/v3.2]
steps:
- uses: actions/checkout@v1
- name: Install dependencies
run: sudo apt-get install git wget curl libssl-dev libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache
- name: Install Python Wheel
run: pip install wheel
- name: Build
env:
GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
IDF_BRANCH: ${{ matrix.idf_branch }}
run: bash ./tools/cron.sh
- name: Upload archive
uses: actions/upload-artifact@v1
with:
name: artifacts
path: dist
9 changes: 3 additions & 6 deletions .github/workflows/main.yml → .github/workflows/push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ on:
- master
- release/*
pull_request:
repository_dispatch:

jobs:

Expand All @@ -20,11 +19,9 @@ jobs:
- name: Install Python Wheel
run: pip install wheel
- name: Build Arduino Libs
env:
TRAVIS_BUILD_DIR: ${{ github.workspace }}
run: bash $TRAVIS_BUILD_DIR/build.sh
- name: Archive libs
run: bash ./build.sh
- name: Upload archive
uses: actions/upload-artifact@v1
with:
name: arduino-libs
name: artifacts
path: dist
23 changes: 23 additions & 0 deletions .github/workflows/repository_dispatch.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
name: Remote Trigger

on: repository_dispatch

jobs:
run:
name: Dispatch Event
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- name: Install dependencies
run: sudo apt-get install git wget curl libssl-dev libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache
- name: Install Python Wheel
run: pip install wheel
- name: Handle Event
env:
GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
run: bash ./tools/repository_dispatch.sh
- name: Upload archive
uses: actions/upload-artifact@v1
with:
name: artifacts
path: dist
8 changes: 5 additions & 3 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,16 @@ if ! [ -x "$(command -v stat)" ]; then
exit 1
fi

# install esp-idf and gcc toolchain
source ./tools/install-esp-idf.sh
if [ $? -ne 0 ]; then exit 1; fi
mkdir -p dist

# update components from git
./tools/update-components.sh
if [ $? -ne 0 ]; then exit 1; fi

# install esp-idf and gcc toolchain
source ./tools/install-esp-idf.sh
if [ $? -ne 0 ]; then exit 1; fi

# build and prepare libs
./tools/build-libs.sh
if [ $? -ne 0 ]; then exit 1; fi
Expand Down
16 changes: 11 additions & 5 deletions tools/archive-build.sh
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
#!/bin/bash

IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD)

idf_version_string=${IDF_BRANCH//\//_}"-$IDF_COMMIT"
archive_path="dist/arduino-esp32-libs-$idf_version_string.tar.gz"
build_archive_path="dist/arduino-esp32-build-$idf_version_string.tar.gz"

mkdir -p dist && \
rm -rf $archive_path && \
cd out && \
tar zcf ../$archive_path * \
&& cd ..
mkdir -p dist && rm -rf "$archive_path" "$build_archive_path"
if [ -d "out" ]; then
cd out && tar zcf "../$archive_path" * && cd ..
fi
if [ -d "build" ]; then
cd build && tar zcf "../$build_archive_path" * && cd ..
fi
59 changes: 54 additions & 5 deletions tools/config.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
#!/bin/bash
IDF_REPO="https://github.com/espressif/esp-idf.git"
IDF_BRANCH="release/v3.2"

IDF_COMPS="$IDF_PATH/components"
IDF_TOOLCHAIN="xtensa-esp32-elf"
IDF_TOOLCHAIN_LINUX_ARMEL="https://dl.espressif.com/dl/xtensa-esp32-elf-linux-armel-1.22.0-87-gb57bad3-5.2.0.tar.gz"
Expand All @@ -9,10 +8,25 @@ IDF_TOOLCHAIN_LINUX64="https://dl.espressif.com/dl/xtensa-esp32-elf-linux64-1.22
IDF_TOOLCHAIN_WIN32="https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip"
IDF_TOOLCHAIN_MACOS="https://dl.espressif.com/dl/xtensa-esp32-elf-osx-1.22.0-80-g6c4433a-5.2.0.tar.gz"

CAMERA_REPO="https://github.com/espressif/esp32-camera.git"
FACE_REPO="https://github.com/espressif/esp-face.git"
if [ -z $IDF_BRANCH ]; then
IDF_BRANCH="release/v3.2"
fi

# Owner of the target ESP32 Arduino repository
AR_USER="espressif"

# The full name of the repository
AR_REPO="$AR_USER/arduino-esp32"

IDF_REPO_URL="https://github.com/espressif/esp-idf.git"
CAMERA_REPO_URL="https://github.com/espressif/esp32-camera.git"
FACE_REPO_URL="https://github.com/espressif/esp-face.git"
AR_REPO_URL="https://github.com/$AR_REPO.git"

if [ -n $GITHUB_TOKEN ]; then
AR_REPO_URL="https://$GITHUB_TOKEN@github.com/$AR_REPO.git"
fi

AR_REPO="https://github.com/espressif/arduino-esp32.git"
AR_ROOT="$PWD"
AR_COMPS="$AR_ROOT/components"
AR_OUT="$AR_ROOT/out"
Expand Down Expand Up @@ -56,3 +70,38 @@ if [[ "$AR_OS" == "macos" ]]; then
export SED="gsed"
export SSTAT="stat -f %z"
fi

function git_commit_exists(){ #git_commit_exists <repo-path> <commit-message>
local repo_path="$1"
local commit_message="$2"
local commits_found=`git -C "$repo_path" log --all --grep="$commit_message" | grep commit`
if [ -n "$commits_found" ]; then echo 1; else echo 0; fi
}

function git_branch_exists(){ # git_branch_exists <repo-path> <branch-name>
local repo_path="$1"
local branch_name="$2"
local branch_found=`git -C "$repo_path" ls-remote --heads origin "$branch_name"`
if [ -n "$branch_found" ]; then echo 1; else echo 0; fi
}

function git_pr_exists(){ # git_pr_exists <branch-name>
local pr_num=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$AR_REPO/pulls?head=$AR_USER:$1&state=open" | jq -r '.[].number'`
if [ ! "$pr_num" == "" ] && [ ! "$pr_num" == "null" ]; then echo 1; else echo 0; fi
}

function git_create_pr(){ # git_create_pr <branch> <title>
local pr_branch="$1"
local pr_title="$2"
local pr_body=""
for component in `ls "$AR_COMPS"`; do
if [ ! $component == "arduino" ] && [ -d "$AR_COMPS/$component/.git" ]; then
pr_body+="$component: "$(git -C "$AR_COMPS/$component" symbolic-ref --short HEAD)" "$(git -C "$AR_COMPS/$component" rev-parse --short HEAD)"\r\n"
fi
done
local pr_data="{\"title\": \"$pr_title\", \"body\": \"$pr_body\", \"head\": \"$AR_USER:$pr_branch\", \"base\": \"master\"}"
git_create_pr_res=`echo "$pr_data" | curl -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" --data @- "https://api.github.com/repos/$AR_REPO/pulls"`
local done_pr=`echo "$git_create_pr_res" | jq -r '.title'`
if [ ! "$done_pr" == "" ] && [ ! "$done_pr" == "null" ]; then echo 1; else echo 0; fi
}

10 changes: 10 additions & 0 deletions tools/cron.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/bin/bash

if [ ! "$GITHUB_EVENT_NAME" == "schedule" ]; then
echo "Wrong event '$GITHUB_EVENT_NAME'!"
exit 1
fi

git checkout "$IDF_BRANCH" #local branches should match what the matrix wants to build
source ./build.sh
bash ./tools/push-to-arduino.sh
96 changes: 86 additions & 10 deletions tools/install-esp-idf.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,102 @@ if ! [ -x "$(command -v $SED)" ]; then
exit 1
fi

#
# CLONE ESP-IDF
#

if [ -z "$IDF_PATH" ]; then
echo "ESP-IDF is not installed! Installing local copy"
idf_was_installed="1"
if ! [ -d esp-idf ]; then
git clone $IDF_REPO -b $IDF_BRANCH
git clone $IDF_REPO_URL -b $IDF_BRANCH
fi
export IDF_PATH="$AR_ROOT/esp-idf"
cd $IDF_PATH
git fetch origin && git pull origin $IDF_BRANCH
git submodule update --init --recursive
python -m pip install -r requirements.txt
cd "$AR_ROOT"
fi

if [ "$IDF_COMMIT" ]; then
git -C $IDF_PATH checkout $IDF_COMMIT
git -C $IDF_PATH submodule update
git -C "$IDF_PATH" checkout "$IDF_COMMIT"
commit_predefined="1"
fi

export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
export IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD)

#
# SETUP ARDUINO DEPLOY
#

if [ "$GITHUB_EVENT_NAME" == "schedule" ] || [ "$GITHUB_EVENT_NAME" == "repository_dispatch" -a "$GITHUB_EVENT_ACTION" == "deploy" ]; then
# format new branch name and pr title
if [ -x $commit_predefined ]; then #commit was not specified at build time
AR_NEW_BRANCH_NAME="idf-$IDF_BRANCH"
AR_NEW_COMMIT_MESSAGE="IDF $IDF_BRANCH $IDF_COMMIT"
AR_NEW_PR_TITLE="IDF $IDF_BRANCH"
else
AR_NEW_BRANCH_NAME="idf-$IDF_COMMIT"
AR_NEW_COMMIT_MESSAGE="IDF $IDF_COMMIT"
AR_NEW_PR_TITLE="$AR_NEW_COMMIT_MESSAGE"
fi

AR_HAS_COMMIT=`git_commit_exists "$AR_COMPS/arduino" "$AR_NEW_COMMIT_MESSAGE"`
AR_HAS_BRANCH=`git_branch_exists "$AR_COMPS/arduino" "$AR_NEW_BRANCH_NAME"`
AR_HAS_PR=`git_pr_exists "$AR_NEW_BRANCH_NAME"`

if [ "$AR_HAS_COMMIT" == "1" ]; then
echo "Commit '$AR_NEW_COMMIT_MESSAGE' Already Exists"
exit 0
fi

if [ "$AR_HAS_BRANCH" == "1" ]; then
echo "Branch '$AR_NEW_BRANCH_NAME' Already Exists"
fi

if [ "$AR_HAS_PR" == "1" ]; then
echo "PR '$AR_NEW_PR_TITLE' Already Exists"
fi

# setup git for pushing
git config --global github.user "$GITHUB_ACTOR"
git config --global user.name "$GITHUB_ACTOR"
git config --global user.email "$GITHUB_ACTOR@github.com"

# create or checkout the branch
if [ ! $AR_HAS_BRANCH == "0" ]; then
echo "Switching to arduino branch '$AR_NEW_BRANCH_NAME'..."
git -C "$AR_COMPS/arduino" checkout $AR_NEW_BRANCH_NAME
else
echo "Creating arduino branch '$AR_NEW_BRANCH_NAME'..."
git -C "$AR_COMPS/arduino" checkout -b $AR_NEW_BRANCH_NAME
fi
if [ $? -ne 0 ]; then
echo "ERROR: Checkout of branch '$AR_NEW_BRANCH_NAME' failed"
exit 1
fi

export AR_NEW_BRANCH_NAME
export AR_NEW_COMMIT_MESSAGE
export AR_NEW_PR_TITLE

export AR_HAS_COMMIT
export AR_HAS_BRANCH
export AR_HAS_PR
fi

#
# UPDATE IDF MODULES
#

if [ -x $idf_was_installed ]; then
git -C $IDF_PATH fetch origin && git -C $IDF_PATH pull origin $IDF_BRANCH
git -C $IDF_PATH submodule update --init --recursive
else
git -C $IDF_PATH submodule update --init --recursive
cd $IDF_PATH && python -m pip install -r requirements.txt && cd "$AR_ROOT"
fi

export IDF_COMMIT=$(git -C $IDF_PATH rev-parse --short HEAD)
export IDF_BRANCH=$(git -C $IDF_PATH symbolic-ref --short HEAD)
#
# INSTALL TOOLCHAIN
#

if ! [ -x "$(command -v $IDF_TOOLCHAIN-gcc)" ]; then
echo "GCC toolchain is not installed! Installing local copy"
Expand Down
2 changes: 1 addition & 1 deletion tools/prepare-libs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ cat pio_start.txt > "$AR_PLATFORMIO_PY"
rm pio_end.txt 1pio_start.txt 2pio_start.txt pio_start.txt

# include dirs
AR_INC="-DESP_PLATFORM -DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\" -DHAVE_CONFIG_H \"-I{compiler.sdk.path}/include/config\""
AR_INC="-DESP_PLATFORM -DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\" -DHAVE_CONFIG_H -DGCC_NOT_5_2_0=0 -DWITH_POSIX \"-I{compiler.sdk.path}/include/config\""
echo " CPPPATH=[" >> "$AR_PLATFORMIO_PY" && echo " join(FRAMEWORK_DIR, \"tools\", \"sdk\", \"include\", \"config\")," >> "$AR_PLATFORMIO_PY"
while [ "$1" != "" ]; do
cpath=$1
Expand Down
59 changes: 59 additions & 0 deletions tools/push-to-arduino.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/bin/bash
source ./tools/config.sh

if [ -x $GITHUB_TOKEN ]; then
echo "ERROR: GITHUB_TOKEN was not defined"
exit 1
fi

if ! [ -d "$AR_COMPS/arduino" ]; then
echo "ERROR: Target arduino folder does not exist!"
exit 1
fi

#
# UPDATE FILES
#

if [ $AR_HAS_COMMIT == "0" ]; then
cd $AR_COMPS/arduino

# make changes to the files
echo "Patching files in branch '$AR_NEW_BRANCH_NAME'..."
rm -rf $AR_COMPS/arduino/tools/sdk
cp -Rf $AR_SDK $AR_COMPS/arduino/tools/sdk
cp -f $AR_ESPTOOL_PY $AR_COMPS/arduino/tools/esptool.py
cp -f $AR_GEN_PART_PY $AR_COMPS/arduino/tools/gen_esp32part.py
cp -f $AR_PLATFORMIO_PY $AR_COMPS/arduino/tools/platformio-build.py
cp -f $AR_PLATFORM_TXT $AR_COMPS/arduino/platform.txt

# did any of the files change?
if [ -n "$(git status --porcelain)" ]; then
echo "Pushing changes to branch '$AR_NEW_BRANCH_NAME'..."
git add . && git commit --message "$AR_NEW_COMMIT_MESSAGE" && git push -u origin $AR_NEW_BRANCH_NAME
if [ $? -ne 0 ]; then
echo "ERROR: Pushing to branch '$AR_NEW_BRANCH_NAME' failed"
exit 1
fi
else
echo "No changes in branch '$AR_NEW_BRANCH_NAME'"
if [ $AR_HAS_BRANCH == "0" ]; then
echo "Delete created branch '$AR_NEW_BRANCH_NAME'"
git branch -d $AR_NEW_BRANCH_NAME
fi
exit 0
fi
fi

#
# CREATE PULL REQUEST
#

if [ "$AR_HAS_PR" == "0" ]; then
pr_created=`git_create_pr "$AR_NEW_BRANCH_NAME" "$AR_NEW_PR_TITLE"`
if [ $pr_created == "0" ]; then
echo "ERROR: Failed to create PR '$AR_NEW_PR_TITLE': "`echo "$git_create_pr_res" | jq -r '.message'`": "`echo "$git_create_pr_res" | jq -r '.errors[].message'`
exit 1
fi
fi
exit 0
Loading

0 comments on commit b78b168

Please sign in to comment.