Skip to content
Permalink
Browse files

Add format and lint to Makefile and CI (#23)

  • Loading branch information...
deliahu committed Feb 28, 2019
1 parent 02e98c5 commit fc2de0636570bc10801f0f6b2bf65835ce009a20
@@ -3,13 +3,15 @@ version: 2
jobs:
test:
docker:
- image: circleci/golang:1.11.5
- image: circleci/python:3.7.2-stretch
steps:
- checkout
- run: wget https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz && sudo tar -C /usr/local -xzf go1.11.5.linux-amd64.tar.gz && rm -rf go*.tar.gz && echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV && echo 'export PATH=$PATH:~/go/bin' >> $BASH_ENV
- run: GO111MODULE=off go get -u -v golang.org/x/lint/golint
- run: sudo pip3 install black
- run:
name: Go Lint
command: make lint-go
name: Lint
command: make lint
- run:
name: License Test
command: make find-missing-license
@@ -18,17 +20,18 @@ jobs:
command: make test-go
test-release:
docker:
- image: circleci/golang:1.11.5
- image: circleci/python:3.7.2-stretch
steps:
- checkout
- run:
name: Version Test
command: make find-missing-version
deploy:
docker:
- image: circleci/golang:1.11.5
- image: circleci/python:3.7.2-stretch
steps:
- run: curl -s "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip" && unzip awscli-bundle.zip && sudo ./awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws && rm -rf awscli-bundle*
- run: wget https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz && sudo tar -C /usr/local -xzf go1.11.5.linux-amd64.tar.gz && rm -rf go*.tar.gz && echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV && echo 'export PATH=$PATH:~/go/bin' >> $BASH_ENV
- run: sudo pip3 install awscli
- checkout
- setup_remote_docker
- run: make build-images
@@ -1,4 +1,3 @@

---
Checklist:
- [ ] Run automated tests (`make test`)
@@ -1,4 +1,3 @@

Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
@@ -98,7 +98,10 @@ aws-clear-bucket:
tools:
@go get -u -v golang.org/x/lint/golint
@go get -u -v github.com/VojtechVitek/rerun/cmd/rerun
@curl -fLSs https://circle.ci/cli | bash
@pip3 install black

format:
@./dev/format.sh

#########
# Tests #
@@ -110,12 +113,12 @@ test:
test-go:
@./build/test.sh go

lint-go:
@./build/golint.sh go

test-python:
@./build/test.sh python

lint:
@./build/lint.sh

###############
# CI Commands #
###############
@@ -17,10 +17,12 @@

set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"

CORTEX_VERSION=master

dir=$1
image=$2

docker build . -f $dir/Dockerfile -t cortexlabs/$image \
-t cortexlabs/$image:$CORTEX_VERSION
docker build "$ROOT" -f $dir/Dockerfile -t cortexlabs/$image \
-t cortexlabs/$image:$CORTEX_VERSION
@@ -17,13 +17,15 @@

set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"

CORTEX_VERSION=master

function build_and_upload() {
set -euo pipefail

os=$1
GOOS=$os GOARCH=amd64 CGO_ENABLED=0 GO111MODULE=on go build -o cortex github.com/cortexlabs/cortex/cli
GOOS=$os GOARCH=amd64 CGO_ENABLED=0 GO111MODULE=on go build -o cortex "$ROOT/cli"
aws s3 cp cortex s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex --only-show-errors
rm cortex
echo "Uploaded CLI to s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex"
@@ -14,7 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.

output=$(find . -type f \

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"

output=$(cd "$ROOT" && find . -type f \
! -path "./vendor/*" \
! -path "./examples/*" \
! -path "./dev/config/*" \
@@ -14,7 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.

output=$(find . -type f \

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"

output=$(cd "$ROOT" && find . -type f \
! -path "./vendor/*" \
! -path "./bin/*" \
! -path "./build/find-missing-version.sh" \

This file was deleted.

Oops, something went wrong.
@@ -0,0 +1,109 @@
#!/bin/bash

# Copyright 2019 Cortex Labs, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"

if ! command -v golint >/dev/null 2>&1; then
echo "golint must be installed"
exit 1
fi

if ! command -v gofmt >/dev/null 2>&1; then
echo "gofmt must be installed"
exit 1
fi

if ! command -v black >/dev/null 2>&1; then
echo "black must be installed"
exit 1
fi

go vet "$ROOT/..."

output=$(golint "$ROOT/..." | grep -v "comment" || true)
if [[ $output ]]; then
echo "$output"
exit 1
fi

output=$(gofmt -s -l "$ROOT")
if [[ $output ]]; then
echo "go files not properly formatted:"
echo "$output"
exit 1
fi

output=$(black --quiet --diff --line-length=100 "$ROOT")
if [[ $output ]]; then
echo "python files not properly formatted:"
echo "$output"
exit 1
fi

# Check for trailing whitespace
output=$(cd "$ROOT" && find . -type f \
! -path "./vendor/*" \
! -path "./bin/*" \
! -path "./.git/*" \
! -name ".*" \
-exec egrep -l " +$" {} \;)
if [[ $output ]]; then
echo "File(s) have lines with trailing whitespace:"
echo "$output"
exit 1
fi

# Check for missing new line at end of file
output=$(cd "$ROOT" && find . -type f \
! -path "./vendor/*" \
! -path "./bin/*" \
! -path "./.git/*" \
! -name ".*" \
-print0 | \
xargs -0 -L1 bash -c 'test "$(tail -c 1 "$0")" && echo "No new line at end of $0"' || true)
if [[ $output ]]; then
echo "$output"
exit 1
fi

# Check for multiple new lines at end of file
output=$(cd "$ROOT" && find . -type f \
! -path "./vendor/*" \
! -path "./bin/*" \
! -path "./.git/*" \
! -name ".*" \
-print0 | \
xargs -0 -L1 bash -c 'test "$(tail -c 2 "$0")" || echo "Multiple new lines at end of $0"' || true)
if [[ $output ]]; then
echo "$output"
exit 1
fi

# Check for new line(s) at beginning of file
output=$(cd "$ROOT" && find . -type f \
! -path "./vendor/*" \
! -path "./bin/*" \
! -path "./.git/*" \
! -name ".*" \
-print0 | \
xargs -0 -L1 bash -c 'test "$(head -c 1 "$0")" || echo "New line at beginning of $0"' || true)
if [[ $output ]]; then
echo "$output"
exit 1
fi
@@ -20,7 +20,7 @@ set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"

function run_go_tests() {
(cd $ROOT && GO111MODULE=on go test ./... && echo -e "\ngo vet..." && GO111MODULE=on go vet ./... && echo "go tests passed")
(cd $ROOT && GO111MODULE=on go test ./... && echo "go tests passed")
}

function run_python_tests() {
@@ -0,0 +1,79 @@
#!/bin/bash

# Copyright 2019 Cortex Labs, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"

if ! command -v gofmt >/dev/null 2>&1; then
echo "gofmt must be installed"
exit 1
fi

if ! command -v black >/dev/null 2>&1; then
echo "black must be installed"
exit 1
fi

gofmt -s -w "$ROOT"

black --quiet --line-length=100 "$ROOT"

# Trim trailing whitespace
if [[ "$OSTYPE" == "darwin"* ]]; then
output=$(cd "$ROOT" && find . -type f \
! -path "./vendor/*" \
! -path "./bin/*" \
! -path "./.git/*" \
! -name ".*" \
-print0 | \
xargs -0 sed -i '' -e's/[[:space:]]*$//')
else
output=$(cd "$ROOT" && find . -type f \
! -path "./vendor/*" \
! -path "./bin/*" \
! -path "./.git/*" \
! -name ".*" \
-print0 | \
xargs -0 sed -i 's/[[:space:]]*$//')
fi

# Add new line to end of file
(cd "$ROOT" && find . -type f \
! -path "./vendor/*" \
! -path "./bin/*" \
! -path "./.git/*" \
! -name ".*" \
-print0 | \
xargs -0 -L1 bash -c 'test "$(tail -c 1 "$0")" && echo "" >> "$0"' || true)

# Remove repeated new lines at end of file
(cd "$ROOT" && find . -type f \
! -path "./vendor/*" \
! -path "./bin/*" \
! -path "./.git/*" \
! -name ".*" \
-print0 | \
xargs -0 -L1 bash -c 'test "$(tail -c 2 "$0")" || (trimmed=$(printf "%s" "$(< $0)") && echo "$trimmed" > "$0")' || true)

# Remove new lines at beginning of file
(cd "$ROOT" && find . -type f \
! -path "./vendor/*" \
! -path "./bin/*" \
! -path "./.git/*" \
! -name ".*" \
-print0 | \
xargs -0 -L1 bash -c 'test "$(head -c 1 "$0")" || (trimmed=$(sed '"'"'/./,$!d'"'"' "$0") && echo "$trimmed" > "$0")' || true)
@@ -65,4 +65,4 @@ oauthlib==3.0.0
httplib2==0.12.0
```

You can install additional PyPI packages and import your own Python packages. See [Python Packages](../advanced/python-packages.md) for more details.
You can install additional PyPI packages and import your own Python packages. See [Python Packages](../advanced/python-packages.md) for more details.
@@ -26,4 +26,4 @@ Cortex is built on top of Kubernetes to limit cloud provider lock-in.

## Can I use Cortex with my existing codebase?

Yes you can. Your custom packages will be installed and made available for use in the Cortex pipeline.
Yes you can. Your custom packages will be installed and made available for use in the Cortex pipeline.
@@ -1 +1 @@
pillow==5.4.1
pillow==5.4.1
@@ -3,4 +3,3 @@ FROM tensorflow/tensorflow:1.12.0-py3
RUN apt-get update -qq && apt-get install -y -q \
zlib1g-dev \
&& apt-get clean -qq && rm -rf /var/lib/apt/lists/*

0 comments on commit fc2de06

Please sign in to comment.
You can’t perform that action at this time.