Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix lint only changes, full lint on merge forwards #50583

Merged
merged 1 commit into from Nov 27, 2018
Merged

Fix lint only changes, full lint on merge forwards #50583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
109 changes: 88 additions & 21 deletions .ci/lint
View file
Open in desktop
Expand Up @@ -3,7 +3,7 @@ pipeline {
options {
timestamps()
ansiColor('xterm')
timeout(time: 1, unit: 'HOURS')
timeout(time: 3, unit: 'HOURS')
dubb-b marked this conversation as resolved.
Show resolved Hide resolved
}
environment {
PYENV_ROOT = "/usr/local/pyenv"
Expand All @@ -14,7 +14,7 @@ pipeline {
stage('github-pending') {
steps {
githubNotify credentialsId: 'test-jenkins-credentials',
description: 'Testing lint...',
description: 'Python lint on changes...',
status: 'PENDING',
context: "jenkins/pr/lint"
}
Expand All @@ -24,12 +24,14 @@ pipeline {
sh '''
# Need -M to detect renames otherwise they are reported as Delete and Add, need -C to detect copies, -C includes -M
# -M is on by default in git 2.9+
git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME" > file-list-status.log
git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" > file-list-status.log
# the -l increase the search limit, lets use awk so we do not need to repeat the search above.
gawk 'BEGIN {FS="\\t"} {if ($1 != "D") {print $NF}}' file-list-status.log > file-list-changed.log
gawk 'BEGIN {FS="\\t"} {if ($1 == "D") {print $NF}}' file-list-status.log > file-list-deleted.log
(git diff --name-status -l99999 -C "origin/$CHANGE_TARGET";echo "---";git diff --name-status -l99999 -C "origin/$BRANCH_NAME";printenv|grep -E '=[0-9a-z]{40,}+$|COMMIT=|BRANCH') > file-list-experiment.log
(git diff --name-status -l99999 -C "origin/$CHANGE_TARGET" "origin/$BRANCH_NAME";echo "---";git diff --name-status -l99999 -C "origin/$BRANCH_NAME";printenv|grep -E '=[0-9a-z]{40,}+$|COMMIT=|BRANCH') > file-list-experiment.log
touch pylint-report-salt.log pylint-report-tests.log
echo 254 > pylint-salt-chg.exit # assume failure
echo 254 > pylint-tests-chg.exit # assume failure
eval "$(pyenv init -)"
pyenv --version
pyenv install --skip-existing 2.7.14
Expand All @@ -41,52 +43,117 @@ pipeline {
archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log,file-list-experiment.log'
}
}
stage('linting') {
failFast false
stage('linting chg') {
parallel {
stage('salt linting') {
stage('lint salt') {
when {
expression { return readFile('file-list-changed.log') =~ /(?i)(^|\n)(salt\/.*\.py|setup\.py)\n/ }
}
steps {
sh '''
eval "$(pyenv init - --no-rehash)"
grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' tox -e pylint-salt | tee pylint-report-salt.log
# tee makes the exit/return code always 0
grep -Ei '^salt/.*\\.py$|^setup\\.py$' file-list-changed.log | (xargs -r '--delimiter=\\n' tox -e pylint-salt;echo "$?" > pylint-salt-chg.exit) | tee pylint-report-salt.log
# remove color escape coding
sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-salt.log
read rc_exit < pylint-salt-chg.exit
dubb-b marked this conversation as resolved.
Show resolved Hide resolved
exit "$rc_exit"
'''
archiveArtifacts artifacts: 'pylint-report-salt.log'
}
}
stage('test linting') {
stage('lint test') {
when {
expression { return readFile('file-list-changed.log') =~ /(?i)(^|\n)tests\/.*\.py\n/ }
}
steps {
sh '''
eval "$(pyenv init - --no-rehash)"
grep -Ei '^tests/.*\\.py$' file-list-changed.log | xargs -r '--delimiter=\\n' tox -e pylint-tests | tee pylint-report-tests.log
# tee makes the exit/return code always 0
grep -Ei '^tests/.*\\.py$' file-list-changed.log | (xargs -r '--delimiter=\\n' tox -e pylint-tests;echo "$?" > pylint-tests-chg.exit) | tee pylint-report-tests.log
# remove color escape coding
sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-tests.log
read rc_exit < pylint-tests-chg.exit
exit "$rc_exit"
'''
archiveArtifacts artifacts: 'pylint-report-tests.log'
}
}
}
post {
always {
step([$class: 'WarningsPublisher',
parserConfigurations: [[
parserName: 'PyLint',
pattern: 'pylint-report*chg.log'
]],
failedTotalAll: '0',
useDeltaValues: false,
canRunOnFailed: true,
usePreviousBuildAsReference: true
])
}
}
}
stage('lint all') {
// perform a full linit if this is a merge forward and the change only lint passed.
when {
expression { return params.BRANCH_NAME =~ /(?i)^merge-/ && readFile('file-list-changed.log') =~ /^0/ }
}
parallel {
stage('begin') {
steps {
githubNotify credentialsId: 'test-jenkins-credentials',
description: 'Python lint on everything...',
status: 'PENDING',
context: "jenkins/pr/lint"
}
}
stage('lint salt') {
steps {
sh '''
eval "$(pyenv init - --no-rehash)"
(tox -e pylint-salt ; echo "$?" > pylint-salt-full.exit) | tee pylint-report-salt-full.log
# remove color escape coding
sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-salt-full.log
read rc_exit < pylint-salt-full.exit
exit "$rc_exit"
'''
archiveArtifacts artifacts: 'pylint-report-salt-full.log'
}
}
stage('lint test') {
steps {
sh '''
eval "$(pyenv init - --no-rehash)"
(tox -e pylint-tests ; echo "$?" > pylint-tests-full.exit) | tee pylint-report-tests-full.log
# remove color escape coding
sed -ri 's/\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' pylint-report-tests-full.log
read rc_exit < pylint-tests-full.exit
exit "$rc_exit"
'''
archiveArtifacts artifacts: 'pylint-report-tests-full.log'
}
}
}
post {
always {
step([$class: 'WarningsPublisher',
parserConfigurations: [[
parserName: 'PyLint',
pattern: 'pylint-report*full.log'
]],
failedTotalAll: '0',
useDeltaValues: false,
canRunOnFailed: true,
usePreviousBuildAsReference: true
])
}
}
}
}
post {
always {
step([$class: 'WarningsPublisher',
parserConfigurations: [[
parserName: 'PyLint',
pattern: 'pylint-report*.log'
]],
failedTotalAll: '0',
useDeltaValues: false,
canRunOnFailed: true,
usePreviousBuildAsReference: true
])
cleanWs()
}
success {
Expand All @@ -97,7 +164,7 @@ pipeline {
}
failure {
githubNotify credentialsId: 'test-jenkins-credentials',
description: 'The lint job has failed',
description: 'The lint test has failed',
status: 'FAILURE',
context: "jenkins/pr/lint"
slackSend channel: "#jenkins-prod-pr",
Expand Down
4 changes: 4 additions & 0 deletions .github/CODEOWNERS
View file
Open in desktop
Validating CODEOWNERS rules …
Expand Up @@ -13,6 +13,7 @@ salt/*/*boto* @saltstack/team-boto

# Team Core
requirements/* @saltstack/team-core
rfcs/* @saltstack/team-core
salt/auth/* @saltstack/team-core
salt/cache/* @saltstack/team-core
salt/cli/* @saltstack/team-core
Expand Down Expand Up @@ -73,3 +74,6 @@ salt/modules/reg.py @saltstack/team-windows
salt/states/reg.py @saltstack/team-windows
tests/*/*win* @saltstack/team-windows
tests/*/test_reg.py @saltstack/team-windows

# Jenkins Integration
.ci/* @saltstack/saltstack-release-engineering @saltstack/team-core @saltstack/team-windows