From 253865818ad055baea7814f4c6a982e0dbb98e4c Mon Sep 17 00:00:00 2001 From: Oleksandr Andriienko Date: Thu, 10 Jun 2021 15:04:43 +0300 Subject: [PATCH] Add scripts to automate golang dependencies check. Signed-off-by: Oleksandr Andriienko --- LICENCE | 21 ++++ createClearDefinedCurations.sh | 182 +++++++++++++++++++++++++++++++++ licenseRetrivier.sh | 66 ++++++++++++ validateDependencies.sh | 171 +++++++++++++++++++++++++++++++ 4 files changed, 440 insertions(+) create mode 100644 LICENCE create mode 100755 createClearDefinedCurations.sh create mode 100755 licenseRetrivier.sh create mode 100755 validateDependencies.sh diff --git a/LICENCE b/LICENCE new file mode 100644 index 0000000..8aa2645 --- /dev/null +++ b/LICENCE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) [year] [fullname] + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/createClearDefinedCurations.sh b/createClearDefinedCurations.sh new file mode 100755 index 0000000..77b2077 --- /dev/null +++ b/createClearDefinedCurations.sh @@ -0,0 +1,182 @@ +#!/bin/bash +# install https://github.com/ericchiang/pup + +findTagSHA() { + page=${1} + repo="${2}" + tag="${3}" + + tagList=$(curl -L -s -H "Authorization: bearer ${GITHUB_TOKEN}" -G -d "per_page=100" -d "page=${page}" "https://api.github.com/repos/${repo}/tags") + + if [[ -n $(echo "${tagList[@]}" | yq -r ".[]") ]]; then + sha=$(echo "${tagList[@]}" | yq -r " .[] | select(.name ==\"${tag}\") | .commit.sha") + if [ -n "${sha}" ]; then + echo "${sha}" + return + fi + + page=$((page+1)) + findTagSHA "${page}" "${repo}" "${tag}" + fi +} + +setUpSourceInfo() { + metaTagName="${1}" + + importMetaTags=$(echo "${goGetHtml}" | pup "meta[name=\"${metaTagName}\"]" | tr -d "\n\r" ) + IFS=$'<' read -rd '' -a metas <<<"$importMetaTags" + packageRoot="" + local sourceLocation + local libName + for meta in "${metas[@]}"; do + if [[ -z "${meta}" ]]; then + continue + fi + metaContent=$(echo "<${meta}" | pup "meta[name=\"${metaTagName}\"] attr{content}") + libName=$(echo "${metaContent}" | tr -d "\n\r" | awk 'BEGIN {FS=" *"} {print $1}') + sourceLocation=$(echo "${metaContent}" | tr -d "\n\r" | awk 'BEGIN {FS=" *"} {print $3}') + if [[ "${initialDepName}" == "${libName}"* ]] || [[ "${initialDepName}" == github.com* ]]; then + dependencyName=${sourceLocation} + packageRoot=${libName} + break + fi + done + + if [ "${initialDepName}" != "${packageRoot}" ] && [[ ${initialDepName} == ${packageRoot}* ]]; then + packageName=${initialDepName#${packageRoot}/} + fi +} + +setUpFullSHA() { + shortSHA="${1}" + repo="${2}" + + shaInfo=$(curl -L -s -H "Authorization: bearer ${GITHUB_TOKEN}" "https://api.github.com/repos/${repo}/commits/${shortSHA}") + sha=$(echo "${shaInfo}" | yq -r ".sha") +} + +findSource() { + dependencyName="${1}" + initialDepName="${1}" + HTTPS="https://" + packageName="" + + rootDepUrl=${dependencyName} + if [[ ${dependencyName} == github.com* ]]; then + rootDepUrl=$(echo "${dependencyName}" | cut -d '/' -f-3) + fi + + goGetHtml=$(curl -sL "${HTTPS}${rootDepUrl}?go-get=1") + setUpSourceInfo "go-import" + if [[ ! $dependencyName == https://github.com* ]]; then + setUpSourceInfo "go-source" + fi + + dependencyName="${dependencyName%/tree*}" + + if [[ ! $dependencyName == https://github.com* ]]; then + echo "dependency ${dependencyName} has got unsupported git provider. We support only 'github' git provider..." + return + fi + + # echo "package root: ${packageRoot}" + # echo "dependency name: ${dependencyName}" + # echo "Package Name: ${packageName}" + + # Cut dependency name to search + dependencyName=${dependencyName%.git} + dependencyName=${dependencyName%/v[0-9]} + dependencyName=${dependencyName#https://} + + dependencyIdentifier=${dependencyName#github.com/} +} + +createPullRequests() { + moduleName="${1}" + pushd "${clearDefFork}" || exit + git add -N . + changedFiles=($(git diff --name-only)) + + for filePath in "${changedFiles[@]}"; do + git checkout master + fileName=$(basename "${filePath}") + fileName="${fileName%.yaml}" + parentDirPath=$(dirname "${filePath}") + parentDir="${parentDirPath##*/}" + + echo "${fileName} and ${parentDir}" + branchName="${parentDir}_${fileName}" + git checkout -B "${branchName}" + git add "${filePath}" + + commitMsg="Defined new revisions for golang dep: github.com/${parentDir}/${fileName}" + git commit -s -m "${commitMsg}" + git push -f origin "${branchName}" + + hub pull-request \ + --base clearlydefined:master \ + --head AndrienkoAleksandr:"${branchName}" -m "${commitMsg}" + done + + popd || exit +} + +createCurration() { + pushd "${clearDefFork}/curations/git/github" || exit + namespace="${dependencyIdentifier%/*}" + name="${dependencyIdentifier#*/}" + provider="github" + type="git" + + curationFolder="${clearDefFork}/curations/git/github/${namespace}" + curationFile="${curationFolder}/${name}.yaml" + + mkdir -p "${curationFolder}" + if [ ! -f "${curationFile}" ]; then + touch "${curationFile}" + yq -rYni "{coordinates : {type : \"${type}\"}}" "${curationFile}" + fi + + yq -rYi "( .coordinates.name ) |= \"${name}\" | + ( .coordinates.namespace ) |= \"${namespace}\" | + ( .coordinates.provider ) |= \"${provider}\" | + ( .coordinates.type ) |= \"${type}\" | + ( .revisions ) += { \"${sha}\": { \"licensed\": { \"declared\": \"${licence}\" } } } | + ." "${curationFile}" + + popd || exit +} + +clearDefFork="/home/user/projects/curated-data" +modules="currations" + +while IFS= read -r line +do + moduleName=$(echo "${line}" | cut -d " " -f1 | cut -d "@" -f1) + revision=$(echo "${line}" | cut -d " " -f1 | cut -d "@" -f2) + licence=$(echo "${line}" | cut -d " " -f2-) + + echo "${moduleName} ${revision} ${licence}" + findSource "${moduleName}" + + if [[ $revision == v*-*-* ]]; then + sha=$(echo "${revision}" | sed -r 's/.*-.*-(.*)/\1/') + setUpFullSHA "${sha}" "${dependencyIdentifier}" + else + if [[ -n "${packageName}" ]] && [[ ! ${packageName} == v[0-9]* ]]; then + revision="${packageName}/${revision}" + fi + revision="${revision%+incompatible}" + sha=$(findTagSHA 0 "${dependencyIdentifier}" "${revision}") + fi + echo "[INFO] URL: https://${dependencyName}/commit/${sha}" + if [ -z "${sha}" ]; then + echo "FAIL" + exit 1 + fi + echo "" + + createCurration +done < "${modules}" + +createPullRequests "${moduleName}" diff --git a/licenseRetrivier.sh b/licenseRetrivier.sh new file mode 100755 index 0000000..a639359 --- /dev/null +++ b/licenseRetrivier.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +#!/bin/bash +# install https://github.com/ericchiang/pup + +findLicense() { + repo="${1}" + repoInfo=$(curl -L -s -H "Authorization: bearer ${GITHUB_TOKEN}" "https://api.github.com/repos/${repo}") + if [ -n "${repoInfo}" ]; then + license=$(echo "${repoInfo}" | jq -r ".license.\"spdx_id\"") + fi +} + +findSource() { + dependencyName="${1}" + HTTPS="https://" + + rootDepUrl=${dependencyName} + if [[ ${dependencyName} == github.com* ]]; then + rootDepUrl=$(echo "${dependencyName}" | cut -d '/' -f-3) + fi + + # todo -L ... + goGetHtml=$(curl -sL "${HTTPS}${rootDepUrl}?go-get=1") + + dependencyURL=$(echo "${goGetHtml}" | pup "meta[name=\"go-import\"] attr{content}" | tr -d "\n\r" | sed -r 's/(.* git )(.*)/\2/') + # if [[ "${dependencyURL}" == ${HTTPS}* ]] ;then + dependencyName="${dependencyURL}" + # fi + + if [[ ! $dependencyName == https://github.com* ]]; then + sourceURL=$(echo "${goGetHtml}" | pup "meta[name=\"go-source\"] attr{content}" | \ + tr -d "\n\r" | \ + cut -d " " -f3) + if [[ ! $sourceURL == https://github.com* ]]; then + echo "dependency ${dependencyName} has got unsupported git provider. We support only 'github' git provider..." + return + fi + + sourceURL="${sourceURL%/tree*}" + # if [[ "${sourceURL}" == ${HTTPS}* ]] ;then + dependencyName="${sourceURL}" + # fi + fi + + # Cut dependency name to search + dependencyName=${dependencyName%.git} + dependencyName=${dependencyName%/v[0-9]} + dependencyName=${dependencyName#https://} + + # Remove github.com on the beggining + dependencyIdentifier=${dependencyName#github.com/} +} + +modules="bad4" + +while IFS= read -r line +do + moduleName=$(echo "${line}" | cut -d " " -f1 | cut -d "@" -f1) + + # echo "Module: ${moduleName}" + findSource "${moduleName}" + findLicense "${dependencyIdentifier}" + echo "${license} for project: ${dependencyURL}" + echo "${line} ${license}" >> "bad5" +done < "${modules}" diff --git a/validateDependencies.sh b/validateDependencies.sh new file mode 100755 index 0000000..9cd0959 --- /dev/null +++ b/validateDependencies.sh @@ -0,0 +1,171 @@ +#!/bin/bash + +# install https://github.com/ericchiang/pup + +findTagSHA() { + page=${1} + repo="${2}" + tag="${3}" + + tagList=$(curl -L -s -H "Authorization: bearer ${GITHUB_TOKEN}" -G -d "per_page=100" -d "page=${page}" "https://api.github.com/repos/${repo}/tags") + + if [[ -n $(echo "${tagList[@]}" | yq -r ".[]") ]]; then + sha=$(echo "${tagList[@]}" | yq -r " .[] | select(.name ==\"${tag}\") | .commit.sha") + if [ -n "${sha}" ]; then + echo "${sha}" + return + fi + + page=$((page+1)) + findTagSHA "${page}" "${repo}" "${tag}" + fi +} + +setUpSourceInfo() { + metaTagName="${1}" + + importMetaTags=$(echo "${goGetHtml}" | pup "meta[name=\"${metaTagName}\"]" | tr -d "\n\r" ) + IFS=$'<' read -rd '' -a metas <<<"$importMetaTags" + packageRoot="" + local sourceLocation + local libName + for meta in "${metas[@]}"; do + if [[ -z "${meta}" ]]; then + continue + fi + metaContent=$(echo "<${meta}" | pup "meta[name=\"${metaTagName}\"] attr{content}") + libName=$(echo "${metaContent}" | tr -d "\n\r" | awk 'BEGIN {FS=" *"} {print $1}') + sourceLocation=$(echo "${metaContent}" | tr -d "\n\r" | awk 'BEGIN {FS=" *"} {print $3}') + if [[ "${initialDepName}" == "${libName}"* ]] || [[ "${initialDepName}" == github.com* ]]; then + dependencyName=${sourceLocation} + packageRoot=${libName} + break + fi + done + + if [ "${initialDepName}" != "${packageRoot}" ] && [[ ${initialDepName} == ${packageRoot}* ]]; then + packageName=${initialDepName#${packageRoot}/} + fi +} + +setUpFullSHA() { + shortSHA="${1}" + repo="${2}" + + shaInfo=$(curl -L -s -H "Authorization: bearer ${GITHUB_TOKEN}" "https://api.github.com/repos/${repo}/commits/${shortSHA}") + sha=$(echo "${shaInfo}" | yq -r ".sha") +} + +findSource() { + dependencyName="${1}" + initialDepName="${1}" + HTTPS="https://" + packageName="" + + rootDepUrl=${dependencyName} + if [[ ${dependencyName} == github.com* ]]; then + rootDepUrl=$(echo "${dependencyName}" | cut -d '/' -f-3) + fi + + goGetHtml=$(curl -sL "${HTTPS}${rootDepUrl}?go-get=1") + setUpSourceInfo "go-import" + if [[ ! $dependencyName == https://github.com* ]]; then + setUpSourceInfo "go-source" + fi + + dependencyName="${dependencyName%/tree*}" + + if [[ ! $dependencyName == https://github.com* ]]; then + echo "Dependency ${initialDepName} => '${dependencyName}' has got unsupported git provider. We support only 'github' git provider..." + return + fi + + # echo "package root: ${packageRoot}" + # echo "dependency name: ${dependencyName}" + # echo "Package Name: ${packageName}" + + # Cut dependency name to search + dependencyName=${dependencyName%.git} + dependencyName=${dependencyName%/v[0-9]} + dependencyName=${dependencyName#https://} + + dependencyIdentifier=${dependencyName#github.com/} +} + + +HandleModule() { + dependencyName="${1}" + + API_DEFINITIONS=https://api.clearlydefined.io/definitions + # API_HARVEST=https://api.clearlydefined.io/harvest + HTTPS="https://" + + echo "================= ${dependencyName} =================" + dependenciesInfo=$(curl -s -G -d 'type=git' -d 'provider=github' -d 'matchCasing=true' -d "pattern=${dependencyIdentifier}" "${API_DEFINITIONS}") + + # depList=( $( echo "${dependenciesInfo}" | jq -r ".[]") ) + readarray -t depList < <(echo "${dependenciesInfo}" | jq -r ".[]") + + for dep in "${depList[@]}"; do + if [[ ${dep} == git* ]]; then + depInfo=$(curl -s "${API_DEFINITIONS}/${dep}") + score=$(echo "${depInfo}" | jq -r ".licensed.toolScore.total") + + if [ "${score}" -ge 75 ]; then + # date=$(curl -s "${API_HARVEST}/${dep}" | \ + # jq -r '.clearlydefined."1.3.0".described.releaseDate') + echo "- ${score} ${dep} ${date}" + fi + fi + done +} + +go list -mod=mod -m all > Modules +modules=$(pwd)/Modules + +i=0 +while IFS= read -r line +do + moduleName=$(echo "${line}" | cut -d " " -f1) + dependencyName="" + + echo "" + echo "------${i}. Module import: ${moduleName} --------" + revision=$(echo "${line}" | cut -d " " -f2) + i=$((i+1)) + + if [[ "${moduleName}" == "${revision}" ]]; then + continue + fi + + replaceVer=$(echo "${line}" | cut -d " " -f5) + if [ -n "${replaceVer}" ]; then + revision="${replaceVer}" + fi + + findSource "${moduleName}" + + if [[ $revision == v*-*-* ]]; then + sha=$(echo "${revision}" | sed -r 's/.*-.*-(.*)/\1/') + setUpFullSHA "${sha}" "${dependencyIdentifier}" + else + if [[ -n "${packageName}" ]] && [[ ! ${packageName} == v[0-9]* ]]; then + revision="${packageName}/${revision}" + fi + revision="${revision%+incompatible}" + sha=$(findTagSHA 0 "${dependencyIdentifier}" "${revision}") + fi + + echo "[INFO] URL: https://${dependencyName}/commit/${sha}" + if [[ -z "${dependencyName}" ]] || [[ -z "${sha}" ]]; then + echo "[FAIL] We can't retrieve sha" + continue + fi + + + echo "==================================================" + HandleModule "${dependencyIdentifier}" + echo "==================================================" + echo "" + +done < "${modules}"