Add scripts to automate golang dependencies check.

Signed-off-by: Oleksandr Andriienko <oandriie@redhat.com>

LICENCE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

createClearDefinedCurations.sh

@@ -0,0 +1,182 @@
+#!/bin/bash
+# install https://github.com/ericchiang/pup
+
+findTagSHA() {
+    page=${1}
+    repo="${2}"
+    tag="${3}"
+
+    tagList=$(curl -L -s -H "Authorization: bearer ${GITHUB_TOKEN}" -G -d "per_page=100" -d "page=${page}" "https://api.github.com/repos/${repo}/tags")
+
+    if [[ -n $(echo "${tagList[@]}" | yq -r ".[]") ]]; then
+        sha=$(echo "${tagList[@]}" | yq -r " .[] | select(.name ==\"${tag}\") | .commit.sha")
+        if [ -n "${sha}" ]; then
+            echo "${sha}"
+            return
+        fi
+
+        page=$((page+1))
+        findTagSHA "${page}" "${repo}" "${tag}"
+    fi
+}
+
+setUpSourceInfo() {
+    metaTagName="${1}"
+
+    importMetaTags=$(echo "${goGetHtml}" | pup "meta[name=\"${metaTagName}\"]" | tr -d "\n\r" )
+    IFS=$'<' read -rd '' -a metas <<<"$importMetaTags"
+    packageRoot=""
+    local sourceLocation
+    local libName
+    for meta in "${metas[@]}"; do
+        if [[ -z "${meta}" ]]; then
+            continue
+        fi
+        metaContent=$(echo "<${meta}" | pup "meta[name=\"${metaTagName}\"] attr{content}")
+        libName=$(echo "${metaContent}" | tr -d "\n\r" | awk 'BEGIN {FS=" *"} {print $1}')
+        sourceLocation=$(echo "${metaContent}" | tr -d "\n\r" | awk 'BEGIN {FS=" *"} {print $3}')
+        if [[ "${initialDepName}" == "${libName}"* ]] || [[ "${initialDepName}" == github.com* ]]; then
+            dependencyName=${sourceLocation}
+            packageRoot=${libName}
+            break
+        fi
+    done
+
+    if [ "${initialDepName}" != "${packageRoot}" ] && [[ ${initialDepName} == ${packageRoot}* ]]; then 
+        packageName=${initialDepName#${packageRoot}/}
+    fi
+}
+
+setUpFullSHA() {
+    shortSHA="${1}"
+    repo="${2}"
+
+    shaInfo=$(curl -L -s -H "Authorization: bearer ${GITHUB_TOKEN}" "https://api.github.com/repos/${repo}/commits/${shortSHA}")
+    sha=$(echo "${shaInfo}" | yq -r ".sha")
+}
+
+findSource() {
+    dependencyName="${1}"
+    initialDepName="${1}"
+    HTTPS="https://"
+    packageName=""
+
+    rootDepUrl=${dependencyName}
+    if [[ ${dependencyName} == github.com* ]]; then
+        rootDepUrl=$(echo "${dependencyName}" | cut -d '/' -f-3)
+    fi
+
+    goGetHtml=$(curl -sL "${HTTPS}${rootDepUrl}?go-get=1")
+    setUpSourceInfo "go-import"
+    if [[ ! $dependencyName == https://github.com* ]]; then
+        setUpSourceInfo "go-source"
+    fi
+
+    dependencyName="${dependencyName%/tree*}"
+
+    if [[ ! $dependencyName == https://github.com* ]]; then
+        echo "dependency ${dependencyName} has got unsupported git provider. We support only 'github' git provider..."
+        return
+    fi
+
+    # echo "package root: ${packageRoot}"
+    # echo "dependency name: ${dependencyName}"
+    # echo "Package Name: ${packageName}"
+
+    # Cut dependency name to search
+    dependencyName=${dependencyName%.git}
+    dependencyName=${dependencyName%/v[0-9]}
+    dependencyName=${dependencyName#https://}
+
+    dependencyIdentifier=${dependencyName#github.com/}
+}
+
+createPullRequests() {
+    moduleName="${1}"
+    pushd "${clearDefFork}" || exit
+    git add -N .
+    changedFiles=($(git diff --name-only))
+
+    for filePath in "${changedFiles[@]}"; do
+        git checkout master
+        fileName=$(basename "${filePath}")
+        fileName="${fileName%.yaml}"
+        parentDirPath=$(dirname "${filePath}")
+        parentDir="${parentDirPath##*/}"
+
+        echo "${fileName} and ${parentDir}"
+        branchName="${parentDir}_${fileName}"
+        git checkout -B "${branchName}"
+        git add "${filePath}"
+
+        commitMsg="Defined new revisions for golang dep: github.com/${parentDir}/${fileName}"
+        git commit -s -m "${commitMsg}"
+        git push -f origin "${branchName}"
+
+        hub pull-request \
+        --base clearlydefined:master \
+        --head AndrienkoAleksandr:"${branchName}" -m "${commitMsg}"
+    done
+
+    popd || exit
+}
+
+createCurration() {
+    pushd "${clearDefFork}/curations/git/github" || exit
+    namespace="${dependencyIdentifier%/*}"
+    name="${dependencyIdentifier#*/}"
+    provider="github"
+    type="git"
+
+    curationFolder="${clearDefFork}/curations/git/github/${namespace}"
+    curationFile="${curationFolder}/${name}.yaml"
+
+    mkdir -p "${curationFolder}"
+    if [ ! -f "${curationFile}" ]; then
+        touch "${curationFile}"
+        yq -rYni "{coordinates : {type : \"${type}\"}}" "${curationFile}"
+    fi
+
+    yq -rYi "( .coordinates.name ) |= \"${name}\" |
+             ( .coordinates.namespace ) |= \"${namespace}\" | 
+             ( .coordinates.provider ) |= \"${provider}\" | 
+             ( .coordinates.type ) |= \"${type}\" | 
+             ( .revisions ) += { \"${sha}\": { \"licensed\": { \"declared\": \"${licence}\" } } } |
+             ." "${curationFile}"
+
+    popd || exit
+}
+
+clearDefFork="/home/user/projects/curated-data"
+modules="currations"
+
+while IFS= read -r line
+do
+    moduleName=$(echo "${line}" | cut -d " " -f1 | cut -d "@" -f1)
+    revision=$(echo "${line}" | cut -d " " -f1  | cut -d "@" -f2)
+    licence=$(echo "${line}" | cut -d " " -f2-)
+
+    echo "${moduleName} ${revision} ${licence}"
+    findSource "${moduleName}"
+
+    if [[ $revision == v*-*-* ]]; then
+        sha=$(echo "${revision}" | sed -r 's/.*-.*-(.*)/\1/')
+        setUpFullSHA "${sha}" "${dependencyIdentifier}"
+    else
+        if [[ -n "${packageName}" ]] && [[ ! ${packageName} == v[0-9]* ]]; then
+            revision="${packageName}/${revision}"
+        fi
+        revision="${revision%+incompatible}"
+        sha=$(findTagSHA 0 "${dependencyIdentifier}" "${revision}")
+    fi
+    echo "[INFO] URL: https://${dependencyName}/commit/${sha}"
+    if [ -z "${sha}" ]; then
+        echo "FAIL"
+        exit 1
+    fi
+    echo ""
+
+    createCurration
+done < "${modules}"
+
+createPullRequests "${moduleName}"

licenseRetrivier.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+#!/bin/bash
+# install https://github.com/ericchiang/pup
+
+findLicense() {
+    repo="${1}"
+    repoInfo=$(curl -L -s -H "Authorization: bearer ${GITHUB_TOKEN}" "https://api.github.com/repos/${repo}")
+    if [ -n "${repoInfo}" ]; then
+        license=$(echo "${repoInfo}" | jq -r ".license.\"spdx_id\"")
+    fi
+}
+
+findSource() {
+    dependencyName="${1}"
+    HTTPS="https://"
+
+    rootDepUrl=${dependencyName}
+    if [[ ${dependencyName} == github.com* ]]; then
+        rootDepUrl=$(echo "${dependencyName}" | cut -d '/' -f-3)
+    fi
+
+    # todo -L ...
+    goGetHtml=$(curl -sL "${HTTPS}${rootDepUrl}?go-get=1")
+
+    dependencyURL=$(echo "${goGetHtml}" | pup "meta[name=\"go-import\"] attr{content}" | tr -d "\n\r" | sed -r 's/(.* git )(.*)/\2/')
+    # if [[ "${dependencyURL}" == ${HTTPS}* ]] ;then
+    dependencyName="${dependencyURL}"
+    # fi
+
+    if [[ ! $dependencyName == https://github.com* ]]; then
+        sourceURL=$(echo "${goGetHtml}" | pup "meta[name=\"go-source\"] attr{content}" | \
+                tr -d "\n\r" | \
+                cut -d " " -f3)
+        if [[ ! $sourceURL == https://github.com* ]]; then
+            echo "dependency ${dependencyName} has got unsupported git provider. We support only 'github' git provider..."
+            return
+        fi
+
+        sourceURL="${sourceURL%/tree*}"
+        # if [[ "${sourceURL}" == ${HTTPS}* ]] ;then
+        dependencyName="${sourceURL}"
+        # fi
+    fi
+
+    # Cut dependency name to search
+    dependencyName=${dependencyName%.git}
+    dependencyName=${dependencyName%/v[0-9]}
+    dependencyName=${dependencyName#https://}  
+
+    # Remove github.com on the beggining
+    dependencyIdentifier=${dependencyName#github.com/}
+}
+
+modules="bad4"
+
+while IFS= read -r line
+do  
+    moduleName=$(echo "${line}" | cut -d " " -f1 | cut -d "@" -f1)
+
+    # echo "Module: ${moduleName}"
+    findSource "${moduleName}"
+    findLicense "${dependencyIdentifier}"
+    echo "${license} for project: ${dependencyURL}"
+    echo "${line} ${license}" >> "bad5"
+done < "${modules}"