diff --git a/.gitignore b/.gitignore index 7e8275d..db8dc42 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ recipes-mirror-replacement.urls recipes-origin.urls _recipes-mirror-replacement.urls _recipes_bare/ +_recipes/ diff --git a/.travis.yml b/.travis.yml index 8128af9..9973154 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,9 +5,7 @@ before_install: - sudo apt-get install git wget -q -y script: - - ./10-retrieve-urls-from-recipes.sh - - DRY_RUN=true ./20-fetch.sh - - PYBOMBS_MIRROR_BASE_URL="http://mirrors.tuna.tsinghua.edu.cnm/pybombs" ./30-replace-recipes.sh + - DRY_RUN=true PYBOMBS_MIRROR_BASE_URL="http://mirrors.tuna.tsinghua.edu.cnm/pybombs" PYBOMBS_MIRROR_WORK_DIR=$(pwd) ./pybombs-mirror.sh notifications: email: diff --git a/10-retrieve-urls-from-recipes.sh b/10-retrieve-urls-from-recipes.sh deleted file mode 100755 index 7ee93e2..0000000 --- a/10-retrieve-urls-from-recipes.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -PYBOMBS_MIRROR_WORK_DIR=$(pwd) -cat recipe-repos.urls | while read REPO_URL REPO_NAME -do - if [ ! -d recipes-origin ]; then - mkdir recipes-origin - fi - - if [ ! -d recipes-origin/${REPO_NAME} ]; then - echo "No recipes-origin/${REPO_NAME} found. Fetching..." - cd recipes-origin/ - echo "git clone ${REPO_URL} ${REPO_NAME}" - git clone ${REPO_URL} ${REPO_NAME} - else - echo "Git updating recipes..." - cd recipes-origin/${REPO_NAME} - echo "Updating ${REPO_NAME}" - git pull - fi - - cd ${PYBOMBS_MIRROR_WORK_DIR} -done - - -if [ -d recipes ]; then - echo "target recipes directory already exists, clean it." - rm -rf ./recipes -fi - -cp -r recipes-origin recipes - -if [ -e pre-replace-upstream.urls ]; then - echo "====================" - echo "pre-replace-upstream.urls found!" - echo "Patching recipes with custom urls" - - cat pre-replace-upstream.urls | while read origin new - do - echo "Replacing ${origin} with ${new}" -# find ./recipes/ -name \*.lwr -exec sed -i "s,${origin},${new},g" {} \; - grep -rl "${origin}" ./recipes/ |grep -v \.git |xargs -r sed -i "s,${origin},${new},g" - done - echo "Patching recipes done!" - echo "====================" - echo -fi - - -echo "====================" -grep -r -E "(git|svn|wget)\+.*$" recipes/ | rev |cut -d' ' -f1 |rev > recipes-origin.urls - -echo "recipes-origin.urls generated successfully!" -echo "====================" -echo - - -if [ -e ignore.urls ]; then - echo "====================" - echo "ignore.urls found!" - echo "Remove ignored urls from recipes-origin.urls" - echo "Removed lines won't be fetched or replaced, mirror users can still fetch that repo directly from upstream." - - - cat ignore.urls | while read url - do - echo "Deleting line contains ${url}" - sed -i "\\,${url},d" recipes-origin.urls - done - echo "recipes-origin.urls patched!" - echo "====================" -fi diff --git a/20-fetch.sh b/20-fetch.sh deleted file mode 100755 index a23f288..0000000 --- a/20-fetch.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/bash - -if [ -z $DRY_RUN ]; then - DRY_RUN=false # when set to true, nothing will be fetched, for debug purpose -fi - -PYBOMBS_MIRROR_WORK_DIR=$(pwd) - - - -if [ ! -e recipes-origin.urls ]; then - echo "No recipes-origin.urls found! Exit." - exit -1 -fi - -if [ -e _recipes-mirror-replacement.urls ]; then - echo "Previous _recipes-mirror-replacement.urls exists, purge it." - rm _recipes-mirror-replacement.urls -fi - -if [ ! -d git ]; then - mkdir git -fi -if [ ! -d svn ]; then - mkdir svn -fi -if [ ! -d wget ]; then - mkdir wget -fi - -if [ -e failed.log ]; then - echo "Previous failed.log found! Roll it to failed.log.1 ." - mv failed.log failed.log.1 - echo -fi - - -cat recipes-origin.urls | sed 's/+/ /' | while read protocol url -do - echo "ENTERING: $PYBOMBS_MIRROR_WORK_DIR" - cd $PYBOMBS_MIRROR_WORK_DIR - - ORIGIN_PYBOMBS_URL="$protocol+$url" - echo "ORGIGIN_PYBOMBS_URL: $ORIGIN_PYBOMBS_URL" - #TARGET_PATH="$protocol/$(basename $url)" - - # If duplication of name like git+git://git.code.sf.net/p/openlte/code --> code happens - # You can use the following name scheme (Not beautiful... indeed...) - - TARGET_PATH="$protocol/$(basename $(dirname $url))_$(basename $url)" - - echo "UPSTREAM: $url" - MIRROR_PYBOMBS_URL="${protocol}+PYBOMBS_MIRROR_BASE_URL/${TARGET_PATH}" - echo "MIRROR_PYBOMBS_URL: ${MIRROR_PYBOMBS_URL}" - - - FETCHING_SUCCESS=true #predefine status - if [ $protocol = "wget" ]; then - if [ -e ./$TARGET_PATH ]; then - echo "./$TARGET_PATH exists, skipping" - else - if [ -e ./${TARGET_PATH}.tmp ]; then - echo "./${TARGET_PATH}.tmp exists.. remove it." - rm ./${TARGET_PATH}.tmp - fi - echo "EXECUTING: wget $url -O ./${TARGET_PATH}.tmp" - if [ ! $DRY_RUN = true ]; then - wget $url --tries=3 -O ./${TARGET_PATH}.tmp - if [ $? -eq 0 ]; then - echo "Fetching done. Renaming ./${TARGET_PATH}.tmp to ./${TARGET_PATH}" - mv ./${TARGET_PATH}.tmp ./${TARGET_PATH} - else - FETCHING_SUCCESS=false - fi - fi - fi - elif [ $protocol = "git" ]; then - if [ -d ./$TARGET_PATH ]; then - echo "./$TARGET_PATH exists, syncing" - cd ./$TARGET_PATH - - if [ ! $DRY_RUN = true ]; then - /usr/bin/timeout -s INT 3600 git remote -v update || FETCHING_SUCCESS=false - git repack -a -d - fi - else - echo "EXECUTING: git clone --mirror $url ./$TARGET_PATH" - if [ ! $DRY_RUN = true ]; then - git clone --mirror $url ./$TARGET_PATH || FETCHING_SUCCESS=false - fi - fi - elif [ $protocol = "svn" ]; then - - if [ ! $DRY_RUN = true ]; then - echo "TODO" - FETCHING_SUCCESS=false - fi - fi - - - cd $PYBOMBS_MIRROR_WORK_DIR - if [ $FETCHING_SUCCESS = true ]; then - echo "${ORIGIN_PYBOMBS_URL} ${MIRROR_PYBOMBS_URL}" >> _recipes-mirror-replacement.urls - echo "${ORIGIN_PYBOMBS_URL} fetching success!" - else - echo "${ORIGIN_PYBOMBS_URL}" >> failed.log - echo "${ORIGIN_PYBOMBS_URL} fetching failed" - fi - - echo "=====================" - echo - -done diff --git a/30-replace-recipes.sh b/30-replace-recipes.sh deleted file mode 100755 index b4c45c0..0000000 --- a/30-replace-recipes.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Currently, all mirrored repo are served through http or https: -# like: -# git+http, svn+http, wget+http -# or: -# git+https, svn+https, wget+https -if [ -z ${PYBOMBS_MIRROR_BASE_URL} ]; then - PYBOMBS_MIRROR_BASE_URL="http://localhost/pybombs" # *NO* tailing / should be added. - echo "No PYBOMBS_MIRROR_BASE_URL passed in, using ${PYBOMBS_MIRROR_BASE_URL}" -fi - -if [ ! -e _recipes-mirror-replacement.urls ]; then - echo "No recipes-mirror-replacement.urls found! Exit." - exit -1 -fi - -if [ ! -d recipes ]; then - echo "No target recipes directory found! Exit." - exit -1 -fi - -echo "Replacing PYBOMBS_MIRROR_BASE_URL with ${PYBOMBS_MIRROR_BASE_URL} in recipes-mirror-replacement.urls .." -cp _recipes-mirror-replacement.urls recipes-mirror-replacement.urls -sed -i "s,PYBOMBS_MIRROR_BASE_URL,${PYBOMBS_MIRROR_BASE_URL},g" recipes-mirror-replacement.urls -echo "Done." - - -cat recipes-mirror-replacement.urls | while read origin new -do - echo "${origin} --> ${new}" -# find ./recipes/ -name \*.lwr -exec sed -i "s,${origin},${new},g" {} \; - grep -rl "${origin}" ./recipes/ |grep -v \.git |xargs -r sed -i "s,${origin},${new},g" - echo -done diff --git a/50-deploy.sh b/50-deploy.sh deleted file mode 100755 index f4b0b27..0000000 --- a/50-deploy.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -PYBOMBS_MIRROR_WORK_DIR=$(pwd) -PYBOMBS_SITE_BASE=/pybombs - -if [ ! -e recipe-repos.urls ]; then - echo "No recipe-repos.urls found! Exit." - exit -1 -fi - -if [ ! -d recipes ]; then - echo "No target recipes directory found! Exit." - exit -1 -fi - - -## Make .git bare repos for recipes. - -if [ -d _recipes_bare ]; then - rm -rf _recipes_bare - echo "Purge previous _recipes_bare directory." -fi - -mkdir _recipes_bare - -cat recipe-repos.urls | while read REPO_URL REPO_NAME -do - if [ -e recipes/${REPO_NAME} ]; then - cd recipes/${REPO_NAME}; - git commit -am "PyBOMBS Mirror Replacement: $(date)" - else - echo "recipes/${REPO_NAME} doesn't exist!" - fi - - cd ${PYBOMBS_MIRROR_WORK_DIR} - git clone --mirror recipes/${REPO_NAME} _recipes_bare/${REPO_NAME}.git -done - -#RSYNC_PARAM="rsync -av --no-perms --no-owner --no-group --delete" -RSYNC_PARAM="-prv --delete" - -rsync $RSYNC_PARAM git/ ${PYBOMBS_SITE_BASE}/git/ -rsync $RSYNC_PARAM wget/ ${PYBOMBS_SITE_BASE}/wget/ -rsync $RSYNC_PARAM svn/ ${PYBOMBS_SITE_BASE}/svn/ -rsync $RSYNC_PARAM _recipes_bare/ ${PYBOMBS_SITE_BASE}/recipes/ -rsync $RSYNC_PARAM recipes-mirror-replacement.urls ${PYBOMBS_SITE_BASE}/ -rsync $RSYNC_PARAM pre-replace-upstream.urls ${PYBOMBS_SITE_BASE}/ - -echo "maybe you need to : sudo chown -R www-data:www-data ${PYBOMBS_SITE_BASE}" diff --git a/README.md b/README.md index bbc2747..c495a2d 100644 --- a/README.md +++ b/README.md @@ -7,25 +7,24 @@ ```bash sudo apt-get install fcgiwrap nginx git svn wget -./10-retrieve-urls-from-recipes.sh -./20-fetch.sh - # Or test drive without actually fetching # DRY_RUN=true ./20-fetch.sh export PYBOMBS_MIRROR_BASE_URL="http://yoursite.example.com/pybombs" -./30-replace-recipes.sh +export DRY_RUN=true +export PYBOMBS_MIRROR_WORK_DIR=/home/scateu/pybombs-mirror-site + +./pybombs-mirror.sh + +cp ./nginx.conf /etc/nginx/sites-available/default -cp ./40-nginx.conf /etc/nginx/sites-available/default sudo /etc/init.d/nginx restart sudo mkdir /pybombs sudo gpasswd -a yourid www-data # then logout and in sudo chown www-data:www-data /pybombs -./50-deploy.sh ``` - - - `recipe-repos.urls` Recipes repos to be fetched. + - `upstream-recipe-repos.urls` Recipes repos to be fetched. - *(optional)* `ignore.urls` defines urls that will be ignored. Users of these URLs will be passed through upstream. - *(optional)* `pre-replace-upstream.urls` defines custom upstreams replacement. In order to gain better syncing speed according to your network condition. @@ -79,18 +78,22 @@ See also: recipes-origin/ + Upstream recipes + | + | [git clone/update] + v + upstream-recipe-repos.urls --> recipes-origin/ | | [Copy] v - recipes/ + _recipes/ | pre-replace-upstream.urls ----> + | [grep & sed] v - recipes/ + _recipes/ | | [grep] v @@ -101,7 +104,7 @@ pre-replace-upstream.urls ----> + v recipes-origin.urls | -20-fetch.sh............................|......................................... +20-fetch...............................|......................................... | +--> svn/ v | [Fetch..] -------+--> wget/ @@ -112,60 +115,25 @@ pre-replace-upstream.urls ----> + _recipes-mirror-replacement.urls failed.log (PYBOMBS_MIRROR_BASE_URL as placeholder) | -30-replace-recipes.sh...........|................................................ +30-replace-_recipes..............|................................................ | [sed] PYBOMBS_MIRROR_BASE_URL -> $PYBOMBS_MIRROR_BASE_URL | v - recipes-mirror-replacement.urls -- [sed] --> recipes/ + recipes-mirror-replacement.urls -- [sed] --> _recipes/ | -50-deploy.sh............................................|........................ +40-deploy...............................................|........................ | [git commit -am ] | [git clone --bare] v - _recipes_bare/ + recipes/ + | | - | [rsync] v DONE. ``` -## 10-retrieve-urls-from-recipes.sh - - - git clone or update repos in `recipe-repos.urls` from upstream into `recipes-origin` directory. - - Copy `recipes-origin` to `recipes` - - Patching recipes in `recipes` directory with custom urls from `pre-replace-upstream.urls` - - Generate `recipes-origin.urls` list from `recipes` directory. In fact, it greps all git/svn/wget urls. - - Remove `ignore.urls` from `recipes-origin.urls` - -## 20-fetch.sh - - Fetch repos one by one from `recipes-origin.urls`. - - You can also set `DRY_RUN=true` to get a test drive without actually fetching data. - -``` -$ DRY_RUN=true ./20-fetch.sh -``` - - - Git: All git repos are cloned with `--mirror` argument into `git/` subdirectory. - - Wget: All wget URLs are fetch into `wget/` subdirectory with 3 max tries. Files are stored without directory hierarchy. - - SVN: All svn repos are cloned into `svn/` subdirectory. - - - Failure: Any failure will be logged in `failed.log`. - -**Output** - - This script will generate a `recipes-mirror-replacement.urls` list containing: - - - -## 30-replace-recipes.sh - - Replace URLs in `recipes/` directory using `recipes-mirror-replacement.urls`, then you can publish this directory to your users. - - ## Build a PyBOMBS totally from upstream PyBOMBS mirror ``` diff --git a/40-nginx.conf b/nginx.conf similarity index 100% rename from 40-nginx.conf rename to nginx.conf diff --git a/pybombs-mirror.sh b/pybombs-mirror.sh new file mode 100755 index 0000000..2e13777 --- /dev/null +++ b/pybombs-mirror.sh @@ -0,0 +1,280 @@ +#!/bin/bash + +if [ -z $PYBOMBS_MIRROR_WORK_DIR ]; then + PYBOMBS_MIRROR_WORK_DIR=$(pwd) +fi + +if [ -z $DRY_RUN ]; then + DRY_RUN=false # when set to true, nothing will be fetched, for debug purpose +fi + +# Currently, all mirrored repo are served through http or https: +# like: +# git+http, svn+http, wget+http +# or: +# git+https, svn+https, wget+https +if [ -z ${PYBOMBS_MIRROR_BASE_URL} ]; then + PYBOMBS_MIRROR_BASE_URL="http://localhost/pybombs" # *NO* tailing / should be added. + echo "No PYBOMBS_MIRROR_BASE_URL passed in, using ${PYBOMBS_MIRROR_BASE_URL}" +fi + + +#======================================================== + +function delete_directory_if_exist() { +if [ -d $1 ]; then + echo "Directory $1 already exists, clean it" + rm -rf $1 +fi +} + +function delete_file_if_exist() { +if [ -e $1 ]; then + echo "File $1 already exists, clean it" + rm $1 +fi +} + +function require_file_existance() { +if [ ! -e $1 ]; then + echo "No file $1 found! Exit." + exit -1 +fi +} + +function require_directory_existance() { +if [ ! -d $1 ]; then + echo "No directory $1 found! Exit." + exit -1 +fi +} + +function make_clean_dir(){ +if [ -d $1 ]; then + rm -rf $1 + echo "Purge existed $1 directory." +fi +echo "mkdir $1" +mkdir $1 +} + + + +#10-retrieve-urls-from-recipes: +#============= + +require_file_existance ${PYBOMBS_MIRROR_WORK_DIR}/upstream-recipe-repos.urls + +cat ${PYBOMBS_MIRROR_WORK_DIR}/upstream-recipe-repos.urls | while read REPO_URL REPO_NAME +do + if [ ! -d ${PYBOMBS_MIRROR_WORK_DIR}/recipes-origin ]; then + mkdir ${PYBOMBS_MIRROR_WORK_DIR}/recipes-origin + fi + + if [ ! -d ${PYBOMBS_MIRROR_WORK_DIR}/recipes-origin/${REPO_NAME} ]; then + echo "No recipes-origin/${REPO_NAME} found. Fetching..." + cd ${PYBOMBS_MIRROR_WORK_DIR}/recipes-origin/ + echo "git clone ${REPO_URL} ${REPO_NAME}" + git clone ${REPO_URL} ${REPO_NAME} + else + echo "Git updating recipes..." + cd ${PYBOMBS_MIRROR_WORK_DIR}/recipes-origin/${REPO_NAME} + echo "Updating ${REPO_NAME}" + git pull + fi + + cd ${PYBOMBS_MIRROR_WORK_DIR} +done + + +delete_directory_if_exist ${PYBOMBS_MIRROR_WORK_DIR}/_recipes +cp -r ${PYBOMBS_MIRROR_WORK_DIR}/recipes-origin ${PYBOMBS_MIRROR_WORK_DIR}/_recipes + +if [ -e ${PYBOMBS_MIRROR_WORK_DIR}/pre-replace-upstream.urls ]; then + echo "====================" + echo "pre-replace-upstream.urls found!" + echo "Patching _recipes with custom urls" + + cat ${PYBOMBS_MIRROR_WORK_DIR}/pre-replace-upstream.urls | while read origin new + do + echo "Replacing ${origin} with ${new}" + grep -rl "${origin}" ${PYBOMBS_MIRROR_WORK_DIR}/_recipes/ |grep -v \.git |xargs -r sed -i "s,${origin},${new},g" + done + echo "Patching _recipes done!" + echo "====================" + echo +fi + + +echo "====================" +grep -r -E "(git|svn|wget)\+.*$" ${PYBOMBS_MIRROR_WORK_DIR}/_recipes/ | rev |cut -d' ' -f1 |rev > ${PYBOMBS_MIRROR_WORK_DIR}/recipes-origin.urls +echo "recipes-origin.urls generated successfully!" +echo "====================" +echo + + +if [ -e ${PYBOMBS_MIRROR_WORK_DIR}/ignore.urls ]; then + echo "====================" + echo "ignore.urls found!" + echo "Remove ignored urls from recipes-origin.urls" + echo "Removed lines won't be fetched or replaced, mirror users can still fetch that repo directly from upstream." + cat ${PYBOMBS_MIRROR_WORK_DIR}/ignore.urls | while read url + do + echo "Deleting line contains ${url}" + sed -i "\\,${url},d" ${PYBOMBS_MIRROR_WORK_DIR}/recipes-origin.urls + done + echo "recipes-origin.urls patched!" + echo "====================" +fi + +#20-fetch: +#============= + +require_file_existance ${PYBOMBS_MIRROR_WORK_DIR}/recipes-origin.urls + +delete_file_if_exist ${PYBOMBS_MIRROR_WORK_DIR}/_recipes-mirror-replacement.urls + +if [ ! -d ${PYBOMBS_MIRROR_WORK_DIR}/git ]; then + mkdir ${PYBOMBS_MIRROR_WORK_DIR}/git +fi +if [ ! -d ${PYBOMBS_MIRROR_WORK_DIR}/svn ]; then + mkdir ${PYBOMBS_MIRROR_WORK_DIR}/svn +fi +if [ ! -d ${PYBOMBS_MIRROR_WORK_DIR}/wget ]; then + mkdir ${PYBOMBS_MIRROR_WORK_DIR}/wget +fi + +if [ -e ${PYBOMBS_MIRROR_WORK_DIR}/failed.log ]; then + echo "Previous failed.log found! Roll it to failed.log.1 ." + mv ${PYBOMBS_MIRROR_WORK_DIR}/failed.log ${PYBOMBS_MIRROR_WORK_DIR}/failed.log.1 + echo +fi + + +cat ${PYBOMBS_MIRROR_WORK_DIR}/recipes-origin.urls | sed 's/+/ /' | while read protocol url +do + echo "=====================" + echo "ENTERING: $PYBOMBS_MIRROR_WORK_DIR" + cd $PYBOMBS_MIRROR_WORK_DIR + + ORIGIN_PYBOMBS_URL="$protocol+$url" + echo "ORGIGIN_PYBOMBS_URL: $ORIGIN_PYBOMBS_URL" + #TARGET_PATH="$protocol/$(basename $url)" + + # If duplication of name like git+git://git.code.sf.net/p/openlte/code --> code happens + # You can use the following name scheme (Not beautiful... indeed...) + + TARGET_PATH="$protocol/$(basename $(dirname $url))_$(basename $url)" + + echo "UPSTREAM: $url" + MIRROR_PYBOMBS_URL="${protocol}+PYBOMBS_MIRROR_BASE_URL/${TARGET_PATH}" + echo "MIRROR_PYBOMBS_URL: ${MIRROR_PYBOMBS_URL}" + + + FETCHING_SUCCESS=true #predefine status + if [ $protocol = "wget" ]; then + if [ -e ${PYBOMBS_MIRROR_WORK_DIR}/$TARGET_PATH ]; then + echo "${PYBOMBS_MIRROR_WORK_DIR}/$TARGET_PATH exists, skipping" + else + if [ -e ${PYBOMBS_MIRROR_WORK_DIR}/${TARGET_PATH}.tmp ]; then + echo "${PYBOMBS_MIRROR_WORK_DIR}/${TARGET_PATH}.tmp exists.. remove it." + rm ${PYBOMBS_MIRROR_WORK_DIR}/${TARGET_PATH}.tmp + fi + echo "EXECUTING: wget $url -O ${PYBOMBS_MIRROR_WORK_DIR}/${TARGET_PATH}.tmp" + if [ ! $DRY_RUN = true ]; then + wget $url --tries=3 -O ${PYBOMBS_MIRROR_WORK_DIR}/${TARGET_PATH}.tmp + if [ $? -eq 0 ]; then + echo "Fetching done. Renaming ${PYBOMBS_MIRROR_WORK_DIR}/${TARGET_PATH}.tmp to ${PYBOMBS_MIRROR_WORK_DIR}/${TARGET_PATH}" + mv ${PYBOMBS_MIRROR_WORK_DIR}/${TARGET_PATH}.tmp ${PYBOMBS_MIRROR_WORK_DIR}/${TARGET_PATH} + else + FETCHING_SUCCESS=false + fi + fi + fi + elif [ $protocol = "git" ]; then + if [ -d ${PYBOMBS_MIRROR_WORK_DIR}/$TARGET_PATH ]; then + echo "${PYBOMBS_MIRROR_WORK_DIR}/$TARGET_PATH exists, syncing" + cd ${PYBOMBS_MIRROR_WORK_DIR}/$TARGET_PATH + + if [ ! $DRY_RUN = true ]; then + /usr/bin/timeout -s INT 3600 git remote -v update || FETCHING_SUCCESS=false + git repack -a -d + fi + else + echo "EXECUTING: git clone --mirror $url ${PYBOMBS_MIRROR_WORK_DIR}/$TARGET_PATH" + if [ ! $DRY_RUN = true ]; then + git clone --mirror $url ${PYBOMBS_MIRROR_WORK_DIR}/$TARGET_PATH || FETCHING_SUCCESS=false + fi + fi + elif [ $protocol = "svn" ]; then + + if [ ! $DRY_RUN = true ]; then + echo "TODO" + FETCHING_SUCCESS=false + fi + fi + + + cd $PYBOMBS_MIRROR_WORK_DIR + if [ $FETCHING_SUCCESS = true ]; then + echo "${ORIGIN_PYBOMBS_URL} ${MIRROR_PYBOMBS_URL}" >> ${PYBOMBS_MIRROR_WORK_DIR}/_recipes-mirror-replacement.urls + echo "${ORIGIN_PYBOMBS_URL} fetching success!" + else + echo "${ORIGIN_PYBOMBS_URL}" >> failed.log + echo "${ORIGIN_PYBOMBS_URL} fetching failed" + fi + + echo "=====================" + echo + +done + +#30-replace-_recipes: +#============= + +require_file_existance ${PYBOMBS_MIRROR_WORK_DIR}/_recipes-mirror-replacement.urls +require_directory_existance ${PYBOMBS_MIRROR_WORK_DIR}/_recipes + +echo "Replacing PYBOMBS_MIRROR_BASE_URL with ${PYBOMBS_MIRROR_BASE_URL} in _recipes-mirror-replacement.urls .." +cp ${PYBOMBS_MIRROR_WORK_DIR}/_recipes-mirror-replacement.urls ${PYBOMBS_MIRROR_WORK_DIR}/_recipes-mirror-replacement.urls +sed -i "s,PYBOMBS_MIRROR_BASE_URL,${PYBOMBS_MIRROR_BASE_URL},g" _recipes-mirror-replacement.urls +echo "Done." + + +cat ${PYBOMBS_MIRROR_WORK_DIR}/_recipes-mirror-replacement.urls | while read origin new +do + echo "${origin} --> ${new}" + grep -rl "${origin}" ${PYBOMBS_MIRROR_WORK_DIR}/_recipes/ |grep -v \.git |xargs -r sed -i "s,${origin},${new},g" + echo +done + +#40-deploy: +#============= + +require_file_existance ${PYBOMBS_MIRROR_WORK_DIR}/upstream-recipe-repos.urls +require_directory_existance ${PYBOMBS_MIRROR_WORK_DIR}/_recipes + +## Make .git bare repos for recipes. + +make_clean_dir ${PYBOMBS_MIRROR_WORK_DIR}/recipes + +cat ${PYBOMBS_MIRROR_WORK_DIR}/upstream-recipe-repos.urls | while read REPO_URL REPO_NAME +do + if [ -e ${PYBOMBS_MIRROR_WORK_DIR}/_recipes/${REPO_NAME} ]; then + cd ${PYBOMBS_MIRROR_WORK_DIR}/_recipes/${REPO_NAME}; + git commit -am "PyBOMBS Mirror Replacement: $(date)" + else + echo "_recipes/${REPO_NAME} doesn't exist!" + fi + + git clone --mirror ${PYBOMBS_MIRROR_WORK_DIR}/_recipes/${REPO_NAME} ${PYBOMBS_MIRROR_WORK_DIR}/recipes/${REPO_NAME}.git +done + +echo "Cleaning up" + +delete_file_if_exist ${PYBOMBS_MIRROR_WORK_DIR}/_recipes-mirror-replacement.urls + + +echo "maybe you need to : sudo chown -R www-data:www-data ${PYBOMBS_SITE_BASE}" + +cd ${PYBOMBS_MIRROR_WORK_DIR} diff --git a/recipe-repos.urls b/upstream-recipe-repos.urls similarity index 100% rename from recipe-repos.urls rename to upstream-recipe-repos.urls