Support for APT authentication. Closes #234 (#235)

README.md

@@ -1,5 +1,5 @@
 <!---
-apt-fast v1.9
+apt-fast v1.10
 Use this just like aptitude or apt-get for faster package downloading.
 
 Copyright: 2008-2012 Matt Parnell, http://www.mattparnell.com
@@ -266,6 +266,13 @@ APTCACHE='/var/cache/apt/archives'
 Directory where apt-get and aptitude download packages.
 
 
+### APT authentication ###
+```sh
+APT_FAST_APT_AUTH=1
+```
+Try to authenticate packages from sources that require login.
+
+
 ### Verbose output ###
 ```sh
 VERBOSE_OUTPUT=y

apt-fast

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# apt-fast v1.9.12
+# apt-fast v1.10.0
 # Use this just like aptitude or apt-get for faster package downloading.
 #
 # Copyright: 2008-2012 Matt Parnell, http://www.mattparnell.com
@@ -10,6 +10,9 @@
 # Public License as published by the Free Software Foundation; either
 # version 3 of the License, or (at your option) any later version.
 #
+
+shopt -s nullglob
+
 [ -n "$DEBUG" ] && set -xv
 
 # Print colored messages.
@@ -70,14 +73,14 @@ TMP__APTMGR="${_APTMGR-${TMP_RANDOM}}"
 TMP_APTCACHE="${APTCACHE-${TMP_RANDOM}}"
 TMP_DLDIR="${DLDIR-${TMP_RANDOM}}"
 TMP_DLLIST="${DLLIST-${TMP_RANDOM}}"
-TMP_LISTDIR="${LISTDIR-${TMP_RANDOM}}"
 TMP__MAXNUM="${MAXNUM-${TMP_RANDOM}}"
 TMP__MAXCONPERSRV="${MAXCONPERSRV-${TMP_RANDOM}}"
 TMP__SPLITCON="${SPLITCON-${TMP_RANDOM}}"
 TMP__MINSPLITSZ=${MINSPLITSZ-${TMP_RANDOM}}
 TMP__PIECEALGO=${PIECEALGO-${TMP_RANDOM}}
 TMP_aptfast_prefix="${aptfast_prefix-${TMP_RANDOM}}"
 TMP_APT_FAST_TIMEOUT="${APT_FAST_TIMEOUT-${TMP_RANDOM}}"
+TMP_APT_FAST_APT_AUTH="${APT_FAST_APT_AUTH-${TMP_RANDOM}}"
 TMP_VERBOSE_OUTPUT="${VERBOSE_OUTPUT-${TMP_RANDOM}}"
 TMP_ftp_proxy="${ftp_proxy-${TMP_RANDOM}}"
 TMP_http_proxy="${http_proxy-${TMP_RANDOM}}"
@@ -93,14 +96,14 @@ if [ "$root" = 1 ] && [ "$UID" != 0 ]; then
             APTCACHE="$TMP_APTCACHE" \
             DLDIR="$TMP_DLDIR" \
             DLLIST="$TMP_DLLIST" \
-            LISTDIR="$TMP_LISTDIR" \
             _MAXNUM="$TMP__MAXNUM" \
             _MAXCONPERSRV="$TMP__MAXCONPERSRV" \
             _SPLITCON="$TMP__SPLITCON" \
             _MINSPLITSZ="$TMP__MINSPLITSZ" \
             _PIECEALGO="$TMP__PIECEALGO" \
             aptfast_prefix="$TMP_aptfast_prefix" \
             APT_FAST_TIMEOUT="$TMP_APT_FAST_TIMEOUT" \
+            APT_FAST_APT_AUTH="$TMP_APT_FAST_APT_AUTH" \
             VERBOSE_OUTPUT="$TMP_VERBOSE_OUTPUT" \
             ftp_proxy="$TMP_ftp_proxy" \
             http_proxy="$TMP_http_proxy" \
@@ -127,14 +130,23 @@ if [ -z "$apt_fast_partial" ]; then
 else
   DLDIR="${apt_fast_partial}"
 fi
-# Currently not needed.
-eval "$(apt-config shell LISTDIR Dir::State::lists/d)"
+
+# Check for apt auth files
+eval "$(apt-config shell NETRC Dir::Etc::netrc/f)"
+eval "$(apt-config shell NETRCDIR Dir::Etc::netrcparts/d)"
+APTAUTHFILES=()
+if [ -f "$NETRC" ]; then
+    APTAUTHFILES=("$NETRC")
+fi
+APTAUTHFILES+=("$NETRCDIR"*)
+
 DLLIST="/tmp/apt-fast.list"
 _MAXNUM=5
 _MAXCONPERSRV=10
 _SPLITCON=8
 _MINSPLITSZ="1M"
 _PIECEALGO="default"
+MIRRORS=()
 
 # Prefix in front of apt-fast output:
 aptfast_prefix=
@@ -153,6 +165,9 @@ APT_FAST_TIMEOUT=60
 # Ask for download confirmation if unset
 DOWNLOADBEFORE=
 
+# Enable APT authentication support
+APT_FAST_APT_AUTH=1
+
 # Formatted package list in download confirmation if unset
 VERBOSE_OUTPUT=
 
@@ -181,14 +196,14 @@ https_proxy=
 [ "$TMP_APTCACHE" = "$TMP_RANDOM" ] || APTCACHE="$TMP_APTCACHE"
 [ "$TMP_DLDIR" = "$TMP_RANDOM" ] || DLDIR="$TMP_DLDIR"
 [ "$TMP_DLLIST" = "$TMP_RANDOM" ] || DLLIST="$TMP_DLLIST"
-[ "$TMP_LISTDIR" = "$TMP_RANDOM" ] || LISTDIR="$TMP_LISTDIR"
 [ "$TMP__MAXNUM" = "$TMP_RANDOM" ] || _MAXNUM="$TMP__MAXNUM"
 [ "$TMP__MAXCONPERSRV" = "$TMP_RANDOM" ] || _MAXCONPERSRV="$TMP__MAXCONPERSRV"
 [ "$TMP__SPLITCON" = "$TMP_RANDOM" ] || _SPLITCON="$TMP__SPLITCON"
 [ "$TMP__MINSPLITSZ" = "$TMP_RANDOM" ] || _MINSPLITSZ="$TMP__MINSPLITSZ"
 [ "$TMP__PIECEALGO" = "$TMP_RANDOM" ] || _PIECEALGO="$TMP__PIECEALGO"
 [ "$TMP_aptfast_prefix" = "$TMP_RANDOM" ] || aptfast_prefix="$TMP_aptfast_prefix"
 [ "$TMP_APT_FAST_TIMEOUT" = "$TMP_RANDOM" ] || APT_FAST_TIMEOUT="$TMP_APT_FAST_TIMEOUT"
+[ "$TMP_APT_FAST_APT_AUTH" = "$TMP_RANDOM" ] || APT_FAST_APT_AUTH="$TMP_APT_FAST_APT_AUTH"
 [ "$TMP_VERBOSE_OUTPUT" = "$TMP_RANDOM" ] || VERBOSE_OUTPUT="$TMP_VERBOSE_OUTPUT"
 [ "$TMP_ftp_proxy" = "$TMP_RANDOM" ] || ftp_proxy="$TMP_ftp_proxy"
 [ "$TMP_http_proxy" = "$TMP_RANDOM" ] || http_proxy="$TMP_http_proxy"
@@ -284,7 +299,7 @@ get_mirrors(){
     for mirror in "${mirrors[@]}"; do
       # Real expension.
       if [[ "$1" == "$mirror"* ]]; then
-        filepath=${1#${mirror}}
+        filepath="${1#"${mirror}"}"
         # Build list for aria download list.
         list="${mirrors[*]}"
         echo -e "${list// /${filepath}\\t}$filepath\n"
@@ -296,6 +311,57 @@ get_mirrors(){
   echo "$1"
 }
 
+AUTH_INFO_PARSED=()
+# Parse apt authentication files.
+# Undefined behavior on whitespaces in host, username or password.
+prepare_auth(){
+  if [ "$APT_FAST_APT_AUTH" -eq 0 ]; then
+    return
+  fi
+  for auth_file in "${APTAUTHFILES[@]}"; do
+    # auth files have netrc syntax, possible multiline entries starting with "machine"
+    auth_info="$(tr '\n' ' ' < "$auth_file" | sed 's/\(\<machine\>\)/\n\1/g' | sed '1d')"
+    while IFS= read -r auth; do
+      machine="$(echo "$auth" | sed 's/.*\<machine\>[ \t]\+\([^ \t]\+\).*/\1/')"
+      login="$(echo "$auth" | sed 's/.*\<login\>[ \t]\+\([^ \t]\+\).*/\1/')"
+      password="$(echo "$auth" | sed 's/.*\<password\>[ \t]\+\([^ \t]\+\).*/\1/')"
+      # if machine does not have protocol, try https://
+      if ! [[ "$machine" =~ ^.*:// ]]; then
+        machine="https://$machine"
+      fi
+      if [ -z "$machine" ] || [ -z "$login" ] || [ -z "$password" ]; then
+        msg "Could not parse apt authentication (skipping): $auth ($auth_file)" "warning"
+        continue
+      fi
+      # use space separated string to convert back to array later
+      AUTH_INFO_PARSED+=("$machine $login $password")
+    done <<< "$auth_info"
+  done
+}
+
+# Gets URI as parameter and tries to add basic http credentials. Will fail on
+# credentials that contain characters that need URL-encoding.
+get_auth(){
+  if [ "$APT_FAST_APT_AUTH" -eq 0 ]; then
+    echo "$1"
+    return
+  fi
+  for auth_info in "${AUTH_INFO_PARSED[@]}"; do
+    # convert to array, don't escape variable here
+    auth_info_arr=($auth_info)
+    machine="${auth_info_arr[0]}"
+    # takes first match
+    if [[ "$1" == "$machine"* ]]; then
+      login="${auth_info_arr[1]}"
+      password="${auth_info_arr[2]}"
+      uri="$(echo "$1" | sed "s|^\([^:]\+://\)|\1$login:$password@|")"
+      echo "$uri"
+      return
+    fi
+  done
+  echo "$1"
+}
+
 # Globals to save package name, version, size and overall size.
 DOWNLOAD_DISPLAY=
 DOWNLOAD_SIZE=0
@@ -333,12 +399,13 @@ get_uris(){
     msg "Package manager quit with exit code." "warning"
     exit
   fi
+  prepare_auth
   while read -r pkg_uri_info
   do
     [ -z "$pkg_uri_info" ] && continue
     ## --print-uris format is:
     # 'fileurl' filename filesize checksum_hint:filechecksum
-    uri="$(echo "$pkg_uri_info" | cut -d' ' -f1 | tr -d "'")"
+    uri="$(get_auth "$(echo "$pkg_uri_info" | cut -d' ' -f1 | tr -d "'")")"
     filename="$(echo "$pkg_uri_info" | cut -d' ' -f2)"
     filesize="$(echo "$pkg_uri_info" | cut -d' ' -f3)"
     checksum_string="$(echo "$pkg_uri_info" | cut -d' ' -f4)"
@@ -508,6 +575,8 @@ while true; do
       d)
         DOWNLOAD_ONLY=true
         ;;
+      *)
+        ;;
     esac
   done
   ((OPTIND++))

apt-fast.conf

@@ -136,6 +136,13 @@
 #APTCACHE=/var/cache/apt/archives
 
 
+# APT authentication support. Uses /etc/apt/auth.conf and /etc/apt/auth.conf.d/* authentication files.
+#
+# Default 1 (enabled)
+#
+#APT_FAST_APT_AUTH=1
+
+
 # apt-fast colors
 # Colors are disabled when not using a terminal.
 #

man/apt-fast.conf.5

@@ -152,6 +152,11 @@ Time in seconds before apt-fast download confirmation dialog times out.
 .br
 Default: 60
 .TP
+\fBAPT_FAST_APT_AUTH\fR
+APT authentication support. Uses /etc/apt/auth.conf and /etc/apt/auth.conf.d/* authentication files.
+.br
+Default: 1 (enabled)
+.TP
 \fBVERBOSE_OUTPUT\fR
 Before download confirmation display aria2 download file if this variable is set. Otherwise display formatted package listing.
 .br