Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

s3 binaries

  • Loading branch information...
commit 1f0e1ad680e111a5df47a563b0eaed23d3f6f256 1 parent be605ae
Terence Lee authored September 23, 2011
79  support/s3/hmac
... ...
@@ -0,0 +1,79 @@
  1
+#!/bin/bash
  2
+# Implement HMAC functionality on top of the OpenSSL digest functions.
  3
+# licensed under the terms of the GNU GPL v2
  4
+# Copyright 2007 Victor Lowther <victor.lowther@gmail.com>
  5
+
  6
+die() {
  7
+  echo $*
  8
+  exit 1
  9
+}
  10
+
  11
+check_deps() {
  12
+  local res=0
  13
+  while [ $# -ne 0 ]; do
  14
+    which "${1}" >& /dev/null || { res=1; echo "${1} not found."; }
  15
+    shift
  16
+  done
  17
+  (( res == 0 )) || die "aborting."
  18
+}
  19
+
  20
+# write a byte (passed as hex) to stdout
  21
+write_byte() {
  22
+  # $1 = byte to write
  23
+  printf "\\x$(printf "%x" ${1})"
  24
+}
  25
+
  26
+# make an hmac pad out of a key.
  27
+# this is not the most secure way of doing it, but it is
  28
+# the most expedient.
  29
+make_hmac_pad() {
  30
+  # using key in file $1 and byte in $2, create the appropriate hmac pad
  31
+  # Pad keys out to $3 bytes
  32
+  # if key is longer than $3, use hash $4 to hash the key first.
  33
+  local x y a size remainder oifs
  34
+  (( remainder = ${3} ))
  35
+  # in case someone else was messing with IFS.
  36
+  for x in $(echo -n "${1}" | od -v -t u1 | cut -b 9-);
  37
+  do
  38
+    write_byte $((${x} ^ ${2}))
  39
+    (( remainder -= 1 ))
  40
+  done
  41
+  for ((y=0; remainder - y ;y++)); do
  42
+    write_byte $((0 ^ ${2}))
  43
+  done
  44
+}
  45
+
  46
+# utility functions for making hmac pads
  47
+hmac_ipad() {
  48
+  make_hmac_pad "${1}" 0x36 ${2} "${3}"
  49
+}
  50
+
  51
+hmac_opad() {
  52
+  make_hmac_pad "${1}" 0x5c ${2} "${3}"
  53
+}
  54
+
  55
+# hmac something
  56
+do_hmac() {
  57
+  # $1 = algo to use.  Must be one that openssl knows about
  58
+  # $2 = keyfile to use
  59
+  # $3 = file to hash.  uses stdin if none is given.
  60
+  # accepts input on stdin, leaves it on stdout.
  61
+  # Output is binary, if you want something else pipe it accordingly.
  62
+  local blocklen keysize x
  63
+  case "${1}" in
  64
+    sha) blocklen=64 ;;
  65
+    sha1) blocklen=64 ;;
  66
+    md5) blocklen=64 ;;
  67
+    md4) blocklen=64 ;;
  68
+    sha256) blocklen=64 ;;
  69
+    sha512) blocklen=128 ;;
  70
+    *) die "Unknown hash ${1} passed to hmac!" ;;
  71
+  esac
  72
+  cat <(hmac_ipad ${2} ${blocklen} "${1}") "${3:--}" | openssl dgst "-${1}" -binary | \
  73
+  cat <(hmac_opad ${2} ${blocklen} "${1}") - | openssl dgst "-${1}" -binary
  74
+}
  75
+
  76
+[[ ${1} ]] || die "Must pass the name of the hash function to use to ${0}".
  77
+
  78
+check_deps od openssl
  79
+do_hmac "${@}"
223  support/s3/s3
... ...
@@ -0,0 +1,223 @@
  1
+#!/bin/bash
  2
+# basic amazon s3 operations
  3
+# Licensed under the terms of the GNU GPL v2
  4
+# Copyright 2007 Victor Lowther <victor.lowther@gmail.com>
  5
+
  6
+set -e
  7
+
  8
+basedir="$( cd -P "$( dirname "$0" )" && pwd )"
  9
+PATH="$basedir:$PATH"
  10
+
  11
+# print a message and bail
  12
+die() {
  13
+  echo $*
  14
+  exit 1
  15
+}
  16
+
  17
+# check to see if the variable name passed exists and holds a value.
  18
+# Die if it does not.
  19
+check_or_die() {
  20
+  [[ ${!1} ]] || die "Environment variable ${1} is not set."
  21
+}
  22
+
  23
+# check to see if we have all the needed S3 variables defined.
  24
+# Bail if we do not.
  25
+check_s3() {
  26
+  local sak x
  27
+  for x in S3_ACCESS_KEY_ID S3_SECRET_ACCESS_KEY; do
  28
+    check_or_die ${x};
  29
+  done
  30
+  sak="$(echo -n $S3_SECRET_ACCESS_KEY | wc -c)"
  31
+  (( ${sak%%[!0-9 ]*} == 40 )) || \
  32
+    die "S3 Secret Access Key is not exactly 40 bytes long.  Please fix it."
  33
+}
  34
+# check to see if our external dependencies exist
  35
+check_dep() {
  36
+  local res=0
  37
+  while [[ $# -ne 0 ]]; do
  38
+    which "${1}" >& /dev/null || { res=1; echo "${1} not found."; }
  39
+    shift
  40
+  done
  41
+  (( res == 0 )) || die "aborting."
  42
+}
  43
+
  44
+check_deps() {
  45
+  check_dep openssl date hmac cat grep curl
  46
+  check_s3
  47
+}
  48
+
  49
+urlenc() {
  50
+  # $1 = string to url encode
  51
+  # output is on stdout
  52
+  # we don't urlencode everything, just enough stuff.
  53
+  echo -n "${1}" |
  54
+  sed 's/%/%25/g
  55
+       s/ /%20/g
  56
+       s/#/%23/g
  57
+       s/\$/%24/g
  58
+       s/\&/%26/g
  59
+       s/+/%2b/g
  60
+       s/,/%2c/g
  61
+       s/:/%3a/g
  62
+       s/;/%3b/g
  63
+       s/?/%3f/g
  64
+       s/@/%40/g
  65
+       s/	/%09/g'
  66
+}
  67
+
  68
+xmldec() {
  69
+  # no parameters.
  70
+  # accept input on stdin, put it on stdout.
  71
+  # patches accepted to get more stuff
  72
+  sed 's/\&quot;/\"/g
  73
+       s/\&amp;/\&/g
  74
+       s/\&lt;/</g
  75
+       s/\&gt;/>/g'
  76
+}
  77
+
  78
+## basic S3 functionality.  x-amz-header functionality is not implemented.
  79
+# make an S3 signature string, which will be output on stdout.
  80
+s3_signature_string() {
  81
+  # $1 = HTTP verb
  82
+  # $2 = date string, must be in UTC
  83
+  # $3 = bucket name, if any
  84
+  # $4 = resource path, if any
  85
+  # $5 = content md5, if any
  86
+  # $6 = content MIME type, if any
  87
+  # $7 = canonicalized headers, if any
  88
+  # signature string will be output on stdout
  89
+  local verr="Must pass a verb to s3_signature_string!"
  90
+  local verb="${1:?verr}"
  91
+  local bucket="${3}"
  92
+  local resource="${4}"
  93
+  local derr="Must pass a date to s3_signature_string!"
  94
+  local date="${2:?derr}"
  95
+  local mime="${6}"
  96
+  local md5="${5}"
  97
+  local headers="${7}"
  98
+  printf "%s\n%s\n%s\n%s\n%s\n%s%s" \
  99
+    "${verb}" "${md5}" "${mime}" "${date}" \
  100
+    "${headers}" "${bucket}" "${resource}" | \
  101
+    hmac sha1 "${S3_SECRET_ACCESS_KEY}" | openssl base64 -e -a
  102
+}
  103
+
  104
+# cheesy, but it is the best way to have multiple headers.
  105
+curl_headers() {
  106
+  # each arg passed will be output on its own line
  107
+  local parms=$#
  108
+  for ((;$#;)); do
  109
+    echo "header = \"${1}\""
  110
+    shift
  111
+  done
  112
+}
  113
+
  114
+s3_curl() {
  115
+  # invoke curl to do all the heavy HTTP lifting
  116
+  # $1 = method (one of GET, PUT, or DELETE. HEAD is not handled yet.)
  117
+  # $2 = remote bucket.
  118
+  # $3 = remote name
  119
+  # $4 = local name.
  120
+  local bucket remote date sig md5 arg inout headers
  121
+  # header handling is kinda fugly, but it works.
  122
+  bucket="${2:+/${2}}/" # slashify the bucket
  123
+  remote="$(urlenc "${3}")" # if you don't, strange things may happen.
  124
+  stdopts="--connect-timeout 10 --fail --silent"
  125
+  [[ $CURL_S3_DEBUG == true ]] && stdopts="${stdopts} --show-error --fail"
  126
+  case "${1}" in
  127
+   GET) arg="-o" inout="${4:--}" # stdout if no $4
  128
+	;;
  129
+   PUT) [[ ${2} ]] || die "PUT can has bucket?"
  130
+        if [[ ! ${3} ]]; then
  131
+	  arg="-X PUT"
  132
+	  headers[${#headers[@]}]="Content-Length: 0"
  133
+	elif [[ -f ${4} ]]; then
  134
+          md5="$(openssl dgst -md5 -binary "${4}"|openssl base64 -e -a)"
  135
+	  arg="-T" inout="${4}"
  136
+	  headers[${#headers[@]}]="x-amz-acl: public-read"
  137
+	  headers[${#headers[@]}]="Expect: 100-continue"
  138
+	else
  139
+	  die "Cannot write non-existing file ${4}"
  140
+        fi
  141
+	;;
  142
+   DELETE) arg="-X DELETE"
  143
+           ;;
  144
+   HEAD) arg="-I" ;;
  145
+   *) die "Unknown verb ${1}.  It probably would not have worked anyways." ;;
  146
+  esac
  147
+  date="$(TZ=UTC date '+%a, %e %b %Y %H:%M:%S %z')"
  148
+  sig=$(s3_signature_string ${1} "${date}" "${bucket}" "${remote}" "${md5}" "" "x-amz-acl:public-read")
  149
+
  150
+  headers[${#headers[@]}]="Authorization: AWS ${S3_ACCESS_KEY_ID}:${sig}"
  151
+  headers[${#headers[@]}]="Date: ${date}"
  152
+  [[ ${md5} ]] && headers[${#headers[@]}]="Content-MD5: ${md5}"
  153
+    curl ${arg} "${inout}" ${stdopts} -o - -K <(curl_headers "${headers[@]}") \
  154
+       "http://s3.amazonaws.com${bucket}${remote}"
  155
+  return $?
  156
+}
  157
+
  158
+s3_put() {
  159
+  # $1 = remote bucket to put it into
  160
+  # $2 = remote name to put
  161
+  # $3 = file to put.  This must be present if $2 is.
  162
+  s3_curl PUT "${1}" "${2}" "${3:-${2}}"
  163
+  return $?
  164
+}
  165
+
  166
+s3_get() {
  167
+  # $1 = bucket to get file from
  168
+  # $2 = remote file to get
  169
+  # $3 = local file to get into. Will be overwritten if it exists.
  170
+  #      If this contains a path, that path must exist before calling this.
  171
+  s3_curl GET "${1}" "${2}" "${3:-${2}}"
  172
+  return $?
  173
+}
  174
+
  175
+s3_test() {
  176
+  # same args as s3_get, but uses the HEAD verb instead of the GET verb.
  177
+  s3_curl HEAD "${1}" "${2}" >/dev/null
  178
+  return $?
  179
+}
  180
+
  181
+# Hideously ugly, but it works well enough.
  182
+s3_buckets() {
  183
+  s3_get |grep -o '<Name>[^>]*</Name>' |sed 's/<[^>]*>//g' |xmldec
  184
+  return $?
  185
+}
  186
+
  187
+# this will only return the first thousand entries, alas
  188
+# Mabye some kind soul can fix this without writing an XML parser in bash?
  189
+# Also need to add xml entity handling.
  190
+s3_list() {
  191
+  # $1 = bucket to list
  192
+  [ "x${1}" == "x" ] && return 1
  193
+  s3_get "${1}" |grep -o '<Key>[^>]*</Key>' |sed 's/<[^>]*>//g'| xmldec
  194
+  return $?
  195
+}
  196
+
  197
+s3_delete() {
  198
+  # $1 = bucket to delete from
  199
+  # $2 = item to delete
  200
+  s3_curl DELETE "${1}" "${2}"
  201
+  return $?
  202
+}
  203
+
  204
+# because this uses s3_list, it suffers from the same flaws.
  205
+s3_rmrf() {
  206
+  # $1 = bucket to delete everything from
  207
+  s3_list "${1}" | while read f; do
  208
+    s3_delete "${1}" "${f}";
  209
+  done
  210
+}
  211
+
  212
+check_deps
  213
+case $1 in
  214
+  put) shift; s3_put "$@" ;;
  215
+  get) shift; s3_get "$@" ;;
  216
+  rm) shift; s3_delete "$@" ;;
  217
+  ls) shift; s3_list "$@" ;;
  218
+  test) shift; s3_test "$@" ;;
  219
+  buckets) s3_buckets ;;
  220
+  rmrf) shift; s3_rmrf "$@" ;;
  221
+  *) die "Unknown command ${1}."
  222
+    ;;
  223
+esac

0 notes on commit 1f0e1ad

Please sign in to comment.
Something went wrong with that request. Please try again.