Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

initial

  • Loading branch information...
commit c8b5a84cff56d4e884530d9f3ea2fe03fcabc56e 0 parents
Bernd Dorn dobe authored
10 README.rst
Source Rendered
... ... @@ -0,0 +1,10 @@
  1 +=====================
  2 +Lovely NGINX examples
  3 +=====================
  4 +
  5 +This project is supposed to hold variaous nginx example configurations
  6 +for specific use cases.
  7 +
  8 +The ``s3`` directory contains information on how to create a
  9 +transparent authenticating proxy to private amazon s3 buckets/objects.
  10 +
10 s3/.gitignore
... ... @@ -0,0 +1,10 @@
  1 +bin/
  2 +develop-eggs/
  3 +*.egg-info/
  4 +.installed.cfg
  5 +*.pyc
  6 +*.pyo
  7 +parts/
  8 +nginx/nginx.conf
  9 +
  10 +
48 s3/README.rst
Source Rendered
... ... @@ -0,0 +1,48 @@
  1 +===========================================
  2 +S3 Authentication Proxy Example using Nginx
  3 +===========================================
  4 +
  5 +This is a short example on how to create a transparent authentication
  6 +proxy to a private s3 bucket.
  7 +
  8 +The main feature used is the set-misc-module's set_hmac_sha1 and
  9 +set_encode_base64 directives. Take a look at the example config file
  10 +template in ``nginx/nginx.conf.in``.
  11 +
  12 +Setup (MAC)
  13 +===========
  14 +
  15 +Your environment will need to contain your AWS credentials in the following
  16 +two environment variables: ``AWS_ACCESS_KEY`` and ``AWS_SECRET_KEY``.
  17 +
  18 +Additional required libraries to the ones need to build nginx are
  19 +openssl and lua.
  20 +
  21 +Also this example uses buildout to download and build nginx with all
  22 +required modules, which requires a python interpreter.
  23 +
  24 +Run bootstrap::
  25 +
  26 + python bootstrap.py --distribute
  27 +
  28 +And run buildout::
  29 +
  30 + ./bin/buildout
  31 +
  32 +If your AWS credentials change after running the buildout initially,
  33 +you need to re-run the buildout.
  34 +
  35 +Testing
  36 +=======
  37 +
  38 +After the build you can start nginx in the foreground with::
  39 +
  40 + ./bin/nginx
  41 +
  42 +The generated config file resides in ``./nginx/nginx.conf``
  43 +
  44 +To issue a get request via curl to your s3 bucket use the following
  45 +url pattern::
  46 +
  47 + curl http://localhost:19999/<bucket-name>/<path-to-object>
  48 +
260 s3/bootstrap.py
... ... @@ -0,0 +1,260 @@
  1 +##############################################################################
  2 +#
  3 +# Copyright (c) 2006 Zope Foundation and Contributors.
  4 +# All Rights Reserved.
  5 +#
  6 +# This software is subject to the provisions of the Zope Public License,
  7 +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
  8 +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
  9 +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  10 +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
  11 +# FOR A PARTICULAR PURPOSE.
  12 +#
  13 +##############################################################################
  14 +"""Bootstrap a buildout-based project
  15 +
  16 +Simply run this script in a directory containing a buildout.cfg.
  17 +The script accepts buildout command-line options, so you can
  18 +use the -c option to specify an alternate configuration file.
  19 +"""
  20 +
  21 +import os, shutil, sys, tempfile, textwrap, urllib, urllib2, subprocess
  22 +from optparse import OptionParser
  23 +
  24 +if sys.platform == 'win32':
  25 + def quote(c):
  26 + if ' ' in c:
  27 + return '"%s"' % c # work around spawn lamosity on windows
  28 + else:
  29 + return c
  30 +else:
  31 + quote = str
  32 +
  33 +# See zc.buildout.easy_install._has_broken_dash_S for motivation and comments.
  34 +stdout, stderr = subprocess.Popen(
  35 + [sys.executable, '-Sc',
  36 + 'try:\n'
  37 + ' import ConfigParser\n'
  38 + 'except ImportError:\n'
  39 + ' print 1\n'
  40 + 'else:\n'
  41 + ' print 0\n'],
  42 + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
  43 +has_broken_dash_S = bool(int(stdout.strip()))
  44 +
  45 +# In order to be more robust in the face of system Pythons, we want to
  46 +# run without site-packages loaded. This is somewhat tricky, in
  47 +# particular because Python 2.6's distutils imports site, so starting
  48 +# with the -S flag is not sufficient. However, we'll start with that:
  49 +if not has_broken_dash_S and 'site' in sys.modules:
  50 + # We will restart with python -S.
  51 + args = sys.argv[:]
  52 + args[0:0] = [sys.executable, '-S']
  53 + args = map(quote, args)
  54 + os.execv(sys.executable, args)
  55 +# Now we are running with -S. We'll get the clean sys.path, import site
  56 +# because distutils will do it later, and then reset the path and clean
  57 +# out any namespace packages from site-packages that might have been
  58 +# loaded by .pth files.
  59 +clean_path = sys.path[:]
  60 +import site
  61 +sys.path[:] = clean_path
  62 +for k, v in sys.modules.items():
  63 + if k in ('setuptools', 'pkg_resources') or (
  64 + hasattr(v, '__path__') and
  65 + len(v.__path__)==1 and
  66 + not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))):
  67 + # This is a namespace package. Remove it.
  68 + sys.modules.pop(k)
  69 +
  70 +is_jython = sys.platform.startswith('java')
  71 +
  72 +setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py'
  73 +distribute_source = 'http://python-distribute.org/distribute_setup.py'
  74 +
  75 +# parsing arguments
  76 +def normalize_to_url(option, opt_str, value, parser):
  77 + if value:
  78 + if '://' not in value: # It doesn't smell like a URL.
  79 + value = 'file://%s' % (
  80 + urllib.pathname2url(
  81 + os.path.abspath(os.path.expanduser(value))),)
  82 + if opt_str == '--download-base' and not value.endswith('/'):
  83 + # Download base needs a trailing slash to make the world happy.
  84 + value += '/'
  85 + else:
  86 + value = None
  87 + name = opt_str[2:].replace('-', '_')
  88 + setattr(parser.values, name, value)
  89 +
  90 +usage = '''\
  91 +[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
  92 +
  93 +Bootstraps a buildout-based project.
  94 +
  95 +Simply run this script in a directory containing a buildout.cfg, using the
  96 +Python that you want bin/buildout to use.
  97 +
  98 +Note that by using --setup-source and --download-base to point to
  99 +local resources, you can keep this script from going over the network.
  100 +'''
  101 +
  102 +parser = OptionParser(usage=usage)
  103 +parser.add_option("-v", "--version", dest="version",
  104 + help="use a specific zc.buildout version")
  105 +parser.add_option("-d", "--distribute",
  106 + action="store_true", dest="use_distribute", default=False,
  107 + help="Use Distribute rather than Setuptools.")
  108 +parser.add_option("--setup-source", action="callback", dest="setup_source",
  109 + callback=normalize_to_url, nargs=1, type="string",
  110 + help=("Specify a URL or file location for the setup file. "
  111 + "If you use Setuptools, this will default to " +
  112 + setuptools_source + "; if you use Distribute, this "
  113 + "will default to " + distribute_source +"."))
  114 +parser.add_option("--download-base", action="callback", dest="download_base",
  115 + callback=normalize_to_url, nargs=1, type="string",
  116 + help=("Specify a URL or directory for downloading "
  117 + "zc.buildout and either Setuptools or Distribute. "
  118 + "Defaults to PyPI."))
  119 +parser.add_option("--eggs",
  120 + help=("Specify a directory for storing eggs. Defaults to "
  121 + "a temporary directory that is deleted when the "
  122 + "bootstrap script completes."))
  123 +parser.add_option("-t", "--accept-buildout-test-releases",
  124 + dest='accept_buildout_test_releases',
  125 + action="store_true", default=False,
  126 + help=("Normally, if you do not specify a --version, the "
  127 + "bootstrap script and buildout gets the newest "
  128 + "*final* versions of zc.buildout and its recipes and "
  129 + "extensions for you. If you use this flag, "
  130 + "bootstrap and buildout will get the newest releases "
  131 + "even if they are alphas or betas."))
  132 +parser.add_option("-c", None, action="store", dest="config_file",
  133 + help=("Specify the path to the buildout configuration "
  134 + "file to be used."))
  135 +
  136 +options, args = parser.parse_args()
  137 +
  138 +# if -c was provided, we push it back into args for buildout's main function
  139 +if options.config_file is not None:
  140 + args += ['-c', options.config_file]
  141 +
  142 +if options.eggs:
  143 + eggs_dir = os.path.abspath(os.path.expanduser(options.eggs))
  144 +else:
  145 + eggs_dir = tempfile.mkdtemp()
  146 +
  147 +if options.setup_source is None:
  148 + if options.use_distribute:
  149 + options.setup_source = distribute_source
  150 + else:
  151 + options.setup_source = setuptools_source
  152 +
  153 +if options.accept_buildout_test_releases:
  154 + args.append('buildout:accept-buildout-test-releases=true')
  155 +args.append('bootstrap')
  156 +
  157 +try:
  158 + import pkg_resources
  159 + import setuptools # A flag. Sometimes pkg_resources is installed alone.
  160 + if not hasattr(pkg_resources, '_distribute'):
  161 + raise ImportError
  162 +except ImportError:
  163 + ez_code = urllib2.urlopen(
  164 + options.setup_source).read().replace('\r\n', '\n')
  165 + ez = {}
  166 + exec ez_code in ez
  167 + setup_args = dict(to_dir=eggs_dir, download_delay=0)
  168 + if options.download_base:
  169 + setup_args['download_base'] = options.download_base
  170 + if options.use_distribute:
  171 + setup_args['no_fake'] = True
  172 + ez['use_setuptools'](**setup_args)
  173 + if 'pkg_resources' in sys.modules:
  174 + reload(sys.modules['pkg_resources'])
  175 + import pkg_resources
  176 + # This does not (always?) update the default working set. We will
  177 + # do it.
  178 + for path in sys.path:
  179 + if path not in pkg_resources.working_set.entries:
  180 + pkg_resources.working_set.add_entry(path)
  181 +
  182 +cmd = [quote(sys.executable),
  183 + '-c',
  184 + quote('from setuptools.command.easy_install import main; main()'),
  185 + '-mqNxd',
  186 + quote(eggs_dir)]
  187 +
  188 +if not has_broken_dash_S:
  189 + cmd.insert(1, '-S')
  190 +
  191 +find_links = options.download_base
  192 +if not find_links:
  193 + find_links = os.environ.get('bootstrap-testing-find-links')
  194 +if find_links:
  195 + cmd.extend(['-f', quote(find_links)])
  196 +
  197 +if options.use_distribute:
  198 + setup_requirement = 'distribute'
  199 +else:
  200 + setup_requirement = 'setuptools'
  201 +ws = pkg_resources.working_set
  202 +setup_requirement_path = ws.find(
  203 + pkg_resources.Requirement.parse(setup_requirement)).location
  204 +env = dict(
  205 + os.environ,
  206 + PYTHONPATH=setup_requirement_path)
  207 +
  208 +requirement = 'zc.buildout'
  209 +version = options.version
  210 +if version is None and not options.accept_buildout_test_releases:
  211 + # Figure out the most recent final version of zc.buildout.
  212 + import setuptools.package_index
  213 + _final_parts = '*final-', '*final'
  214 + def _final_version(parsed_version):
  215 + for part in parsed_version:
  216 + if (part[:1] == '*') and (part not in _final_parts):
  217 + return False
  218 + return True
  219 + index = setuptools.package_index.PackageIndex(
  220 + search_path=[setup_requirement_path])
  221 + if find_links:
  222 + index.add_find_links((find_links,))
  223 + req = pkg_resources.Requirement.parse(requirement)
  224 + if index.obtain(req) is not None:
  225 + best = []
  226 + bestv = None
  227 + for dist in index[req.project_name]:
  228 + distv = dist.parsed_version
  229 + if _final_version(distv):
  230 + if bestv is None or distv > bestv:
  231 + best = [dist]
  232 + bestv = distv
  233 + elif distv == bestv:
  234 + best.append(dist)
  235 + if best:
  236 + best.sort()
  237 + version = best[-1].version
  238 +if version:
  239 + requirement = '=='.join((requirement, version))
  240 +cmd.append(requirement)
  241 +
  242 +if is_jython:
  243 + import subprocess
  244 + exitcode = subprocess.Popen(cmd, env=env).wait()
  245 +else: # Windows prefers this, apparently; otherwise we would prefer subprocess
  246 + exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env]))
  247 +if exitcode != 0:
  248 + sys.stdout.flush()
  249 + sys.stderr.flush()
  250 + print ("An error occurred when trying to install zc.buildout. "
  251 + "Look above this message for any errors that "
  252 + "were output by easy_install.")
  253 + sys.exit(exitcode)
  254 +
  255 +ws.add_entry(eggs_dir)
  256 +ws.require(requirement)
  257 +import zc.buildout.buildout
  258 +zc.buildout.buildout.main(args)
  259 +if not options.eggs: # clean up temporary egg directory
  260 + shutil.rmtree(eggs_dir)
93 s3/buildout.cfg
... ... @@ -0,0 +1,93 @@
  1 +[buildout]
  2 +dump-picked-versions-file = missing_versions.cfg
  3 +versions = versions
  4 +parts = nginx
  5 + nginx-conf
  6 + nginx-cmd
  7 +extensions = lovely.buildouthttp
  8 + buildout.dumppickedversions
  9 +unzip = true
  10 +include-site-packages = false
  11 +
  12 +[versions]
  13 +echo_module = v0.35
  14 +upload_module = 2.2
  15 +ngx_dev_kit = v0.2.14rc1
  16 +lua_module = v0.1.5rc3
  17 +set_misc_module = v0.21rc2
  18 +distribute = 0.6.14
  19 +hexagonit.recipe.cmmi = 1.5.0
  20 +hexagonit.recipe.download = 1.5.0
  21 +zc.buildout = 1.5.2
  22 +lovely.buildouthttp = 0.4.0
  23 +gocept.recipe.env = 1.0
  24 +lovely.recipe = 1.0.0
  25 +z3c.recipe.filetemplate = 2.1.0
  26 +zc.recipe.egg = 1.3.2
  27 +
  28 +[echo_module]
  29 +recipe = hexagonit.recipe.download
  30 +url=http://github.com/agentzh/echo-nginx-module/tarball/${versions:echo_module}
  31 +strip-top-level-dir=true
  32 +
  33 +[upload_module]
  34 +recipe = hexagonit.recipe.download
  35 +url = https://github.com/scompt/nginx-upload-module/tarball/${versions:upload_module}
  36 +strip-top-level-dir = true
  37 +
  38 +[ngx_dev_kit]
  39 +recipe = hexagonit.recipe.download
  40 +url = https://github.com/simpl/ngx_devel_kit/tarball/${versions:ngx_dev_kit}
  41 +strip-top-level-dir = true
  42 +
  43 +[lua_module]
  44 +recipe = hexagonit.recipe.download
  45 +url = https://github.com/chaoslawful/lua-nginx-module/tarball/${versions:lua_module}
  46 +strip-top-level-dir = true
  47 +
  48 +[set_misc_module]
  49 +recipe = hexagonit.recipe.download
  50 +url = https://github.com/agentzh/set-misc-nginx-module/tarball/${versions:set_misc_module}
  51 +strip-top-level-dir = true
  52 +
  53 +[nginx]
  54 +recipe = hexagonit.recipe.cmmi
  55 +url = http://sysoev.ru/nginx/nginx-0.8.54.tar.gz
  56 +configure-options = --with-debug
  57 + --add-module=${echo_module:destination}
  58 + --add-module=${upload_module:destination}
  59 + --add-module=${ngx_dev_kit:destination}
  60 + --add-module=${lua_module:destination}
  61 + --add-module=${set_misc_module:destination}
  62 + --with-cc-opt="-D NGX_HAVE_CASELESS_FILESYSTEM=0"
  63 + --with-http_stub_status_module
  64 + --with-http_ssl_module
  65 + --without-mail_pop3_module
  66 + --without-mail_imap_module
  67 + --without-mail_smtp_module
  68 + --without-http_geo_module
  69 + --without-http_memcached_module
  70 + --without-http_empty_gif_module
  71 + --without-http_autoindex_module
  72 + --without-http_fastcgi_module
  73 + --without-http_scgi_module
  74 + --without-http_uwsgi_module
  75 +
  76 +[env]
  77 +recipe = gocept.recipe.env
  78 +
  79 +[nginx-conf]
  80 +recipe = z3c.recipe.filetemplate
  81 +files = nginx/nginx.conf
  82 +bucket = extfile.testing.sl.assets
  83 +aws_secret_key = ${env:AWS_SECRET_KEY}
  84 +aws_access_key = ${env:AWS_ACCESS_KEY}
  85 +
  86 +[nginx-cmd]
  87 +recipe = lovely.recipe:mkfile
  88 +path = ${buildout:bin-directory}/nginx
  89 +mode = 0755
  90 +command = ${nginx:location}/sbin/nginx -c ${buildout:directory}/nginx/nginx.conf -p ${nginx:location}/
  91 +content =
  92 + #!/bin/sh
  93 + ${nginx-cmd:command} $@
1  s3/missing_versions.cfg
... ... @@ -0,0 +1 @@
  1 +[versions]
53 s3/nginx/nginx.conf.in
... ... @@ -0,0 +1,53 @@
  1 +# -*- mode: perl -*-
  2 +
  3 +worker_processes 1;
  4 +daemon "off";
  5 +
  6 +events {
  7 + worker_connections 1024;
  8 +}
  9 +
  10 +
  11 +http {
  12 + server {
  13 + listen 19999;
  14 + server_name "localhost";
  15 +
  16 + location ~ '^/([^/]+)/(.*)$' {
  17 + set $bucket $1;
  18 + set $key $2;
  19 +
  20 + # Setup AWS Authorization header
  21 + set $aws_signature '';
  22 +
  23 + # the only reason we need lua is to get the current date
  24 + set_by_lua $now "return ngx.cookie_time(ngx.time())";
  25 +
  26 + #the access key
  27 + set $aws_access_key '${aws_access_key}';
  28 + set $aws_secret_key '${aws_secret_key}';
  29 +
  30 + # the actual string to be signed
  31 + # see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/RESTAuthentication.html
  32 + set $string_to_sign "$request_method\n\n\n\nx-amz-date:$now\n/$bucket/$key";
  33 +
  34 + # create the hmac signature
  35 + set_hmac_sha1 $aws_signature $aws_secret_key $string_to_sign;
  36 + # encode the signature with base64
  37 + set_encode_base64 $aws_signature $aws_signature;
  38 + proxy_set_header x-amz-date $now;
  39 + proxy_set_header Authorization "AWS $aws_access_key:$aws_signature";
  40 +
  41 + rewrite .* /$key break;
  42 +
  43 + # we need to set the host header here in order to find the bucket
  44 + proxy_set_header Host $bucket.s3.amazonaws.com;
  45 + rewrite .* /$key break;
  46 +
  47 + # another solution would be to use the bucket in the url
  48 + # rewrite .* /$bucket/$key break;
  49 +
  50 + proxy_pass http://s3.amazonaws.com;
  51 + }
  52 + }
  53 +}

0 comments on commit c8b5a84

Please sign in to comment.
Something went wrong with that request. Please try again.