From 0a9f29f154b56c6f5087a22a6167bd22ae71ad12 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Mon, 17 Sep 2018 14:40:47 +0100 Subject: [PATCH 1/4] Fixing error code to return correct error rather than 500. 500 was being thrown due to a missing method _jsonify. --- iiif_validator/validator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/iiif_validator/validator.py b/iiif_validator/validator.py index 4f7e8ef..8482061 100644 --- a/iiif_validator/validator.py +++ b/iiif_validator/validator.py @@ -456,8 +456,10 @@ def empty_response(self, *args, **kwargs): def error(self, error, message=None): """Returns the error response.""" - return self._jsonify({"error": error.status_code, - "message": error.body or message}, "") + data = json.dumps({"error": error.status_code, "message": error.body or message}) + # add content-type and CORS headers to error + self.after_request() + return data def get_error_handler(self): """Customized errors""" From 1c4caf42b18b233a301ccef9d6916e9fa120b272 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Thu, 20 Sep 2018 11:10:57 +0100 Subject: [PATCH 2/4] Catching hacking attempts and hiding from the log --- .ebextensions/http/conf.d/dontlog.conf | 19 ++ .ebextensions/http/conf/httpd.conf | 353 +++++++++++++++++++++++++ 2 files changed, 372 insertions(+) create mode 100644 .ebextensions/http/conf.d/dontlog.conf create mode 100644 .ebextensions/http/conf/httpd.conf diff --git a/.ebextensions/http/conf.d/dontlog.conf b/.ebextensions/http/conf.d/dontlog.conf new file mode 100644 index 0000000..d99a19d --- /dev/null +++ b/.ebextensions/http/conf.d/dontlog.conf @@ -0,0 +1,19 @@ +RewriteEngine On +RedirectMatch ^/$ https://iiif.io/api/image/validator/ + +SetEnvIf Request_URI "^/glen_test.html$" dontlog +SetEnvIf Request_URI "^/Nmap/.*$" dontlog +SetEnvIf Request_URI "^/.*Yealink.*$" dontlog +SetEnvIf Request_URI "^/.*.php$" dontlog +SetEnvIf Request_URI "^/.*.cfg$" dontlog +SetEnvIf Request_URI "^/.*.aspx$" dontlog +SetEnvIf Request_URI "^/manager/html$" dontlog +SetEnvIf Request_URI "^/.well-known/security.txt$" dontlog +SetEnvIf Request_URI "^/js/czjl.js$" dontlog +SetEnvIf Request_URI "^/data/admin/.*$" dontlog +SetEnvIf Request_URI "^/sdk$" dontlog +SetEnvIf Request_URI "^/graphs/$" dontlog +SetEnvIf Request_URI "^/english/$" dontlog +SetEnvIf Request_URI "^/evox/about$" dontlog +SetEnvIf Request_URI "^/v3.0/web/assets/css/site.css$" dontlog +SetEnvIf Request_URI "^/user/register/$" dontlog diff --git a/.ebextensions/http/conf/httpd.conf b/.ebextensions/http/conf/httpd.conf new file mode 100644 index 0000000..98696a5 --- /dev/null +++ b/.ebextensions/http/conf/httpd.conf @@ -0,0 +1,353 @@ +# +# This is the main Apache HTTP server configuration file. It contains the +# configuration directives that give the server its instructions. +# See for detailed information. +# In particular, see +# +# for a discussion of each configuration directive. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so 'log/access_log' +# with ServerRoot set to '/www' will be interpreted by the +# server as '/www/log/access_log', where as '/log/access_log' will be +# interpreted as '/log/access_log'. + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# Do not add a slash at the end of the directory path. If you point +# ServerRoot at a non-local disk, be sure to specify a local disk on the +# Mutex directive, if file-based mutexes are used. If you wish to share the +# same ServerRoot for multiple httpd daemons, you will need to change at +# least PidFile. +# +ServerRoot "/etc/httpd" + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the +# directive. +# +# Change this to Listen on specific IP addresses as shown below to +# prevent Apache from glomming onto all bound IP addresses. +# +#Listen 12.34.56.78:80 +Listen 80 + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Statically compiled modules (those listed by `httpd -l') do not need +# to be loaded here. +# +# Example: +# LoadModule foo_module modules/mod_foo.so +# +Include conf.modules.d/*.conf + +# +# If you wish httpd to run as a different user or group, you must run +# httpd as root initially and it will switch. +# +# User/Group: The name (or #number) of the user/group to run httpd as. +# It is usually good practice to create a dedicated user and group for +# running httpd, as with most system services. +# +User apache +Group apache + +# 'Main' server configuration +# +# The directives in this section set up the values used by the 'main' +# server, which responds to any requests that aren't handled by a +# definition. These values also provide defaults for +# any containers you may define later in the file. +# +# All of these directives may appear inside containers, +# in which case these default settings will be overridden for the +# virtual host being defined. +# + +# +# ServerAdmin: Your address, where problems with the server should be +# e-mailed. This address appears on some server-generated pages, such +# as error documents. e.g. admin@your-domain.com +# +ServerAdmin root@localhost + +# +# ServerName gives the name and port that the server uses to identify itself. +# This can often be determined automatically, but we recommend you specify +# it explicitly to prevent problems during startup. +# +# If your host doesn't have a registered DNS name, enter its IP address here. +# +#ServerName www.example.com:80 + +# +# Deny access to the entirety of your server's filesystem. You must +# explicitly permit access to web content directories in other +# blocks below. +# + + AllowOverride none + Require all denied + + +# +# Note that from this point forward you must specifically allow +# particular features to be enabled - so if something's not working as +# you might expect, make sure that you have specifically enabled it +# below. +# + +# +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +DocumentRoot "/var/www/html" + +# +# Relax access to content within /var/www. +# + + AllowOverride None + # Allow open access: + Require all granted + + +# Further relax access to the default document root: + + # + # Possible values for the Options directive are "None", "All", + # or any combination of: + # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews + # + # Note that "MultiViews" must be named *explicitly* --- "Options All" + # doesn't give it to you. + # + # The Options directive is both complicated and important. Please see + # http://httpd.apache.org/docs/2.4/mod/core.html#options + # for more information. + # + Options Indexes FollowSymLinks + + # + # AllowOverride controls what directives may be placed in .htaccess files. + # It can be "All", "None", or any combination of the keywords: + # Options FileInfo AuthConfig Limit + # + AllowOverride None + + # + # Controls who can get stuff from this server. + # + Require all granted + + +# +# DirectoryIndex: sets the file that Apache will serve if a directory +# is requested. +# + + DirectoryIndex index.html + + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# + + Require all denied + + +# +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +# +ErrorLog "logs/error_log" + +# +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +# +LogLevel warn + + + # + # The following directives define some format nicknames for use with + # a CustomLog directive (see below). + # + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + # You need to enable mod_logio.c to use %I and %O + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + # + # The location and format of the access logfile (Common Logfile Format). + # If you do not define any access logfiles within a + # container, they will be logged here. Contrariwise, if you *do* + # define per- access logfiles, transactions will be + # logged therein and *not* in this file. + # + #CustomLog "logs/access_log" common + + # + # If you prefer a logfile with access, agent, and referer information + # (Combined Logfile Format) you can use the following directive. + # + CustomLog "logs/access_log" combined env=!dontlog + + + + # + # Redirect: Allows you to tell clients about documents that used to + # exist in your server's namespace, but do not anymore. The client + # will make a new request for the document at its new location. + # Example: + # Redirect permanent /foo http://www.example.com/bar + + # + # Alias: Maps web paths into filesystem paths and is used to + # access content that does not live under the DocumentRoot. + # Example: + # Alias /webpath /full/filesystem/path + # + # If you include a trailing / on /webpath then the server will + # require it to be present in the URL. You will also likely + # need to provide a section to allow access to + # the filesystem path. + + # + # ScriptAlias: This controls which directories contain server scripts. + # ScriptAliases are essentially the same as Aliases, except that + # documents in the target directory are treated as applications and + # run by the server when requested rather than as documents sent to the + # client. The same rules about trailing "/" apply to ScriptAlias + # directives as to Alias. + # + ScriptAlias /cgi-bin/ "/var/www/cgi-bin/" + + + +# +# "/var/www/cgi-bin" should be changed to whatever your ScriptAliased +# CGI directory exists, if you have that configured. +# + + AllowOverride None + Options None + Require all granted + + + + # + # TypesConfig points to the file containing the list of mappings from + # filename extension to MIME-type. + # + TypesConfig /etc/mime.types + + # + # AddType allows you to add to or override the MIME configuration + # file specified in TypesConfig for specific file types. + # + #AddType application/x-gzip .tgz + # + # AddEncoding allows you to have certain browsers uncompress + # information on the fly. Note: Not all browsers support this. + # + #AddEncoding x-compress .Z + #AddEncoding x-gzip .gz .tgz + # + # If the AddEncoding directives above are commented-out, then you + # probably should define those extensions to indicate media types: + # + AddType application/x-compress .Z + AddType application/x-gzip .gz .tgz + + # + # AddHandler allows you to map certain file extensions to "handlers": + # actions unrelated to filetype. These can be either built into the server + # or added with the Action directive (see below) + # + # To use CGI scripts outside of ScriptAliased directories: + # (You will also need to add "ExecCGI" to the "Options" directive.) + # + #AddHandler cgi-script .cgi + + # For type maps (negotiated resources): + #AddHandler type-map var + + # + # Filters allow you to process content before it is sent to the client. + # + # To parse .shtml files for server-side includes (SSI): + # (You will also need to add "Includes" to the "Options" directive.) + # + AddType text/html .shtml + AddOutputFilter INCLUDES .shtml + + +# +# Specify a default charset for all content served; this enables +# interpretation of all content as UTF-8 by default. To use the +# default browser choice (ISO-8859-1), or to allow the META tags +# in HTML content to override this choice, comment out this +# directive: +# +AddDefaultCharset UTF-8 + + + # + # The mod_mime_magic module allows the server to use various hints from the + # contents of the file itself to determine its type. The MIMEMagicFile + # directive tells the module where the hint definitions are located. + # + MIMEMagicFile conf/magic + + +# +# Customizable error responses come in three flavors: +# 1) plain text 2) local redirects 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# EnableMMAP and EnableSendfile: On systems that support it, +# memory-mapping or the sendfile syscall may be used to deliver +# files. This usually improves server performance, but must +# be turned off when serving from networked-mounted +# filesystems or if support for these functions is otherwise +# broken on your system. +# Defaults if commented: EnableMMAP On, EnableSendfile Off +# +#EnableMMAP off +EnableSendfile on + +# Supplemental configuration +# +# Load config files in the "/etc/httpd/conf.d" directory, if any. +IncludeOptional conf.d/*.conf From 47956d06ea5eb57c9674a0e06599f14a3edb3ff2 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Thu, 20 Sep 2018 11:11:28 +0100 Subject: [PATCH 3/4] Adding docker testing infrastructure to test AWS deployment. --- Dockerfile | 37 +++++++++++++++++++++++++++++ docker-files/conf.modules.d.tar.gz | Bin 0 -> 2117 bytes docker-files/wsgi.conf | 27 +++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-files/conf.modules.d.tar.gz create mode 100644 docker-files/wsgi.conf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5b9eaec --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +FROM httpd:2.4.33 + +RUN apt-get update +RUN apt-get -y install python3.4 python3-pip libapache2-mod-wsgi-py3 libxml2-dev libxslt1-dev lib32z1-dev libjpeg-dev libmagic-dev python-dev vim + +RUN mkdir -p /opt/python/current/app +COPY . /opt/python/current/app +WORKDIR /opt/python/current/app +RUN pip3 install -r requirements.txt + +RUN ln -s /usr/local/apache2/conf/ /etc/httpd +RUN ln -s /usr/local/apache2/modules /etc/httpd/modules +RUN ln -s /usr/lib/apache2/modules/mod_wsgi.so /etc/httpd/modules/mod_wsgi.so +RUN mkdir /var/run/httpd +RUN ln -s /var/run/httpd /etc/httpd/run +RUN mkdir /var/www +RUN ln -s /usr/local/apache2/htdocs /var/www/html +RUN ln -s /usr/local/apache2/logs /var/log/httpd +RUN ln -s /var/log/httpd /etc/httpd/logs +RUN mkdir /etc/httpd/conf.d + +COPY .ebextensions/http/conf/httpd.conf /etc/httpd/httpd.conf +RUN sed -i 's/User apache/User daemon/g' /etc/httpd/httpd.conf +RUN sed -i 's/Group apache/Group daemon/g' /etc/httpd/httpd.conf +COPY .ebextensions/http/conf.d/* /etc/httpd/conf.d +COPY docker-files/wsgi.conf /etc/httpd/conf.d + +WORKDIR /etc/httpd/ +COPY docker-files/conf.modules.d.tar.gz /tmp/ +RUN tar zxvf /tmp/conf.modules.d.tar.gz + + + +#RUN rm /etc/nginx/conf.d/*.conf +#COPY .ebextensions/nginx/conf.d/*.conf /etc/nginx/conf.d/ +#COPY .ebextensions/nginx/conf.d/elasticbeanstalk /etc/nginx/conf.d/elasticbeanstalk +EXPOSE 80 diff --git a/docker-files/conf.modules.d.tar.gz b/docker-files/conf.modules.d.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..0d89478adfea8d35f305e4fe7fd2d314d553d0e9 GIT binary patch literal 2117 zcmV-L2)g$liwFP*CZbya1MON{kE1pe-sk)ZcjvW}dnVGfs??^pZ_AUCv8p_|9jH?Q?}t39SxIqE+2b?phdxi7h15#VsXks&la?x%aSa@^x<+(vw-{N=ql8Z%g|P_3H71J zNC))A#;?Br52$39O#sjM|GVXM)%ySSYN7ML#d^s9t|E^Ahm)`Uzf?rT#^e8`SO8<# zXh67RVgTL&{E(GL&R| zsh6culqQAOJ0B?^{*oOQ;YKKS(N9^wiH!yYGpbq=m3fEpE8T0 zG_lL^oT~$)M4j&Vc^d801kWS(c^yxdjP zRmBfe{}@3wol-pat09-b(XD{Xu40uHT%fuJanxCupvwBXtD2)N1-R@gJ{7!Vs0Xg0 z9zkiRpzz~%KpmQ&f~rH>AF&dhRt-6XI!t1O8tM^ZbTRdHM^)$BIxpkiG~S|p8os#( zaFjC|mZ-7;TxFiYlhQ3701qfi!wHUR&bC5vs-rx{=)exKWSLNmqxDsT^(cQRn1QJ9 znx#eiw8|68Yp6%qm7W@|h8zM^bzm?&#)}+_xs+6>jAff5r5s(kBLxpxCTPU!z%H?N zmYz`8fjvYR*6VIv9nTOjt1vjlHwrFUQB|Qr3D-%(r5ebZ$o1Si!Ly?}bT=9GhNf=b% z=^k8;$XU}dh=nehHQY@?c+TYg2sDe5#iIOO-|L>d%Zilg(IlbZzw%p^lsd19zSMfR zT2T+QV*y0Ly50PNv3rszRoqV|`X?Gw;~U4K+)g49%E^4Z)HXh;Z#7O-mL4S_1q42e zp?SywHs0Ze?BGvU+9^L&sp7ZqOQAa!dU8!7*F1{8MoF>{N*)0>Gy2VQ)&CZj{!bsw z&c{C2%~8Kn&)uTFC#$*k!Pp~A*gn@A@<>Zg^S1MIc<4jVXDeu`aqtha%HT<14Etyj zjzGGH%`Y{WGuxT5Ox>n=WTa@<>~B4+nWOJn4d{DR%lyn8{2A%{|Foi4e*ZsR&3gWS zG5G(hNY_VKXwO2QhnMy#E=P`bNw+q&glq)HrJ!nZl%{rWPW|Xj1gQ*{ z3W%bVP138YZav|{jb2Zf3T?JBF?JKZ)`P(~Jl>DA?(qyDQ8LP35?DIDMC~0%-PHzPv2C8)kB`W|O%8(?2gnz6odhV2#^mmBrC0!7C44C{{i z?;M0<|Fh}sPF~spoU{K0B>S`ZdNp6qXY1uu+y8QLH`xC*xS>!}<6wfbQHSCH=4Uv1cy z`m?Ay+QMOIEbYE)i3Y(;(VidY9duwnq1pudppyGsf9!>|8v1Jff$F!b=LTxj`~yMW zAIF|gTzgu+VNQN_+}Co3v5QH6L*G}oAW8~oO7jr!_a1AY^txH=XV(fSnLR-ggKy=L z0!ofODb4}KX;DE>-O}PXRh?L008x^z;QTC&;xq`9_!r?_9Hq*W3nP6Cvy-ZVtxEmW vMq_B_4Km0egA6jrAcG7t$RL9ZGRPo<3^K?dgA6jr|6Be9Yj~G<08jt`r}I90 literal 0 HcmV?d00001 diff --git a/docker-files/wsgi.conf b/docker-files/wsgi.conf new file mode 100644 index 0000000..3bc0fd0 --- /dev/null +++ b/docker-files/wsgi.conf @@ -0,0 +1,27 @@ +LoadModule wsgi_module modules/mod_wsgi.so +WSGISocketPrefix run/wsgi +WSGIRestrictEmbedded On + + + +Alias /static/ /opt/python/current/app/static/ + +Order allow,deny +Allow from all + + + +WSGIScriptAlias / /opt/python/current/app/iiif-validator.py + + + + Require all granted + + +WSGIDaemonProcess wsgi processes=1 threads=15 display-name=%{GROUP} \ + python-path=/opt/python/current/app user=daemon group=daemon \ + home=/opt/python/current/app +WSGIProcessGroup wsgi + + +LogFormat "%h (%{X-Forwarded-For}i) %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined From 2662e3c695d7d744899d6dd4bc6a9d908ff18570 Mon Sep 17 00:00:00 2001 From: Glen Robson Date: Thu, 20 Sep 2018 11:30:43 +0100 Subject: [PATCH 4/4] Returning correct http error codes rather than just 200 with textual body --- iiif_validator/validator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/iiif_validator/validator.py b/iiif_validator/validator.py index 8482061..94ec7a6 100644 --- a/iiif_validator/validator.py +++ b/iiif_validator/validator.py @@ -375,7 +375,7 @@ def handle_test(self, testname): all_tests = testSuite.list_tests(version) return json.dumps(all_tests) if not testSuite.has_test(testname): - return "No such test: %s" % testname + abort(404,"No such test: %s" % testname) server = request.query.get('server', '') server = server.strip() @@ -392,7 +392,7 @@ def handle_test(self, testname): else: auth = "" if not server: - return "Missing mandatory parameter: server" + abort(400, "Missing mandatory parameter: server") if server[-1] == '/': server = server[:-1] @@ -409,7 +409,7 @@ def handle_test(self, testname): identifier = request.query.get('identifier', '') identifier = identifier.strip() if not identifier: - return "Missing mandatory parameter: identifier" + abort(400, "Missing mandatory parameter: identifier") try: result = ImageAPI(identifier, server, prefix, scheme, auth, version)