From dac00451820993181ae93506566ac4270e03afd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Sat, 25 Aug 2018 14:06:25 +0200 Subject: [PATCH] Make this elsable Fixes https://github.com/pyvec/python.cz/issues/263 --- .gitignore | 4 + .travis.yml | 15 ++-- README.md | 13 +--- deployment/README.md | 102 ------------------------- deployment/deploy.sh | 22 ------ deployment/id_rsa_pyvec_deployment.enc | Bin 3248 -> 0 bytes deployment/update.sh | 28 ------- pythoncz/__init__.py | 4 + pythoncz/templates/_base.html | 1 - pythoncz/templates/meta_redirect.html | 19 +++++ pythoncz/views.py | 95 +++++++++++++++++++---- pythoncz_tests/views_test.py | 64 +++++----------- runserver.py | 14 +--- setup.py | 2 + 14 files changed, 147 insertions(+), 236 deletions(-) delete mode 100644 deployment/README.md delete mode 100755 deployment/deploy.sh delete mode 100644 deployment/id_rsa_pyvec_deployment.enc delete mode 100755 deployment/update.sh create mode 100644 pythoncz/templates/meta_redirect.html diff --git a/.gitignore b/.gitignore index e603a349..01d66724 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,7 @@ pip-log.txt /cache/ .cache .pytest_cache/ + +# Elsa +_build/ +/talks-archive/ diff --git a/.travis.yml b/.travis.yml index d893682a..550f186a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,15 +6,20 @@ install: - "pip install .[tests]" script: - "pytest" + - "test $TRAVIS_PULL_REQUEST == false || export DISABLE_GITHUB_ISSUES_FETCH=true" + - "python runserver.py freeze" after_success: - "coveralls" deploy: - provider: "script" - script: "deployment/deploy.sh" + provider: script + skip_cleanup: true + script: 'python runserver.py deploy --no-freeze --push' on: - branch: "master" - python: "3.4" - repo: "pyvec/python.cz" + branch: master + repo: pyvec/python.cz notifications: slack: secure: "s/PCU4+geUozmw3AJguPYpv/YFNZVYmQ30A3DZpbSOE/9HDEgqbaF/hONY1VhHriRBG1EJ2uSezbxjOveyJPV+UMGJFMPU4c4Mw4AHxyROSnJe/MpOKPd8QRgbtYWYwy5UI47bzxotGoN49Mk0EjqhaFk9DFb14et9YPwMDbrVY=" +env: + global: + secure: R1+KSJV1oOikNBvyil0m/FMace5Y4lVC8WTlIpspZZ30BDkJCQ0eTnXy7gFvuEa4wBmVSqRKnRm+LKf44wXRjyKy/KRwUIzuy9AmzzQMe/PuvWSQDz7zf1dvG7GlR0kTJlgJ7s0+Vh4uqEpjIQnvtn+xkLguYVs3QXmTUvxLMnA= diff --git a/README.md b/README.md index ddb42160..979a17a9 100644 --- a/README.md +++ b/README.md @@ -20,22 +20,17 @@ The site uses GitHub API. For certian pages to work correctly, you need to set t ```sh $ export GITHUB_TOKEN=... -$ python runserver.py +$ python runserver.py --help ``` ### Deployment -The site gets automatically deployed after any push to the `master` branch. See [documentation in the `deployment` directory](deployment/README.md). - -- **Hosting:** [Rosti.cz](https://rosti.cz/)
- Access: [Pyvec](http://pyvec.org/) +The site gets automatically deployed after any push to the `master` branch. +It is frozen by [Elsa](https://github.com/pyvec/elsa). A cron job redeploys +it daily, so any content of dynamic nature isn't outdated. - **Domain:** bestowed by [KRAXNET](http://www.kraxnet.cz/)
Access: e-mail request to [KRAXNET](http://www.kraxnet.cz/) - -- **Monitoring:** [UptimeRobot](https://uptimerobot.com/)
- Access: [@honzajavorek](http://github.com/honzajavorek) - - **Analytics:** [Google Analytics](http://www.google.com/analytics/)
Access: [@honzajavorek](http://github.com/honzajavorek), [@encukou](http://github.com/encukou), [@martinbilek](http://github.com/martinbilek), [@benabraham](http://github.com/benabraham) diff --git a/deployment/README.md b/deployment/README.md deleted file mode 100644 index 1dfd0f96..00000000 --- a/deployment/README.md +++ /dev/null @@ -1,102 +0,0 @@ - -# Deployment at [Rosti.cz](https://rosti.cz/) - -## Initial Setup - -- Register at [Rosti.cz](https://rosti.cz/), create an app with SSH access. -- Clear default contents: - - ``` - $ rm -rf /srv/app - ``` - -- Clone the repository: - - ``` - $ git clone https://github.com/pyvec/python.cz /srv/app - ``` - -- Install: - - ``` - $ cd /srv/app - $ pip install -r requirements.txt - ``` - -- Set `GITHUB_TOKEN` environment value. Get [GitHub Personal Access Token](https://github.com/settings/tokens) (with no scopes) and edit `/srv/conf/supervisor.d/python.conf`: - - ``` - [program:app] - command=... - environment=GITHUB_TOKEN="123a123dc..." - ... - ``` - -- Restart [Supervisor](http://supervisord.org/): - - ``` - $ supervisorctl restart app - ``` - - The `app.py` file in the root of the project is used as WSGI endpoint. - -- The app should be up and running (e.g. [pythoncz-0375.rostiapp.cz](http://pythoncz-0375.rostiapp.cz/)). - -## Continuous Deployment - -Continuous deployment means your app gets automatically deployed if continuous integration (CI) build was successful. When turned on only for the `master` branch, the app gets deployed every time someone pushes to `master`, including merged GitHub Pull Requests. - -[Travis CI](http://travis-ci.org/) supports continuous deployment out of the box [for various PaaS services](http://docs.travis-ci.com/user/deployment/). [Rosti.cz](https://rosti.cz/) is not among them, but [it's possible to setup also your own deployment](http://docs.travis-ci.com/user/deployment/script/) using the `script` keyword. - -### Setup, step by step - -- Create an SSH key which is going to be used for deployment. Make sure it is *without passphrase* as you, obviously, won't be able to interactively type in the password at the end of your TravisCI builds. - - ``` - $ ssh-keygen -t rsa -b 4096 -C "info@pyvec.org" - ... - $ ssh-add ~/.ssh/id_rsa_pyvec_deployment - ``` - -- Upload the public key to the production server. First, you need to prepare the server so it has `authorized_keys` available - follow instructions from the [Rosti.cz documentation](https://docs.rosti.cz/base/#ssh). Then upload the public key: - - ``` - $ ssh-copy-id -i ~/.ssh/id_rsa_pyvec_deployment.pub app@pluto.rosti.cz -p 10365 # see Rosti.cz administration for username, host, port... - ``` - -- Encrypt the private key. See [documentation for encrypting files](http://docs.travis-ci.com/user/encrypting-files/). Please be aware that the encryption is repository-specific. Even if using the same private key for multiple projects, it needs to be re-encrypted individually for each of them. - - ``` - $ gem install travis - $ travis login --auto - $ travis encrypt-file id_rsa_pyvec_deployment - ``` - - Follow instructions of the `travis encrypt-file` output. You need to add decrypting command to your build. This repository has it in the `deployment/deploy.sh` script. - -- Make sure the deploy is going to happen. Update `.travis.yml` with custom deployment settings: - - ```yaml - deploy: - provider: "script" - script: "deployment/deploy.sh" - on: - branch: "master" - python: "3.4" - repo: "pyvec/python.cz" - ``` - - This triggers `deploy.sh` script only for `master` branch with Python 3.4. - -- The `deploy.sh` script, executed within the TravisCI build, does three things: - - 1. Decrypts the private key (see above). - 2. Connects to the production machine via SSH and executes the `update.sh` script. - 3. Deletes the decrypted private key. - -- The `update.sh` script, executed on the production machine, does four things: - - 1. Deletes all source code of the current app. - 2. Uses `git` to get the latest source code in the `master` branch. - 3. Installs dependencies (plain simple `pip install -r requirements.txt`). - 4. Restarts the app ([specific to Rosti.cz](https://docs.rosti.cz/apps/python/#supervisor)). diff --git a/deployment/deploy.sh b/deployment/deploy.sh deleted file mode 100755 index 8914ae8f..00000000 --- a/deployment/deploy.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -# Automatic TravisCI Deployment -# -# TravisCI runs this script in case the build on master branch was successful. -# See repository README for details. - - -# This script should be executed on the CI machine. -if [ -z "$CI" ]; then - exit -fi - -# Decrypting private key -openssl aes-256-cbc -K $encrypted_3814399f38ed_key -iv $encrypted_3814399f38ed_iv -in deployment/id_rsa_pyvec_deployment.enc -out deployment/id_rsa_pyvec_deployment -d -chmod 600 deployment/id_rsa_pyvec_deployment - -# Run the 'update.sh' script remotely. -ssh 'app@pluto.rosti.cz' -p '10365' -o 'StrictHostKeyChecking no' -i 'deployment/id_rsa_pyvec_deployment' '/srv/app/deployment/update.sh' - -# Remove decrypted private key -rm deployment/id_rsa_pyvec_deployment diff --git a/deployment/id_rsa_pyvec_deployment.enc b/deployment/id_rsa_pyvec_deployment.enc deleted file mode 100644 index 8cf974246c6019e7667573e662c6c67d77b7cc2d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3248 zcmV;h3{UgIND}7c!!)+Dc>op~EhtfWLcFwz<{Mcq&J8=fx7HTe7`JcUiY=xWY~9|L zsWpx9a-?uJsAq@CxmHw5^EWOfju?hBDs<+JfB`AD6`LRYEf& zACn&pU$hVJVxdp;3n&S>MGoJ)B@`<}DNZ$5&#>BW`I$jDc3&4OE@3FuW6rt_g}bW^>6{((^Xc zCshs}hN#)~B%E_%l#V%1iAFOGuj&)k*f1y`kjsDrh)dPd0;TwB+)mJ8n2^OxgvB}K z-4C;7EfO^j1C7JmJD6_N?5&Z;=QScLT9UGZ|5^gyhBnO$J^%2|xDlG7@Y(4Zxv}=D zdfSM8E90g-MI61tpT}dS2Lu%yS9=1$7-RvfLafrI;4gSX2jR5(%5plXCCFXnN3l0E zXIQBke%NC+9dG;1o&ecs+5QCe_A@PUGozZb#nB;S@8)m4WZ1{th79U)9u`$I$0%Ln zVNZLfhl}$RC<|SvfZ;V;+l}(}(E9Q1W2(RZ0;2Q`m0y+Q7ktZ?lAfAhC-q$U$Gw3HXFzrcM9P94p#nZQ zXLyrp`vpp{>oJY{n^ZjjJ(OeP618f!t^a#9H0k)Gav;gHs5FHeGV7=-xSJ zthPpoBZM05p#jdnUIU886BYNuCOj~V(@9SC*vny z4)=`g^HZ~RjKnwG>Cm8U&?phGghXpfdhO0ExeT3gi5lMf%&@PY3SBfD@a5Cu-k=Op z5MZmGxp;^`B+&4;{^;a&ghPi*m|q9_K*VFv)o$-CjUg$%U%5pTR3z{E7BW$dX+ANO zfx`a(r-r}y5To=+A=`2rM?*&{Qnts~k&1Uq`(Qvw8WBIFm4HXD{%+pJQL<}JF?&Q_NsoRio z{Agf>kj;QDH;3EPK9Q0)!7vu0ePB z2*~$Cn+F4SYVyylOrN`N)GW2g})9kjloyjtO zYSiD~dtYWqGLwz=k?hNr!J-&BO*P;87F%(Xxi~tx52Lf&R}craP&9i+NV^;d6~m~h z0WPPigQ-$Y9z>5PI_E2y7@uv!hZHxwUZ=x@_^41kT>fI_>6G2@R2*U)-%ay;Ln0!p&!$rilhE&1rC>j@a(6 z6Ks_L3cRoycWbK3gq}6lq^QE6uv3#dTW~zKcy!TPw7PhdAzYlC><3P8UjPlQ7N(u& z_j}`mml#@mb*ZAKx(3{VQO^d3!(7(-ilMsZ3mN5A@WCC;Hvd_*`A@<()}M6?jCoyH zZ^I|F#;qmJ=~sl(?T{rsx8%zg7U#&bj3E+e@M&toy19>qpc>P$WcESxoQRf3(Mhz z00c@EX!b5Kl5T9aLgNdF&psALhAh1ce&bd9ug_CLq`#n!RFURzh(2zc^~aiuGE(+4|_S@b<_`9 z(_V=`v^^VDK`3W`I}FQTi8z@_5nq4SLMnPdh$sEVRfb{M*`8AUV)kGZeCXyR+|m|Gc@z^or!7V?vds)} z6OOi$ytO}bIFl8sYY*ZZx{!A>l)%RSUC4WTqQr?RZe?4d`2<_7pLai-g(lN0=(npF zUm2;Ls6%j4CFu<2HIBM7{opN0nQ(0lfoG7QGR&kgp=n&vJ?>1@a}^7_pJx z&*UXu&?)IT-NBCS<>)J+))*^ZuHlhC9>Xr}URt2Fw||%HkqY%KLlF9K9^BFYOYg>` zgeG~Ce`AF}a+$u;PU&lox?7er*1!Dz2!Gwgb)hPF%bVa(#Dc`gfDAN^Z}n_Go@Tat+ywH)Z-JN{jb7lC6+%R#Advo}hATtnl#xKx#kOhvBHl=Qp_mnob+^M_P@OhUd{wkoFa zodHpnDlO)NC4!sL!0OuHqWeer--t=)1Q-0uIg!qTb`gW{D4Sm^N1!jkd4}_!45zJ9 zF4d||Er+LM;n7djZ-5@%y|%*#4}O)!cWo{_+brO#O8!Yg4ll79&`Z9HHBRsn5*)!4 zDgJ&MXe6bNh@&6*I8vs~D>ry2AAJcnp&@00eg@#PA@O`lEH+}@OqkAcb2WGqNBgFN zR)Tvjvmwkb!%6!3D8p;yNAAou`I`1wx5Y|Q)yH79Ay}bSahPI z*q-5$PYoshC|q%X-|XBgb<8VSt251RejZ7$};1)-@oJWmrHrKWU`0j}Ro~a4cY0l$WTY zHP1olluv2J&_kulYfLV93USecv25MgI)=5j>$i^u;iK@0AAg2DJdl3UxFPGA}2^eSz-0B1%4ub`Y-=ROwqT~uTn>%ibNkgV%*mj+&p2* zY#sayux%ZH+TQfn>X|VgIRrB-t8qQIqx*%}A-a-;AAd4+Jyinfo@pyvec.org, tweetujte na @pyvec.
Ilustrace kreslí Honza Javorek. - Stránky hostujeme zdarma u Roští.cz. Doménu python.cz zapůjčil KRAXNET. Děkujeme!

diff --git a/pythoncz/templates/meta_redirect.html b/pythoncz/templates/meta_redirect.html new file mode 100644 index 00000000..7c460b55 --- /dev/null +++ b/pythoncz/templates/meta_redirect.html @@ -0,0 +1,19 @@ +{% extends '_base.html' %} +{% block head %} + + + + +{% endblock %} + +{% block content %} +

Přesměrování

+
+
+

+ Obsah této stránky byl přesunut na novou adresu: + {{ url }}. +

+
+
+{% endblock %} diff --git a/pythoncz/views.py b/pythoncz/views.py index f8f172c7..30b02143 100644 --- a/pythoncz/views.py +++ b/pythoncz/views.py @@ -1,9 +1,12 @@ +import os +import subprocess +from fnmatch import fnmatch from urllib.parse import quote_plus as url_quote_plus from flask import (render_template as _render_template, url_for, - redirect, request, make_response) + request, make_response, send_from_directory) -from pythoncz import app +from pythoncz import app, freezer from pythoncz.models import jobs, photos, beginners, github @@ -65,26 +68,31 @@ def jobs_en(): @app.route('/zapojse/') def get_involved_cs(): + disabled = os.getenv('DISABLE_GITHUB_ISSUES_FETCH', False) try: + if disabled: + raise Exception('DISABLE_GITHUB_ISSUES_FETCH is set') issues = github.get_issues(app.config['GITHUB_ORGANIZATIONS']) return render_template('get_involved_cs.html', issues=issues) except Exception as e: template = render_template('get_involved_cs.html', issues=[], error=e) - return make_response(template, 500) + code = 200 if disabled else 500 + return make_response(template, code) -# Subdomain redirect - -@app.route('/', subdomain='www') -def subdomain_redirect(): - return redirect(url_for('index_cs')) +# Redirects of legacy stuff +def redirect(url, code=None): + """Return a response with a Meta redirect, code is unused""" -# Redirects of legacy stuff + # With static pages, we can't use HTTP redirects. + # Return a page wit instead. + # + # When Frozen-Flask gets support for redirects + # (https://github.com/Frozen-Flask/Frozen-Flask/issues/81), + # this should be revisited. -@app.route('/index.html') -def index_legacy(): - return redirect(url_for('index_cs'), code=301) + return render_template('meta_redirect.html', url=url) @app.route('/english.html') @@ -97,12 +105,69 @@ def pyladies(target): return redirect('http://pyladies.cz/v1/' + target, code=301) +@freezer.register_generator +def pyladies(): + # This is hardcoded because it doesn't change & it's easier to hardcode it + targets = ( + 's001-install/', + 's002-hello-world/', + 's003-looping/', + 's004-strings/', + 's005-modules/', + 's006-lists/', + 's007-cards/', + 's008-cards2/', + 's009-git/', + 's010-data/', + 's011-dicts/', + 's012-pyglet/', + 's014-class/', + 's015-asteroids/', + 's016-micropython/', + ) + yield from ({'target': t} for t in targets) + + @app.route('/pyladies/') def pyladies_index(): - return redirect('http://pyladies.cz', code=301) + return redirect('http://pyladies.cz/', code=301) + + +# Talks, this used to redirect, but that's not possible with *.pdf HTML files +talks_dir = 'talks-archive' + + +@app.before_first_request +def clone_talks(): + try: + os.chdir(talks_dir) + except FileNotFoundError: + subprocess.run(('git', 'clone', + 'https://github.com/pyvec/talks-archive', + '--depth', '1')) + os.chdir(talks_dir) + else: + subprocess.run(('git', 'fetch', 'origin'), check=True) + subprocess.run(('git', 'reset', '--hard', 'origin/master'), check=True) + finally: + os.chdir('..') @app.route('/talks/') def talks(target): - base_url = 'https://github.com/pyvec/talks-archive/raw/master/' - return redirect(base_url + target, code=301) + # sends from pythoncz directory, hence ../ + return send_from_directory(f'../{talks_dir}', target) + + +@freezer.register_generator +def talks(): + ignore = ['.travis.yml', '.gitignore'] + for name, dirs, files in os.walk(talks_dir): + if '.git' in dirs: + dirs.remove('.git') + for file in files: + if file == '.git': + continue + if not any(fnmatch(file, ig) for ig in ignore): + path = os.path.relpath(os.path.join(name, file), talks_dir) + yield {'target': path} diff --git a/pythoncz_tests/views_test.py b/pythoncz_tests/views_test.py index 557c845b..927bb89a 100644 --- a/pythoncz_tests/views_test.py +++ b/pythoncz_tests/views_test.py @@ -198,49 +198,27 @@ def get_issues(self, *args, **kwargs): assert url in html, "URL for filing a bug report isn't present in the HTML" -def test_index_legacy_redirect(test_client): - response = test_client.get('/index.html') - - assert response.status_code == 301 - assert response.headers['location'] == url_for('index_cs', _external=True) - - def test_index_en_legacy_redirect(test_client): response = test_client.get('/english.html') - - assert response.status_code == 301 - assert response.headers['location'] == url_for('index_en', _external=True) - - -def test_pyladies_redirect(test_client): - response = test_client.get('/pyladies/foo/bar/baz.html') - - assert response.status_code == 301 - assert response.headers['location'] == ( - 'http://pyladies.cz/v1/foo/bar/baz.html' - ) - - -def test_pyladies_index_trailing_slash_redirect(test_client): - response = test_client.get('/pyladies/') - - assert response.status_code == 301 - assert response.headers['location'] == 'http://pyladies.cz' - - -def test_pyladies_index_without_trailing_slash_redirect(test_client): - response = test_client.get('/pyladies') - - assert response.status_code == 301 - assert response.headers['location'] == ( - url_for('pyladies_index', _external=True) - ) - - -def test_talks_redirect(test_client): - response = test_client.get('/talks/foo/bar/baz.html') - - assert response.status_code == 301 - assert response.headers['location'] == ( - 'https://github.com/pyvec/talks-archive/raw/master/foo/bar/baz.html' + url = url_for('index_en') + html = response.get_data(as_text=True) + head = html[:html.find('')] + assert ''.format(url) in head + + +@pytest.mark.parametrize('suffix', ('', 's001-install/')) +def test_pyladies_redirect(test_client, suffix): + response = test_client.get('/pyladies/' + suffix) + url = 'http://pyladies.cz/' + if suffix: + url += 'v1/' + suffix + html = response.get_data(as_text=True) + head = html[:html.find('')] + assert ''.format(url) in head + + +def test_talks_pdf_download(test_client): + response = test_client.get( + '/talks/brno-2013-11-28-veros-kaplan-postgis.pdf' ) + assert response.headers['content-type'] == 'application/pdf' diff --git a/runserver.py b/runserver.py index 763e67fe..36a6ac31 100755 --- a/runserver.py +++ b/runserver.py @@ -1,15 +1,7 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- - - -import os - -from pythoncz import app +from pythoncz import app, freezer if __name__ == '__main__': - host = '0.0.0.0' - port = int(os.environ.get('PORT', 5000)) - - app.config['SERVER_NAME'] = '{host}:{port}'.format(host=host, port=port) - app.run(host=host, port=port, debug=True) + from elsa import cli + cli(app, freezer=freezer, base_url='https://python.cz') diff --git a/setup.py b/setup.py index e186e6aa..24ca521a 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,9 @@ install_requires = [ + 'elsa==0.1.5', 'flask==1.0.2', + 'frozen-Flask==0.15', 'requests==2.19.1', 'czech-sort==0.4', 'pyyaml==3.13',