From 086ca2848f1147c71395689d8488162aa67590c4 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 12:30:45 -0400 Subject: [PATCH 01/14] Updated readme file with more helpful help --- readme.md | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/readme.md b/readme.md index 948f474..3f59f4b 100644 --- a/readme.md +++ b/readme.md @@ -2,21 +2,48 @@ A server that listens for webhook posts from GitHub, generates a website with Jekyll, and moves it somewhere to be published. Use this to run your own GitHub Pages-style web server. Great for when you need to serve your websites behind a firewall, need extra server-level features like HTTP basic auth (see below for an NGINX config with basic auth), or want to host your site directly on a CDN or file host like S3. It's cutomizable with two user-configurable shell scripts and a config file. +*This guide is tested on Ubuntu 14.0* + +## Dependencies Installation + +First install main dependencies + + $: sudo apt-get update + $: sudo apt-get install git nodejs ruby ruby1.9.1-dev npm + +To keep server running we use Forever: + + $: sudo npm install -g forever + +We also need Jekyll and Nginx + + $: sudo gem install jekyll + $: sudo gem install rdiscount + $: sudo gem install json + $: sudo apt-get install nginx + ## Installation -- run `$ npm install` to install app dependencies -- Set a [Web hook]() on your GitHub repository that points to your jekyll-hook server `http://example.com:8080/hooks/jekyll/:branch`, where `:branch` is the branch you want to publish. Usually this is `gh-pages` or `master` for `*.github.com` / `*.github.io` repositories. +Clone the repo + + $: git clone https://github.com/developmentseed/jekyll-hook.git + +Install dependencies: + $: npm install + +Set a [Web hook]() on your GitHub repository that points to your jekyll-hook server `http://example.com:8080/hooks/jekyll/:branch`, where `:branch` is the branch you want to publish. Usually this is `gh-pages` or `master` for `*.github.com` / `*.github.io` repositories. ## Configuration -Adjust `build.sh` and `publish.sh` to suit your workflow. By default, they generate a site with Jekyll and publish it to an NGINX web directory. +Copy `config.sample.json` to `config.json` in the root directory and customize: -Copy `config.sample.json` to `config.json` in the root directory and customize. + $: cp config.sample.json config.json + $: vim config.json Configuration attributes: -- `gh_server` The GitHub server from which to pull code +- `gh_server` The GitHub server from which to pull code, e.g. github.com - `temp` A directory to store code and site files - `scripts` - `build` A script to run to build the site @@ -24,9 +51,12 @@ Configuration attributes: - `email` Optional. Settings for sending email alerts - `user` Sending email account's user name (e.g. `example@gmail.com`) - `password` Sending email account's password - - `host` SMTP host for sending email account (e.g. `smtp.gmail.com`) + - `host` SMTP host for sending email account (e.g. `smtp.gmail.com`) - `ssl` `true` or `false` for SSL - `accounts` An array of accounts or organizations whose repositories can be used with this server + +You can also adjust `build.sh` and `publish.sh` to suit your workflow. By default, they generate a site with Jekyll and publish it to an NGINX web directory. + ## Usage - run as executable: `$ ./jekyll-hook.js` From c5b18e5b5786f03d5d395854cde0241bd27244e0 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 12:31:46 -0400 Subject: [PATCH 02/14] Github sends payload in json format now. Removing json.PARSER conversion --- jekyll-hook.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jekyll-hook.js b/jekyll-hook.js index 5154fd6..42b505c 100755 --- a/jekyll-hook.js +++ b/jekyll-hook.js @@ -20,7 +20,7 @@ app.post('/hooks/jekyll/:branch', function(req, res) { // Queue request handler tasks.defer(function(req, res, cb) { - var data = JSON.parse(req.body.payload); + var data = req.body; var branch = req.params.branch; var params = []; From 805ba85ff3008bfcb6e146eddfd1d3ba9c5fd21b Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 13:06:18 -0400 Subject: [PATCH 03/14] If the repo is private, ssh key is needed, if it's public a simple https request is enough --- config.sample.json | 10 +++++----- jekyll-hook.js | 9 ++++++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/config.sample.json b/config.sample.json index 1ad82cf..dee6f1b 100644 --- a/config.sample.json +++ b/config.sample.json @@ -1,18 +1,18 @@ { "gh_server": "github.com", "temp": "/home/ubuntu/jekyll-hook", + "public-repo": True, "scripts": { "build": "./scripts/build.sh", "publish": "./scripts/publish.sh" }, "email": { - "user": "", - "password": "", - "host": "", + "user": "", + "password": "", + "host": "", "ssl": true }, "accounts": [ "developmentseed", - "mapbox" ] -} \ No newline at end of file +} diff --git a/jekyll-hook.js b/jekyll-hook.js index 42b505c..32359e2 100755 --- a/jekyll-hook.js +++ b/jekyll-hook.js @@ -47,7 +47,14 @@ app.post('/hooks/jekyll/:branch', function(req, res) { /* repo */ params.push(data.repo); /* branch */ params.push(data.branch); /* owner */ params.push(data.owner); - /* giturl */ params.push('git@' + config.gh_server + ':' + data.owner + '/' + data.repo + '.git'); + + /* giturl */ + if (config.public-repo) { + params.push('https://' + config.gh_server + '/' + data.owner + '/' + data.repo + '.git'); + } else { + params.push('git@' + config.gh_server + ':' + data.owner + '/' + data.repo + '.git'); + } + /* source */ params.push(config.temp + '/' + data.owner + '/' + data.repo + '/' + data.branch + '/' + 'code'); /* build */ params.push(config.temp + '/' + data.owner + '/' + data.repo + '/' + data.branch + '/' + 'site'); From f4a6440b71d58db09c800c20b6bde4f8a7a275e2 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 13:09:05 -0400 Subject: [PATCH 04/14] Added an activate key for the email in config and code --- config.sample.json | 3 ++- jekyll-hook.js | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/config.sample.json b/config.sample.json index dee6f1b..9b5c601 100644 --- a/config.sample.json +++ b/config.sample.json @@ -1,12 +1,13 @@ { "gh_server": "github.com", "temp": "/home/ubuntu/jekyll-hook", - "public-repo": True, + "public-repo": true, "scripts": { "build": "./scripts/build.sh", "publish": "./scripts/publish.sh" }, "email": { + "isActivated": false, "user": "", "password": "", "host": "", diff --git a/jekyll-hook.js b/jekyll-hook.js index 32359e2..80db6fd 100755 --- a/jekyll-hook.js +++ b/jekyll-hook.js @@ -112,13 +112,15 @@ function run(file, params, cb) { } function send(body, subject, data) { - if (config.email && data.pusher.email) { - var message = { - text: body, - from: config.email.user, - to: data.pusher.email, - subject: subject - }; - mailer.send(message, function(err) { if (err) console.warn(err); }); + if (config.email.isActivated) { + if (config.email && data.pusher.email) { + var message = { + text: body, + from: config.email.user, + to: data.pusher.email, + subject: subject + }; + mailer.send(message, function(err) { if (err) console.warn(err); }); + } } } From 05ac6ca2fdd6aa08917240b8b46610f6d0a237c6 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 13:11:37 -0400 Subject: [PATCH 05/14] Updated build script with newly added jekyll build command --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index 4d689db..9eb4fe2 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -25,5 +25,5 @@ cd - # Run jekyll cd $source -jekyll $source $build --no-server --no-auto +jekyll build -s $source -d $build cd - From 8b1dcb3b4353ff72ac041c7d95f4dcedf488e513 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 13:12:36 -0400 Subject: [PATCH 06/14] Updated publish script with the updated nginx file location --- scripts/publish.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/publish.sh b/scripts/publish.sh index 82e728b..34928a4 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -13,8 +13,8 @@ source=$5 build=$6 # Set the path of the hosted site -site="/usr/share/nginx/www/$repo" +site="/usr/share/nginx/html/$repo" # Remove old site files, move new ones in place -rm -rf $site -mv $build $site +sudo rm -rf $site +sudo mv $build $site From 671fb1a650b0e5ae62e425b348742043a4069b95 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 13:43:11 -0400 Subject: [PATCH 07/14] Updated readme/help --- readme.md | 106 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 60 insertions(+), 46 deletions(-) diff --git a/readme.md b/readme.md index 3f59f4b..eaaf462 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,12 @@ # jekyll-hook -A server that listens for webhook posts from GitHub, generates a website with Jekyll, and moves it somewhere to be published. Use this to run your own GitHub Pages-style web server. Great for when you need to serve your websites behind a firewall, need extra server-level features like HTTP basic auth (see below for an NGINX config with basic auth), or want to host your site directly on a CDN or file host like S3. It's cutomizable with two user-configurable shell scripts and a config file. +A server that listens for webhook posts from GitHub, generates a website with +Jekyll, and moves it somewhere to be published. Use this to run your own GitHub +Pages-style web server. Great for when you need to serve your websites behind a +firewall, need extra server-level features like HTTP basic auth (see below for an +NGINX config with basic auth), or want to host your site directly on a CDN or +file host like S3. It's cutomizable with two user-configurable shell scripts +and a config file. *This guide is tested on Ubuntu 14.0* @@ -11,15 +17,17 @@ First install main dependencies $: sudo apt-get update $: sudo apt-get install git nodejs ruby ruby1.9.1-dev npm +Symlink nodejs to node + + $: sudo ln -s /usr/bin/nodejs /usr/bin/node + To keep server running we use Forever: $: sudo npm install -g forever We also need Jekyll and Nginx - $: sudo gem install jekyll - $: sudo gem install rdiscount - $: sudo gem install json + $: sudo gem install jekyll rdiscount json $: sudo apt-get install nginx ## Installation @@ -30,9 +38,15 @@ Clone the repo Install dependencies: - $: npm install + $: cd jekyll-hook + jekyll-hook $: npm install + +If you receive an error similar to this `npm ERR! Error: EACCES, mkdir +'/home/ubuntu/tmp/npm-2223-4myn3niN'` run: + + $: sudo chown -R ubuntu:ubuntu /home/ubuntu/tmp -Set a [Web hook]() on your GitHub repository that points to your jekyll-hook server `http://example.com:8080/hooks/jekyll/:branch`, where `:branch` is the branch you want to publish. Usually this is `gh-pages` or `master` for `*.github.com` / `*.github.io` repositories. +*You should replace `ubuntu` with your username* ## Configuration @@ -45,21 +59,55 @@ Configuration attributes: - `gh_server` The GitHub server from which to pull code, e.g. github.com - `temp` A directory to store code and site files +- `public-repo` Whether the repo is public or private (default is public) - `scripts` - `build` A script to run to build the site - `publish` A script to run to publish the site - `email` Optional. Settings for sending email alerts + - `isActivated` If set to true email will be sent after each trigger - `user` Sending email account's user name (e.g. `example@gmail.com`) - `password` Sending email account's password - `host` SMTP host for sending email account (e.g. `smtp.gmail.com`) - `ssl` `true` or `false` for SSL -- `accounts` An array of accounts or organizations whose repositories can be used with this server +- `accounts` An array of accounts or organizations whose repositories can be used +with this server + +You can also adjust `build.sh` and `publish.sh` to suit your workflow. By default, +they generate a site with Jekyll and publish it to an NGINX web directory. + +## Webhook Setup on Github + +Set a [Web hook](https://developer.github.com/webhooks/) on your GitHub repository +that points to your jekyll-hook server `http://example.com:8080/hooks/jekyll/:branch`, where `:branch` is the branch you want to publish. Usually this is `gh-pages` or `master` for `*.github.com` / `*.github.io` repositories. + +## Configure a webserver (nginx) + +The default `publish.sh` is setup for nginx and copies `_site` folder to `/usr/share/nginx/html/rep_name`. + +If you would like to copy the website to another location, make sure to update +nginx virtual hosts which is located at `/etc/nginx/nginx/site-available` on Ubuntu 14. + +You also need to update `publish.sh` + +For more information Google or [read this](https://www.digitalocean.com/community/tutorials/how-to-configure-the-nginx-web-server-on-a-virtual-private-server): + +## Launch + + $: ./jekyll-hook.js -You can also adjust `build.sh` and `publish.sh` to suit your workflow. By default, they generate a site with Jekyll and publish it to an NGINX web directory. +To launch in background run: -## Usage + $: forever start jekyll-hook.js -- run as executable: `$ ./jekyll-hook.js` +To kill the background job: + +``` + $: forever list + info: Forever processes running + data: uid command script forever pid logfile uptime + data: [0] ZQMF /usr/bin/nodejs jekyll-hook.js 4166 4168 /home/ubuntu/.forever/ZQMF.log 0:0:1:22.176 + $: forever stop 0 +``` ## Publishing content @@ -87,39 +135,5 @@ server { Replace this script with whatever you need for your particular hosting environment. -You probably want to configure your server to only respond POST requests from GitHub's public IP addresses, found on the webhooks settings page. - -## Dependencies - -Here's a sample script to install the approriate dependencies on an Ubuntu server: - -```sh -#!/bin/sh - -# Install node and depencencies -sudo apt-get update -y -sudo apt-get install python-software-properties python g++ make -y -# On Ubuntu 12.10 and greater, add-apt-repository is provided by the software-properties-common package -#sudo apt-get install software-properties-common -y -sudo add-apt-repository ppa:chris-lea/node.js -y -sudo apt-get update -y -sudo apt-get install nodejs -y - -# Forever to keep server running -sudo npm install -g forever - -# Git -sudo apt-get install git -y - -# Ruby -sudo apt-get install ruby1.8 -y -sudo apt-get install rubygems -y - -# Jekyll -sudo gem install jekyll --version "0.12.0" -sudo gem install rdiscount -- version "1.6.8" -sudo gem install json --version "1.6.1" - -# Nginx for static content -sudo apt-get install nginx -y -``` +You probably want to configure your server to only respond POST requests from GitHub's +public IP addresses, found on the webhooks settings page. From 0f6bc4396eaefeea13af3bdce676e1133d9e064c Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 13:44:18 -0400 Subject: [PATCH 08/14] Removed sudo from publish script --- scripts/publish.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/publish.sh b/scripts/publish.sh index 34928a4..38b2143 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -16,5 +16,7 @@ build=$6 site="/usr/share/nginx/html/$repo" # Remove old site files, move new ones in place -sudo rm -rf $site -sudo mv $build $site +# On amazon EC2 use sudo if nginx html forlder has root ownership + +rm -rf $site +mv $build $site From 2ed380b7678d07c176af8e7fee2075511c67db4c Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 13:45:29 -0400 Subject: [PATCH 09/14] Removing v1 build script --- scripts/build-v1.sh | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100755 scripts/build-v1.sh diff --git a/scripts/build-v1.sh b/scripts/build-v1.sh deleted file mode 100755 index 0e85e39..0000000 --- a/scripts/build-v1.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -set -e - -# This script is meant to be run automatically -# as part of the jekyll-hook application. -# https://github.com/developmentseed/jekyll-hook - -repo=$1 -branch=$2 -owner=$3 -giturl=$4 -source=$5 -build=$6 - -# Check to see if repo exists. If not, git clone it -if [ ! -d $source ]; then - git clone $giturl $source -fi - -# Git checkout appropriate branch, pull latest code -cd $source -git checkout $branch -git pull origin $branch -cd - - -# Run jekyll -cd $source -jekyll build --source $source --destination $build -cd - From 38f970c6d3086bc7a5fa76722c12ba563fa38f95 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 14:02:08 -0400 Subject: [PATCH 10/14] omitted an extra comma --- config.sample.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.sample.json b/config.sample.json index 9b5c601..59a4c41 100644 --- a/config.sample.json +++ b/config.sample.json @@ -14,6 +14,6 @@ "ssl": true }, "accounts": [ - "developmentseed", + "developmentseed" ] } From 6b1c513513d20eb68d78299b0d19fb404d10014c Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 14:04:34 -0400 Subject: [PATCH 11/14] Fixed issue with config item naming --- config.sample.json | 2 +- jekyll-hook.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.sample.json b/config.sample.json index 59a4c41..01da368 100644 --- a/config.sample.json +++ b/config.sample.json @@ -1,7 +1,7 @@ { "gh_server": "github.com", "temp": "/home/ubuntu/jekyll-hook", - "public-repo": true, + "public_repo": true, "scripts": { "build": "./scripts/build.sh", "publish": "./scripts/publish.sh" diff --git a/jekyll-hook.js b/jekyll-hook.js index 80db6fd..a5d61fa 100755 --- a/jekyll-hook.js +++ b/jekyll-hook.js @@ -49,7 +49,7 @@ app.post('/hooks/jekyll/:branch', function(req, res) { /* owner */ params.push(data.owner); /* giturl */ - if (config.public-repo) { + if (config.public_repo) { params.push('https://' + config.gh_server + '/' + data.owner + '/' + data.repo + '.git'); } else { params.push('git@' + config.gh_server + ':' + data.owner + '/' + data.repo + '.git'); From dfa0e4dd18bc637d883f2264ed6eb8680e99ded4 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 14:16:11 -0400 Subject: [PATCH 12/14] minor fix in readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index eaaf462..1aad748 100644 --- a/readme.md +++ b/readme.md @@ -99,7 +99,7 @@ To launch in background run: $: forever start jekyll-hook.js -To kill the background job: +To kill or restart the background job: ``` $: forever list From 3df8dcf63b875f16629d8c428b4a136c7004d5bd Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 15:05:10 -0400 Subject: [PATCH 13/14] Another fix in the readme file --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 1aad748..464d906 100644 --- a/readme.md +++ b/readme.md @@ -45,6 +45,7 @@ If you receive an error similar to this `npm ERR! Error: EACCES, mkdir '/home/ubuntu/tmp/npm-2223-4myn3niN'` run: $: sudo chown -R ubuntu:ubuntu /home/ubuntu/tmp + $: npm install *You should replace `ubuntu` with your username* From 54d4b6cc127a786506cbf4b0abc0ee8e8cc55c56 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 17 Jul 2014 15:52:43 -0400 Subject: [PATCH 14/14] Added S3 help to readme --- readme.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/readme.md b/readme.md index 464d906..44d91e9 100644 --- a/readme.md +++ b/readme.md @@ -112,6 +112,19 @@ To kill or restart the background job: ## Publishing content +### S3 + +To publish the site on Amazon S3, you need to install S3cmd. On Ubuntu run: + + $: sudo apt-get install s3cmd + $: s3cmd --configure + +For more information [read this](http://xmodulo.com/2013/06/how-to-access-amazon-s3-cloud-storage-from-command-line-in-linux.html). + +`scripts/publish-s3.sh` does the rest of the job for you. Just make sure to add your bucket name there. + +### More details on build.sh + The stock `build.sh` copies rendered site files to subdirectories under a web server's `www` root directory. For instance, use this script and NGINX with the following configuration file to serve static content behind HTTP basic authentication: ```