Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
339 lines (287 sloc) 11.2 KB
/**
* Copyright 2018 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
import fs from 'fs';
import bodyParser from 'body-parser';
import url from 'url';
const URL = url.URL;
// import stream from 'stream';
import express from 'express';
import firebaseAdmin from 'firebase-admin';
import puppeteer from 'puppeteer';
import nunjucks from 'nunjucks';
import GoogleAnalytics from 'universal-analytics';
import * as prerender from './ssr.mjs';
import Twitter from './public/twitter.mjs';
import * as feeds from './public/feeds.mjs';
import * as util from './public/util.mjs';
import * as dbHelper from './public/firebaseHelper.mjs';
import RSSFeed from './public/rss.mjs';
import * as ga from './analytics.mjs';
import * as ytAnalytics from './analytics-youtube.mjs';
const PORT = process.env.PORT || 8080;
const GA_ACCOUNT = 'UA-114661299-1';
const twitter = new Twitter('ChromiumDev');
// Async route handlers are wrapped with this to catch rejected promise errors.
const catchAsyncErrors = fn => (req, res, next) => {
Promise.resolve(fn(req, res, next)).catch(next);
};
async function doSSR(url, req) {
// Ignore random query params on the URL. Use only the ones we care about.
url = new URL(url);
if ('tweets' in req.query) {
url.searchParams.set('tweets', '');
}
if ('year' in req.query) {
url.searchParams.set('year', req.query.year);
}
const html = await prerender.ssr(url.href, {
useCache: 'nocache' in req.query ? false : true,
inlineStyles: 'noinline' in req.query ? false : true,
inlineScripts: 'noinline' in req.query ? false : true,
onlyCriticalRequests: 'noreduce' in req.query ? false : true,
reuseChrome: 'reusechrome' in req.query ? true : false,
headless: 'noheadless' in req.query ? false : true,
});
return html;
}
async function getPosts(req, res) {
const year = req.params.year || util.currentYear;
// Pad values if missing leading '0'.
const month = req.params.month ? req.params.month.padStart(2, '0') : null;
const day = req.params.day ? req.params.day.padStart(2, '0') : null;
const maxResults = req.query.maxresults ? Number(req.query.maxresults) : null;
// Record GA pageview.
const visitor = GoogleAnalytics(GA_ACCOUNT, {https: true});
visitor.pageview(req.originalUrl).send();
const rssPosts = await feeds.collectRSSFeeds();
const posts = util.uniquePosts(
await dbHelper.getPosts(year, month, day, rssPosts, maxResults));
return posts;
}
async function addAnalyticsData(posts, uid) {
if (!uid) {
return posts;
}
try {
const user = await firebaseAdmin.auth().getUser(uid);
if (user.customClaims.admin) {
const urlMap = await ga.updateAnalyticsData();
const ytVideoIdMap = await ytAnalytics.updateAnalyticsData();
const titleMap = new Map([
...ga.Analytics.toTitleMap(urlMap),
...ytAnalytics.YoutubeAnalytics.toTitleMap(ytVideoIdMap),
]);
posts = posts.map(post => {
const urlMatch = urlMap.get(new URL(post.url).pathname);
const titleMatch = titleMap.get(post.title);
const match = urlMatch || titleMatch;
if (match) {
// Normalize between GA and YT properties.
const pageviews = match.pageviews || (match.statistics && match.statistics.viewCount);
return Object.assign({pageviews}, post);
}
return post;
});
}
} catch (err) {
console.error(err);
// Noop. Pass through posts without analytics data.
}
return posts;
}
dbHelper.setApp(firebaseAdmin.initializeApp({
// credential: firebaseAdmin.credential.applicationDefault()
credential: firebaseAdmin.credential.cert(
JSON.parse(fs.readFileSync('./serviceAccountKey.json')))
}));
const app = express();
nunjucks.configure(['./views'], {
autoescape: true,
express: app,
watch: process.env.DEV || false,
});
app.use(function forceSSL(req, res, next) {
const fromCron = req.get('X-Appengine-Cron');
if (!fromCron && req.hostname !== 'localhost' && req.get('X-Forwarded-Proto') === 'http') {
return res.redirect(`https://${req.hostname}${req.url}`);
}
next();
});
app.use(function addRequestHelpers(req, res, next) {
req.getCurrentUrl = () => `${req.protocol}://${req.get('host')}${req.originalUrl}`;
req.getOrigin = () => {
let protocol = 'https';
if (req.hostname === 'localhost') {
protocol = 'http';
}
return `${protocol}://${req.get('host')}`;
};
next();
});
// app.use(bodyParser.urlencoded({extended: true}));
app.use(bodyParser.json());
// Handle index.html page dynamically.
app.get('/', async (req, res, next) => {
// Serve prerendered page to search crawlers.
const ua = req.get('User-Agent');
if (ua && (ua.match(/googlebot|bingbot/i) && !ua.match(/HeadlessChrome/i))) {
const html = await doSSR(req.getOrigin(), req);
// res.append('Link', `<${url}/styles.css>; rel=preload; as=style`); // Push styles.
return res.status(200).send(html);
}
return res.render('./posts.html');//, {title: 'Posts'});
});
app.use(express.static('public', {extensions: ['html', 'htm']}));
app.use(express.static('node_modules'));
// app.use(express.static('node_modules/firebase'));
// app.use(function cors(req, res, next) {
// res.set('Access-Control-Allow-Origin', '*');
// // res.set('Content-Type', 'application/json;charset=utf-8');
// // res.set('Cache-Control', 'public, max-age=300, s-maxage=600');
// next();
// });
// Admin handlers --------------------------------------------------------------
app.get('/admin/charts', async (req, res) => {
res.render('./admin/charts.html');
});
app.get('/admin/stats', async (req, res) => {
res.render('./admin/stats.html');
});
app.post('/admin/user/update/:uid', async (req, res) => {
const uid = req.params.uid;
if (!uid) {
return res.status(400).json({error: 'Missing uid'});
}
const user = await firebaseAdmin.auth().getUser(uid);
const isGoogler = user.email.endsWith('@google.com');
await firebaseAdmin.auth().setCustomUserClaims(uid, {admin: isGoogler});
res.status(200).json(user.customClaims || {});
});
app.get('/admin/update/feeds', async (req, res) => {
if (!req.get('X-Appengine-Cron')) {
return res.status(403).send('Sorry, handler runs from GAE cron.');
}
res.status(200).json(await feeds.updateFeeds());
});
app.get('/admin/update/tweets/:username', async (req, res) => {
if (!req.get('X-Appengine-Cron')) {
return res.status(403).send('Sorry, handler runs from GAE cron.');
}
const username = req.params.username;
res.status(200).json(await twitter.updateTweets(username));
});
app.get('/admin/update/analytics', async (req, res) => {
if (!req.get('X-Appengine-Cron')) {
return res.status(403).send('Sorry, handler runs from GAE cron.');
}
await ga.updateAnalyticsData(true);
await ytAnalytics.updateAnalyticsData(true);
res.status(200).send('Done');
});
app.get('/admin/update/rendercache', async (req, res) => {
if (!req.get('X-Appengine-Cron')) {
return res.status(403).send('Sorry, handler runs from GAE cron.');
}
const browser = await puppeteer.launch({args: ['--disable-dev-shm-usage']});
const url = new URL(req.getOrigin());
// Re-render main page and a few years back.
prerender.clearCache();
await prerender.ssr(url.href, {useCache: false, existingBrowser: browser});
await prerender.ssr(`${url}?year=${util.currentYear - 1}`,
{useCache: false, existingBrowser: browser});
await prerender.ssr(`${url}?year=${util.currentYear - 2}`,
{useCache: false, existingBrowser: browser});
await prerender.ssr(`${url}?year=${util.currentYear - 3}`,
{useCache: false, existingBrowser: browser});
await browser.close();
res.status(200).send('Render cache updated!');
});
// -----------------------------------------------------------------------------
// Client-side version, 3G Slow:
// FP: 4s, FCP: 11s
// SSR render, 3G Slow:
// FP/FCP: 2.3s, 8.37s faster!
app.get('/ssr', catchAsyncErrors(async (req, res) => {
const tic = Date.now();
const html = await doSSR(req.getOrigin(), req);
res.set('Server-Timing', `Prerender;dur=${Date.now() - tic};desc="Headless render time (ms)"`);
res.status(200).send(html);
}));
app.get('/tweets/:username', async (req, res) => {
const username = req.params.username;
res.status(200).json(await twitter.getTweets(username));
});
app.post('/posts', async (req, res) => {
res.set('Access-Control-Allow-Origin', '*');
res.set('Access-Control-Allow-Headers', 'Content-Type');
await dbHelper.newPost(req.body);
res.status(200).send('Success!');
});
// app.delete('/posts/:year?/:month?/:idx?', async (req, res) => {
// const year = req.params.year;r
// const month = req.params.month.padStart(2, '0');
// const itemsIdx = req.params.idx;
// await db.deletePost(year, month, itemsIdx);
// res.status(200).send('Success!');
// });
app.get('/posts/:year?/:month?/:day?', async (req, res) => {
let posts = await getPosts(req, res);
const format = req.query.format || null;
if (format === 'rss') {
const feedUrl = req.getCurrentUrl();
const xml = (new RSSFeed(feedUrl)).create(posts);
res.set('Content-Type', 'application/rss+xml');
return res.status(200).send(xml);
}
// Zest in analytics data if user is admin.
posts = await addAnalyticsData(posts, req.query.uid);
// TODO: monitor updates to other years. e.g. If the server is running when
// a new year occurs, it will need to be restarted to pick up updates to that
// new year.
// TODO: figure out way to realtime update tweets and RSS posts.
// Note: this setups a single realtime monitor (e.g. not one per request).
dbHelper.monitorRealtimeUpdateToPosts(util.currentYear, async changes => {
const origin = req.getOrigin();
prerender.deleteCacheItemsFromOrigin(origin);
// Note: this is wasteful. We're proactively "precaching" the page again so
// the next time it's requested, first load is fast. Otherwise, the user
// that loads runs into the cache miss will a pef hit.
await prerender.ssr(origin, {useCache: false});
});
res.status(200).send(posts);
});
app.listen(PORT, async () => {
console.log(`App listening on port ${PORT}`);
console.log('Press Ctrl+C to quit.');
// Initially populate caches on server bootup. In parallel.
feeds.updateFeeds();
// twitter.updateTweets();
ga.updateAnalyticsData();
ytAnalytics.updateAnalyticsData();
});
// Make sure node server process stops if we get a terminating signal.
function processTerminator(sig) {
if (typeof sig === 'string') {
process.exit(1);
}
console.log('%s: Node server stopped.', Date(Date.now()));
}
['SIGHUP', 'SIGINT', 'SIGQUIT', 'SIGILL', 'SIGTRAP', 'SIGABRT',
'SIGBUS', 'SIGFPE', 'SIGUSR1', 'SIGSEGV', 'SIGUSR2', 'SIGTERM'
].forEach(sig => {
process.once(sig, () => processTerminator(sig));
});
You can’t perform that action at this time.