Skip to content
Permalink
Browse files

Merge pull request #72 from halkeye/sitemap

Add a sitemap to get google indexing the plugin site
  • Loading branch information
olblak committed Dec 5, 2019
2 parents b204164 + 7e0e7e8 commit c2c50cf4a507abbfd84b27e4abd258e0af860710
Showing with 275 additions and 61 deletions.
  1. +70 −0 create-site-map.js
  2. +1 −1 package.json
  3. +2 −0 public/robots.txt
  4. +13 −1 server.js
  5. +2 −1 webpack/webpack.config.js
  6. +187 −58 yarn.lock
@@ -0,0 +1,70 @@
/* eslint-disable no-console */
const unirest = require('unirest');

const pluginSite = process.env.REST_API_URL || 'https://plugins.jenkins.io/api';

function formatDate(date) {
var d = new Date(date),
month = `${ d.getMonth() + 1}`,
day = `${ d.getDate()}`,
year = d.getFullYear();

if (month.length < 2) month = `0${ month}`;
if (day.length < 2) day = `0${ day}`;

return [year, month, day].join('-');
}


const loggerInfo = console.info.bind(console, '[create-site-map.js][createSiteMap]');

function getPage(page) {
const url = `${pluginSite}/plugins/?limit=100&page=${page}`;
loggerInfo(`Fetching '${url}'`);
return unirest.get(url).then(results => {
if (results.statusCode !== 200) {
throw results.body;
}
return results.body;
});
}

function createSiteMap() {
const plugins = [];
loggerInfo('Starting to fetch plugins for sitemap');

const handlePromise = pluginsContainer => {
for (const plugin of pluginsContainer.plugins) {
plugins.push(plugin);
}
if(pluginsContainer.page <= pluginsContainer.pages) {
return getPage(pluginsContainer.page+1).then(handlePromise);
}
return null;
};

return getPage(1).then(handlePromise).then(() => {
return `<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
${plugins.map(plugin => `<url>
<loc>https://plugins.jenkins.io/${plugin.name}</loc>
<lastmod>${formatDate(new Date(plugin.releaseTimestamp))}</lastmod>
</url>`
).join('')}
</urlset>`;
});
}

if (require.main == module) {
createSiteMap()
.then(console.log)
.then(() => {
console.log('done');
process.exit(0);
})
.catch(console.error);
}

module.exports = {
createSiteMap
};
@@ -74,7 +74,7 @@
"redux-logger": "^2.6.1",
"redux-thunk": "^2.1.0",
"reselect": "^2.5.4",
"unirest": "^0.5.1"
"unirest": "^0.6.0"
},
"devDependencies": {
"babel": "^6.5.2",
@@ -1,2 +1,4 @@
User-agent: *
Disallow:

Sitemap: https://plugins.jenkins.io/sitemap.xml
@@ -17,9 +17,12 @@ import cheerio from 'cheerio';
import schedule from 'node-schedule';
import { cleanTitle, defaultPluginSiteTitle, pluginSiteTitleSuffix } from './app/commons/helper';

import {createSiteMap} from './create-site-map.js';

const app = express();
const port = 5000;
const port = process.env.PORT || 5000;
const jsPath = '/assets/js';
let sitemap = '';

// Using helmet to secure Express with various HTTP headers
app.use(helmet());
@@ -80,6 +83,7 @@ const downloadHeader = () => {
}

downloadHeader();
createSiteMap().then(s => { sitemap = s });

const getPluginSiteVersion = () => {
const file = './GIT_COMMIT';
@@ -92,6 +96,11 @@ const getPluginSiteVersion = () => {

const pluginSiteVersion = getPluginSiteVersion();

app.get('/sitemap.xml', (req, res) => {
res.type('application/xml');
res.send(sitemap);
});

app.get('*', (req, res, next) => {
match({ routes: routes, location: req.url }, (error, redirectLocation, renderProps) => {
if (error) {
@@ -150,5 +159,8 @@ app.listen(port, (error) => {
schedule.scheduleJob('*/15 * * * *', () => {
downloadHeader();
});
schedule.scheduleJob('15 */3 * * *', () => {
createSiteMap().then(s => { sitemap = s });
});
}
});
@@ -20,7 +20,8 @@ const plugins = [
// This is needed for React to properly do production builds
'process.env': JSON.stringify({
debug: !release,
NODE_ENV: release ? 'production' : 'development'
NODE_ENV: release ? 'production' : 'development',
PORT: process.env.PORT || 5000,
}),
__PRODUCTION__: release,
__REST_API_URL__: JSON.stringify(process.env.REST_API_URL || "https://plugins.jenkins.io/api"),

0 comments on commit c2c50cf

Please sign in to comment.
You can’t perform that action at this time.