angular.js/gdocs.js at v0.10.2 · angular/angular.js · GitHub

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Newer

Older

100755 254 lines (229 sloc) 7.289 kb

Raw Normal view History

7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		1	#!/usr/bin/env node
887da56 enhancing gdocs.js to work with nested collections IgorMinar authored Mar 1, 2011
		2
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		3	var http = require('http');
		4	var https = require('https');
		5	var fs = require('fs');
		6
887da56 enhancing gdocs.js to work with nested collections IgorMinar authored Mar 2, 2011
		7	var collections = {
29d36e9 feat(gdocs): better error handling mhevery authored Oct 5, 2011
		8	'guide': 'https://docs.google.com/feeds/default/private/full/folder%3A0B9PsajIPqzmANGUwMGVhZmYtMTk1ZC00NTdmLWIxMDAtZGI5YWNlZjQ2YjZl/contents',
		9	'api': 'https://docs.google.com/feeds/default/private/full/folder%3A0B7Ovm8bUYiUDYjMwYTc2YWUtZTgzYy00YjIxLThlZDYtYWJlOTFlNzE2NzEw/contents',
		10	'tutorial': 'https://docs.google.com/feeds/default/private/full/folder%3A0B9PsajIPqzmAYWMxYWE3MzYtYzdjYS00OGQxLWJhZjItYzZkMzJiZTRhZjFl/contents',
		11	'cookbook': 'https://docs.google.com/feeds/default/private/full/folder%3A0B7Ovm8bUYiUDNzkxZWM5ZTItN2M5NC00NWIxLTg2ZDMtMmYwNDY1NWM1MGU4/contents',
		12	'misc': 'https://docs.google.com/feeds/default/private/full/folder%3A0B7Ovm8bUYiUDZjVlNmZkYzQtMjZlOC00NmZhLWI5MjAtMGRjZjlkOGJkMDBi/contents'
d6e4636 Couple of missing semi-colons vojtajina authored Jun 7, 2011
		13	};
887da56 enhancing gdocs.js to work with nested collections IgorMinar authored Mar 2, 2011
		14
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		15	console.log('Google Docs...');
		16
		17	var flag = process && process.argv[2];
29d36e9 feat(gdocs): better error handling mhevery authored Oct 5, 2011
		18	if (flag == '--login') {
		19	var username = process.argv[3];
		20	if (username) {
		21	askPassword(function(password){
		22	login(username, password);
		23	});
		24	} else {
		25	console.log('Missing username!');
		26	}
		27	} else if (flag == '--fetch') {
887da56 enhancing gdocs.js to work with nested collections IgorMinar authored Mar 2, 2011
		28	var collection = process.argv[3];
		29	if (collection) {
		30	fetch(collection, collections[collection]);
		31	} else {
		32	for (collection in collections)
		33	fetch(collection, collections[collection]);
		34	}
29d36e9 feat(gdocs): better error handling mhevery authored Oct 5, 2011
		35	} else {
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		36	help();
29d36e9 feat(gdocs): better error handling mhevery authored Oct 5, 2011
		37	}
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		38
		39	function help(){
		40	console.log('Synopsys');
887da56 enhancing gdocs.js to work with nested collections IgorMinar authored Mar 2, 2011
		41	console.log('gdocs.js --login <username>');
		42	console.log('gdocs.js --fetch [<docs collection>]');
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		43	process.exit(-1);
		44	};
		45
		46
91f9efe gdocs.js should store files under docs/content/[collection]/ IgorMinar authored May 9, 2011
		47	function fetch(collection, url){
		48	console.log('fetching a list of docs in collection ' + collection + '...');
887da56 enhancing gdocs.js to work with nested collections IgorMinar authored Mar 2, 2011
		49	request('GET', url, {
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		50	headers: {
		51	'Gdata-Version': '3.0',
		52	'Authorization': 'GoogleLogin auth=' + getAuthToken()
		53	}
		54	},
		55	function(chunk){
		56	var entries = chunk.split('<entry');
		57	entries.shift();
		58	entries.forEach(function(entry){
		59	var title = entry.match(/<title>(.*?)<\/title>/)[1];
		60	if (title.match(/\.ngdoc$/)) {
		61	var exportUrl = entry.match(/<content type='text\/html' src='(.*?)'\/>/)[1];
91f9efe gdocs.js should store files under docs/content/[collection]/ IgorMinar authored May 10, 2011
		62	download(collection, title, exportUrl);
c763b00 feat(gdocs.js): output usefull error messages when not logged in. mhevery authored Aug 31, 2011
		63	};
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		64	});
		65	}
		66	);
		67	}
		68
91f9efe gdocs.js should store files under docs/content/[collection]/ IgorMinar authored May 10, 2011
		69	function download(collection, name, url) {
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		70	console.log('Downloading:', name, '...');
		71	request('GET', url + '&exportFormat=txt',
		72	{
		73	headers: {
		74	'Gdata-Version': '3.0',
		75	'Authorization': 'GoogleLogin auth=' + getAuthToken()
		76	}
		77	},
		78	function(data){
		79	data = data.replace('\ufeff', '');
		80	data = data.replace(/\r\n/mg, '\n');
		81
a4dd9ca fix comment stripping IgorMinar authored Jun 7, 2011
		82	// strip out all text annotations
		83	data = data.replace(/\[[a-zA-Z]{1,2}\]/mg, '');
		84
22dee3e gdocs.js - add docos style comment stripping IgorMinar authored May 19, 2011
		85	// strip out all docos comments
d6e4636 Couple of missing semi-colons vojtajina authored Jun 7, 2011
		86	data = data.replace(/^[^\s_]+:\n\S+[\S\s]*$/m, '');
22dee3e gdocs.js - add docos style comment stripping IgorMinar authored May 20, 2011
		87
fd6e5e3 replace smart-quotes with regular quotes mhevery authored Feb 18, 2011
		88	// fix smart-quotes
		89	data = data.replace(/[“”]/g, '"');
		90	data = data.replace(/[‘’]/g, "'");
		91
		92
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		93	data = data + '\n';
d428c99 Replaced double line break with single when text is fetched from Google ... Di Peng authored Jun 14, 2011
		94
		95	//this should be a bug in Google Doc API, hence need to remove this once the bug is fixed
		96	data = data.replace(/\n\n/g, '\n');
		97
91f9efe gdocs.js should store files under docs/content/[collection]/ IgorMinar authored May 10, 2011
		98	fs.writeFileSync('docs/content/' + collection + '/' + name, reflow(data, 100));
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		99	}
		100	);
		101	}
		102
		103	/**
		104	* token=$(curl
		105	* -s https://www.google.com/accounts/ClientLogin
		106	* -d Email=...username...
		107	* -d Passwd=...password...
		108	* -d accountType=GOOGLE
		109	* -d service=writely
		110	* -d Gdata-version=3.0 \| cut -d "=" -f 2)
		111	*/
		112	function login(username, password){
		113	request('POST', 'https://www.google.com/accounts/ClientLogin',
		114	{
		115	data: {
		116	Email: username,
		117	Passwd: password,
		118	accountType: 'GOOGLE',
		119	service: 'writely',
		120	'Gdata-version': '3.0'
		121	},
		122	headers: {
		123	'Content-type': 'application/x-www-form-urlencoded'
		124	}
		125	},
		126	function(chunk){
		127	var token;
		128	chunk.split('\n').forEach(function(line){
		129	var parts = line.split('=');
		130	if (parts[0] == 'Auth') {
		131	token = parts[1];
		132	}
		133	});
		134	if (token) {
		135	fs.writeFileSync('tmp/gdocs.auth', token);
		136	console.log("logged in, token saved in 'tmp/gdocs.auth'");
		137	} else {
		138	console.log('failed to log in');
		139	}
		140	}
		141	);
		142	}
		143
		144	function getAuthToken(){
c763b00 feat(gdocs.js): output usefull error messages when not logged in. mhevery authored Aug 31, 2011
		145	var pwdFile = 'tmp/gdocs.auth';
		146	try {
		147	fs.statSync(pwdFile);
		148	return fs.readFileSync(pwdFile);
		149	} catch (e) {
		150	console.log('Please log in first...');
		151	}
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		152	}
		153
		154	function request(method, url, options, response) {
		155	var url = url.match(/http(s?):\/\/(.+?)(\/.*)/);
29d36e9 feat(gdocs): better error handling mhevery authored Oct 5, 2011
		156	var isHttps = url[1];
		157	var request = (isHttps ? https : http).request({
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		158	host: url[2],
		159	port: (url[1] ? 443 : 80),
		160	path: url[3],
		161	method: method
		162	}, function(res){
c763b00 feat(gdocs.js): output usefull error messages when not logged in. mhevery authored Aug 31, 2011
		163	switch (res.statusCode) {
29d36e9 feat(gdocs): better error handling mhevery authored Oct 5, 2011
		164	case 200:
c763b00 feat(gdocs.js): output usefull error messages when not logged in. mhevery authored Aug 31, 2011
		165	var data = [];
		166	res.setEncoding('utf8');
29d36e9 feat(gdocs): better error handling mhevery authored Oct 5, 2011
		167	res.on('end', function (){ response(data.join('')); });
		168	res.on('close', function (){ response(data.join('')); }); // https
		169	res.on('data', function (chunk) { data.push(chunk); });
		170	res.on('error', function (e){ console.log(e); });
c763b00 feat(gdocs.js): output usefull error messages when not logged in. mhevery authored Aug 31, 2011
		171	break;
29d36e9 feat(gdocs): better error handling mhevery authored Oct 5, 2011
		172	case 401:
c763b00 feat(gdocs.js): output usefull error messages when not logged in. mhevery authored Aug 31, 2011
		173	console.log('Eror: Login credentials expired! Please login.');
		174	break;
29d36e9 feat(gdocs): better error handling mhevery authored Oct 5, 2011
		175	default:
		176	var data = [];
		177	console.log('ERROR: ', res.statusCode);
		178	console.log('REQUEST URL: ', url[0]);
		179	console.log('REQUEST POST: ', options.data);
		180	console.log('REQUEST HEADERS: ', options.headers);
		181	console.log('RESPONSE HEADERS: ', res.headers);
		182	res.on('end', function (){ console.log('BODY: ', data.join('')); });
		183	res.on('close', function (){ console.log('BODY: ', data.join('')); }); // https
		184	res.on('data', function (chunk) { data.push(chunk); });
		185	res.on('error', function (e){ console.log(e); });
c763b00 feat(gdocs.js): output usefull error messages when not logged in. mhevery authored Aug 31, 2011
		186	}
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		187	});
		188	for(var header in options.headers) {
		189	request.setHeader(header, options.headers[header]);
		190	}
		191	if (options.data)
		192	request.write(encodeData(options.data));
		193	request.on('end', function(){
		194	console.log('end');
		195	});
		196	request.end();
		197	}
		198
		199	function encodeData(obj) {
		200	var pairs = [];
		201	for(var key in obj) {
		202	pairs.push(key + '=' + obj[key]);
		203	}
		204	return pairs.join('&') + '\n';
		205	}
		206
		207	function askPassword(callback) {
		208	var stdin = process.openStdin(),
		209	stdio = process.binding("stdio");
		210
		211	stdio.setRawMode();
		212
		213	console.log('Enter your password:');
d6e4636 Couple of missing semi-colons vojtajina authored Jun 7, 2011
		214	var password = "";
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		215	stdin.on("data", function (c) {
		216	c = c + "";
		217	switch (c) {
		218	case "\n": case "\r": case "\u0004":
		219	stdio.setRawMode(false);
		220	stdin.pause();
		221	callback(password);
		222	break;
		223	case "\u0003":
		224	process.exit();
		225	break;
		226	default:
		227	password += c;
		228	break;
		229	}
		230	})
		231
		232	}
		233
		234	function reflow(text, margin) {
		235	var lines = [];
		236	text.split(/\n/).forEach(function(line) {
		237	var col = 0;
		238	var reflowLine = '';
		239	function flush(){
08e3b1e gdocs.js should strip trailing whitespace in imported docs IgorMinar authored Feb 22, 2011
		240	reflowLine = reflowLine.replace(/\s*$/, '');
7a54d27 script for dowlnoading docs from google docs mhevery authored Feb 17, 2011
		241	lines.push(reflowLine);
		242	reflowLine = '';
		243	col = 0;
		244	}
		245	line.replace(/\s\S\s*/g, function(chunk){
		246	if (col + chunk.length > margin) flush();
		247	reflowLine += chunk;
		248	col += chunk.length;
		249	});
		250	flush();
		251	});
		252	return lines.join('\n');
		253	}

Something went wrong with that request. Please try again.