Permalink
Browse files

Initial import

  • Loading branch information...
0 parents commit 844323626a20b7a8142e1681d702f94d3a09d6e7 @mixu committed Sep 17, 2013
Showing with 8,363 additions and 0 deletions.
  1. +1 −0 .gitignore
  2. +34 −0 Makefile
  3. +136 −0 bookgen/generate.js
  4. +24 −0 generate.js
  5. +47 −0 input/0_index.md
  6. +257 −0 input/1_intro.md
  7. +299 −0 input/2_abstractions.md
  8. +339 −0 input/3_time.md
  9. +352 −0 input/4_replication.md
  10. +486 −0 input/5_eventual.md
  11. +62 −0 input/6_appendix.md
  12. +29 −0 layouts/default/footer.html
  13. +61 −0 layouts/default/header.html
  14. +14 −0 layouts/default/index-insert.html
  15. +296 −0 output/abstractions.html
  16. +132 −0 output/appendix.html
  17. +10 −0 output/assets/assert.css
  18. BIN output/assets/bgnoise.png
  19. +90 −0 output/assets/ebook.css
  20. +18 −0 output/assets/jquery-1.6.1.min.js
  21. +52 −0 output/assets/prettify.css
  22. +28 −0 output/assets/prettify.js
  23. +71 −0 output/assets/prettify_coffee.css
  24. +550 −0 output/assets/printable.css
  25. +23 −0 output/assets/quote_colors.js
  26. +35 −0 output/assets/runner.js
  27. +603 −0 output/assets/style.css
  28. +53 −0 output/assets/sunburst.css
  29. +1,356 −0 output/ebook.html
  30. +412 −0 output/eventual.html
  31. BIN output/images/CAP.png
  32. BIN output/images/CAP_choice.png
  33. BIN output/images/barroso_holzle.png
  34. BIN output/images/chandra_failure_detectors.png
  35. BIN output/images/epoch.png
  36. BIN output/images/format_epub.png
  37. BIN output/images/format_html.png
  38. BIN output/images/format_mobi.png
  39. BIN output/images/format_pdf.png
  40. BIN output/images/git-icon.png
  41. BIN output/images/global-clock.png
  42. BIN output/images/google-transact09.png
  43. BIN output/images/local-clock.png
  44. BIN output/images/news_120.jpg
  45. BIN output/images/oltp_overhead.png
  46. BIN output/images/part-repl.png
  47. BIN output/images/pbs.png
  48. BIN output/images/replication-async.png
  49. BIN output/images/replication-both.png
  50. BIN output/images/replication-sync.png
  51. BIN output/images/replication.pptx
  52. BIN output/images/statediagram.png
  53. BIN output/images/system-model.png
  54. BIN output/images/system-of-2.png
  55. BIN output/images/system-of-3.png
  56. BIN output/images/vector_clock.svg.png
  57. +131 −0 output/index.html
  58. +281 −0 output/intro.html
  59. BIN output/mixu-distributed-systems-book.epub
  60. BIN output/mixu-distributed-systems-book.mobi
  61. +329 −0 output/replication.html
  62. +1,356 −0 output/single-page.html
  63. +329 −0 output/time.html
  64. +19 −0 package.json
  65. +48 −0 readme.md
@@ -0,0 +1 @@
+node_modules/
@@ -0,0 +1,34 @@
+build:
+ @node generate.js
+
+ebook:
+ @echo "\n... generating $@"
+ ebook-convert output/ebook.html output/mixu-distributed-systems-book.mobi \
+ --max-levels 0 \
+ --chapter "//*[@class = 'chapter']" \
+ --chapter-mark=none \
+ --sr1-search "<blockquote>" \
+ --sr1-replace "<hr><blockquote>" \
+ --sr2-search "</blockquote>" \
+ --sr2-replace "</blockquote><hr>" \
+ --page-breaks-before='/' \
+ --linearize-tables \
+ --authors "Mikito Takada" \
+ --language en \
+ --output-profile kindle
+ @echo "\n... generating $@"
+ ebook-convert output/ebook.html output/mixu-distributed-systems-book.epub \
+ --max-levels 0 \
+ --chapter "//*[@class = 'chapter']" \
+ --chapter-mark=none \
+ --sr1-search "<blockquote>" \
+ --sr1-replace "<hr><blockquote>" \
+ --sr2-search "</blockquote>" \
+ --sr2-replace "</blockquote><hr>" \
+ --page-breaks-before='/' \
+ --linearize-tables \
+ --authors "Mikito Takada" \
+ --no-default-epub-cover \
+ --language en
+
+.PHONY: build ebook
@@ -0,0 +1,136 @@
+var fs = require('fs'),
+ path = require('path'),
+ existsSync = (fs.existsSync ? fs.existsSync : path.existsSync),
+ marked = require('marked'),
+ util = require('util');
+
+var header = fs.readFileSync('./layouts/default/header.html').toString(),
+ footer = fs.readFileSync('./layouts/default/footer.html').toString();
+
+var BookGen = function() { };
+
+BookGen.generate = function(config) {
+ // get all the files in input
+ if(!Array.isArray(config.input.files)) {
+ // iterate the path and add all files
+ fs.readdir(config.input.files, function (err, files) {
+ if (err) throw err;
+ var basename = path.basename(config.input.files);
+ BookGen.processFiles(config, files.map(function(relname) {
+ return path.normalize( basename + relname);
+ }));
+ });
+ } else {
+ BookGen.processFiles(config, config.input.files);
+ }
+};
+
+BookGen.processFiles = function(config, files) {
+ if(config.input.order == 'sort') {
+ // sort the files
+ files.sort();
+ }
+ if(config.input.index) {
+ // move the index file first
+ var pos = files.indexOf(config.input.index);
+ if(pos > -1) {
+ files.splice(pos, 1);
+ files.unshift(config.input.index);
+ }
+ }
+
+ files = files.filter(function(name) {
+ return !fs.statSync(name).isDirectory();
+ });
+ // concatenate the files
+ console.log(files);
+
+ var full = files
+ .sort(function(a, b) { return a.localeCompare(b); })
+ .map(function(infile, index) {
+ // add an anchor so that the epub links work
+ return '<a name="'+path.basename(infile, '.md').replace(/^[^a-z]*/, '')+'"></a>' +
+ BookGen.writeFile(infile, index, config);
+ }).join('<div style="page-break-after: always;"></div>');
+
+ // write a single page version as well
+ fs.writeFile(config.output+'single-page.html',
+ header.replace('assets/style.css', 'assets/printable.css')
+ .replace(/{{prev}}/g, 'index.html')
+ .replace(/{{next}}/g, 'index.html') +
+ // change links to single page format
+ full.replace('href="index.html"', 'href="#index"')
+ .replace('href="intro.html"', 'href="#intro"')
+ .replace('href="abstractions.html"', 'href="#abstractions"')
+ .replace('href="time.html"', 'href="#time"')
+ .replace('href="replication.html"', 'href="#replication"')
+ .replace('href="eventual.html"', 'href="#eventual"')
+ .replace('href="appendix.html"', 'href="#appendix"') +
+ footer
+ .replace(/{{prev}}/g, 'index.html')
+ .replace(/{{next}}/g, 'index.html')
+ );
+ fs.writeFile(config.output+'ebook.html',
+ header.replace(/<link[^>]+>/g, '')
+ .replace(/{{prev}}/g, 'index.html')
+ .replace(/{{next}}/g, 'index.html')
+ .replace('<script type="text/javascript" src="assets/quote_colors.js"></script>', '')
+ .replace('</head>', '<link type="text/css" rel="stylesheet" href="assets/ebook.css"/></head>') +
+ // change links to single page format
+ full.replace('href="index.html"', 'href="#index"')
+ .replace('href="intro.html"', 'href="#intro"')
+ .replace('href="abstractions.html"', 'href="#abstractions"')
+ .replace('href="time.html"', 'href="#time"')
+ .replace('href="replication.html"', 'href="#replication"')
+ .replace('href="eventual.html"', 'href="#eventual"')
+ .replace('href="appendix.html"', 'href="#appendix"') +
+ footer
+ .replace(/{{prev}}/g, 'index.html')
+ .replace(/{{next}}/g, 'index.html')
+ );
+};
+
+BookGen.writeFile = function(infile, index, config) {
+
+ console.log(infile)
+ var tokens = marked.lexer(fs.readFileSync(infile).toString());
+ var content = marked
+ .parser(tokens)
+ .replace(/<(ul|ol)>/g, '<$1 class="list">')
+ .replace(/<pre><code[^>]*>([\s\S]*?)<\/code><\/pre>/mg, '<pre>$1</pre>')
+ .replace(/<p><img([^>]*)>\s*<\/p>/g, '<p class="img-container"><img$1></p>')
+ .replace(/%chapter_number%\.?/g, index+'.');
+
+ var links = {
+ 'index': { prev: 'index.html', next: 'intro.html' },
+ 'intro': { prev: 'index.html', next: 'abstractions.html' },
+ 'abstractions': { prev: 'intro.html', next: 'time.html' },
+ 'time': { prev: 'abstractions.html', next: 'replication.html' },
+ 'replication': { prev: 'time.html', next: 'eventual.html' },
+ 'eventual': { prev: 'replication.html', next: 'appendix.html' },
+ 'appendix': { prev: 'eventual.html', next: 'appendix.html' },
+ };
+
+
+ // replace until the first alpha character
+ var outName = path.basename(infile, '.md').replace(/^[^a-z]*/, '');
+
+ fs.writeFileSync(config.output + outName + '.html',
+ header
+ .replace(/{{title}}/g, config.titles[outName +'.md' ] || 'Distributed systems: algorithms and artefacts')
+ .replace(/{{prev}}/g, (links[outName] ? links[outName].prev : ''))
+ .replace(/{{next}}/g, (links[outName] ? links[outName].next : ''))
+ // special download header
+ .replace('<!-- index-insert -->', (outName == 'index' ? fs.readFileSync('./layouts/default/index-insert.html') : '')) +
+ content +
+ footer
+ .replace(/{{prev}}/g, (links[outName] ? links[outName].prev : ''))
+ .replace(/{{next}}/g, (links[outName] ? links[outName].next : ''))
+ );
+
+ prev = outName+'.html';
+
+ return content;
+};
+
+module.exports = BookGen;
@@ -0,0 +1,24 @@
+var BookGen = require('./bookgen/generate.js');
+
+var config = {
+ output: __dirname + '/output/',
+
+ input: {
+
+ dir: __dirname + '/input/',
+
+ files: __dirname + '/input/',
+
+ // specify exact order later on, when single page v is generated
+
+ index: 'index.html'
+ },
+
+ titles: {
+ },
+
+ layout: __dirname + '/layouts/default/'
+
+};
+
+BookGen.generate(config);
@@ -0,0 +1,47 @@
+## Introduction
+
+I wanted a text that would bring together the ideas behind many of the more recent distributed systems - systems such as Amazon's Dynamo, Google's BigTable and MapReduce, Apache's Hadoop and so on.
+
+In this text I've tried to provide a more accessible introduction to distributed systems. To me, that means two things: introducing the key concepts that you will need in order to [have a good time](https://www.google.com/search?q=super+cool+ski+instructor) reading more serious texts, and providing a narrative that covers things in enough detail that you get a gist of what's going on without getting stuck on details. It's 2013, you've got the Internet, and you can selectively read more about the topics you find most interesting.
+
+In my view, much of distributed programming is about dealing with the implications of two consequences of distribution:
+
+- that information travels at the speed of light
+- that independent things fail independently*
+
+In other words, that the core of distributed programming is dealing with distance (duh!) and having more than one thing (duh!). These constraints define a space of possible system designs, and my hope is that after reading this you'll have a better sense of how distance, time and consistency models interact.
+
+This text is focused on distributed programming and systems concepts you'll need to understand commercial systems in the data center. It would be madness to attempt to cover everything. You'll learn many key protocols and algorithms (covering, for example, many of the most cited papers in the discipline), including some new exciting ways to look at eventual consistency that haven't still made it into college textbooks - such as CRDTs and the CALM theorem.
+
+I hope you like it! If you want to say thanks, follow me on [Github](https://github.com/mixu/) (or [Twitter](http://twitter.com/mikitotakada)). And if you spot an error, [file a pull request on Github](https://github.com/mixu/distsysbook/issues).
+
+---
+
+# 1. Basics
+
+[The first chapter](intro.html) covers distributed systems at a high level by introducing a number of important terms and concepts. It covers high level goals, such as scalability, availability, performance, latency and fault tolerance; how those are hard to achieve, and how abstractions and models as well as partitioning and replication come into play.
+
+# 2. Up and down the level of abstraction
+
+[The second chapter](abstractions.html) dives deeper into abstractions and impossibility results. It starts with a Nietzsche quote, and then introduces system models and the many assumptions that are made in a typical system model. It then discusses the CAP theorem and summarizes the FLP impossibility result. It then turns to the implications of the CAP theorem, one of which is that one ought to explore other consistency models. A number of consistency models are then discussed.
+
+# 3. Time and order
+
+A big part of understanding distributed systems is about understanding time and order. To the extent that we fail to understand and model time, our systems will fail. [The third chapter](time.html) discusses time and order, and clocks as well as the various uses of time, order and clocks (such as vector clocks and failure detectors).
+
+# 4. Replication: preventing divergence
+
+The [fourth chapter](replication.html) introduces the replication problem, and the two basic ways in which it can be performed. It turns out that most of the relevant characteristics can be discussed with just this simple characterization. Then, replication methods for maintaining single-copy consistency are discussed from the least fault tolerant (2PC) to Paxos.
+
+# 5. Replication: accepting divergence
+
+The [fifth chapter](eventual.html) discussed replication with weak consistency guarantees. It introduces a basic reconciliation scenario, where partitioned replicas attempt to reach agreement. It then discusses Amazon's Dynamo as an example of a system design with weak consistency guarantees. Finally, two perspectives on disorderly programming are discussed: CRDTs and the CALM theorem.
+
+# Appendix
+
+[The appendix](appendix.html) covers recommendations for further reading.
+
+---
+
+<p class="footnote">*: This is a [lie](http://en.wikipedia.org/wiki/Statistical_independence). [This post by Jay Kreps elaborates](http://blog.empathybox.com/post/19574936361/getting-real-about-distributed-system-reliability).
+</p>
Oops, something went wrong.

0 comments on commit 8443236

Please sign in to comment.