From e381bfb1b7ef1ea9fa2424c844628768f0d0d947 Mon Sep 17 00:00:00 2001 From: Alexander Falzberger Date: Thu, 2 Apr 2020 13:00:59 +0200 Subject: [PATCH] Update mdbook to Timely 0.11.1 and Differential 0.11.0 (#267) * Fixing some typos in mdbook and readme files * Fixing introductory examples in mdbook * Updating mdbook to timely 0.11.1 and differential 0.11.0 --- mdbook/src/chapter_0/chapter_0_0.md | 4 +- mdbook/src/chapter_0/chapter_0_1.md | 24 ++++---- mdbook/src/chapter_0/chapter_0_2.md | 90 ++++++++++++++--------------- mdbook/src/chapter_0/chapter_0_3.md | 2 +- mdbook/src/chapter_2/chapter_2_3.md | 4 +- mdbook/src/chapter_2/chapter_2_4.md | 6 +- mdbook/src/chapter_2/chapter_2_5.md | 4 +- mdbook/src/chapter_2/chapter_2_6.md | 2 +- mdbook/src/chapter_2/chapter_2_7.md | 6 +- mdbook/src/chapter_3/chapter_3_2.md | 2 +- mdbook/src/chapter_4/chapter_4_1.md | 2 +- mdbook/src/chapter_5/chapter_5_3.md | 2 - mdbook/src/chapter_a/chapter_a_2.md | 10 ++-- mdbook/src/chapter_a/chapter_a_3.md | 14 ++--- 14 files changed, 85 insertions(+), 87 deletions(-) diff --git a/mdbook/src/chapter_0/chapter_0_0.md b/mdbook/src/chapter_0/chapter_0_0.md index f4fc713155..a8ea8c5232 100644 --- a/mdbook/src/chapter_0/chapter_0_0.md +++ b/mdbook/src/chapter_0/chapter_0_0.md @@ -21,8 +21,8 @@ Instead, edit your `Cargo.toml` file, which tells Rust about your dependencies, authors = ["Your Name "] [dependencies] - timely = "0.7" - differential-dataflow = "0.7" + timely = "0.11.1" + differential-dataflow = "0.11.0" Echidnatron% You should only need to add those last two lines there, which bring in dependencies on both [timely dataflow](https://github.com/TimelyDataflow/timely-dataflow) and [differential dataflow](https://github.com/TimelyDataflow/differential-dataflow). We will be using both of those. diff --git a/mdbook/src/chapter_0/chapter_0_1.md b/mdbook/src/chapter_0/chapter_0_1.md index 999d24d3f4..60b16e51ad 100644 --- a/mdbook/src/chapter_0/chapter_0_1.md +++ b/mdbook/src/chapter_0/chapter_0_1.md @@ -27,7 +27,7 @@ If you are following along at home, put this in your `src/main.rs` file. // create a new collection from our input. let manages = input.to_collection(scope); - // if (m2, m1) and (m1, p), then output (m1, m2, p) + // if (m2, m1) and (m1, p), then output (m1, (m2, p)) manages .map(|(m2, m1)| (m1, m2)) .join(&manages) @@ -50,7 +50,7 @@ If you are following along at home, put this in your `src/main.rs` file. This program has a bit of boilerplate, but at its heart it defines a new input `manages` and then joins it with itself, once the fields have been re-ordered. The intent is as stated in the comment: ```rust,no_run - // if (m2, m1) and (m1, p), then output (m1, m2, p) + // if (m2, m1) and (m1, p), then output (m1, (m2, p)) ``` We want to report each pair `(m2, p)`, and we happen to also produce as evidence the `m1` connecting them. @@ -59,16 +59,16 @@ When we execute this program we get to see the skip-level reports for the small Echidnatron% cargo run -- 10 Running `target/debug/my_project` - ((0, 0, 0), (Root, 0), 1) - ((0, 0, 1), (Root, 0), 1) - ((1, 0, 2), (Root, 0), 1) - ((1, 0, 3), (Root, 0), 1) - ((2, 1, 4), (Root, 0), 1) - ((2, 1, 5), (Root, 0), 1) - ((3, 1, 6), (Root, 0), 1) - ((3, 1, 7), (Root, 0), 1) - ((4, 2, 8), (Root, 0), 1) - ((4, 2, 9), (Root, 0), 1) + ((0, (0, 0)), 0, 1) + ((0, (0, 1)), 0, 1) + ((1, (0, 2)), 0, 1) + ((1, (0, 3)), 0, 1) + ((2, (1, 4)), 0, 1) + ((2, (1, 5)), 0, 1) + ((3, (1, 6)), 0, 1) + ((3, (1, 7)), 0, 1) + ((4, (2, 8)), 0, 1) + ((4, (2, 9)), 0, 1) Echidnatron% This is a bit crazy, but what we are seeing is many triples of the form diff --git a/mdbook/src/chapter_0/chapter_0_2.md b/mdbook/src/chapter_0/chapter_0_2.md index ac9a910b3e..2fb18f8c29 100644 --- a/mdbook/src/chapter_0/chapter_0_2.md +++ b/mdbook/src/chapter_0/chapter_0_2.md @@ -35,65 +35,65 @@ We do this for each of the non-boss employees and get to see a bunch of outputs. Echidnatron% cargo run -- 10 Running `target/debug/my_project` - ((0, 0, 0), (Root, 0), 1) - ((0, 0, 1), (Root, 0), 1) - ((0, 0, 2), (Root, 2), 1) - ((1, 0, 2), (Root, 0), 1) - ((1, 0, 2), (Root, 2), -1) - ((1, 0, 3), (Root, 0), 1) - ((1, 0, 4), (Root, 4), 1) - ((1, 0, 5), (Root, 5), 1) - ((2, 0, 4), (Root, 2), 1) - ((2, 0, 4), (Root, 4), -1) - ((2, 0, 5), (Root, 2), 1) - ((2, 0, 5), (Root, 5), -1) - ((2, 0, 6), (Root, 6), 1) - ((2, 0, 7), (Root, 7), 1) - ((2, 0, 8), (Root, 8), 1) - ((2, 1, 4), (Root, 0), 1) - ((2, 1, 4), (Root, 2), -1) - ((2, 1, 5), (Root, 0), 1) - ((2, 1, 5), (Root, 2), -1) - ((3, 1, 6), (Root, 0), 1) - ((3, 1, 6), (Root, 6), -1) - ((3, 1, 7), (Root, 0), 1) - ((3, 1, 7), (Root, 7), -1) - ((3, 1, 9), (Root, 9), 1) - ((4, 1, 8), (Root, 4), 1) - ((4, 1, 8), (Root, 8), -1) - ((4, 1, 9), (Root, 4), 1) - ((4, 1, 9), (Root, 9), -1) - ((4, 2, 8), (Root, 0), 1) - ((4, 2, 8), (Root, 4), -1) - ((4, 2, 9), (Root, 0), 1) - ((4, 2, 9), (Root, 4), -1) + ((0, (0, 0)), 0, 1) + ((0, (0, 1)), 0, 1) + ((0, (0, 2)), 2, 1) + ((1, (0, 2)), 0, 1) + ((1, (0, 2)), 2, -1) + ((1, (0, 3)), 0, 1) + ((1, (0, 4)), 4, 1) + ((1, (0, 5)), 5, 1) + ((2, (0, 4)), 2, 1) + ((2, (0, 4)), 4, -1) + ((2, (0, 5)), 2, 1) + ((2, (0, 5)), 5, -1) + ((2, (0, 6)), 6, 1) + ((2, (0, 7)), 7, 1) + ((2, (0, 8)), 8, 1) + ((2, (1, 4)), 0, 1) + ((2, (1, 4)), 2, -1) + ((2, (1, 5)), 0, 1) + ((2, (1, 5)), 2, -1) + ((3, (1, 6)), 0, 1) + ((3, (1, 6)), 6, -1) + ((3, (1, 7)), 0, 1) + ((3, (1, 7)), 7, -1) + ((3, (1, 9)), 9, 1) + ((4, (1, 8)), 4, 1) + ((4, (1, 8)), 8, -1) + ((4, (1, 9)), 4, 1) + ((4, (1, 9)), 9, -1) + ((4, (2, 8)), 0, 1) + ((4, (2, 8)), 4, -1) + ((4, (2, 9)), 0, 1) + ((4, (2, 9)), 4, -1) Echidnatron% Gaaaaaaah! What in the !#$!? -It turns out our input changes result in output changes. Let's try and break this down and make some sense. If we group the columns by time, those `(Root, _)` fields, we see a bit more structure. +It turns out our input changes result in output changes. Let's try and break this down and make some sense. If we group the columns by time, the second element of the tuples, we see a bit more structure. -1. The `(Root, 0)` entries are exactly the same as for our prior computation, where we just loaded the data. +1. The entries with time `0` are exactly the same as for our prior computation, where we just loaded the data. -2. There aren't any `(Root, 1)` entries (go check). That is because the input didn't change in our first step, because 1/2 == 1/3 == 0. Since the input didn't change, the output doesn't change. +2. There aren't any entries at time `1` (go check). That is because the input didn't change in our first step, because 1/2 == 1/3 == 0. Since the input didn't change, the output doesn't change. 3. The other times are more complicated. -Let's look at times `(Root, 4)`. +Let's look at the entries for time `4`. - ((1, 0, 4), (Root, 4), 1) - ((2, 0, 4), (Root, 4), -1) - ((4, 1, 8), (Root, 4), 1) - ((4, 1, 9), (Root, 4), 1) - ((4, 2, 8), (Root, 4), -1) - ((4, 2, 9), (Root, 4), -1) + ((1, (0, 4)), 4, 1) + ((2, (0, 4)), 4, -1) + ((4, (1, 8)), 4, 1) + ((4, (1, 9)), 4, 1) + ((4, (2, 8)), 4, -1) + ((4, (2, 9)), 4, -1) There is a bit going on here. Four's manager changed from two to one, and while their skip-level manager remained zero the explanation changed. The first two lines record this change. The next four lines record the change in the skip-level manager of four's reports, eight and nine. -At the end, `(Root, 9)`, things are a bit simpler because we have reached the employees with no reports, and so the only changes are their skip-level manager, without any implications for other people. +At the end, time `9`, things are a bit simpler because we have reached the employees with no reports, and so the only changes are their skip-level manager, without any implications for other people. - ((3, 1, 9), (Root, 9), 1) - ((4, 1, 9), (Root, 9), -1) + ((3, (1, 9)), 9, 1) + ((4, (1, 9)), 9, -1) Oof. Well, we probably *could* have figured these things out by hand, right? diff --git a/mdbook/src/chapter_0/chapter_0_3.md b/mdbook/src/chapter_0/chapter_0_3.md index 68fe1f2167..c8999e5fb4 100644 --- a/mdbook/src/chapter_0/chapter_0_3.md +++ b/mdbook/src/chapter_0/chapter_0_3.md @@ -68,7 +68,7 @@ Instead of loading all of our changes and only waiting for the result, we can lo // create a new collection from an input session. let manages = input.to_collection(scope); - // if (m2, m1) and (m1, p), then output (m1, m2, p) + // if (m2, m1) and (m1, p), then output (m1, (m2, p)) manages .map(|(m2, m1)| (m1, m2)) .join(&manages) diff --git a/mdbook/src/chapter_2/chapter_2_3.md b/mdbook/src/chapter_2/chapter_2_3.md index 7bb9210026..0a7782b66b 100644 --- a/mdbook/src/chapter_2/chapter_2_3.md +++ b/mdbook/src/chapter_2/chapter_2_3.md @@ -14,7 +14,7 @@ This collection likely has at most one copy of each record, unless perhaps any m Importantly, `concat` doesn't do the hard work of ensuring that there is only one physical of each element. If we inspect the output of the `concat` above, we might see - ((0,0), (Root, 0), 1) - ((0,0), (Root, 0), 1) + ((0, 0), 0, 1) + ((0, 0), 0, 1) Although these are two updates to the same element at the same time, `concat` is a bit lazy (read: efficient) and doesn't do the hard work until we ask it. For that, we'll need the `consolidate` operator. \ No newline at end of file diff --git a/mdbook/src/chapter_2/chapter_2_4.md b/mdbook/src/chapter_2/chapter_2_4.md index a7d1d97825..f61cebce6d 100644 --- a/mdbook/src/chapter_2/chapter_2_4.md +++ b/mdbook/src/chapter_2/chapter_2_4.md @@ -15,8 +15,8 @@ As an example, if we were to inspect we might see two copies of the same element: - ((0,0), (Root, 0), 1) - ((0,0), (Root, 0), 1) + ((0, 0), 0, 1) + ((0, 0), 0, 1) However, by introducing `consolidate` @@ -30,6 +30,6 @@ However, by introducing `consolidate` we are guaranteed to see at most one `(0,0)` update at each time: - ((0,0), (Root, 0), 2) + ((0, 0), 0, 2) The `consolidate` operator is mostly useful before `inspect`ing data, but it can also be important for efficiency; knowing when to spend the additional computation to consolidate the representation of your data is an advanced topic! diff --git a/mdbook/src/chapter_2/chapter_2_5.md b/mdbook/src/chapter_2/chapter_2_5.md index 2b513637c8..48a45ba794 100644 --- a/mdbook/src/chapter_2/chapter_2_5.md +++ b/mdbook/src/chapter_2/chapter_2_5.md @@ -1,6 +1,6 @@ ## The Join Operator -The `join` operator takes two input collections, each of which must have records with a `(key, value)` structure, and must have the same type of `key`. For each pair of elements with matching key, one from each input, the join operator produces the output `(key, value1, value2)`. +The `join` operator takes two input collections, each of which must have records with a `(key, value)` structure, and must have the same type of `key`. For each pair of elements with matching key, one from each input, the join operator produces the output `(key, (value1, value2))`. Our example from earlier uses a join to match up pairs `(m2, m1)` and `(m1, p)` when the `m1` is in common. To do this, we first have to switch the records in the first collection around, so that they are keyed by `m1` instead of `m2`. @@ -11,4 +11,4 @@ Our example from earlier uses a join to match up pairs `(m2, m1)` and `(m1, p)` .inspect(|x| println!("{:?}", x)); ``` -The join operator multiplies frequencies, so if a record `(key, val1)` has multiplicity five, and a matching record `(key, val2)` has multiplicity three, the output result will be `(key, val1, val2)` with multiplicity fifteen. +The join operator multiplies frequencies, so if a record `(key, val1)` has multiplicity five, and a matching record `(key, val2)` has multiplicity three, the output result will be `(key, (val1, val2))` with multiplicity fifteen. diff --git a/mdbook/src/chapter_2/chapter_2_6.md b/mdbook/src/chapter_2/chapter_2_6.md index bc33104430..ec8fbb0e5d 100644 --- a/mdbook/src/chapter_2/chapter_2_6.md +++ b/mdbook/src/chapter_2/chapter_2_6.md @@ -11,7 +11,7 @@ For example, to produce for each manager their managee with the lowest identifie // Each element of input is a `(&Value, Count)` for index in 1 .. input.len() { - if input[min_index] > input[index].0 { + if input[min_index].0 > input[index].0 { min_index = index; } } diff --git a/mdbook/src/chapter_2/chapter_2_7.md b/mdbook/src/chapter_2/chapter_2_7.md index 4ab8b47c2d..f593e1ff78 100644 --- a/mdbook/src/chapter_2/chapter_2_7.md +++ b/mdbook/src/chapter_2/chapter_2_7.md @@ -10,7 +10,7 @@ As an example, we can take our `manages` relation and determine for all employee transitive .map(|(mk, m1)| (m1, mk)) .join(&transitive) - .map(|(m1, mk, p)| (mk, p)) + .map(|(m1, (mk, p))| (mk, p)) .concat(&transitive) .distinct() }); @@ -34,12 +34,12 @@ In the example above, we could rewrite manages // transitive contains (manager, person) for many hops. .iterate(|transitive| { - let manages = manages.enter(transivite.scope()); + let manages = manages.enter(transitive.scope()); transitive .map(|(mk, m1)| (m1, mk)) .join(&manages) - .map(|(m1, mk, p)| (mk, p)) + .map(|(m1, (mk, p))| (mk, p)) .concat(&manages) .distinct() }); diff --git a/mdbook/src/chapter_3/chapter_3_2.md b/mdbook/src/chapter_3/chapter_3_2.md index 1096378008..f71fcef36e 100644 --- a/mdbook/src/chapter_3/chapter_3_2.md +++ b/mdbook/src/chapter_3/chapter_3_2.md @@ -13,7 +13,7 @@ For example, recall our example of interacting with our management computation, // create a new collection from an input session. let manages = input.to_collection(scope); - // if (m2, m1) and (m1, p), then output (m1, m2, p) + // if (m2, m1) and (m1, p), then output (m1, (m2, p)) manages .map(|(m2, m1)| (m1, m2)) .join(&manages) diff --git a/mdbook/src/chapter_4/chapter_4_1.md b/mdbook/src/chapter_4/chapter_4_1.md index 738aa54d1a..fadce2b492 100644 --- a/mdbook/src/chapter_4/chapter_4_1.md +++ b/mdbook/src/chapter_4/chapter_4_1.md @@ -18,7 +18,7 @@ Let's write this computation starting from a collection `edges`, using different let labels = labels.enter(inner.scope()); let edges = edges.enter(inner.scope()); inner.join(&edges) - .map(|(_src,lbl,dst)| (dst,lbl)) + .map(|(_src,(lbl,dst))| (dst,lbl)) .concat(&labels) .reduce(|_dst, lbls, out| { let min_lbl = diff --git a/mdbook/src/chapter_5/chapter_5_3.md b/mdbook/src/chapter_5/chapter_5_3.md index 27c5602deb..7d41bbb0c3 100644 --- a/mdbook/src/chapter_5/chapter_5_3.md +++ b/mdbook/src/chapter_5/chapter_5_3.md @@ -2,8 +2,6 @@ Arrangements have the additional appealing property that they can be shared not only within a dataflow, but *across* dataflows. -Imagine we take our `knows` collection from before, and want to make it available for others to use - Imagine we want to build and maintain a relatively large and continually changing collection. But we want to do this in a way that allows an arbitrary number of subsequent queries to access the collection at almost no additional cost. The following example demonstrates going from an interactive input session (`input`) to an arrangement (`trace`) returned from the dataflow and available for use by others. diff --git a/mdbook/src/chapter_a/chapter_a_2.md b/mdbook/src/chapter_a/chapter_a_2.md index bfe7f6ba10..fab3500b1c 100644 --- a/mdbook/src/chapter_a/chapter_a_2.md +++ b/mdbook/src/chapter_a/chapter_a_2.md @@ -5,7 +5,7 @@ Differential dataflow works great using multiple threads and computers. It even For this to work out, we'll want to ask each worker to load up a fraction of the input. If we just run the same code with multiple workers, then each of the workers will run ```rust,ignore - for person in 0 .. people { + for person in 0 .. size { input.insert((person/2, person)); } ``` @@ -16,7 +16,7 @@ Instead, each timely dataflow worker has methods `index()` and `peers()`, which ```rust,ignore let mut person = worker.index(); - while person < people { + while person < size { input.insert((person/2, person)); person += worker.peers(); } @@ -25,12 +25,12 @@ Instead, each timely dataflow worker has methods `index()` and `peers()`, which We can also make the same changes to the code that supplies the change, where each worker is responsible for those people whose number equals `worker.index()` modulo `worker.peers()`. ```rust,ignore - let mut person = index; - while person < people { + let mut person = worker.index(); + while person < size { input.remove((person/2, person)); input.insert((person/3, person)); input.advance_to(person); - person += peers; + person += worker.peers(); } ``` diff --git a/mdbook/src/chapter_a/chapter_a_3.md b/mdbook/src/chapter_a/chapter_a_3.md index 6ee7564405..b87d2e55ff 100644 --- a/mdbook/src/chapter_a/chapter_a_3.md +++ b/mdbook/src/chapter_a/chapter_a_3.md @@ -9,7 +9,7 @@ Instead of loading all of our changes and only waiting for the result, we can lo // create a new collection from an input session. let manages = input.to_collection(scope); - // if (m2, m1) and (m1, p), then output (m1, m2, p) + // if (m2, m1) and (m1, p), then output (m1, (m2, p)) manages .map(|(m2, m1)| (m1, m2)) .join(&manages) @@ -22,7 +22,7 @@ We can then use this probe to limit the introduction of new data, by waiting for ```rust,ignore let mut person = worker.index(); - while person < people { + while person < size { input.insert((person/2, person)); person += worker.peers(); } @@ -31,7 +31,7 @@ We can then use this probe to limit the introduction of new data, by waiting for input.advance_to(1); input.flush(); while probe.less_than(&input.time()) { worker.step(); } - println!("{:?}\tdata loaded", timer.elapsed()); + println!("{:?}\tdata loaded", worker.timer().elapsed()); ``` These four new lines are each important, especially the one that prints things out. The other three do a bit of magic that get timely dataflow to work for us until we are certain that inputs have been completely processed. @@ -40,15 +40,15 @@ We can make the same changes for the interactive loading, but we'll synchronize ```rust,ignore // make changes, but await completion. - let mut person = 1 + index; - while person < people { + let mut person = 1 + worker.index(); + while person < size { input.remove((person/2, person)); input.insert((person/3, person)); input.advance_to(person); input.flush(); while probe.less_than(&input.time()) { worker.step(); } - println!("{:?}\tstep {} complete", timer.elapsed(), person); - person += peers; + println!("{:?}\tstep {} complete", worker.timer().elapsed(), person); + person += worker.peers(); } ```