Permalink
Browse files

Problem: the concept of "items"

It originally was a conversion of "issues" into something that
can be somewhat more generic. However, with further exploration of
different domains it became rather clear that it doesn't fit many of
them well and can only be artifically bolted on. That's not great.

Solution: transition into a flat records namespace

In order to avoid this problem, the idea is to move out all records to
.sit/records into a single flat namespace and make the transition
(almost) seamless by retaining items for a little while but converting
records in them into "text link" files to the real records.
  • Loading branch information...
yrashk committed Aug 2, 2018
1 parent 3d5fa17 commit 243cbd25708a6ab9bcf5fe3b4b614394fb1d17c5
@@ -2,6 +2,7 @@ language: rust
rust:
- stable
- nightly
build_script: make test
cache: cargo
matrix:
include:

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.
@@ -1,3 +1,5 @@
.PHONY: test
osx: target/x86_64-apple-darwin/release/sit target/x86_64-apple-darwin/release/sit-web
linux: target/x86_64-unknown-linux-musl/release/sit target/x86_64-unknown-linux-musl/release/sit-web
@@ -12,3 +14,11 @@ target/x86_64-unknown-linux-musl/release/sit target/x86_64-unknown-linux-musl/re
sed -i s/sha256://g ._docker_linux
docker run -u `id -u`:`id -g` -v `pwd`:/sit -w /sit -t `cat ._docker_linux` sh -c "cargo build --release --target=x86_64-unknown-linux-musl && strip target/x86_64-unknown-linux-musl/release/sit target/x86_64-unknown-linux-musl/release/sit-web"
rm -f ._docker_linux
test:
# Test without deprecated-item-api
cd sit-core && cargo test --no-default-features --features="sha-1 blake2 duktape-reducers duktape-mmap duktape-require"
# Test
cargo test
# Test sit without deprecated-items
cd sit && cargo test --no-default-features
@@ -24,7 +24,7 @@ install:
build: false
test_script:
- cargo test
- make test
on_failure:
- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
BIN -4.84 KB (74%) doc/overview.png
Binary file not shown.
@@ -2,11 +2,7 @@
class Repository {
hashing_algorithm : Blake2(N) | SHA-1
encoding: Base32
id_generator: UUIDv4
version: 1
}
class Item {
id : repository.id_generator::type
version: 2
}
class Record {
hash : repository.hashing_algorithm::type
@@ -15,7 +11,6 @@ class File {
name : String
content : Binary
}
Repository "1" --> "many" Item: contains
Item "1" --> "many" Record: contains
Repository "1" --> "many" Record: contains
Record "1" --> "many" File: contains
@enduml
@@ -19,6 +19,8 @@ data-encoding = "2.1"
data-encoding-macro = "0.1"
glob = "0.2"
lazy_static = "1.0"
itertools = "0.7"
walkdir = "2"
blake2 = { version = "0.7", optional = true }
sha-1 = { version = "0.7", optional = true }
uuid = { version = "0.5", features = ["v4"], optional = true }
@@ -37,10 +39,11 @@ cc = "1.0"
include_dir = "0.1.5"
[features]
default = ["blake2", "sha-1", "uuid", "duktape-reducers", "duktape-mmap", "duktape-require"]
default = ["blake2", "sha-1", "uuid", "duktape-reducers", "duktape-mmap", "duktape-require", "deprecated-item-api"]
duktape-require = []
duktape-reducers = ["duktape", "cesu8"]
duktape = []
duktape-mmap = ["memmap"]
windows7 = []
git = ["git2"]
deprecated-item-api = []
@@ -112,6 +112,16 @@ impl HashingAlgorithm {
}
}
/// Returns the output length of the hash
pub fn len(&self) -> usize {
match self {
#[cfg(feature = "blake2")]
&HashingAlgorithm::Blake2b { size } => size,
#[cfg(feature = "sha-1")]
&HashingAlgorithm::SHA1 => 20,
}
}
}
#[cfg(test)]
@@ -2,58 +2,21 @@
use serde_json::{Map, Value};
use super::Reducer;
use record::{File, OrderedFiles};
#[derive(Debug, Error)]
pub enum ReductionError<Err: ::std::error::Error + ::std::fmt::Debug> {
ImplementationError(Err)
}
use record::{RecordOwningContainer, RecordContainerReduction};
/// Because of SIT's extensible nature, item can
/// be used to represent a wild variety of entities, such
/// as issue, documents, accounts, etc.
pub trait Item: Sized {
/// Error type used by the implementation
type Error: ::std::error::Error + ::std::fmt::Debug;
/// Record type used by the implementation
type Record : super::Record;
/// Type used to list records that can be referenced as a slice of records
type Records : IntoIterator<Item=Self::Record>;
/// Iterator over lists of records
type RecordIter : Iterator<Item=Self::Records>;
/// Issue must have an ID, ideally human-readable
pub trait Item: RecordOwningContainer {
/// Item must have an ID, ideally human-readable
fn id(&self) -> &str;
/// Iterates through the tree of records
fn record_iter(&self) -> Result<Self::RecordIter, Self::Error>;
/// Creates and returns a new record.
///
/// Will reference all dangling records as its parent, unless
/// `link_parents` is set to `false`
fn new_record<'f, F: File + 'f, I: Into<OrderedFiles<'f, F>>>(&self, files: I, link_parents: bool)
-> Result<Self::Record, Self::Error> where F::Read: 'f;
}
/// [`Issue`] trait extension that defines and implements default reduction algorithms
///
/// [`Issue`]: trait.Issue.html
pub trait ItemReduction: Item {
/// Reduces item with a given [`Reducer`]
///
/// Will insert item's `id` into the initial state
///
/// [`Reducer`]: ../reducers/trait.Reducer.html
fn reduce_with_reducer<R: Reducer<State=Map<String, Value>, Item=Self::Record>>(&self, reducer: &mut R) -> Result<Map<String, Value>, ReductionError<Self::Error>> {
let records = self.record_iter()?;
let mut state: Map<String, Value> = Default::default();
impl<T> RecordContainerReduction for T where T: Item {
fn initialize_state(&self, mut state: Map<String, Value>) -> Map<String, Value> {
state.insert("id".into(), Value::String(self.id().into()));
Ok(records.fold(state, |acc, recs|
recs.into_iter().fold(acc, |acc, rec| reducer.reduce(acc, &rec))))
state
}
}
impl<T> ItemReduction for T where T: Item {}
@@ -38,6 +38,9 @@ extern crate digest;
extern crate relative_path;
extern crate itertools;
extern crate walkdir;
// Crates necessary for testing
#[cfg(test)] #[macro_use] extern crate assert_matches;
#[cfg(test)] #[macro_use] extern crate proptest;
@@ -46,9 +49,12 @@ extern crate relative_path;
pub mod path;
pub mod hash;
pub mod encoding;
#[cfg(feature = "deprecated-item-api")]
pub mod id;
pub mod repository;
#[cfg(feature = "deprecated-item-api")]
pub mod item;
#[cfg(feature = "deprecated-item-api")]
pub use item::Item;
pub mod record;
pub use record::Record;
@@ -1,7 +1,8 @@
//! Record is an immutable collection of files
use std::io::{self, Read};
use hash::Hasher;
use hash::{Hasher, HashingAlgorithm};
use std::path::PathBuf;
/// Record's file
///
@@ -240,6 +241,16 @@ mod ordered_files_tests {
}
}
/// Returns string split by `length` characters
pub(crate) fn split_path<S: AsRef<str>>(s: S, length: usize) -> PathBuf {
use itertools::Itertools;
let mut path = s.as_ref().chars().chunks(length).into_iter()
.fold(PathBuf::new(), |acc, chunk| acc.join(chunk.into_iter().collect::<String>()));
path.pop();
path.join(s.as_ref())
}
/// Record is an immutable collection of files
pub trait Record {
/// Implementation's type for reading files
@@ -261,13 +272,112 @@ pub trait Record {
/// [`hash`]: struct.Record.html#hash
fn encoded_hash(&self) -> Self::Str;
/// Returns encoded record hash path split by `length` characters
fn split_path(&self, length: usize) -> PathBuf {
split_path(self.encoded_hash(), length)
}
/// Returns enclosing item's ID
#[cfg(feature = "deprecated-item-api")]
fn item_id(&self) -> Self::Str;
/// Returns an iterator over files in the record
fn file_iter(&self) -> Self::Iter;
/// Returns true if the integrity of the record is intact
fn integrity_intact(&self, hashing_algorithm: &HashingAlgorithm) -> bool {
let mut hasher = hashing_algorithm.hasher();
let ordered_files = OrderedFiles::from(self.file_iter());
match ordered_files.hash(&mut *hasher) {
Ok(_) => {
let hash = hasher.result_box();
self.hash().as_ref() == hash.as_slice()
},
_ => {
false
}
}
}
}
pub trait RecordContainer {
/// Error type used by the implementation
type Error: ::std::error::Error + ::std::fmt::Debug;
/// Record type used by the implementation
type Record : super::Record;
/// Type used to list records that can be referenced as a slice of records
type Records : IntoIterator<Item=Self::Record>;
/// Iterator over lists of records
type Iter : Iterator<Item=Self::Records>;
/// Iterates through the tree of records
fn record_iter(&self) -> Result<Self::Iter, Self::Error>;
fn fixed_roots<S: Into<String>, I: IntoIterator<Item = S>>(&self, roots: I) ->
FixedRootsRecordContainer<Self> where Self: Sized {
FixedRootsRecordContainer {
container: self,
roots: roots.into_iter().map(|s| s.into()).collect(),
}
}
}
pub struct FixedRootsRecordContainer<'a, RC: RecordContainer + 'a> {
container: &'a RC,
roots: Vec<String>,
}
impl<'a, RC: RecordContainer + 'a> RecordContainer for FixedRootsRecordContainer<'a, RC> {
type Error = RC::Error;
type Record = RC::Record;
type Records = Vec<RC::Record>;
type Iter = FixedRootsRecordIterator<RC>;
fn record_iter(&self) -> Result<Self::Iter, Self::Error> {
Ok(FixedRootsRecordIterator {
iter: self.container.record_iter()?,
known: vec![],
roots: self.roots.clone(),
})
}
}
pub struct FixedRootsRecordIterator<RC: RecordContainer> {
iter: RC::Iter,
known: Vec<String>,
roots: Vec<String>,
}
impl<RC: RecordContainer> Iterator for FixedRootsRecordIterator<RC> {
type Item = Vec<RC::Record>;
fn next(&mut self) -> Option<Self::Item> {
match self.iter.next() {
None => None,
Some(value) => {
let records: Vec<_> = value.into_iter()
.filter(|record|
self.roots.iter().any(|root| root == record.encoded_hash().as_ref()) ||
record.file_iter().filter(|&(ref name, _)| name.as_ref().starts_with(".prev/"))
.any(|(name, _)| self.known.iter().any(|known| known == &name.as_ref()[6..])))
.collect();
for r in records.iter() {
self.known.push(r.encoded_hash().as_ref().into());
}
Some(records)
}
}
}
}
pub trait RecordOwningContainer: RecordContainer {
/// Creates and returns a new record.
///
/// Will reference all dangling records as its parent, unless
/// `link_parents` is set to `false`
fn new_record<'f, F: File + 'f, I: Into<OrderedFiles<'f, F>>>(&self, files: I, link_parents: bool)
-> Result<Self::Record, Self::Error> where F::Read: 'f;
}
use serde_json::{Value as JsonValue, Map as JsonMap};
use serde::Serializer;
@@ -326,4 +436,40 @@ pub trait RecordExt: Record {
}
impl<T> RecordExt for T where T: Record {}
impl<T> RecordExt for T where T: Record {}
use reducers::Reducer;
#[derive(Debug, Error)]
pub enum ReductionError<Err: ::std::error::Error + ::std::fmt::Debug> {
ImplementationError(Err)
}
/// Default reduction algorithm
///
pub trait RecordContainerReduction: RecordContainer {
fn initialize_state(&self, state: JsonMap<String, JsonValue>) -> JsonMap<String, JsonValue> {
state
}
/// Reduces item with a given [`Reducer`]
///
/// [`Reducer`]: ../reducers/trait.Reducer.html
fn reduce_with_reducer<R: Reducer<State=JsonMap<String, JsonValue>, Item=Self::Record>>(&self, reducer: &mut R) -> Result<JsonMap<String, JsonValue>, ReductionError<Self::Error>> {
let state: JsonMap<String, JsonValue> = Default::default();
let state = self.initialize_state(state);
self.reduce_with_reducer_and_state(reducer, state)
}
/// Reduces item with a given [`Reducer`] and state
///
/// [`Reducer`]: ../reducers/trait.Reducer.html
fn reduce_with_reducer_and_state<R: Reducer<State=JsonMap<String, JsonValue>, Item=Self::Record>>(&self, reducer: &mut R, state: JsonMap<String, JsonValue>) -> Result<JsonMap<String, JsonValue>, ReductionError<Self::Error>> {
let records = self.record_iter()?;
Ok(records.fold(state, |acc, recs|
recs.into_iter().fold(acc, |acc, rec| reducer.reduce(acc, &rec))))
}
}
impl<'a, RC> RecordContainerReduction for FixedRootsRecordContainer<'a, RC> where RC: RecordContainer {}
Oops, something went wrong.

0 comments on commit 243cbd2

Please sign in to comment.