Skip to content

Commit

Permalink
Makes ion-hash a feature in ion-rs
Browse files Browse the repository at this point in the history
  • Loading branch information
popematt committed Mar 13, 2023
1 parent f49659e commit dda6d3f
Show file tree
Hide file tree
Showing 14 changed files with 125 additions and 130 deletions.
7 changes: 6 additions & 1 deletion .github/workflows/rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,16 @@ jobs:
with:
command: build
args: --verbose --workspace
- name: Cargo Test
- name: Cargo Test (default/no features)
uses: actions-rs/cargo@v1
with:
command: test
args: --verbose --workspace
- name: Cargo Test (all features)
uses: actions-rs/cargo@v1
with:
command: test
args: --verbose --workspace --all-features
- name: Rustfmt Check
uses: actions-rs/cargo@v1
with:
Expand Down
2 changes: 1 addition & 1 deletion .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
path = ion-tests
url = https://github.com/amazon-ion/ion-tests.git
[submodule "ion-hash-test"]
path = ion-hash/ion-hash-test
path = ion-hash-test
url = https://github.com/amazon-ion/ion-hash-test.git
6 changes: 6 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ members = [
"ion-hash"
]

[features]
default = []
ion-hash = ["digest"]

[dependencies]
base64 = "0.12"
bigdecimal = "0.3.0"
Expand All @@ -40,6 +44,8 @@ num-integer = "0.1.44"
num-traits = "0.2"
arrayvec = "0.7"
smallvec = "1.9.0"
digest = { version = "0.9", optional = true }
sha2 = { version = "0.9", optional = true }

[dev-dependencies]
rstest = "0.16.0"
Expand Down
8 changes: 6 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,14 @@

A Rust implementation of the [Amazon Ion][spec] data format.

Includes the feature `ion-hash` which is an implementation of [Ion Hash][ion-hash-spec].

***This package is considered experimental, under active/early development, and the API is subject to change.***

## Development

This project uses a submodule to pull in [Ion Tests][ion-tests]. The easiest way to pull
everything in is to clone the repository recursively:
This project uses a submodule to pull in [Ion Tests][ion-tests] and [Ion Hash Tests][ion-hash-tests].
The easiest way to pull everything in is to clone the repository recursively:

```bash
$ git clone --recursive https://github.com/amazon-ion/ion-rust
Expand All @@ -40,3 +42,5 @@ $ cargo test --workspace
[spec]: https://amazon-ion.github.io/ion-docs/docs/spec.html
[ion-tests]: https://github.com/amazon-ion/ion-tests
[bindgen-req]: https://rust-lang.github.io/rust-bindgen/requirements.html
[ion-hash-spec]: https://amazon-ion.github.io/ion-hash/docs/spec.html
[ion-hash-tests]: https://github.com/amazon-ion/ion-hash-tests
7 changes: 1 addition & 6 deletions ion-hash/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,10 @@ exclude = [
"**/ion-tests/iontestdata/**",
"*.pdf"
]
version = "0.1.0"
version = "0.1.1"
edition = "2021"
# We need at least 1.65 for GATs
# https://blog.rust-lang.org/2022/11/03/Rust-1.65.0.html
rust-version = "1.65"

[dependencies]
ion-rs = { path="..", version = "0.15" }
num-bigint = "0.4.3"
digest = "0.9"
sha2 = "0.9"
thiserror = "1.0"
16 changes: 2 additions & 14 deletions ion-hash/README.md
Original file line number Diff line number Diff line change
@@ -1,19 +1,7 @@
# `ion-hash-rust`

[![Crate](https://img.shields.io/crates/v/ion-hash.svg)](https://crates.io/crates/ion-hash)
[![Docs](https://docs.rs/ion-hash/badge.svg)](https://docs.rs/ion-hash)
[![License](https://img.shields.io/crates/l/ion-hash)](https://crates.io/crates/ion-hash)
[![CI Build](https://github.com/amazon-ion/ion-rust/workflows/CI%20Build/badge.svg)](https://github.com/amazon-ion/ion-rust/actions?query=workflow%3A%22CI+Build%22)
***This package is deprecated in favor of the `ion-hash` feature in `ion-rs >= 0.16.0`.***

A Rust implementation of [Ion Hash][spec].

***This package is considered experimental, under active/early development, and the API is subject to change.***

## Development

See [Ion Rust][ion-rust] for details. This crate is currently developed in concert with Ion Rust
as a [Cargo workspace][cargo-workspace].
See [Ion Rust][ion-rust] for details.

[ion-rust]: https://github.com/amazon-ion/ion-rust
[spec]: https://amazon-ion.github.io/ion-hash/docs/spec.html
[cargo-workspace]: https://doc.rust-lang.org/cargo/reference/workspaces.html
75 changes: 1 addition & 74 deletions ion-hash/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,76 +1,3 @@
// Copyright Amazon.com, Inc. or its affiliates.

//! Implements the [Ion Hash specification](https://amazon-ion.github.io/ion-hash/docs/spec.html)
//!
//! ## Examples
//! ```rust
//! use ion_rs::value::reader::{self, ElementReader};
//! use ion_rs::result::IonResult;
//!
//! # fn main() -> IonResult<()> {
//! let loader = reader::element_reader();
//! let elem = loader.iterate_over(b"\"hello world\"")?.next().unwrap()?;
//! let digest = ion_hash::sha256(&elem);
//! println!("{:?}", digest);
//! # Ok(())
//! # }
//! ```

use digest::{FixedOutput, Output, Reset, Update};
use ion_rs::result::IonResult;

// TODO: Make sha2 an optional dependency.
use sha2::Sha256;

use element_hasher::ElementHasher;
use ion_rs::value::owned::Element;

mod element_hasher;
mod representation;
mod type_qualifier;

/// Utility to hash an [`Element`] using SHA-256 as the hash function.
// TODO: Make this conditional on some feature flag
pub fn sha256(elem: &Element) -> IonResult<Output<Sha256>> {
Sha256::hash_element(elem)
}

/// Bytes markers as per the spec.
struct Markers;
impl Markers {
/// single byte begin marker
const B: u8 = 0x0B;
/// single byte end marker
const E: u8 = 0x0E;
/// single byte escape
const ESC: u8 = 0x0C;
}

/// This trait mostly exists to extend the [`Digest`] trait to support Ion Hash.
/// See [`hash_element`].
pub trait IonHasher {
type Output;

/// Returns the Ion Hash of the given [`Element`].
fn hash_element(elem: &Element) -> IonResult<Self::Output>;
}

/// Implements [`IonHasher`] for any type that implements `Digest`.
///
/// Note: the `Digest` trait is implemented for types that implement a set of
/// traits. You should read the `D` generic as `Digest`. The reason we list the
/// subtraits here is that implementors have much less work to do if they
/// implement the subtraits only, as the blanket impls in the `digest` crate take
/// care of assembly.
impl<D> IonHasher for D
where
D: Update + FixedOutput + Reset + Clone + Default,
{
type Output = digest::Output<D>;

/// Provides Ion hash over arbitrary [`IonElement`] instances with a given
/// [`Digest`] algorithm.
fn hash_element(elem: &Element) -> IonResult<Self::Output> {
ElementHasher::new(D::default()).hash_element(elem)
}
}
// Intentionally empty
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@

use std::io;

use crate::value::owned::Element;
use crate::IonResult;
use digest::{FixedOutput, Output, Reset, Update};
use ion_rs::result::IonResult;
use ion_rs::value::owned::Element;

use crate::representation::RepresentationEncoder;
use crate::type_qualifier::TypeQualifier;
use crate::Markers;
use crate::ion_hash::representation::RepresentationEncoder;
use crate::ion_hash::type_qualifier::TypeQualifier;
use crate::ion_hash::Markers;

pub(crate) struct ElementHasher<D>
where
Expand Down
76 changes: 76 additions & 0 deletions src/ion_hash/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
// Copyright Amazon.com, Inc. or its affiliates.

//! Implements the [Ion Hash specification](https://amazon-ion.github.io/ion-hash/docs/spec.html)
//!
//! ## Examples
//! ```rust
//! use ion_rs::value::reader::{self, ElementReader};
//! use ion_rs::result::IonResult;
//! use ion_rs::ion_hash;
//!
//! # fn main() -> IonResult<()> {
//! let loader = reader::element_reader();
//! let elem = loader.iterate_over(b"\"hello world\"")?.next().unwrap()?;
//! let digest = ion_hash::sha256(&elem);
//! println!("{:?}", digest);
//! # Ok(())
//! # }
//! ```

use digest::{self, FixedOutput, Output, Reset, Update};

use crate::value::owned::Element;
use crate::IonResult;
use element_hasher::ElementHasher;

mod element_hasher;
mod representation;
mod type_qualifier;

#[cfg(feature = "sha2")]
use sha2::Sha256;
/// Utility to hash an [`Element`] using SHA-256 as the hash function.
#[cfg(feature = "sha2")]
pub fn sha256(elem: &Element) -> IonResult<Output<Sha256>> {
Sha256::hash_element(elem)
}

/// Bytes markers as per the spec.
struct Markers;
impl Markers {
/// single byte begin marker
const B: u8 = 0x0B;
/// single byte end marker
const E: u8 = 0x0E;
/// single byte escape
const ESC: u8 = 0x0C;
}

/// This trait mostly exists to extend the [`Digest`] trait to support Ion Hash.
/// See [`hash_element`].
pub trait IonHasher {
type Output;

/// Returns the Ion Hash of the given [`Element`].
fn hash_element(elem: &Element) -> IonResult<Self::Output>;
}

/// Implements [`IonHasher`] for any type that implements [Digest].
///
/// Note: the `Digest` trait is implemented for types that implement a set of
/// traits. You should read the `D` generic as `Digest`. The reason we list the
/// subtraits here is that implementors have much less work to do if they
/// implement the subtraits only, as the blanket impls in the `digest` crate take
/// care of assembly.
impl<D> IonHasher for D
where
D: Update + FixedOutput + Reset + Clone + Default,
{
type Output = digest::Output<D>;

/// Provides Ion hash over arbitrary [`IonElement`] instances with a given
/// [`Digest`] algorithm.
fn hash_element(elem: &Element) -> IonResult<Self::Output> {
ElementHasher::new(D::default()).hash_element(elem)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@
//! instead. This implementation fills in that gap, and is focused on coverage
//! and not speed.

use crate::element_hasher::ElementHasher;
use crate::type_qualifier::type_qualifier_symbol;
use crate::binary::{self, decimal::DecimalBinaryEncoder, timestamp::TimestampBinaryEncoder};
use crate::ion_hash::element_hasher::ElementHasher;
use crate::ion_hash::type_qualifier::type_qualifier_symbol;
use crate::types::decimal::Decimal;
use crate::types::integer::Integer;
use crate::value::owned::{Element, Sequence, Struct};
use crate::{result::IonResult, types::timestamp::Timestamp, IonType, Symbol};
use digest::{FixedOutput, Output, Reset, Update};
use ion_rs::binary::{self, decimal::DecimalBinaryEncoder, timestamp::TimestampBinaryEncoder};
use ion_rs::types::decimal::Decimal;
use ion_rs::types::integer::Integer;
use ion_rs::value::owned::{Element, Sequence, Struct};
use ion_rs::{result::IonResult, types::timestamp::Timestamp, IonType, Symbol};

pub(crate) trait RepresentationEncoder {
fn update_with_representation(&mut self, elem: &Element) -> IonResult<()> {
Expand Down
16 changes: 3 additions & 13 deletions ion-hash/src/type_qualifier.rs → src/ion_hash/type_qualifier.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,9 @@

use std::slice;

///! Implements "type qualifiers" (TQ) as per the [spec][spec].
///!
///! [spec]: https://amazon-ion.github.io/ion-hash/docs/spec.html.
use ion_rs::types::integer::Integer;
use ion_rs::value::owned::{Element, Sequence, Struct};
///! Implements "type qualifiers" (TQ) as per the [spec][spec].
///!
///! [spec]: https://amazon-ion.github.io/ion-hash/docs/spec.html.
use ion_rs::{
binary::IonTypeCode,
types::{decimal::Decimal, timestamp::Timestamp},
IonType, Symbol,
};
use crate::binary::IonTypeCode;
use crate::value::owned::{Element, Sequence, Struct};
use crate::{Decimal, Integer, IonType, Symbol, Timestamp};
use num_bigint::Sign;

// For many types, the qualifier is either 'null' or 'not null'. That's what
Expand Down
3 changes: 3 additions & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ pub mod text;
pub mod types;
pub mod value;

#[cfg(feature = "ion-hash")]
pub mod ion_hash;

mod catalog;
pub mod constants;
pub mod ion_eq;
Expand Down
15 changes: 8 additions & 7 deletions ion-hash/tests/ion_hash_tests.rs → tests/ion_hash_tests.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
// Copyright Amazon.com, Inc. or its affiliates.
#![cfg(feature = "ion-hash")]

use digest::consts::U4096;
use digest::{FixedOutput, Reset, Update};
use ion_hash::IonHasher;
use ion_rs::ion_hash::IonHasher;
use ion_rs::result::{illegal_operation, IonResult};
use ion_rs::types::integer::IntAccess;
use ion_rs::value::owned::{Element, Struct};
Expand Down Expand Up @@ -73,7 +74,7 @@ impl Reset for IdentityDigest {
}

fn without_trailing_zeros(data: &[u8]) -> &[u8] {
if data.len() == 0 {
if data.is_empty() {
return data;
}

Expand All @@ -86,7 +87,7 @@ fn without_trailing_zeros(data: &[u8]) -> &[u8] {
&data[0..=index]
}

const IGNORE_LIST: &[&'static str] = &[
const IGNORE_LIST: &[&str] = &[
// Uses md5 (not identity)
r#"{Metrics:{'Event.Catchup':[{Value:0,Unit:ms}],'FanoutCache.Time':[{Value:1,Unit:ms}]}}"#,
// ion-rust doesn't have full support for unknown symbols yet, so we can't properly read these test cases.
Expand Down Expand Up @@ -158,7 +159,7 @@ fn test_all(elems: Vec<Element>) -> IonHashTestResult<()> {
let loaded = element_reader().read_all(&bytes)?;
let elem = loaded
.into_iter()
.nth(0)
.next()
.expect("10n test case should have a single element (there were none)");
test_case(annotated_test_name, &elem, expect)
}
Expand Down Expand Up @@ -218,7 +219,7 @@ fn test_case(

if expected_string != actual_string {
Err(IonHashTestError::TestFailed {
test_case_name: test_case_name.to_string(),
test_case_name,
message: Some(format!(
"expected: {}\nwas: {}",
expected_string, actual_string
Expand Down Expand Up @@ -247,11 +248,11 @@ fn expected_hash(struct_: &Struct) -> IonResult<Vec<u8>> {
let bytes = seq_to_bytes(expectation);

match method {
"digest" | "final_digest" => return Ok(bytes),
"digest" | "final_digest" => Ok(bytes),
_ => illegal_operation(format!("unknown expectation `{}`", method))?,
}
} else {
illegal_operation(format!("expected at least expectation!"))?
illegal_operation("expected at least expectation!")?
}
}

Expand Down

0 comments on commit dda6d3f

Please sign in to comment.