Skip to content
Permalink
Browse files

0.2 work (#71)

0.2 release

* upstream tracking version
* generalisation of utf8parsing
* generalisation of stage1
* api update
* perf improvements
* tape parsing
  • Loading branch information
Licenser committed Dec 1, 2019
1 parent c0567c3 commit 0deaa233cb8dfa5ee04bf2f62af309d9e49acfcf
@@ -29,8 +29,15 @@ jobs:
- name: Build
env:
RUSTFLAGS: ${{ matrix.rustflags }}
run: cargo build --verbose ${{ matrix.features }}
run: cargo build ${{ matrix.features }}
- name: Run tests
env:
RUSTFLAGS: ${{ matrix.rustflags }}
run: cargo test --verbose ${{ matrix.features }}
run: cargo test ${{ matrix.features }}
- name: Run tests (alloc)
if: matrix.rust == 'nightly' && matrix.features == ''
env:
RUSTFLAGS: ${{ matrix.rustflags }}
run: cargo test --features alloc


@@ -7,8 +7,8 @@ jobs:
strategy:
matrix:
rustflags:
- '-C target-cpu=native'
- '-C target-cpu=native -C target-feature=-avx2'
- "-C target-cpu=native"
- "-C target-cpu=native -C target-feature=-avx2"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
@@ -40,14 +40,14 @@ jobs:
run: bash <(curl https://raw.githubusercontent.com/xd009642/tarpaulin/master/travis-install.sh)
- name: run tarpaulin
env:
RUSTFLAGS: '-C target-cpu=native -C target-feature=-avx2'
RUSTFLAGS: "-C target-cpu=native -C target-feature=-avx2"
PROPTEST_CASES: 512
run: cargo tarpaulin -v --features no-inline --out Xml -- --test-threads=1 && cp cobertura.xml ssl.xml
run: cargo tarpaulin -v --features no-inline --out Xml && cp cobertura.xml sse.xml
- uses: codecov/codecov-action@v1.0.2
with:
token: ${{secrets.CODECOV_TOKEN}} #required
file: ./ssl.xml
flags: ssl
file: ./sse.xml
flags: e
tarpaulin-avx2:
runs-on: ubuntu-latest
steps:
@@ -62,13 +62,13 @@ jobs:
run: bash <(curl https://raw.githubusercontent.com/xd009642/tarpaulin/master/travis-install.sh)
- name: run tarpaulin
env:
RUSTFLAGS: '-C target-cpu=native'
RUSTFLAGS: "-C target-cpu=native"
PROPTEST_CASES: 512
run: cargo tarpaulin -v --features no-inline --out Xml -- --test-threads=1 && cp cobertura.xml avx.xml
run: cargo tarpaulin -v --features no-inline --out Xml && cp cobertura.xml avx.xml
- uses: codecov/codecov-action@v1.0.2
with:
token: ${{secrets.CODECOV_TOKEN}} #required
file: ./avx.xml
file: ./avx.xml
flags: avx2
tarpaulin-sse-known-key:
runs-on: ubuntu-latest
@@ -84,14 +84,14 @@ jobs:
run: bash <(curl https://raw.githubusercontent.com/xd009642/tarpaulin/master/travis-install.sh)
- name: run tarpaulin
env:
RUSTFLAGS: '-C target-cpu=native -C target-feature=-avx2'
RUSTFLAGS: "-C target-cpu=native -C target-feature=-avx2"
PROPTEST_CASES: 512
run: cargo tarpaulin -v --features no-inline known-key --out Xml -- --test-threads=1 && cp cobertura.xml ssl.xml
run: cargo tarpaulin -v --features no-inline known-key --out Xml && cp cobertura.xml sse-known-key.xml
- uses: codecov/codecov-action@v1.0.2
with:
token: ${{secrets.CODECOV_TOKEN}} #required
file: ./ssl.xml
flags: ssl-known-key
file: ./sse-known-key.xml
flags: sseKnownKey
tarpaulin-avx2-known-key:
runs-on: ubuntu-latest
steps:
@@ -106,31 +106,11 @@ jobs:
run: bash <(curl https://raw.githubusercontent.com/xd009642/tarpaulin/master/travis-install.sh)
- name: run tarpaulin
env:
RUSTFLAGS: '-C target-cpu=native'
RUSTFLAGS: "-C target-cpu=native"
PROPTEST_CASES: 512
run: cargo tarpaulin -v --features no-inline known-key --out Xml -- --test-threads=1 && cp cobertura.xml avx.xml
run: cargo tarpaulin -v --features no-inline known-key --out Xml && cp cobertura.xml avx2-known-key.xml
- uses: codecov/codecov-action@v1.0.2
with:
token: ${{secrets.CODECOV_TOKEN}} #required
file: ./avx.xml
flags: avx2-known-key
# tarpaulin-neon:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v1
# - name: Install deps
# run: sudo apt-get update && sudo apt-get install -y libssl-dev pkg-config cmake zlib1g-dev
# - uses: actions-rs/toolchain@v1
# with:
# toolchain: nightly
# override: true
# - name: install tarpaulin
# run: cargo install cargo-tarpaulin -f
# - name: run tarpaulin
# env:
# RUSTFLAGS: '-C target-cpu=native'
# run: cargo tarpaulin -v --out Xml
# - uses: codecov/codecov-action@v1.0.2
# with:
# token: ${{secrets.CODECOV_TOKEN}} #required
# file: ./cobertura.xml
file: ./avx2-known-key.xml
flags: avx2KnownKey
@@ -29,6 +29,9 @@ simd-lite = { version = "0.1.0", optional = true }
# for float comparison
float-cmp = "0.5"

# because dev dependenceis can't be optional ...
alloc_counter = {version = "0.0.3", optional = true}




@@ -49,21 +52,30 @@ harness = false

[features]
default = ["swar-number-parsing", "serde_impl"]
# Support for ARM NEON SIMD
neon = ["simd-lite"]
# used for enabeling known keys in favour of a slower
# hasher that is not protected against hash collision
# attacks
known-key = [ "halfbrown/fxhash" ]
# use 8 number at once parsing strategy
swar-number-parsing = []
# serde compatibility
serde_impl = [ "serde", "serde_json", "halfbrown/serde" ]
# Support for ARM NEON SIMD
neon = ["simd-lite"]
# for testing allocations
alloc = ["alloc_counter"]
# don't inline code - used for debugging
no-inline = []
# uses safe slice access ([]) instead of get_unsafe
# **for debugging**
safe = []
# also bench serde in the benchmarks
bench-serde = []
# use branch hints - requires nightly :(
hints = [] # requires nightly
# for perf testing, used by the example
perf = ["perfcnt", "getopts", "colored"]
known-key = [ "halfbrown/fxhash" ]



[[example]]
@@ -15,6 +15,10 @@

## readme (for real!)

### simdjson version

**Currently tracking version 0.2.x of simdjson upstream (work in progress, feedback welcome!).**

### CPU target

To be able to take advantage of simdjson your system needs to be SIMD compatible. This means to compile with native cpu support and the given features. Look at [The cargo config in this repository](.cargo/config) to get an example.
@@ -1,2 +1,5 @@
ignore:
- "src/neon/**" # We currently can't run coverage on arm so we're excluding it
- "examples/*" # examples
- "benches/*" # examples
- "simd-fuzz-target/*" # fuzz testing
@@ -2,8 +2,8 @@ use std::env;

#[cfg(feature = "perf")]
mod int {
const ROUNDS: usize = 10000;
const WARMUP: usize = 2000;
const ROUNDS: usize = 2000;
const WARMUP: usize = 200;
use colored::*;
use perfcnt::linux::{HardwareEventType, PerfCounterBuilderLinux};
use perfcnt::{AbstractPerfCounter, PerfCounter};
@@ -11,36 +11,46 @@ use crate::Deserializer;
pub use crate::Result;

impl<'de> Deserializer<'de> {
// Allow it to keep in sync with upstream
#[allow(clippy::if_not_else)]
#[allow(
clippy::if_not_else,
mutable_transmutes,
clippy::transmute_ptr_to_ptr,
clippy::too_many_lines,
clippy::cast_ptr_alignment,
clippy::cast_possible_wrap,
clippy::if_not_else,
clippy::too_many_lines
)]
#[cfg_attr(not(feature = "no-inline"), inline(always))]
pub fn parse_str_(&mut self) -> Result<&'de str> {
pub(crate) fn parse_str_<'invoke>(
input: &'de [u8],
buffer: &'invoke mut [u8],
mut idx: usize,
) -> Result<&'de str> {
use ErrorType::*;
let input: &mut [u8] = unsafe { std::mem::transmute(input) };
// Add 1 to skip the initial "
let idx = self.iidx + 1;
idx += 1;
let mut padding = [0_u8; 32];
//let mut read: usize = 0;

// we include the terminal '"' so we know where to end
// This is safe since we check sub's lenght in the range access above and only
// create sub sliced form sub to `sub.len()`.

let src: &[u8] = unsafe { &self.input.get_unchecked(idx..) };
let src: &[u8] = unsafe { input.get_unchecked(idx..) };
let mut src_i: usize = 0;
let mut len = src_i;
loop {
let v: __m256i = if src.len() >= src_i + 32 {
// This is safe since we ensure src is at least 32 wide
#[allow(clippy::cast_ptr_alignment)]
unsafe {
_mm256_loadu_si256(src.as_ptr().add(src_i) as *const __m256i)
}
unsafe { _mm256_loadu_si256(src.as_ptr().add(src_i) as *const __m256i) }
} else {
unsafe {
padding
.get_unchecked_mut(..src.len() - src_i)
.clone_from_slice(src.get_unchecked(src_i..));
// This is safe since we ensure src is at least 32 wide
#[allow(clippy::cast_ptr_alignment)]
_mm256_loadu_si256(padding.as_ptr() as *const __m256i)
}
};
@@ -50,11 +60,9 @@ impl<'de> Deserializer<'de> {
let bs_bits: u32 = unsafe {
static_cast_u32!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(
v,
#[allow(clippy::cast_possible_wrap)]
_mm256_set1_epi8(b'\\' as i8)
)))
};
#[allow(clippy::cast_possible_wrap)]
let quote_mask = unsafe { _mm256_cmpeq_epi8(v, _mm256_set1_epi8(b'"' as i8)) };
let quote_bits = unsafe { static_cast_u32!(_mm256_movemask_epi8(quote_mask)) };
if (bs_bits.wrapping_sub(1) & quote_bits) != 0 {
@@ -72,63 +80,54 @@ impl<'de> Deserializer<'de> {

len += quote_dist as usize;
unsafe {
let v = self.input.get_unchecked(idx..idx + len) as *const [u8] as *const str;
let v = input.get_unchecked(idx..idx + len) as *const [u8] as *const str;
return Ok(&*v);
}

// we compare the pointers since we care if they are 'at the same spot'
// not if they are the same value
}
if (quote_bits.wrapping_sub(1) & bs_bits) != 0 {
if (quote_bits.wrapping_sub(1) & bs_bits) == 0 {
// they are the same. Since they can't co-occur, it means we encountered
// neither.
src_i += 32;
len += 32;
} else {
// Move to the 'bad' character
let bs_dist: u32 = bs_bits.trailing_zeros();
len += bs_dist as usize;
src_i += bs_dist as usize;
break;
} else {
// they are the same. Since they can't co-occur, it means we encountered
// neither.
src_i += 32;
len += 32;
}
}

let mut dst_i: usize = 0;
let dst: &mut [u8] = &mut self.strings;

// To be more conform with upstream
loop {
let v: __m256i = if src.len() >= src_i + 32 {
// This is safe since we ensure src is at least 32 wide
#[allow(clippy::cast_ptr_alignment)]
unsafe {
_mm256_loadu_si256(src.as_ptr().add(src_i) as *const __m256i)
}
unsafe { _mm256_loadu_si256(src.as_ptr().add(src_i) as *const __m256i) }
} else {
unsafe {
padding
.get_unchecked_mut(..src.len() - src_i)
.clone_from_slice(src.get_unchecked(src_i..));
// This is safe since we ensure src is at least 32 wide
#[allow(clippy::cast_ptr_alignment)]
_mm256_loadu_si256(padding.as_ptr() as *const __m256i)
}
};

#[allow(clippy::cast_ptr_alignment)]
unsafe {
_mm256_storeu_si256(dst.as_mut_ptr().add(dst_i) as *mut __m256i, v)
};
unsafe { _mm256_storeu_si256(buffer.as_mut_ptr().add(dst_i) as *mut __m256i, v) };

// store to dest unconditionally - we can overwrite the bits we don't like
// later
let bs_bits: u32 = unsafe {
static_cast_u32!(_mm256_movemask_epi8(_mm256_cmpeq_epi8(
v,
#[allow(clippy::cast_possible_wrap)]
_mm256_set1_epi8(b'\\' as i8)
)))
};
#[allow(clippy::cast_possible_wrap)]
let quote_mask = unsafe { _mm256_cmpeq_epi8(v, _mm256_set1_epi8(b'"' as i8)) };
let quote_bits = unsafe { static_cast_u32!(_mm256_movemask_epi8(quote_mask)) };
if (bs_bits.wrapping_sub(1) & quote_bits) != 0 {
@@ -146,12 +145,11 @@ impl<'de> Deserializer<'de> {

dst_i += quote_dist as usize;
unsafe {
self.input
input
.get_unchecked_mut(idx + len..idx + len + dst_i)
.clone_from_slice(&self.strings.get_unchecked(..dst_i));
let v = self.input.get_unchecked(idx..idx + len + dst_i) as *const [u8]
as *const str;
self.str_offset += dst_i as usize;
.clone_from_slice(&buffer.get_unchecked(..dst_i));
let v =
input.get_unchecked(idx..idx + len + dst_i) as *const [u8] as *const str;
return Ok(&*v);
}

@@ -170,14 +168,14 @@ impl<'de> Deserializer<'de> {
dst_i += bs_dist as usize;
let (o, s) = if let Ok(r) =
handle_unicode_codepoint(unsafe { src.get_unchecked(src_i..) }, unsafe {
dst.get_unchecked_mut(dst_i..)
buffer.get_unchecked_mut(dst_i..)
}) {
r
} else {
return Err(self.error(ErrorType::InvlaidUnicodeCodepoint));
return Err(Self::raw_error(src_i, 'u', InvlaidUnicodeCodepoint));
};
if o == 0 {
return Err(self.error(ErrorType::InvlaidUnicodeCodepoint));
return Err(Self::raw_error(src_i, 'u', InvlaidUnicodeCodepoint));
};
// We moved o steps forword at the destiation and 6 on the source
src_i += s;
@@ -190,10 +188,10 @@ impl<'de> Deserializer<'de> {
let escape_result: u8 =
unsafe { *ESCAPE_MAP.get_unchecked(escape_char as usize) };
if escape_result == 0 {
return Err(self.error(ErrorType::InvalidEscape));
return Err(Self::raw_error(src_i, escape_char as char, InvalidEscape));
}
unsafe {
*dst.get_unchecked_mut(dst_i + bs_dist as usize) = escape_result;
*buffer.get_unchecked_mut(dst_i + bs_dist as usize) = escape_result;
}
src_i += bs_dist as usize + 2;
dst_i += bs_dist as usize + 1;

0 comments on commit 0deaa23

Please sign in to comment.
You can’t perform that action at this time.