Skip to content

Commit

Permalink
Wrong type of quantization coefficient fix (microsoft#80)
Browse files Browse the repository at this point in the history
* Test is failing because of no difference in this seed

* Correct seed with difference
  • Loading branch information
Melirius committed Jun 11, 2024
1 parent 16668fe commit d8b56d0
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 6 deletions.
10 changes: 5 additions & 5 deletions src/structs/lepton_encoder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -718,7 +718,7 @@ fn roundtrip_large_coef() {
&block,
&block,
[65535; 64],
0xe9a6f36fcaf42727,
0xcac19d7e86aece1b,
&EnabledFeatures::compat_lepton_vector_read(),
);
}
Expand All @@ -728,10 +728,10 @@ fn roundtrip_large_coef() {
fn roundtrip_random_seed() {
use rand::Rng;

// the 22 seed is a choice that doesn't overflow the DC coefficient
// the 127 seed is a choice that doesn't overflow the DC coefficient
// since the encoder is somewhat picky if the DC estimate overflows
// it also has different behavior for 32 and 16 bit codepath
let mut rng = crate::helpers::get_rand_from_seed([22; 32]);
let mut rng = crate::helpers::get_rand_from_seed([127; 32]);

let arr = [0i16; 64];

Expand All @@ -748,7 +748,7 @@ fn roundtrip_random_seed() {
&above_left,
&here,
qt,
0x8f043f2ae83c2d5a,
0xe3c687262f0df4f5,
&EnabledFeatures::compat_lepton_scalar_read(),
);

Expand All @@ -759,7 +759,7 @@ fn roundtrip_random_seed() {
&above_left,
&here,
qt,
0x99675a04115a3b3b,
0xdbacb31b714489fc,
&EnabledFeatures::compat_lepton_vector_read(),
);

Expand Down
2 changes: 1 addition & 1 deletion src/structs/probability_tables.rs
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ impl ProbabilityTables {
let uncertainty2_val = (far_afield_value >> 3) as i16;

return PredictDCResult {
predicted_dc: ((avgmed / i32::from(q_transposed.get_coefficient(0))) + 4) >> 3,
predicted_dc: ((avgmed / i32::from(q_transposed.get_coefficient(0) as u16)) + 4) >> 3,
uncertainty: uncertainty_val,
uncertainty2: uncertainty2_val,
advanced_predict_dc_pixels_sans_dc: pixels_sans_dc,
Expand Down

0 comments on commit d8b56d0

Please sign in to comment.