hjson-rust fails with Rust 1.48

[sophiajt]

We've been using hjson in Nushell, and it's been great. Definitely an improvement to have a more-flexible json format.

With the latest Rust 1.48, though, we ran into an issue where one of the dependencies is leaving memory uninitialized.

Here's the full backtrace:

---- commands::math::compound_where_paren stdout ----
=== stderr
thread 'main' panicked at 'attempted to leave type `linked_hash_map::Node<std::string::String, value::Value>` uninitialized, which is invalid', /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/mem/mod.rs:658:9
stack backtrace:
   0: rust_begin_unwind
             at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:483
   1: core::panicking::panic_fmt
             at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/panicking.rs:85
   2: core::panicking::panic
             at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/panicking.rs:50
   3: core::mem::uninitialized
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/mem/mod.rs:658
   4: linked_hash_map::LinkedHashMap<K,V,S>::insert
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/linked-hash-map-0.3.0/src/lib.rs:203
   5: <linked_hash_map::serde::LinkedHashMapVisitor<K,V> as serde::de::Visitor>::visit_map
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/linked-hash-map-0.3.0/src/serde.rs:72
   6: <<serde_hjson::value::Value as serde::de::Deserialize>::deserialize::ValueVisitor as serde::de::Visitor>::visit_map
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-hjson-0.9.1/src/value.rs:441
   7: serde_hjson::de::Deserializer<Iter>::parse_value
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-hjson-0.9.1/src/de.rs:151
   8: <serde_hjson::de::Deserializer<Iter> as serde::de::Deserializer>::deserialize
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-hjson-0.9.1/src/de.rs:443
   9: <serde_hjson::value::Value as serde::de::Deserialize>::deserialize
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-hjson-0.9.1/src/value.rs:446
  10: <serde_hjson::de::SeqVisitor<Iter> as serde::de::SeqVisitor>::visit
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-hjson-0.9.1/src/de.rs:537
  11: <serde::de::impls::VecVisitor<T> as serde::de::Visitor>::visit_seq
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-0.8.23/src/de/impls.rs:433
  12: <<serde_hjson::value::Value as serde::de::Deserialize>::deserialize::ValueVisitor as serde::de::Visitor>::visit_seq
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-hjson-0.9.1/src/value.rs:432
  13: serde_hjson::de::Deserializer<Iter>::parse_value
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-hjson-0.9.1/src/de.rs:147
  14: <serde_hjson::de::Deserializer<Iter> as serde::de::Deserializer>::deserialize
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-hjson-0.9.1/src/de.rs:443
  15: <serde_hjson::value::Value as serde::de::Deserialize>::deserialize
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-hjson-0.9.1/src/value.rs:446
  16: serde_hjson::de::from_iter
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-hjson-0.9.1/src/de.rs:813
  17: serde_hjson::de::from_slice
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-hjson-0.9.1/src/de.rs:846
  18: serde_hjson::de::from_str
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-hjson-0.9.1/src/de.rs:853
  19: nu_cli::commands::from_json::from_json_string_to_value
             at /home/jonathan/Source/nushell/crates/nu-cli/src/commands/from_json.rs:71
  20: nu_cli::commands::from_json::from_json::{{closure}}
             at /home/jonathan/Source/nushell/crates/nu-cli/src/commands/from_json.rs:114
  21: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:80
  22: <nu_cli::commands::from_json::FromJSON as nu_cli::commands::command::WholeStreamCommand>::run::__run::{{closure}}
             at /home/jonathan/Source/nushell/crates/nu-cli/src/commands/from_json.rs:36
  23: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:80
  24: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:119
  25: nu_cli::commands::command::Command::run::{{closure}}
             at /home/jonathan/Source/nushell/crates/nu-cli/src/commands/command.rs:321
  26: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:80
  27: nu_cli::evaluation_context::EvaluationContext::run_command::{{closure}}
             at /home/jonathan/Source/nushell/crates/nu-cli/src/evaluation_context.rs:159
  28: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:80
  29: nu_cli::commands::classified::internal::run_internal_command::{{closure}}
             at /home/jonathan/Source/nushell/crates/nu-cli/src/commands/classified/internal.rs:29
  30: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:80
  31: nu_cli::commands::classified::block::run_pipeline::{{closure}}
             at /home/jonathan/Source/nushell/crates/nu-cli/src/commands/classified/block.rs:82
  32: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:80
  33: nu_cli::commands::classified::block::run_block::{{closure}}
             at /home/jonathan/Source/nushell/crates/nu-cli/src/commands/classified/block.rs:55
  34: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:80
  35: nu_cli::cli::process_line::{{closure}}
             at /home/jonathan/Source/nushell/crates/nu-cli/src/cli.rs:1036
  36: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:80
  37: nu_cli::cli::cli::{{closure}}
             at /home/jonathan/Source/nushell/crates/nu-cli/src/cli.rs:479
  38: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:80
  39: futures_executor::local_pool::block_on::{{closure}}
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.5/src/local_pool.rs:317
  40: futures_executor::local_pool::run_executor::{{closure}}
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.5/src/local_pool.rs:87
  41: std::thread::local::LocalKey<T>::try_with
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:272
  42: std::thread::local::LocalKey<T>::with
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:248
  43: futures_executor::local_pool::run_executor
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.5/src/local_pool.rs:83
  44: futures_executor::local_pool::block_on
             at /home/jonathan/.cargo/registry/src/github.com-1ecc6299db9ec823/futures-executor-0.3.5/src/local_pool.rs:317
  45: nu::main
             at /home/jonathan/Source/nushell/src/main.rs:171
  46: core::ops::function::FnOnce::call_once
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:227
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

thread 'commands::math::compound_where_paren' panicked at 'assertion failed: `(left == right)`
  left: `""`,
 right: `"[{\"a\":2,\"b\":1},{\"a\":2,\"b\":2}]"`', crates/nu-cli/tests/commands/math/mod.rs:257:5
stack backtrace:
   0: rust_begin_unwind
             at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:483
   1: std::panicking::begin_panic_fmt
             at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/std/src/panicking.rs:437
   2: main::commands::math::compound_where_paren
             at ./tests/commands/math/mod.rs:257
   3: main::commands::math::compound_where_paren::{{closure}}
             at ./tests/commands/math/mod.rs:249
   4: core::ops::function::FnOnce::call_once
             at /home/jonathan/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:227
   5: core::ops::function::FnOnce::call_once
             at /rustc/7eac88abb2e57e752f3302f02be5f3ce3d7adfb4/library/core/src/ops/function.rs:227
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

Assuming I'm reading that correctly, the issue is in linked_hash_map. I looked on their repo and it looks like it's no longer maintained.

Any ideas if there's a fix for this? We'd definitely love to keep using this package.

[cheradenine]

what's the story with your nu_hjson fork, @jonathandturner ? I can't find the source and it looks like it is also on an older version of serde (0.8.23). From the issues list here there seems to be no viable hjson package for rust currently.

[Yarn]

You can avoid the issue with

serde-hjson = { version = "0.9", default-features = false }

this will disable preservation of key order

---

I made a version that patches the issue (it was and still is UB but it seems to work for now).
https://github.com/Yarn/hjson-rust/tree/unitialized_fix
https://github.com/Yarn/linked-hash-map/tree/hjson_fix

you can use it with

serde-hjson = { git = "https://github.com/Yarn/hjson-rust.git", rev = "ae3a336153d0c4f757beb848d2059123be00cb75" }

---

My workaround for the older version of serde is to read the file to a serde_hjson::Value then using an old version of serde_json to write json to a string then reading that string using a new version of serde_json

in Cargo.toml

serde-hjson = { git = "https://github.com/Yarn/hjson-rust.git", rev = "ae3a336153d0c4f757beb848d2059123be00cb75" }
serde_json = "1.0"
serde_json_old = { package = "serde_json", version = "0.8.6" }

in your code assuming Thing implements serde 1.0 deserialize

let mut f = std::fs::File::open(path).unwrap();
let a: serde_hjson::Value = serde_hjson::from_reader(f).unwrap();
let b = serde_json_old::to_string(&a).unwrap();
let thing: Thing = serde_json::from_str(&b).unwrap();

[dessalines]

@Nutomic has tried swapping out linked_hash_map (the dead library causing this error) for hashlink (a new, well maintained one). However unfortunately it has this error:

            #[inline]
            fn visit_map<'de, V>(&mut self, visitor: V) -> Result<Value, V::Error>
                where V: de::MapVisitor,
                // where V: de::MapAccess<'de>,
            {
                let values = try!(MapVisitor::default().visit_map(visitor));
                Ok(Value::Object(values))
            }

error:

error[E0599]: no method named `visit_map` found for struct `LinkedHashMapVisitor<_, _>` in the current scope
   --> src/value.rs:442:57
    |
442 |                 let values = try!(MapVisitor::default().visit_map(visitor));
    |                                                         ^^^^^^^^^ method not found in `LinkedHashMapVisitor<_, _>`
    |
    = help: items from traits can only be used if the trait is in scope
    = note: the following trait is implemented but not in scope; perhaps add a `use` for it:
            `use serde::de::Visitor;`

This is because hashlink requires serde::de::MapAccess rather than serde::de::MapVisitor, which unfortunately only newer versions of serde have.

I tried upgrading the serde version from 0.8.0, but it causes ~100 errors that I'm not familiar enough with serde to fix.

edit: [here's the code: Nutomic/hjson-rust#use-hashlink

[Yarn]

#24

sometimes the solution is in the obvious place

[trobro]

#24 has been merged, closing this.