From 448949b09085f4b708718f508da4ef8ff4d2ce69 Mon Sep 17 00:00:00 2001 From: Pasi Miettinen Date: Fri, 15 Mar 2019 15:11:36 +0200 Subject: [PATCH 1/4] Python: Sort JSON by keys --- python/sbp/client/loggers/json_logger.py | 51 +++++++++++++----------- python/sbp/msg.py | 4 +- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/python/sbp/client/loggers/json_logger.py b/python/sbp/client/loggers/json_logger.py index 0df26d2886..3e654d57c8 100644 --- a/python/sbp/client/loggers/json_logger.py +++ b/python/sbp/client/loggers/json_logger.py @@ -16,13 +16,17 @@ import warnings -class JSONLogger(BaseLogger): +class JSONLoggerBase(BaseLogger): """ - JSONLogger + JSONLoggerBase - The :class:`JSONLogger` logs JSON records. + The :class:`JSONLoggerBase` base class for JSON loggers. """ + def __init__(self, handle, tags={}, dispatcher=None, sort_keys=False): + BaseLogger.__init__(self, handle, tags, dispatcher) + self._sort_keys=sort_keys + def fmt_msg(self, data, **metadata): if metadata: metadata.update(self.tags) @@ -31,10 +35,25 @@ def fmt_msg(self, data, **metadata): return data + def __call__(self, msg, **metadata): + output = self.dump(msg, **metadata) + if output: + self.handle.write(output + "\n") + + +class JSONLogger(JSONLoggerBase): + """ + JSONLogger + + The :class:`JSONLogger` logs JSON records. + """ + def dump(self, msg, **metadata): try: data = self.dispatch(msg).to_json_dict() - return json.dumps(self.fmt_msg(data, **metadata), allow_nan=False) + return json.dumps(self.fmt_msg(data, **metadata), + allow_nan=False, + sort_keys=self._sort_keys) except (ValueError, UnicodeDecodeError): try: warn = "Bad values in JSON encoding for msg_type %d for msg %s" \ @@ -44,27 +63,14 @@ def dump(self, msg, **metadata): except (ValueError, UnicodeDecodeError): return None - def __call__(self, msg, **metadata): - output = self.dump(msg, **metadata) - if output: - self.handle.write(output + "\n") - -class JSONBinLogger(BaseLogger): +class JSONBinLogger(JSONLoggerBase): """ JSONBinLogger The :class:`JSONLogger` logs JSON records without expanding the fields. """ - def fmt_msg(self, data, **metadata): - if metadata: - metadata.update(self.tags) - metadata['data'] = data - return metadata - - return data - def dump(self, msg, **metadata): try: data = { @@ -75,7 +81,9 @@ def dump(self, msg, **metadata): 'payload': base64.standard_b64encode(msg.payload).decode('ascii'), 'crc': msg.crc } - return json.dumps(self.fmt_msg(data, **metadata), allow_nan=False) + return json.dumps(self.fmt_msg(data, **metadata), + allow_nan=False, + sort_keys=self._sort_keys) except (ValueError, UnicodeDecodeError): try: warn = "Bad values in JSON encoding for msg_type %d for msg %s" \ @@ -85,11 +93,6 @@ def dump(self, msg, **metadata): except (ValueError, UnicodeDecodeError): return None - def __call__(self, msg, **metadata): - output = self.dump(msg, **metadata) - if output: - self.handle.write(output + "\n") - class JSONLogIterator(LogIterator): """ diff --git a/python/sbp/msg.py b/python/sbp/msg.py index af2335df19..c44b5b535f 100755 --- a/python/sbp/msg.py +++ b/python/sbp/msg.py @@ -156,12 +156,12 @@ def to_binary(self): ret += struct.pack(" Date: Fri, 15 Mar 2019 18:06:54 +0200 Subject: [PATCH 2/4] sbp2prettyjason --- haskell/main/SBP2PRETTYJSON.hs | 36 ++++++++++++++++++++++++++++++++++ haskell/sbp.cabal | 16 +++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 haskell/main/SBP2PRETTYJSON.hs diff --git a/haskell/main/SBP2PRETTYJSON.hs b/haskell/main/SBP2PRETTYJSON.hs new file mode 100644 index 0000000000..acb7aa261c --- /dev/null +++ b/haskell/main/SBP2PRETTYJSON.hs @@ -0,0 +1,36 @@ +{-# LANGUAGE NoImplicitPrelude #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | +-- Module: SBP2PRETTYJSON +-- Copyright: Copyright (C) 2019 Swift Navigation, Inc. +-- License: LGPL-3 +-- Maintainer: Pasi Miettinen +-- Stability: experimental +-- Portability: portable +-- +-- SBP to JSON tool - reads SBP binary from stdin and sends SBP JSON +-- to stdout. + +import BasicPrelude hiding (map) +import Control.Monad.Trans.Resource +import Data.Aeson.Encode.Pretty +import Data.ByteString.Lazy hiding (ByteString, map) +import Data.Conduit +import Data.Conduit.Binary +import Data.Conduit.List +import Data.Conduit.Serialization.Binary +import SwiftNav.SBP +import System.IO + +-- | Encode a SBPMsg to a line of JSON. +encodeLine :: SBPMsg -> ByteString +encodeLine v = toStrict $ encodePretty' (defConfig {confIndent = Spaces 0, confCompare = compare}) v <> "\n" + +main :: IO () +main = + runResourceT $ + sourceHandle stdin + =$= conduitDecode + =$= map encodeLine + $$ sinkHandle stdout diff --git a/haskell/sbp.cabal b/haskell/sbp.cabal index 511278f40a..32cf09cba1 100644 --- a/haskell/sbp.cabal +++ b/haskell/sbp.cabal @@ -54,6 +54,7 @@ library hs-source-dirs: src ghc-options: -Wall build-depends: aeson + , aeson-pretty , array , base >= 4.8 && < 5 , base64-bytestring @@ -82,6 +83,21 @@ executable sbp2json , sbp default-language: Haskell2010 +executable sbp2prettyjson + hs-source-dirs: main + main-is: SBP2PRETTYJSON.hs + ghc-options: -threaded -rtsopts -with-rtsopts=-N -Wall + build-depends: aeson-pretty + , base + , basic-prelude + , binary-conduit + , bytestring + , conduit + , conduit-extra + , resourcet + , sbp + default-language: Haskell2010 + executable json2sbp hs-source-dirs: main main-is: JSON2SBP.hs From e432f916a3052fa1a3fdff6bc8bc2cff73c81aff Mon Sep 17 00:00:00 2001 From: Pasi Miettinen Date: Mon, 18 Mar 2019 14:33:45 +0200 Subject: [PATCH 3/4] Use compact endoding with Python json.dumps() --- python/sbp/client/loggers/json_logger.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/sbp/client/loggers/json_logger.py b/python/sbp/client/loggers/json_logger.py index 3e654d57c8..001e6409f8 100644 --- a/python/sbp/client/loggers/json_logger.py +++ b/python/sbp/client/loggers/json_logger.py @@ -53,7 +53,8 @@ def dump(self, msg, **metadata): data = self.dispatch(msg).to_json_dict() return json.dumps(self.fmt_msg(data, **metadata), allow_nan=False, - sort_keys=self._sort_keys) + sort_keys=self._sort_keys, + separators=(',',':')) except (ValueError, UnicodeDecodeError): try: warn = "Bad values in JSON encoding for msg_type %d for msg %s" \ @@ -83,7 +84,8 @@ def dump(self, msg, **metadata): } return json.dumps(self.fmt_msg(data, **metadata), allow_nan=False, - sort_keys=self._sort_keys) + sort_keys=self._sort_keys, + separators=(',',':')) except (ValueError, UnicodeDecodeError): try: warn = "Bad values in JSON encoding for msg_type %d for msg %s" \ From bc795424e2ee56151460c57f24c870d1c1bfad6d Mon Sep 17 00:00:00 2001 From: Pasi Miettinen Date: Mon, 18 Mar 2019 14:26:29 +0200 Subject: [PATCH 4/4] Add sbp2prettyjson cmd arguments --- haskell/main/SBP2PRETTYJSON.hs | 25 ++++++++++++++++++++----- haskell/sbp.cabal | 2 +- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/haskell/main/SBP2PRETTYJSON.hs b/haskell/main/SBP2PRETTYJSON.hs index acb7aa261c..716fee9e39 100644 --- a/haskell/main/SBP2PRETTYJSON.hs +++ b/haskell/main/SBP2PRETTYJSON.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE DeriveDataTypeable #-} +{-# OPTIONS_GHC -fno-cse #-} {-# LANGUAGE NoImplicitPrelude #-} {-# LANGUAGE OverloadedStrings #-} @@ -12,7 +14,7 @@ -- SBP to JSON tool - reads SBP binary from stdin and sends SBP JSON -- to stdout. -import BasicPrelude hiding (map) +import BasicPrelude hiding (map, print) import Control.Monad.Trans.Resource import Data.Aeson.Encode.Pretty import Data.ByteString.Lazy hiding (ByteString, map) @@ -21,16 +23,29 @@ import Data.Conduit.Binary import Data.Conduit.List import Data.Conduit.Serialization.Binary import SwiftNav.SBP +import System.Console.CmdArgs import System.IO +data Cfg = Cfg {skiporder :: Bool + ,spaces :: Int} + deriving (Show, Data, Typeable) + +defaultCfg :: Cfg +defaultCfg = Cfg{skiporder = False + ,spaces = 0} + -- | Encode a SBPMsg to a line of JSON. -encodeLine :: SBPMsg -> ByteString -encodeLine v = toStrict $ encodePretty' (defConfig {confIndent = Spaces 0, confCompare = compare}) v <> "\n" +encodeLine :: Cfg -> SBPMsg -> ByteString +encodeLine c v = toStrict + $ encodePretty' + (defConfig {confIndent = Spaces (spaces c), + confCompare = if (skiporder c) then mempty else compare}) v <> "\n" main :: IO () -main = +main = do + cfg <- cmdArgs defaultCfg runResourceT $ sourceHandle stdin =$= conduitDecode - =$= map encodeLine + =$= map (encodeLine cfg) $$ sinkHandle stdout diff --git a/haskell/sbp.cabal b/haskell/sbp.cabal index 32cf09cba1..c14fd23c1b 100644 --- a/haskell/sbp.cabal +++ b/haskell/sbp.cabal @@ -54,7 +54,6 @@ library hs-source-dirs: src ghc-options: -Wall build-depends: aeson - , aeson-pretty , array , base >= 4.8 && < 5 , base64-bytestring @@ -92,6 +91,7 @@ executable sbp2prettyjson , basic-prelude , binary-conduit , bytestring + , cmdargs , conduit , conduit-extra , resourcet