Skip to content

Commit

Permalink
Add JSON Lines (NDJSON) message serialization (#5048)
Browse files Browse the repository at this point in the history
## Summary

This adds `json-lines` (https://jsonlines.org/ or http://ndjson.org/) as
an output format.

I'm sure you already know, but

* JSONL is more greppable (each record is a single line) than the pretty
JSON
* JSONL is faster to ingest piecewise (and/or in parallel) than JSON

## Test Plan

Snapshot test in the new module :)
  • Loading branch information
akx committed Jun 13, 2023
1 parent e1fd396 commit 7b4dde0
Show file tree
Hide file tree
Showing 8 changed files with 84 additions and 27 deletions.
53 changes: 28 additions & 25 deletions crates/ruff/src/message/json.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use std::io::Write;

use serde::ser::SerializeSeq;
use serde::{Serialize, Serializer};
use serde_json::json;
use serde_json::{json, Value};

use ruff_diagnostics::Edit;
use ruff_python_ast::source_code::SourceCode;
Expand Down Expand Up @@ -38,37 +38,40 @@ impl Serialize for ExpandedMessages<'_> {
let mut s = serializer.serialize_seq(Some(self.messages.len()))?;

for message in self.messages {
let source_code = message.file.to_source_code();

let fix = message.fix.as_ref().map(|fix| {
json!({
"applicability": fix.applicability(),
"message": message.kind.suggestion.as_deref(),
"edits": &ExpandedEdits { edits: fix.edits(), source_code: &source_code },
})
});

let start_location = source_code.source_location(message.start());
let end_location = source_code.source_location(message.end());
let noqa_location = source_code.source_location(message.noqa_offset);

let value = json!({
"code": message.kind.rule().noqa_code().to_string(),
"message": message.kind.body,
"fix": fix,
"location": start_location,
"end_location": end_location,
"filename": message.filename(),
"noqa_row": noqa_location.row
});

let value = message_to_json_value(message);
s.serialize_element(&value)?;
}

s.end()
}
}

pub(crate) fn message_to_json_value(message: &Message) -> Value {
let source_code = message.file.to_source_code();

let fix = message.fix.as_ref().map(|fix| {
json!({
"applicability": fix.applicability(),
"message": message.kind.suggestion.as_deref(),
"edits": &ExpandedEdits { edits: fix.edits(), source_code: &source_code },
})
});

let start_location = source_code.source_location(message.start());
let end_location = source_code.source_location(message.end());
let noqa_location = source_code.source_location(message.noqa_offset);

json!({
"code": message.kind.rule().noqa_code().to_string(),
"message": message.kind.body,
"fix": fix,
"location": start_location,
"end_location": end_location,
"filename": message.filename(),
"noqa_row": noqa_location.row
})
}

struct ExpandedEdits<'a> {
edits: &'a [Edit],
source_code: &'a SourceCode<'a, 'a>,
Expand Down
39 changes: 39 additions & 0 deletions crates/ruff/src/message/json_lines.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
use std::io::Write;

use crate::message::json::message_to_json_value;
use crate::message::{Emitter, EmitterContext, Message};

#[derive(Default)]
pub struct JsonLinesEmitter;

impl Emitter for JsonLinesEmitter {
fn emit(
&mut self,
writer: &mut dyn Write,
messages: &[Message],
_context: &EmitterContext,
) -> anyhow::Result<()> {
let mut w = writer;
for message in messages {
serde_json::to_writer(&mut w, &message_to_json_value(message))?;
w.write_all(b"\n")?;
}
Ok(())
}
}

#[cfg(test)]
mod tests {
use crate::message::json_lines::JsonLinesEmitter;
use insta::assert_snapshot;

use crate::message::tests::{capture_emitter_output, create_messages};

#[test]
fn output() {
let mut emitter = JsonLinesEmitter::default();
let content = capture_emitter_output(&mut emitter, &create_messages());

assert_snapshot!(content);
}
}
2 changes: 2 additions & 0 deletions crates/ruff/src/message/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ pub use github::GithubEmitter;
pub use gitlab::GitlabEmitter;
pub use grouped::GroupedEmitter;
pub use json::JsonEmitter;
pub use json_lines::JsonLinesEmitter;
pub use junit::JunitEmitter;
pub use pylint::PylintEmitter;
use ruff_diagnostics::{Diagnostic, DiagnosticKind, Fix};
Expand All @@ -24,6 +25,7 @@ mod github;
mod gitlab;
mod grouped;
mod json;
mod json_lines;
mod junit;
mod pylint;
mod text;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
source: crates/ruff/src/message/jsonlines.rs
expression: content
---
{"code":"F401","message":"`os` imported but unused","fix":{"applicability":"Suggested","message":"Remove unused import: `os`","edits":[{"content":"","location":{"row":1,"column":1},"end_location":{"row":2,"column":1}}]},"location":{"row":1,"column":8},"end_location":{"row":1,"column":10},"filename":"fib.py","noqa_row":1}
{"code":"F841","message":"Local variable `x` is assigned to but never used","fix":{"applicability":"Suggested","message":"Remove assignment to unused variable `x`","edits":[{"content":"","location":{"row":6,"column":5},"end_location":{"row":6,"column":10}}]},"location":{"row":6,"column":5},"end_location":{"row":6,"column":6},"filename":"fib.py","noqa_row":6}
{"code":"F821","message":"Undefined name `a`","fix":null,"location":{"row":1,"column":4},"end_location":{"row":1,"column":5},"filename":"undef.py","noqa_row":1}

1 change: 1 addition & 0 deletions crates/ruff/src/settings/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@ impl FromStr for PatternPrefixPair {
pub enum SerializationFormat {
Text,
Json,
JsonLines,
Junit,
Grouped,
Github,
Expand Down
5 changes: 4 additions & 1 deletion crates/ruff_cli/src/printer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use ruff::linter::FixTable;
use ruff::logging::LogLevel;
use ruff::message::{
AzureEmitter, Emitter, EmitterContext, GithubEmitter, GitlabEmitter, GroupedEmitter,
JsonEmitter, JunitEmitter, PylintEmitter, TextEmitter,
JsonEmitter, JsonLinesEmitter, JunitEmitter, PylintEmitter, TextEmitter,
};
use ruff::notify_user;
use ruff::registry::{AsRule, Rule};
Expand Down Expand Up @@ -184,6 +184,9 @@ impl Printer {
SerializationFormat::Json => {
JsonEmitter::default().emit(writer, &diagnostics.messages, &context)?;
}
SerializationFormat::JsonLines => {
JsonLinesEmitter::default().emit(writer, &diagnostics.messages, &context)?;
}
SerializationFormat::Junit => {
JunitEmitter::default().emit(writer, &diagnostics.messages, &context)?;
}
Expand Down
2 changes: 1 addition & 1 deletion docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ Options:
--ignore-noqa
Ignore any `# noqa` comments
--format <FORMAT>
Output serialization format for violations [env: RUFF_FORMAT=] [possible values: text, json, junit, grouped, github, gitlab, pylint, azure]
Output serialization format for violations [env: RUFF_FORMAT=] [possible values: text, json, json-lines, junit, grouped, github, gitlab, pylint, azure]
--target-version <TARGET_VERSION>
The minimum Python version that should be supported [possible values: py37, py38, py39, py310, py311]
--config <CONFIG>
Expand Down
1 change: 1 addition & 0 deletions ruff.schema.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 7b4dde0

Please sign in to comment.