Skip to content

Commit eda6229

Browse files
d10xad10xa
authored
Refactoring LogViewerStream (#35)
1 parent a1f2b54 commit eda6229

File tree

8 files changed

+508
-112
lines changed

8 files changed

+508
-112
lines changed

‎json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/FilterComponents.scala‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package ru.d10xa.jsonlogviewer
2+
3+
import ru.d10xa.jsonlogviewer.config.ResolvedConfig
4+
import ru.d10xa.jsonlogviewer.decline.Config
5+
import ru.d10xa.jsonlogviewer.formatout.{ColorLineFormatter, RawFormatter}
6+
7+
final case class FilterComponents(
8+
timestampFilter: TimestampFilter,
9+
parseResultKeys: ParseResultKeys,
10+
logLineFilter: LogLineFilter,
11+
fuzzyFilter: FuzzyFilter,
12+
outputLineFormatter: OutputLineFormatter
13+
)
14+
15+
object FilterComponents {
16+
17+
def fromConfig(resolvedConfig: ResolvedConfig): FilterComponents = {
18+
val timestampFilter = TimestampFilter()
19+
val parseResultKeys = ParseResultKeys(resolvedConfig)
20+
val logLineFilter = LogLineFilter(resolvedConfig, parseResultKeys)
21+
val fuzzyFilter = new FuzzyFilter(resolvedConfig)
22+
23+
val outputLineFormatter = resolvedConfig.formatOut match {
24+
case Some(Config.FormatOut.Raw) => RawFormatter()
25+
case Some(Config.FormatOut.Pretty) | None =>
26+
ColorLineFormatter(
27+
resolvedConfig,
28+
resolvedConfig.feedName,
29+
resolvedConfig.excludeFields
30+
)
31+
}
32+
33+
FilterComponents(
34+
timestampFilter,
35+
parseResultKeys,
36+
logLineFilter,
37+
fuzzyFilter,
38+
outputLineFormatter
39+
)
40+
}
41+
}

‎json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/FilterPipeline.scala‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package ru.d10xa.jsonlogviewer
2+
3+
import cats.effect.IO
4+
import fs2.Stream
5+
import ru.d10xa.jsonlogviewer.config.ResolvedConfig
6+
import scala.util.{Failure, Success, Try}
7+
8+
object FilterPipeline {
9+
10+
// Filter order: rawFilter -> parse -> grep -> query -> fuzzy -> timestamp -> format
11+
def applyFilters(
12+
stream: Stream[IO, String],
13+
parser: LogLineParser,
14+
components: FilterComponents,
15+
resolvedConfig: ResolvedConfig
16+
): Stream[IO, String] = {
17+
stream
18+
.filter(
19+
rawFilter(_, resolvedConfig.rawInclude, resolvedConfig.rawExclude)
20+
)
21+
.map(parser.parse)
22+
.filter(components.logLineFilter.grep)
23+
.filter(components.logLineFilter.logLineQueryPredicate)
24+
.filter(components.fuzzyFilter.test)
25+
.through(
26+
components.timestampFilter.filterTimestampAfter(resolvedConfig.timestampAfter)
27+
)
28+
.through(
29+
components.timestampFilter.filterTimestampBefore(
30+
resolvedConfig.timestampBefore
31+
)
32+
)
33+
.map(formatWithSafety(_, components.outputLineFormatter))
34+
}
35+
36+
private def rawFilter(
37+
str: String,
38+
include: Option[List[String]],
39+
exclude: Option[List[String]]
40+
): Boolean = {
41+
import scala.util.matching.Regex
42+
val includeRegexes: List[Regex] = include.getOrElse(Nil).map(_.r)
43+
val excludeRegexes: List[Regex] = exclude.getOrElse(Nil).map(_.r)
44+
val includeMatches = includeRegexes.isEmpty || includeRegexes.exists(
45+
_.findFirstIn(str).isDefined
46+
)
47+
val excludeMatches = excludeRegexes.forall(_.findFirstIn(str).isEmpty)
48+
includeMatches && excludeMatches
49+
}
50+
51+
private def formatWithSafety(
52+
parseResult: ParseResult,
53+
formatter: OutputLineFormatter
54+
): String =
55+
Try(formatter.formatLine(parseResult)) match {
56+
case Success(formatted) => formatted.toString
57+
case Failure(_) => parseResult.raw
58+
}
59+
}

‎json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/LogLineParserFactory.scala‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package ru.d10xa.jsonlogviewer
2+
3+
import ru.d10xa.jsonlogviewer.config.ResolvedConfig
4+
import ru.d10xa.jsonlogviewer.csv.CsvLogLineParser
5+
import ru.d10xa.jsonlogviewer.decline.Config.FormatIn
6+
import ru.d10xa.jsonlogviewer.logfmt.LogfmtLogLineParser
7+
8+
object LogLineParserFactory {
9+
10+
def createNonCsvParser(resolvedConfig: ResolvedConfig): LogLineParser = {
11+
val jsonPrefixPostfix = JsonPrefixPostfix(JsonDetector())
12+
resolvedConfig.formatIn match {
13+
case Some(FormatIn.Logfmt) =>
14+
LogfmtLogLineParser(resolvedConfig)
15+
case Some(FormatIn.Csv) =>
16+
throw new IllegalStateException(
17+
"CSV format requires header line, use createCsvParser instead"
18+
)
19+
case Some(FormatIn.Json) | None =>
20+
JsonLogLineParser(resolvedConfig, jsonPrefixPostfix)
21+
}
22+
}
23+
24+
def createCsvParser(
25+
resolvedConfig: ResolvedConfig,
26+
headerLine: String
27+
): LogLineParser = {
28+
CsvLogLineParser(resolvedConfig, headerLine)
29+
}
30+
}

‎json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/LogViewerStream.scala‎

Lines changed: 13 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,11 @@ import fs2.*
66
import fs2.Pull
77
import ru.d10xa.jsonlogviewer.config.ConfigResolver
88
import ru.d10xa.jsonlogviewer.config.ResolvedConfig
9-
import ru.d10xa.jsonlogviewer.csv.CsvLogLineParser
109
import ru.d10xa.jsonlogviewer.decline.yaml.ConfigYaml
1110
import ru.d10xa.jsonlogviewer.decline.Config
1211
import ru.d10xa.jsonlogviewer.decline.Config.FormatIn
13-
import ru.d10xa.jsonlogviewer.formatout.ColorLineFormatter
14-
import ru.d10xa.jsonlogviewer.formatout.RawFormatter
15-
import ru.d10xa.jsonlogviewer.logfmt.LogfmtLogLineParser
1612
import ru.d10xa.jsonlogviewer.shell.Shell
1713
import ru.d10xa.jsonlogviewer.shell.ShellImpl
18-
import scala.util.matching.Regex
19-
import scala.util.Failure
20-
import scala.util.Success
21-
import scala.util.Try
2214

2315
object LogViewerStream {
2416

@@ -116,43 +108,18 @@ object LogViewerStream {
116108
)
117109
)
118110
} else {
119-
IO.pure(makeNonCsvLogLineParser(resolvedConfig))
111+
IO.pure(LogLineParserFactory.createNonCsvParser(resolvedConfig))
120112
}
121113

122114
Stream.eval(getParser).flatMap { parser =>
123-
val timestampFilter = TimestampFilter()
124-
val parseResultKeys = ParseResultKeys(resolvedConfig)
125-
val logLineFilter = LogLineFilter(resolvedConfig, parseResultKeys)
126-
val fuzzyFilter = new FuzzyFilter(resolvedConfig)
127-
128-
val outputLineFormatter = resolvedConfig.formatOut match {
129-
case Some(Config.FormatOut.Raw) => RawFormatter()
130-
case Some(Config.FormatOut.Pretty) | None =>
131-
ColorLineFormatter(
132-
resolvedConfig,
133-
resolvedConfig.feedName,
134-
resolvedConfig.excludeFields
135-
)
136-
}
137-
138-
Stream
139-
.emit(line)
140-
.filter(
141-
rawFilter(_, resolvedConfig.rawInclude, resolvedConfig.rawExclude)
142-
)
143-
.map(parser.parse)
144-
.filter(logLineFilter.grep)
145-
.filter(logLineFilter.logLineQueryPredicate)
146-
.filter(fuzzyFilter.test)
147-
.through(
148-
timestampFilter.filterTimestampAfter(resolvedConfig.timestampAfter)
149-
)
150-
.through(
151-
timestampFilter.filterTimestampBefore(
152-
resolvedConfig.timestampBefore
153-
)
154-
)
155-
.map(formatWithSafety(_, outputLineFormatter))
115+
val components = FilterComponents.fromConfig(resolvedConfig)
116+
117+
FilterPipeline.applyFilters(
118+
Stream.emit(line),
119+
parser,
120+
components,
121+
resolvedConfig
122+
)
156123
}
157124
}
158125

@@ -162,80 +129,14 @@ object LogViewerStream {
162129
): Stream[IO, String] =
163130
lines.pull.uncons1.flatMap {
164131
case Some((headerLine, rest)) =>
165-
val csvHeaderParser = CsvLogLineParser(resolvedConfig, headerLine)
166-
167-
val timestampFilter = TimestampFilter()
168-
val parseResultKeys = ParseResultKeys(resolvedConfig)
169-
val logLineFilter = LogLineFilter(resolvedConfig, parseResultKeys)
170-
val fuzzyFilter = new FuzzyFilter(resolvedConfig)
171-
172-
val outputLineFormatter = resolvedConfig.formatOut match {
173-
case Some(Config.FormatOut.Raw) => RawFormatter()
174-
case Some(Config.FormatOut.Pretty) | None =>
175-
ColorLineFormatter(
176-
resolvedConfig,
177-
resolvedConfig.feedName,
178-
resolvedConfig.excludeFields
179-
)
180-
}
132+
val csvHeaderParser = LogLineParserFactory.createCsvParser(resolvedConfig, headerLine)
133+
val components = FilterComponents.fromConfig(resolvedConfig)
181134

182-
rest
183-
.filter(
184-
rawFilter(_, resolvedConfig.rawInclude, resolvedConfig.rawExclude)
185-
)
186-
.map(csvHeaderParser.parse)
187-
.filter(logLineFilter.grep)
188-
.filter(logLineFilter.logLineQueryPredicate)
189-
.filter(fuzzyFilter.test)
190-
.through(
191-
timestampFilter.filterTimestampAfter(resolvedConfig.timestampAfter)
192-
)
193-
.through(
194-
timestampFilter.filterTimestampBefore(
195-
resolvedConfig.timestampBefore
196-
)
197-
)
198-
.map(formatWithSafety(_, outputLineFormatter))
135+
FilterPipeline
136+
.applyFilters(rest, csvHeaderParser, components, resolvedConfig)
199137
.pull
200138
.echo
201139
case None =>
202140
Pull.done
203141
}.stream
204-
205-
private def formatWithSafety(
206-
parseResult: ParseResult,
207-
formatter: OutputLineFormatter
208-
): String =
209-
Try(formatter.formatLine(parseResult)) match {
210-
case Success(formatted) => formatted.toString
211-
case Failure(_) => parseResult.raw
212-
}
213-
214-
def makeNonCsvLogLineParser(
215-
resolvedConfig: ResolvedConfig
216-
): LogLineParser = {
217-
val jsonPrefixPostfix = JsonPrefixPostfix(JsonDetector())
218-
resolvedConfig.formatIn match {
219-
case Some(FormatIn.Logfmt) => LogfmtLogLineParser(resolvedConfig)
220-
case Some(FormatIn.Csv) =>
221-
throw new IllegalStateException(
222-
"method makeNonCsvLogLineParser does not support csv"
223-
)
224-
case _ => JsonLogLineParser(resolvedConfig, jsonPrefixPostfix)
225-
}
226-
}
227-
228-
def rawFilter(
229-
str: String,
230-
include: Option[List[String]],
231-
exclude: Option[List[String]]
232-
): Boolean = {
233-
val includeRegexes: List[Regex] = include.getOrElse(Nil).map(_.r)
234-
val excludeRegexes: List[Regex] = exclude.getOrElse(Nil).map(_.r)
235-
val includeMatches = includeRegexes.isEmpty || includeRegexes.exists(
236-
_.findFirstIn(str).isDefined
237-
)
238-
val excludeMatches = excludeRegexes.forall(_.findFirstIn(str).isEmpty)
239-
includeMatches && excludeMatches
240-
}
241142
}

‎json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/FilterComponentsTest.scala‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
package ru.d10xa.jsonlogviewer
2+
3+
import munit.FunSuite
4+
import ru.d10xa.jsonlogviewer.decline.{Config, FieldNamesConfig}
5+
import ru.d10xa.jsonlogviewer.formatout.{ColorLineFormatter, RawFormatter}
6+
7+
class FilterComponentsTest extends FunSuite with TestResolvedConfig {
8+
9+
override def baseResolvedConfig = super.baseResolvedConfig.copy(
10+
feedName = Some("test-feed")
11+
)
12+
13+
test("fromConfig should create all filter components") {
14+
val components = FilterComponents.fromConfig(baseResolvedConfig)
15+
16+
assertNotEquals(components.timestampFilter, null, "TimestampFilter should be created")
17+
assertNotEquals(components.parseResultKeys, null, "ParseResultKeys should be created")
18+
assertNotEquals(components.logLineFilter, null, "LogLineFilter should be created")
19+
assertNotEquals(components.fuzzyFilter, null, "FuzzyFilter should be created")
20+
assertNotEquals(components.outputLineFormatter, null, "OutputLineFormatter should be created")
21+
}
22+
23+
test("fromConfig should create RawFormatter when formatOut is Raw") {
24+
val config = baseResolvedConfig.copy(formatOut = Some(Config.FormatOut.Raw))
25+
val components = FilterComponents.fromConfig(config)
26+
27+
assert(
28+
components.outputLineFormatter.isInstanceOf[RawFormatter],
29+
"Should create RawFormatter for Raw format"
30+
)
31+
}
32+
33+
test("fromConfig should create ColorLineFormatter when formatOut is Pretty") {
34+
val config = baseResolvedConfig.copy(formatOut = Some(Config.FormatOut.Pretty))
35+
val components = FilterComponents.fromConfig(config)
36+
37+
assert(
38+
components.outputLineFormatter.isInstanceOf[ColorLineFormatter],
39+
"Should create ColorLineFormatter for Pretty format"
40+
)
41+
}
42+
43+
test("fromConfig should create ColorLineFormatter when formatOut is None") {
44+
val config = baseResolvedConfig.copy(formatOut = None)
45+
val components = FilterComponents.fromConfig(config)
46+
47+
assert(
48+
components.outputLineFormatter.isInstanceOf[ColorLineFormatter],
49+
"Should create ColorLineFormatter when formatOut is None (default)"
50+
)
51+
}
52+
53+
test("fromConfig should initialize ParseResultKeys with correct config") {
54+
val config = baseResolvedConfig.copy(
55+
fieldNames = FieldNamesConfig(
56+
timestampFieldName = "custom_ts",
57+
levelFieldName = "custom_level",
58+
messageFieldName = "custom_msg",
59+
stackTraceFieldName = "custom_stack",
60+
loggerNameFieldName = "custom_logger",
61+
threadNameFieldName = "custom_thread"
62+
)
63+
)
64+
val components = FilterComponents.fromConfig(config)
65+
66+
// Verify that ParseResultKeys was created (opaque object, can't test internals)
67+
assertNotEquals(
68+
components.parseResultKeys,
69+
null,
70+
"ParseResultKeys should be created with custom config"
71+
)
72+
}
73+
74+
test("fromConfig should create components multiple times with same config") {
75+
val components1 = FilterComponents.fromConfig(baseResolvedConfig)
76+
val components2 = FilterComponents.fromConfig(baseResolvedConfig)
77+
78+
// Components should be created each time (not cached)
79+
assert(
80+
components1 ne components2,
81+
"Each call should create new FilterComponents"
82+
)
83+
}
84+
}

0 commit comments

Comments
 (0)