work towards generic grouping
All checks were successful
CI / build (push) Successful in 48s
CI / checks (push) Successful in 1m1s
CI / build-nixpkgs (push) Successful in 42s

This commit is contained in:
Robin Appelman 2025-08-09 17:27:25 +02:00
commit 3943407b9b
13 changed files with 309 additions and 202 deletions

View file

@ -1,28 +1,26 @@
use crate::grouping::group_lines_by;
use crate::grouping::LogGrouping;
use crate::logfile::{LogFile, LogLine, LogLineNumber};
use crate::logs::ParsedLogs;
use crate::matcher::MatchResult;
use crate::timegraph::{SparkLine, TimeGraph};
use logsmash_data::{LoggingStatementWithPathPrefix, StatementList};
use regex::{escape, Regex, RegexBuilder};
use serde_json::Error as JsonError;
use std::cell::OnceCell;
use std::fmt::Display;
use std::ops::RangeInclusive;
use time::OffsetDateTime;
pub struct App<'logs> {
pub lines: &'logs ParsedLogs<'logs>,
pub log_statements: StatementList,
pub matches: Vec<LogMatch<'logs>>,
pub all: LogMatch<'logs>,
pub unmatched: LogMatch<'logs>,
pub matches: Vec<LogGrouping<'logs, MatchResult>>,
pub all: LogGrouping<'logs, MatchResult>,
pub log_file: &'logs LogFile,
pub unmatched_count: usize,
}
impl<'logs> App<'logs> {
pub fn match_lines(&self) -> usize {
let unmatched_line_count = if self.unmatched.count == 0 { 0 } else { 1 };
self.matches.len() + 1 + unmatched_line_count
self.matches.len() + 1
}
pub fn get_source_line(&self, index: LogLineNumber) -> Option<&'logs str> {
@ -47,75 +45,8 @@ impl<'logs> App<'logs> {
self.lines.errors().len()
}
pub fn time_range(&self) -> (OffsetDateTime, OffsetDateTime) {
(self.lines.first().time, self.lines.last().time)
}
}
pub struct LogMatch<'logs> {
pub result: Option<MatchResult>,
pub count: usize,
pub all: LineSet<'logs>,
pub grouped: Vec<LineSet<'logs>>,
}
impl<'logs> LogMatch<'logs> {
pub fn new(result: Option<MatchResult>, lines: Vec<&'logs LogLine<'logs>>) -> Self {
let count = lines.len();
let grouped = group_lines_by(lines.iter().copied(), LogLine::identity);
let all = LineSet::new(lines);
LogMatch {
result,
count,
grouped,
all,
}
}
pub fn sparkline(&self, app: &App) -> &SparkLine {
self.all.sparkline(app)
}
pub fn histogram(&self, app: &App) -> &TimeGraph {
self.all.histogram(app)
}
pub fn row_count(&self) -> usize {
self.result.as_ref().map(|res| res.count()).unwrap_or(1)
}
pub fn statements<'a>(
&'a self,
) -> impl Iterator<Item = &'a LoggingStatementWithPathPrefix> + use<'a> {
self.result.iter().flat_map(|res| res.iter())
}
pub fn matches(&self, filter: &Filter) -> bool {
if filter.is_empty() {
return true;
}
self.statements().any(|statement| {
filter.parts().all(|filter_part| {
filter_part.is_match(statement.statement.pattern)
|| filter_part.is_match(statement.statement.path)
|| filter_part.is_match(statement.path_prefix)
|| statement
.statement
.placeholders
.iter()
.any(|placeholder| filter_part.is_match(placeholder))
|| statement
.statement
.exception
.filter(|exception| filter_part.is_match(exception))
.is_some()
})
})
}
pub fn count(&self) -> usize {
self.count
pub fn time_range(&self) -> RangeInclusive<OffsetDateTime> {
self.lines.first().time..=self.lines.last().time
}
}
@ -134,15 +65,14 @@ impl<'logs> LineSet<'logs> {
}
}
pub fn sparkline(&self, app: &App) -> &SparkLine {
pub fn sparkline(&self, time_range: RangeInclusive<OffsetDateTime>) -> &SparkLine {
self.sparkline
.get_or_init(|| self.histogram(app).sparkline())
.get_or_init(|| self.histogram(time_range).sparkline())
}
pub fn histogram(&self, app: &App) -> &TimeGraph {
pub fn histogram(&self, time_range: RangeInclusive<OffsetDateTime>) -> &TimeGraph {
self.histogram.get_or_init(|| {
let (min_time, max_time) = app.time_range();
let mut histogram = TimeGraph::new(min_time, max_time);
let mut histogram = TimeGraph::new(*time_range.start(), *time_range.end());
for line in self.lines.iter() {
histogram.add(line.time);
}

View file

@ -1,6 +1,99 @@
use crate::app::LineSet;
use crate::app::{Filter, LineSet};
use crate::logfile::LogLine;
use crate::timegraph::{SparkLine, TimeGraph};
use ratatui::layout::{Alignment, Constraint};
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::ops::RangeInclusive;
use time::OffsetDateTime;
pub trait GroupedLine {
const HEADER: &'static [(&'static str, Alignment)];
const WIDTHS: &'static [Constraint];
}
#[allow(clippy::len_without_is_empty)]
pub trait GroupingResult {
type Lines: GroupedLine;
fn len(&self) -> usize;
fn lines<'a>(&'a self) -> impl Iterator<Item = &'a Self::Lines> + use<'a, Self>
where
<Self as GroupingResult>::Lines: 'a;
fn matches(&self, filter: &Filter) -> bool;
fn render(&self) -> impl Iterator<Item = Cow<str>>;
}
pub struct LogGrouping<'logs, G> {
pub name: Option<&'static str>,
pub result: Option<G>,
pub count: usize,
pub all: LineSet<'logs>,
pub grouped: Vec<LineSet<'logs>>,
}
impl<'logs, G: GroupingResult> LogGrouping<'logs, G> {
pub fn new(result: G, lines: Vec<&'logs LogLine<'logs>>) -> Self {
let count = lines.len();
let grouped = group_lines_by(lines.iter().copied(), LogLine::identity);
let all = LineSet::new(lines);
LogGrouping {
name: None,
result: Some(result),
count,
grouped,
all,
}
}
pub fn named(name: &'static str, lines: Vec<&'logs LogLine<'logs>>) -> Self {
let count = lines.len();
let grouped = group_lines_by(lines.iter().copied(), LogLine::identity);
let all = LineSet::new(lines);
LogGrouping {
name: Some(name),
result: None,
count,
grouped,
all,
}
}
pub fn sparkline(&self, time_range: RangeInclusive<OffsetDateTime>) -> &SparkLine {
self.all.sparkline(time_range)
}
pub fn histogram(&self, time_range: RangeInclusive<OffsetDateTime>) -> &TimeGraph {
self.all.histogram(time_range)
}
pub fn row_count(&self) -> usize {
self.result.as_ref().map(|res| res.len()).unwrap_or(1)
}
pub fn iter(&self) -> impl Iterator<Item = &LineSet<'logs>> {
self.grouped.iter()
}
pub fn matches(&self, filter: &Filter) -> bool {
if filter.is_empty() {
return true;
}
match &self.result {
Some(result) => result.matches(filter),
_ => true,
}
}
pub fn count(&self) -> usize {
self.count
}
}
pub fn group_lines_by<'logs, I, F, K>(indices: I, f: F) -> Vec<LineSet<'logs>>
where

View file

@ -1,4 +1,4 @@
use crate::app::{App, LogMatch};
use crate::app::App;
use crate::error::LogError;
use crate::logfile::{LogFile, LogLineNumber};
use crate::logs::ParsedLogs;
@ -29,6 +29,7 @@ mod matcher;
mod timegraph;
mod ui;
use crate::grouping::LogGrouping;
#[cfg(target_env = "musl")]
use tikv_jemallocator::Jemalloc;
use time::format_description::{parse_owned, parse_strftime_owned};
@ -151,21 +152,25 @@ fn main() -> MainResult {
matched_lines.sort_by_key(|(_, lines)| lines.len());
matched_lines.reverse();
let all = LogMatch::new(None, parsed_log.find_lines(|_| true).collect());
let unmatched = LogMatch::new(None, unmatched_lines);
let all = LogGrouping::<MatchResult>::named("All", parsed_log.find_lines(|_| true).collect());
let unmatched = LogGrouping::<MatchResult>::named("Unmatched", unmatched_lines);
let matches = matched_lines
let mut matches: Vec<_> = matched_lines
.into_par_iter()
.map(|(result, lines)| LogMatch::new(Some(result), lines))
.map(|(result, lines)| LogGrouping::new(result, lines))
.collect();
let unmatched_count = unmatched.count();
if unmatched_count > 0 {
matches.push(unmatched);
}
let app = App {
lines: &parsed_log,
log_statements: statements,
matches,
unmatched,
all,
log_file: &log_file,
unmatched_count,
};
if args.profile {

View file

@ -1,7 +1,12 @@
use crate::app::Filter;
use crate::grouping::{GroupedLine, GroupingResult};
use crate::logfile::logline::{Exception, LogLine};
use crate::logfile::LineNumber;
use itertools::Either;
use logsmash_data::{LogLevel, LoggingStatementWithPathPrefix, StatementList};
use ratatui::layout::{Alignment, Constraint};
use std::borrow::Cow;
use std::fmt::Write;
use std::hash::{Hash, Hasher};
use std::iter::once;
use std::ops::Range;
@ -206,6 +211,73 @@ impl From<Vec<LoggingStatementWithPathPrefix>> for MatchResult {
}
}
impl GroupedLine for LoggingStatementWithPathPrefix {
const HEADER: &'static [(&'static str, Alignment)] = &[
("Statement", Alignment::Left),
("File", Alignment::Left),
("Line", Alignment::Right),
];
const WIDTHS: &'static [Constraint] = &[
Constraint::Percentage(70),
Constraint::Percentage(30),
Constraint::Min(6),
Constraint::Length(10),
Constraint::Min(10),
];
}
impl GroupingResult for MatchResult {
type Lines = LoggingStatementWithPathPrefix;
fn len(&self) -> usize {
self.count()
}
fn lines<'a>(&'a self) -> impl Iterator<Item = &'a Self::Lines> + use<'a>
where
<Self as GroupingResult>::Lines: 'a,
{
self.iter()
}
fn matches(&self, filter: &Filter) -> bool {
if filter.is_empty() {
return true;
}
self.iter().any(|statement| {
filter.parts().all(|filter_part| {
filter_part.is_match(statement.statement.pattern)
|| filter_part.is_match(statement.statement.path)
|| filter_part.is_match(statement.path_prefix)
|| statement
.statement
.placeholders
.iter()
.any(|placeholder| filter_part.is_match(placeholder))
|| statement
.statement
.exception
.filter(|exception| filter_part.is_match(exception))
.is_some()
})
})
}
fn render(&self) -> impl Iterator<Item = Cow<str>> {
let mut message = String::new();
let mut paths = String::new();
let mut lines = String::new();
for statement in self.lines() {
writeln!(&mut message, "{}", statement.message()).ok();
writeln!(&mut paths, "{}", statement.path()).ok();
writeln!(&mut lines, "{}", statement.line()).ok();
}
[message.into(), paths.into(), lines.into()].into_iter()
}
}
#[derive(Default, Copy, Clone)]
pub struct SingleMatchState<'a> {
pattern: &'a [u8],

View file

@ -1,5 +1,6 @@
use hdrhistogram::Histogram;
use ratatui::text::Text;
use std::borrow::Cow;
use std::cmp::max;
use time::OffsetDateTime;
@ -83,4 +84,12 @@ impl<'a> From<&'a SparkLine> for Text<'a> {
}
}
impl<'a> From<&'a SparkLine> for Cow<'a, str> {
fn from(value: &'a SparkLine) -> Self {
// SAFETY: we only put bytes into the buffer from encode_utf8
let str = unsafe { str::from_utf8_unchecked(&value.bytes).trim_end_matches(char::from(0)) };
str.into()
}
}
const SPARKS: &[char] = &[' ', '▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];

View file

@ -27,7 +27,7 @@ pub fn footer<'a>(app: &App<'a>, params: FooterParams<'a>) -> Table<'a> {
Table::new(
[Row::new([
Text::from(help(page)),
Text::from(format!("{} unmatched items", app.unmatched.count())),
Text::from(format!("{} unmatched items", app.unmatched_count)),
Text::from(format!("{} parse errors", app.error_count())),
])],
widths,

59
src/ui/grouping_list.rs Normal file
View file

@ -0,0 +1,59 @@
use crate::app::Filter;
use crate::grouping::{GroupedLine, GroupingResult, LogGrouping};
use crate::ui::style::TABLE_HEADER_STYLE;
use crate::ui::table::ScrollbarTable;
use ratatui::prelude::*;
use ratatui::widgets::{Cell, Row};
use std::borrow::Cow;
use std::iter::once;
use std::ops::RangeInclusive;
use time::OffsetDateTime;
pub fn grouping_list<'a, G: GroupingResult>(
all: &'a LogGrouping<'a, G>,
items: &'a [LogGrouping<'a, G>],
time_range: RangeInclusive<OffsetDateTime>,
filter: &Filter,
) -> ScrollbarTable<'a> {
let header = G::Lines::HEADER
.iter()
.copied()
.chain([("Time", Alignment::Left), ("Count", Alignment::Left)]);
let header = header
.map(|(text, align)| Text::from(text).alignment(align))
.map(Cell::from)
.collect::<Row>()
.style(TABLE_HEADER_STYLE)
.height(1);
ScrollbarTable::new(
once(all)
.chain(items.iter().filter(|result| result.matches(filter)))
.map(|result| grouped_row(result, time_range.clone())),
G::Lines::WIDTHS,
)
.header(header)
}
fn grouped_row<'a, G: GroupingResult>(
grouping: &'a LogGrouping<'a, G>,
time_range: RangeInclusive<OffsetDateTime>,
) -> Row<'a> {
if let Some(match_result) = &grouping.result {
let grouping_columns = match_result.render();
let columns = grouping_columns.chain([
Cow::from(grouping.sparkline(time_range)),
Cow::from(grouping.count().to_string()),
]);
Row::new(columns).height(match_result.len() as u16)
} else {
Row::new([
Text::from(grouping.name.unwrap_or_default()),
Text::from(""),
Text::from(""),
Text::from(grouping.sparkline(time_range)),
Text::from(grouping.count().to_string()),
])
}
}

View file

@ -1,80 +0,0 @@
use crate::app::{App, Filter, LogMatch};
use crate::ui::style::TABLE_HEADER_STYLE;
use crate::ui::table::ScrollbarTable;
use itertools::Either;
use ratatui::prelude::*;
use ratatui::widgets::{Cell, Row};
use std::fmt::Write;
use std::iter::{empty, once};
pub fn match_list<'a>(app: &'a App<'a>, filter: &Filter) -> ScrollbarTable<'a> {
let header = [
Text::from("Statement"),
Text::from("File"),
Text::from("Line").alignment(Alignment::Right),
Text::from("Time"),
Text::from("Count"),
]
.into_iter()
.map(Cell::from)
.collect::<Row>()
.style(TABLE_HEADER_STYLE)
.height(1);
let widths = [
Constraint::Percentage(70),
Constraint::Percentage(30),
Constraint::Min(6),
Constraint::Length(10),
Constraint::Min(10),
];
let all = log_row(&app.all, app, "All lines");
let unmatched = if app.unmatched.count() == 0 {
Either::Right(empty())
} else {
Either::Left(once(log_row(&app.unmatched, app, "Unmatched lines")))
};
ScrollbarTable::new(
once(all)
.chain(
app.matches
.iter()
.filter(|result| result.matches(filter))
.map(|result| log_row(result, app, "")),
)
.chain(unmatched),
widths,
)
.header(header)
}
fn log_row<'a>(result: &'a LogMatch, app: &'a App, name: &'static str) -> Row<'a> {
if let Some(match_result) = &result.result {
let mut message = String::new();
let mut paths = String::new();
let mut lines = String::new();
for statement in match_result.iter() {
writeln!(&mut message, "{}", statement.message()).ok();
writeln!(&mut paths, "{}", statement.path()).ok();
writeln!(&mut lines, "{}", statement.line()).ok();
}
Row::new([
Text::from(message),
Text::from(paths),
Text::from(lines).alignment(Alignment::Right),
Text::from(result.sparkline(app)),
Text::from(result.count().to_string()),
])
.height(match_result.count() as u16)
} else {
Row::new([
Text::from(name),
Text::from(""),
Text::from(""),
Text::from(result.sparkline(app)),
Text::from(result.count().to_string()),
])
}
}

View file

@ -1,11 +1,12 @@
use crate::app::App;
use crate::error::UiError;
use crate::matcher::MatchResult;
use crate::ui::error_list::error_list;
use crate::ui::footer::footer;
use crate::ui::grouped_logs::grouped_logs;
use crate::ui::grouping_list::grouping_list;
use crate::ui::histogram::UiHistogram;
use crate::ui::input::handle_events;
use crate::ui::match_list::match_list;
use crate::ui::single_log::single_log;
use crate::ui::single_match::grouped_lines;
use crate::ui::state::{
@ -27,9 +28,9 @@ use std::panic::{set_hook, take_hook};
mod error_list;
mod footer;
mod grouped_logs;
mod grouping_list;
mod histogram;
mod input;
mod match_list;
mod single_log;
mod single_match;
mod state;
@ -116,16 +117,18 @@ fn ui(frame: &mut Frame, app: &App, state: &mut UiState) {
}) => {
let selected = table_state.selected();
let histogram = if selected == 0 {
app.all.histogram(app)
} else if selected < app.matches.len() + 1 {
let log_match = &app.matches[selected - 1];
log_match.histogram(app)
app.all.histogram(app.time_range())
} else {
app.unmatched.histogram(app)
let log_match = &app.matches[selected - 1];
log_match.histogram(app.time_range())
};
frame.render_widget(UiHistogram::new(histogram), layout[0]);
frame.render_stateful_widget(match_list(app, filter), layout[1], table_state);
frame.render_stateful_widget(
grouping_list::<MatchResult>(&app.all, &app.matches, app.time_range(), filter),
layout[1],
table_state,
);
frame.render_widget(footer(app, state.footer_params()), layout[2]);
}
UiState::Match(MatchState {
@ -141,7 +144,10 @@ fn ui(frame: &mut Frame, app: &App, state: &mut UiState) {
&result.grouped[selected - 1]
};
frame.render_widget(UiHistogram::new(selected_group.histogram(app)), layout[0]);
frame.render_widget(
UiHistogram::new(selected_group.histogram(app.time_range())),
layout[0],
);
frame.render_stateful_widget(
grouped_lines(app, result, filter),
layout[1],

View file

@ -1,4 +1,6 @@
use crate::app::{App, Filter, LineSet, LogMatch};
use crate::app::{App, Filter, LineSet};
use crate::grouping::LogGrouping;
use crate::matcher::MatchResult;
use crate::ui::style::TABLE_HEADER_STYLE;
use crate::ui::table::{ScrollbarTable, ScrollbarTableState};
use ratatui::buffer::Buffer;
@ -7,10 +9,12 @@ use ratatui::prelude::StatefulWidget;
use ratatui::text::Text;
use ratatui::widgets::{Cell, Row};
use std::iter::once;
use std::ops::RangeInclusive;
use time::OffsetDateTime;
pub fn grouped_lines<'a>(
app: &'a App<'a>,
log_match: &'a LogMatch,
log_match: &'a LogGrouping<'a, MatchResult>,
filter: &'a Filter,
) -> SingleMatchTable<'a> {
SingleMatchTable {
@ -22,7 +26,7 @@ pub fn grouped_lines<'a>(
pub struct SingleMatchTable<'a> {
app: &'a App<'a>,
log_match: &'a LogMatch<'a>,
log_match: &'a LogGrouping<'a, MatchResult>,
filter: &'a Filter,
}
@ -59,7 +63,7 @@ impl StatefulWidget for SingleMatchTable<'_> {
Text::from("All lines"),
Text::from(""),
Text::from(""),
Text::from(self.log_match.sparkline(self.app)),
Text::from(self.log_match.sparkline(self.app.time_range())),
Text::from(self.log_match.count().to_string()),
]))
.chain(
@ -68,7 +72,11 @@ impl StatefulWidget for SingleMatchTable<'_> {
.filter(|group| group.matches(self.filter))
.enumerate()
.map(|(i, group)| {
group_row(self.app, group, i.abs_diff(state.selected()) < 100)
group_row(
self.app.time_range(),
group,
i.abs_diff(state.selected()) < 100,
)
}),
),
widths,
@ -78,7 +86,11 @@ impl StatefulWidget for SingleMatchTable<'_> {
}
}
fn group_row<'a>(app: &'a App, group: &'a LineSet, is_in_view: bool) -> Row<'a> {
fn group_row<'a>(
time_range: RangeInclusive<OffsetDateTime>,
group: &'a LineSet,
is_in_view: bool,
) -> Row<'a> {
if is_in_view {
let line = group.lines[0];
@ -86,7 +98,7 @@ fn group_row<'a>(app: &'a App, group: &'a LineSet, is_in_view: bool) -> Row<'a>
Text::from(line.level.as_str()),
Text::from(line.app.as_ref()),
Text::from(line.display()),
Text::from(group.sparkline(app)),
Text::from(group.sparkline(time_range)),
Text::from(group.len().to_string()),
])
} else {

View file

@ -1,6 +1,8 @@
use crate::app::{App, Filter, LogMatch, EMPTY_FILTER};
use crate::app::{App, Filter, EMPTY_FILTER};
use crate::error::ParseError;
use crate::grouping::LogGrouping;
use crate::logfile::logline::{FullLogLine, LogLine};
use crate::matcher::MatchResult;
use crate::ui::footer::FooterParams;
use crate::ui::input::{PopMode, UiEvent};
use crate::ui::table::ScrollbarTableState;
@ -45,19 +47,16 @@ impl<'a> MatchListState<'a> {
fn enter(self, selected: usize, app: &'a App) -> UiState<'a> {
let result = if selected == 0 {
&app.all
} else if selected <= app.matches.len() {
if self.filter.is_empty() {
&app.matches[selected - 1]
} else {
app.matches
.iter()
.filter(|log_match| log_match.matches(&self.filter))
.nth(selected - 1)
.unwrap_or(&app.unmatched)
}
} else if self.filter.is_empty() {
&app.matches[selected - 1]
} else {
&app.unmatched
app.matches
.iter()
.filter(|log_match| log_match.matches(&self.filter))
.nth(selected - 1)
.unwrap_or(app.matches.last().unwrap())
};
let table_state = ScrollbarTableState::new(result.grouped.len() + 1);
UiState::Match(MatchState {
result,
@ -77,7 +76,7 @@ impl PartialEq for MatchListState<'_> {
#[derive(Clone)]
pub struct MatchState<'a> {
pub result: &'a LogMatch<'a>,
pub result: &'a LogGrouping<'a, MatchResult>,
pub table_state: ScrollbarTableState,
pub previous: Box<UiState<'a>>,
pub filter: Filter,