robin/logsmash
1
0
Fork 0
mirror of https://codeberg.org/icewind/logsmash.git synced 2026-06-03 18:14:11 +02:00
Code Issues Projects Releases 3 Packages Wiki Activity Actions

grouped lines

Browse source
This commit is contained in:
Robin Appelman 2024-07-25 19:59:46 +02:00
commit 483bb5691d
12 changed files with 158 additions and 92 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Cargo.lock generated
View file

@ -26,6 +26,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
dependencies = [
"cfg-if",
"getrandom",
"once_cell",
"version_check",
"zerocopy",
@ -203,6 +204,7 @@ dependencies = [
name = "cloud-log-analyser"
version = "0.1.0"
dependencies = [
"ahash",
"clap",
"cloud-log-analyser-data",
"hdrhistogram",

1
Cargo.toml
View file

@ -19,6 +19,7 @@ ratatui = "0.27.0"
tinystr = { version = "0.7.6", features = ["serde"] }
time = { version = "0.3.36", features = ["serde", "serde-well-known"] }
hdrhistogram = "7.5.4"
ahash = "0.8.11"
[profile.dev.package."*"]
opt-level = 3

2
data/src/types.rs
View file

@ -1,7 +1,7 @@
use serde::Deserialize;
use std::fmt::{Display, Formatter};
#[derive(Debug, Default, PartialEq, Clone, Copy, Deserialize)]
#[derive(Debug, Default, PartialEq, Clone, Copy, Deserialize, Hash)]
#[serde(from = "i64")]
pub enum LogLevel {
Debug,

59
src/app.rs
View file

@ -2,43 +2,53 @@ use crate::logline::LogLine;
use crate::matcher::MatchResult;
use crate::timegraph::TimeGraph;
use cloud_log_analyser_data::StatementList;
use std::collections::BTreeMap;
use time::OffsetDateTime;
pub struct App {
pub first_date: OffsetDateTime,
pub last_date: OffsetDateTime,
pub lines: Vec<LogLine>,
pub histogram: TimeGraph,
pub log_statements: StatementList,
pub matches: Vec<LogMatch>,
pub error_count: usize,
pub unmatched: Vec<usize>,
pub unmatched_histogram: TimeGraph,
pub all: LogMatch,
pub unmatched: LogMatch,
}
impl App {
pub fn match_lines(&self) -> usize {
let unmatched_line_count = if self.unmatched.is_empty() { 0 } else { 1 };
let unmatched_line_count = if self.unmatched.lines.is_empty() {
0
} else {
1
};
self.matches.len() + 1 + unmatched_line_count
}
}
pub struct LogMatch {
pub result: MatchResult,
pub result: Option<MatchResult>,
pub lines: Vec<usize>,
pub histogram: TimeGraph,
pub grouped: Vec<GroupedLines>,
}
impl LogMatch {
pub fn new(result: MatchResult, lines: Vec<usize>, all_lines: &[LogLine]) -> Self {
pub fn new(result: Option<MatchResult>, lines: Vec<usize>, all_lines: &[LogLine]) -> Self {
let min_time = all_lines[0].time;
let max_time = all_lines.last().unwrap().time;
let mut histogram = TimeGraph::new(min_time, max_time);
for line in lines.iter().map(|line| &all_lines[*line]) {
histogram.add(line.time);
}
let grouped = group_lines(all_lines, lines.iter().copied());
LogMatch {
result,
lines,
histogram,
grouped,
}
}
}
@ -48,3 +58,40 @@ impl LogMatch {
self.lines.len()
}
}
fn group_lines<I: Iterator<Item = usize>>(all_lines: &[LogLine], indices: I) -> Vec<GroupedLines> {
let mut map: BTreeMap<u64, Vec<usize>> = BTreeMap::new();
for (i, line) in indices.map(|i| (i, &all_lines[i])) {
map.entry(line.index()).or_default().push(i);
}
let mut list: Vec<_> = map
.into_values()
.map(|lines| GroupedLines::new(lines, all_lines))
.collect();
list.sort_by_key(|list| list.len());
list.reverse();
list
}
pub struct GroupedLines {
pub lines: Vec<usize>,
pub histogram: TimeGraph,
}
impl GroupedLines {
pub fn new(lines: Vec<usize>, all_lines: &[LogLine]) -> Self {
let min_time = all_lines[0].time;
let max_time = all_lines.last().unwrap().time;
let mut histogram = TimeGraph::new(min_time, max_time);
for line in lines.iter().map(|line| &all_lines[*line]) {
histogram.add(line.time);
}
GroupedLines { lines, histogram }
}
pub fn len(&self) -> usize {
self.lines.len()
}
}

14
src/logline.rs
View file

@ -1,5 +1,7 @@
use ahash::AHasher;
use cloud_log_analyser_data::LogLevel;
use serde::Deserialize;
use std::hash::{Hash, Hasher};
use time::OffsetDateTime;
use tinystr::TinyAsciiStr;
@ -23,6 +25,18 @@ impl LogLine {
.unwrap_or(self.version.as_str());
major.parse().ok()
}
pub fn index(&self) -> u64 {
let mut hasher = AHasher::default();
self.message.hash(&mut hasher);
self.level.hash(&mut hasher);
self.exception
.as_ref()
.map(|e| e.exception.as_str())
.hash(&mut hasher);
self.app.hash(&mut hasher);
hasher.finish()
}
}
#[derive(Deserialize, Debug)]

24
src/main.rs
View file

@ -3,7 +3,6 @@ use crate::error::LogError;
use crate::logfile::LogFile;
use crate::logline::LogLine;
use crate::matcher::{MatchResult, Matcher};
use crate::timegraph::TimeGraph;
use crate::ui::run_ui;
use clap::Parser;
use cloud_log_analyser_data::{get_statements, MAX_VERSION};
@ -84,27 +83,26 @@ fn main() -> MainResult {
matched_lines.sort_by_key(|(_, lines)| lines.len());
matched_lines.reverse();
let histogram = TimeGraph::generate(&parsed_lines);
let all = LogMatch::new(
None,
parsed_lines.iter().enumerate().map(|(i, _)| i).collect(),
&parsed_lines,
);
let unmatched = LogMatch::new(None, unmatched_lines, &parsed_lines);
let matches = matched_lines
.into_iter()
.map(|(result, lines)| LogMatch::new(result, lines, &parsed_lines))
.map(|(result, lines)| LogMatch::new(Some(result), lines, &parsed_lines))
.collect();
let min_time = parsed_lines[0].time;
let max_time = parsed_lines.last().unwrap().time;
let mut unmatched_histogram = TimeGraph::new(min_time, max_time);
for lines in unmatched_lines.iter().map(|line| &parsed_lines[*line]) {
unmatched_histogram.add(lines.time);
}
let app = App {
first_date: parsed_lines[0].time,
last_date: parsed_lines.last().unwrap().time,
lines: parsed_lines,
histogram,
log_statements: statements,
matches,
unmatched: unmatched_lines,
unmatched_histogram,
unmatched,
all,
error_count,
};

12
src/timegraph.rs
View file

@ -1,4 +1,3 @@
use crate::logline::LogLine;
use hdrhistogram::Histogram;
use time::OffsetDateTime;
@ -23,17 +22,6 @@ impl TimeGraph {
}
}
pub fn generate(lines: &[LogLine]) -> Self {
let min_time = lines[0].time;
let max_time = lines.last().unwrap().time;
let mut histogram = TimeGraph::new(min_time, max_time);
for line in lines {
histogram.add(line.time);
}
histogram
}
pub fn add(&mut self, time: OffsetDateTime) {
self.histogram
.record(time.unix_timestamp() as u64 - self.start + 1)

2
src/ui/footer.rs
View file

@ -19,7 +19,7 @@ pub fn footer(app: &App, page: UiPage) -> Table {
Table::new(
[Row::new([
help(page).to_string(),
format!("{} unmatched items", app.unmatched.len()),
format!("{} unmatched items", app.unmatched.lines.len()),
format!("{} parse errors", app.error_count),
])],
widths,

64
src/ui/match_list.rs
View file

@ -29,28 +29,16 @@ pub fn match_list(app: &App) -> Table {
Constraint::Min(10),
];
let all = Row::new([
Text::from("All lines"),
Text::from(String::new()),
Text::from(String::new()).alignment(Alignment::Right),
Text::from(sparkline(&app.histogram.counts(10))),
Text::from(app.lines.len().to_string()),
]);
let unmatched = if app.unmatched.is_empty() {
let all = log_row(&app.all, &app, "All lines");
let unmatched = if app.unmatched.lines.is_empty() {
Either::Right(empty())
} else {
Either::Left(once(Row::new([
Text::from("Unmatched lines"),
Text::from(String::new()),
Text::from(String::new()).alignment(Alignment::Right),
Text::from(sparkline(&app.unmatched_histogram.counts(10))),
Text::from(app.unmatched.len().to_string()),
])))
Either::Left(once(log_row(&app.unmatched, &app, "Unmatched lines")))
};
Table::new(
once(all)
.chain(app.matches.iter().map(|result| log_row(result, app)))
.chain(app.matches.iter().map(|result| log_row(result, app, "")))
.chain(unmatched),
widths,
)
@ -59,22 +47,32 @@ pub fn match_list(app: &App) -> Table {
.highlight_spacing(HighlightSpacing::Always)
}
fn log_row<'a>(result: &LogMatch, app: &'a App) -> Row<'a> {
let mut message = String::new();
let mut paths = String::new();
let mut lines = String::new();
for index in result.result.iter() {
let statement = app.log_statements.get(index).expect("invalid match index");
writeln!(&mut message, "{}", statement.message()).unwrap();
writeln!(&mut paths, "{}", statement.path()).unwrap();
writeln!(&mut lines, "{}", statement.line).unwrap();
fn log_row<'a>(result: &LogMatch, app: &'a App, name: &'static str) -> Row<'a> {
if let Some(match_result) = &result.result {
let mut message = String::new();
let mut paths = String::new();
let mut lines = String::new();
for index in match_result.iter() {
let statement = app.log_statements.get(index).expect("invalid match index");
writeln!(&mut message, "{}", statement.message()).unwrap();
writeln!(&mut paths, "{}", statement.path()).unwrap();
writeln!(&mut lines, "{}", statement.line).unwrap();
}
Row::new([
Text::from(message),
Text::from(paths),
Text::from(lines).alignment(Alignment::Right),
Text::from(sparkline(&result.histogram.counts(10))),
Text::from(result.count().to_string()),
])
.height(match_result.len() as u16)
} else {
Row::new([
Text::from(name),
Text::from(""),
Text::from(""),
Text::from(sparkline(&result.histogram.counts(10))),
Text::from(result.count().to_string()),
])
}
Row::new([
Text::from(message),
Text::from(paths),
Text::from(lines).alignment(Alignment::Right),
Text::from(sparkline(&result.histogram.counts(10))),
Text::from(result.count().to_string()),
])
.height(result.result.len() as u16)
}

24
src/ui/mod.rs
View file

@ -80,12 +80,12 @@ fn ui(frame: &mut Frame, app: &App, state: &mut UiState) {
UiState::MatchList { table_state } => {
let selected = table_state.selected().unwrap_or(0);
let histogram = if selected == 0 {
&app.histogram
&app.all.histogram
} else if selected < app.matches.len() + 1 {
let log_match = &app.matches[selected - 1];
&log_match.histogram
} else {
&app.unmatched_histogram
&app.unmatched.histogram
};
frame.render_widget(UiHistogram::new(histogram), layout[0]);
frame.render_stateful_widget(match_list(app), layout[1], table_state);
@ -96,19 +96,25 @@ fn ui(frame: &mut Frame, app: &App, state: &mut UiState) {
table_state,
} => {
let log_match = &app.matches[*index];
let lines = log_match.lines.iter().map(|i| &app.lines[*i]);
frame.render_widget(UiHistogram::new(&log_match.histogram), layout[0]);
frame.render_stateful_widget(grouped_lines(lines), layout[1], table_state);
let selected_group = &log_match.grouped[table_state.selected().unwrap_or_default()];
frame.render_widget(UiHistogram::new(&selected_group.histogram), layout[0]);
frame.render_stateful_widget(grouped_lines(app, log_match), layout[1], table_state);
frame.render_widget(footer(app, page), layout[2]);
}
UiState::All { table_state } => {
frame.render_stateful_widget(grouped_lines(app.lines.iter()), layout[1], table_state);
let selected_group = &app.all.grouped[table_state.selected().unwrap_or_default()];
frame.render_widget(UiHistogram::new(&selected_group.histogram), layout[0]);
frame.render_stateful_widget(grouped_lines(app, &app.all), layout[1], table_state);
frame.render_widget(footer(app, page), layout[2]);
}
UiState::Unmatched { table_state } => {
let lines = app.unmatched.iter().map(|i| &app.lines[*i]);
frame.render_stateful_widget(grouped_lines(lines), layout[1], table_state);
let selected_group = &app.unmatched.grouped[table_state.selected().unwrap_or_default()];
frame.render_widget(UiHistogram::new(&selected_group.histogram), layout[0]);
frame.render_stateful_widget(
grouped_lines(app, &app.unmatched),
layout[1],
table_state,
);
frame.render_widget(footer(app, page), layout[2]);
}
}

40
src/ui/single_match.rs
View file

@ -1,35 +1,47 @@
use crate::logline::LogLine;
use crate::ui::style::{TABLE_HEADER_STYLE, TABLE_SELECTED_STYLE, TIME_FORMAT};
use crate::app::{App, GroupedLines, LogMatch};
use crate::ui::histogram::sparkline;
use crate::ui::style::{TABLE_HEADER_STYLE, TABLE_SELECTED_STYLE};
use ratatui::layout::Constraint;
use ratatui::text::Text;
use ratatui::widgets::{Cell, HighlightSpacing, Row, Table};
use time::format_description::well_known::Iso8601;
pub fn grouped_lines<'a, I: Iterator<Item = &'a LogLine> + 'a>(lines: I) -> Table<'a> {
let header = ["Level", "App", "Message", "Date"]
.into_iter()
.map(Cell::from)
.collect::<Row>()
.style(TABLE_HEADER_STYLE)
.height(1);
pub fn grouped_lines(app: &App, log_match: &LogMatch) -> Table<'static> {
let grouped = &log_match.grouped;
let header = [
Text::from("Level"),
Text::from("App"),
Text::from("Message"),
Text::from("Time"),
Text::from("Count"),
]
.into_iter()
.map(Cell::from)
.collect::<Row>()
.style(TABLE_HEADER_STYLE)
.height(1);
let widths = [
Constraint::Min(10),
Constraint::Min(20),
Constraint::Percentage(100),
Constraint::Min(30),
Constraint::Length(10),
Constraint::Min(10),
];
let table = Table::new(lines.map(|line| log_row(line)), widths)
let table = Table::new(grouped.iter().map(|group| group_row(app, group)), widths)
.header(header)
.highlight_style(TABLE_SELECTED_STYLE)
.highlight_spacing(HighlightSpacing::Always);
table
}
fn log_row(line: &LogLine) -> Row {
fn group_row(app: &App, group: &GroupedLines) -> Row<'static> {
let line = &app.lines[group.lines[0]];
Row::new([
line.level.as_str().to_string(),
line.app.to_string(),
line.message.clone(),
line.time.format(&Iso8601::<TIME_FORMAT>).unwrap(),
sparkline(&group.histogram.counts(10)),
group.len().to_string(),
])
}

6
src/ui/state.rs
View file

@ -51,9 +51,9 @@ impl UiState {
fn table_count(&self, app: &App) -> usize {
match self {
UiState::MatchList { .. } => app.match_lines(),
UiState::Match { selected, .. } => app.matches[*selected].count(),
UiState::All { .. } => app.lines.len(),
UiState::Unmatched { .. } => app.unmatched.len(),
UiState::Match { selected, .. } => app.matches[*selected].grouped.len(),
UiState::All { .. } => app.all.grouped.len(),
UiState::Unmatched { .. } => app.unmatched.grouped.len(),
UiState::Quit => 0,
}
}