grouped lines

Cargo.lock

@@ -26,6 +26,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
  "cfg-if",
+ "getrandom",
  "once_cell",
  "version_check",
  "zerocopy",
@@ -203,6 +204,7 @@ dependencies = [
 name = "cloud-log-analyser"
 version = "0.1.0"
 dependencies = [
+ "ahash",
  "clap",
  "cloud-log-analyser-data",
  "hdrhistogram",

Cargo.toml

@@ -19,6 +19,7 @@ ratatui = "0.27.0"
 tinystr = { version = "0.7.6", features = ["serde"] }
 time = { version = "0.3.36", features = ["serde", "serde-well-known"] }
 hdrhistogram = "7.5.4"
+ahash = "0.8.11"
 
 [profile.dev.package."*"]
 opt-level = 3

data/src/types.rs

@@ -1,7 +1,7 @@
 use serde::Deserialize;
 use std::fmt::{Display, Formatter};
 
-#[derive(Debug, Default, PartialEq, Clone, Copy, Deserialize)]
+#[derive(Debug, Default, PartialEq, Clone, Copy, Deserialize, Hash)]
 #[serde(from = "i64")]
 pub enum LogLevel {
     Debug,

src/app.rs

@@ -2,43 +2,53 @@ use crate::logline::LogLine;
 use crate::matcher::MatchResult;
 use crate::timegraph::TimeGraph;
 use cloud_log_analyser_data::StatementList;
+use std::collections::BTreeMap;
+use time::OffsetDateTime;
 
 pub struct App {
+    pub first_date: OffsetDateTime,
+    pub last_date: OffsetDateTime,
     pub lines: Vec<LogLine>,
-    pub histogram: TimeGraph,
     pub log_statements: StatementList,
     pub matches: Vec<LogMatch>,
     pub error_count: usize,
-    pub unmatched: Vec<usize>,
+    pub all: LogMatch,
-    pub unmatched_histogram: TimeGraph,
+    pub unmatched: LogMatch,
 }
 
 impl App {
     pub fn match_lines(&self) -> usize {
-        let unmatched_line_count = if self.unmatched.is_empty() { 0 } else { 1 };
+        let unmatched_line_count = if self.unmatched.lines.is_empty() {
+            0
+        } else {
+            1
+        };
         self.matches.len() + 1 + unmatched_line_count
     }
 }
 
 pub struct LogMatch {
-    pub result: MatchResult,
+    pub result: Option<MatchResult>,
     pub lines: Vec<usize>,
     pub histogram: TimeGraph,
+    pub grouped: Vec<GroupedLines>,
 }
 
 impl LogMatch {
-    pub fn new(result: MatchResult, lines: Vec<usize>, all_lines: &[LogLine]) -> Self {
+    pub fn new(result: Option<MatchResult>, lines: Vec<usize>, all_lines: &[LogLine]) -> Self {
         let min_time = all_lines[0].time;
         let max_time = all_lines.last().unwrap().time;
         let mut histogram = TimeGraph::new(min_time, max_time);
         for line in lines.iter().map(|line| &all_lines[*line]) {
             histogram.add(line.time);
         }
+        let grouped = group_lines(all_lines, lines.iter().copied());
 
         LogMatch {
             result,
             lines,
             histogram,
+            grouped,
         }
     }
 }
@@ -48,3 +58,40 @@ impl LogMatch {
         self.lines.len()
     }
 }
+
+fn group_lines<I: Iterator<Item = usize>>(all_lines: &[LogLine], indices: I) -> Vec<GroupedLines> {
+    let mut map: BTreeMap<u64, Vec<usize>> = BTreeMap::new();
+
+    for (i, line) in indices.map(|i| (i, &all_lines[i])) {
+        map.entry(line.index()).or_default().push(i);
+    }
+
+    let mut list: Vec<_> = map
+        .into_values()
+        .map(|lines| GroupedLines::new(lines, all_lines))
+        .collect();
+    list.sort_by_key(|list| list.len());
+    list.reverse();
+    list
+}
+
+pub struct GroupedLines {
+    pub lines: Vec<usize>,
+    pub histogram: TimeGraph,
+}
+
+impl GroupedLines {
+    pub fn new(lines: Vec<usize>, all_lines: &[LogLine]) -> Self {
+        let min_time = all_lines[0].time;
+        let max_time = all_lines.last().unwrap().time;
+        let mut histogram = TimeGraph::new(min_time, max_time);
+        for line in lines.iter().map(|line| &all_lines[*line]) {
+            histogram.add(line.time);
+        }
+        GroupedLines { lines, histogram }
+    }
+
+    pub fn len(&self) -> usize {
+        self.lines.len()
+    }
+}

src/logline.rs

@@ -1,5 +1,7 @@
+use ahash::AHasher;
 use cloud_log_analyser_data::LogLevel;
 use serde::Deserialize;
+use std::hash::{Hash, Hasher};
 use time::OffsetDateTime;
 use tinystr::TinyAsciiStr;
 
@@ -23,6 +25,18 @@ impl LogLine {
             .unwrap_or(self.version.as_str());
         major.parse().ok()
     }
+
+    pub fn index(&self) -> u64 {
+        let mut hasher = AHasher::default();
+        self.message.hash(&mut hasher);
+        self.level.hash(&mut hasher);
+        self.exception
+            .as_ref()
+            .map(|e| e.exception.as_str())
+            .hash(&mut hasher);
+        self.app.hash(&mut hasher);
+        hasher.finish()
+    }
 }
 
 #[derive(Deserialize, Debug)]

src/main.rs

@@ -3,7 +3,6 @@ use crate::error::LogError;
 use crate::logfile::LogFile;
 use crate::logline::LogLine;
 use crate::matcher::{MatchResult, Matcher};
-use crate::timegraph::TimeGraph;
 use crate::ui::run_ui;
 use clap::Parser;
 use cloud_log_analyser_data::{get_statements, MAX_VERSION};
@@ -84,27 +83,26 @@ fn main() -> MainResult {
     matched_lines.sort_by_key(|(_, lines)| lines.len());
     matched_lines.reverse();
 
-    let histogram = TimeGraph::generate(&parsed_lines);
+    let all = LogMatch::new(
+        None,
+        parsed_lines.iter().enumerate().map(|(i, _)| i).collect(),
+        &parsed_lines,
+    );
+    let unmatched = LogMatch::new(None, unmatched_lines, &parsed_lines);
 
     let matches = matched_lines
         .into_iter()
-        .map(|(result, lines)| LogMatch::new(result, lines, &parsed_lines))
+        .map(|(result, lines)| LogMatch::new(Some(result), lines, &parsed_lines))
         .collect();
 
-    let min_time = parsed_lines[0].time;
-    let max_time = parsed_lines.last().unwrap().time;
-    let mut unmatched_histogram = TimeGraph::new(min_time, max_time);
-    for lines in unmatched_lines.iter().map(|line| &parsed_lines[*line]) {
-        unmatched_histogram.add(lines.time);
-    }
-
     let app = App {
+        first_date: parsed_lines[0].time,
+        last_date: parsed_lines.last().unwrap().time,
         lines: parsed_lines,
-        histogram,
         log_statements: statements,
         matches,
-        unmatched: unmatched_lines,
+        unmatched,
-        unmatched_histogram,
+        all,
         error_count,
     };
 

src/timegraph.rs

@@ -1,4 +1,3 @@
-use crate::logline::LogLine;
 use hdrhistogram::Histogram;
 use time::OffsetDateTime;
 
@@ -23,17 +22,6 @@ impl TimeGraph {
         }
     }
 
-    pub fn generate(lines: &[LogLine]) -> Self {
-        let min_time = lines[0].time;
-        let max_time = lines.last().unwrap().time;
-        let mut histogram = TimeGraph::new(min_time, max_time);
-
-        for line in lines {
-            histogram.add(line.time);
-        }
-        histogram
-    }
-
     pub fn add(&mut self, time: OffsetDateTime) {
         self.histogram
             .record(time.unix_timestamp() as u64 - self.start + 1)

src/ui/footer.rs

@@ -19,7 +19,7 @@ pub fn footer(app: &App, page: UiPage) -> Table {
     Table::new(
         [Row::new([
             help(page).to_string(),
-            format!("{} unmatched items", app.unmatched.len()),
+            format!("{} unmatched items", app.unmatched.lines.len()),
             format!("{} parse errors", app.error_count),
         ])],
         widths,

src/ui/match_list.rs

@@ -29,28 +29,16 @@ pub fn match_list(app: &App) -> Table {
         Constraint::Min(10),
     ];
 
-    let all = Row::new([
+    let all = log_row(&app.all, &app, "All lines");
-        Text::from("All lines"),
+    let unmatched = if app.unmatched.lines.is_empty() {
-        Text::from(String::new()),
-        Text::from(String::new()).alignment(Alignment::Right),
-        Text::from(sparkline(&app.histogram.counts(10))),
-        Text::from(app.lines.len().to_string()),
-    ]);
-    let unmatched = if app.unmatched.is_empty() {
         Either::Right(empty())
     } else {
-        Either::Left(once(Row::new([
+        Either::Left(once(log_row(&app.unmatched, &app, "Unmatched lines")))
-            Text::from("Unmatched lines"),
-            Text::from(String::new()),
-            Text::from(String::new()).alignment(Alignment::Right),
-            Text::from(sparkline(&app.unmatched_histogram.counts(10))),
-            Text::from(app.unmatched.len().to_string()),
-        ])))
     };
 
     Table::new(
         once(all)
-            .chain(app.matches.iter().map(|result| log_row(result, app)))
+            .chain(app.matches.iter().map(|result| log_row(result, app, "")))
             .chain(unmatched),
         widths,
     )
@@ -59,11 +47,12 @@ pub fn match_list(app: &App) -> Table {
     .highlight_spacing(HighlightSpacing::Always)
 }
 
-fn log_row<'a>(result: &LogMatch, app: &'a App) -> Row<'a> {
+fn log_row<'a>(result: &LogMatch, app: &'a App, name: &'static str) -> Row<'a> {
+    if let Some(match_result) = &result.result {
         let mut message = String::new();
         let mut paths = String::new();
         let mut lines = String::new();
-    for index in result.result.iter() {
+        for index in match_result.iter() {
             let statement = app.log_statements.get(index).expect("invalid match index");
             writeln!(&mut message, "{}", statement.message()).unwrap();
             writeln!(&mut paths, "{}", statement.path()).unwrap();
@@ -76,5 +65,14 @@ fn log_row<'a>(result: &LogMatch, app: &'a App) -> Row<'a> {
             Text::from(sparkline(&result.histogram.counts(10))),
             Text::from(result.count().to_string()),
         ])
-    .height(result.result.len() as u16)
+        .height(match_result.len() as u16)
+    } else {
+        Row::new([
+            Text::from(name),
+            Text::from(""),
+            Text::from(""),
+            Text::from(sparkline(&result.histogram.counts(10))),
+            Text::from(result.count().to_string()),
+        ])
+    }
 }

src/ui/mod.rs

@@ -80,12 +80,12 @@ fn ui(frame: &mut Frame, app: &App, state: &mut UiState) {
         UiState::MatchList { table_state } => {
             let selected = table_state.selected().unwrap_or(0);
             let histogram = if selected == 0 {
-                &app.histogram
+                &app.all.histogram
             } else if selected < app.matches.len() + 1 {
                 let log_match = &app.matches[selected - 1];
                 &log_match.histogram
             } else {
-                &app.unmatched_histogram
+                &app.unmatched.histogram
             };
             frame.render_widget(UiHistogram::new(histogram), layout[0]);
             frame.render_stateful_widget(match_list(app), layout[1], table_state);
@@ -96,19 +96,25 @@ fn ui(frame: &mut Frame, app: &App, state: &mut UiState) {
             table_state,
         } => {
             let log_match = &app.matches[*index];
-            let lines = log_match.lines.iter().map(|i| &app.lines[*i]);
+            let selected_group = &log_match.grouped[table_state.selected().unwrap_or_default()];
-
+            frame.render_widget(UiHistogram::new(&selected_group.histogram), layout[0]);
-            frame.render_widget(UiHistogram::new(&log_match.histogram), layout[0]);
+            frame.render_stateful_widget(grouped_lines(app, log_match), layout[1], table_state);
-            frame.render_stateful_widget(grouped_lines(lines), layout[1], table_state);
             frame.render_widget(footer(app, page), layout[2]);
         }
         UiState::All { table_state } => {
-            frame.render_stateful_widget(grouped_lines(app.lines.iter()), layout[1], table_state);
+            let selected_group = &app.all.grouped[table_state.selected().unwrap_or_default()];
+            frame.render_widget(UiHistogram::new(&selected_group.histogram), layout[0]);
+            frame.render_stateful_widget(grouped_lines(app, &app.all), layout[1], table_state);
             frame.render_widget(footer(app, page), layout[2]);
         }
         UiState::Unmatched { table_state } => {
-            let lines = app.unmatched.iter().map(|i| &app.lines[*i]);
+            let selected_group = &app.unmatched.grouped[table_state.selected().unwrap_or_default()];
-            frame.render_stateful_widget(grouped_lines(lines), layout[1], table_state);
+            frame.render_widget(UiHistogram::new(&selected_group.histogram), layout[0]);
+            frame.render_stateful_widget(
+                grouped_lines(app, &app.unmatched),
+                layout[1],
+                table_state,
+            );
             frame.render_widget(footer(app, page), layout[2]);
         }
     }

src/ui/single_match.rs

@@ -1,11 +1,19 @@
-use crate::logline::LogLine;
+use crate::app::{App, GroupedLines, LogMatch};
-use crate::ui::style::{TABLE_HEADER_STYLE, TABLE_SELECTED_STYLE, TIME_FORMAT};
+use crate::ui::histogram::sparkline;
+use crate::ui::style::{TABLE_HEADER_STYLE, TABLE_SELECTED_STYLE};
 use ratatui::layout::Constraint;
+use ratatui::text::Text;
 use ratatui::widgets::{Cell, HighlightSpacing, Row, Table};
-use time::format_description::well_known::Iso8601;
 
-pub fn grouped_lines<'a, I: Iterator<Item = &'a LogLine> + 'a>(lines: I) -> Table<'a> {
+pub fn grouped_lines(app: &App, log_match: &LogMatch) -> Table<'static> {
-    let header = ["Level", "App", "Message", "Date"]
+    let grouped = &log_match.grouped;
+    let header = [
+        Text::from("Level"),
+        Text::from("App"),
+        Text::from("Message"),
+        Text::from("Time"),
+        Text::from("Count"),
+    ]
     .into_iter()
     .map(Cell::from)
     .collect::<Row>()
@@ -16,20 +24,24 @@ pub fn grouped_lines<'a, I: Iterator<Item = &'a LogLine> + 'a>(lines: I) -> Tabl
         Constraint::Min(10),
         Constraint::Min(20),
         Constraint::Percentage(100),
-        Constraint::Min(30),
+        Constraint::Length(10),
+        Constraint::Min(10),
     ];
-    let table = Table::new(lines.map(|line| log_row(line)), widths)
+    let table = Table::new(grouped.iter().map(|group| group_row(app, group)), widths)
         .header(header)
         .highlight_style(TABLE_SELECTED_STYLE)
         .highlight_spacing(HighlightSpacing::Always);
     table
 }
 
-fn log_row(line: &LogLine) -> Row {
+fn group_row(app: &App, group: &GroupedLines) -> Row<'static> {
+    let line = &app.lines[group.lines[0]];
+
     Row::new([
         line.level.as_str().to_string(),
         line.app.to_string(),
         line.message.clone(),
-        line.time.format(&Iso8601::<TIME_FORMAT>).unwrap(),
+        sparkline(&group.histogram.counts(10)),
+        group.len().to_string(),
     ])
 }

src/ui/state.rs

@@ -51,9 +51,9 @@ impl UiState {
     fn table_count(&self, app: &App) -> usize {
         match self {
             UiState::MatchList { .. } => app.match_lines(),
-            UiState::Match { selected, .. } => app.matches[*selected].count(),
+            UiState::Match { selected, .. } => app.matches[*selected].grouped.len(),
-            UiState::All { .. } => app.lines.len(),
+            UiState::All { .. } => app.all.grouped.len(),
-            UiState::Unmatched { .. } => app.unmatched.len(),
+            UiState::Unmatched { .. } => app.unmatched.grouped.len(),
             UiState::Quit => 0,
         }
     }