robin/logsmash
1
0
Fork 0
mirror of https://codeberg.org/icewind/logsmash.git synced 2026-06-03 10:04:12 +02:00
Code Issues Projects Releases 3 Packages Wiki Activity Actions

add app data

Browse source
This commit is contained in:
Robin Appelman 2024-07-23 18:49:05 +02:00
commit bc0bcf936c
10 changed files with 201 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

47
data/src/data.rs
View file

@ -1,21 +1,62 @@
use crate::{LoggingStatement, LoggingStatementWithPathPrefix};
// mod server_24;
// mod server_25;
// mod server_26;
// mod server_27;
// mod server_28;
mod files_accesscontrol_1_19;
mod files_antivirus_5_5;
mod server_29;
pub const MIN_VERSION: u32 = 24;
pub const MAX_VERSION: u32 = 29;
pub fn get_statements(name: &str, version: u32) -> &[crate::LoggingStatement] {
match (name, version) {
pub fn get_statements(version: u32) -> StatementList {
let server = match version {
// ("server", 24) => server_24::STATEMENTS,
// ("server", 25) => server_25::STATEMENTS,
// ("server", 26) => server_26::STATEMENTS,
// ("server", 27) => server_27::STATEMENTS,
// ("server", 28) => server_28::STATEMENTS,
("server", 29) => server_29::STATEMENTS,
29 => server_29::STATEMENTS,
_ => server_29::STATEMENTS,
};
StatementList::new(vec![
("", server),
(
"/apps/files_accesscontrol",
files_accesscontrol_1_19::STATEMENTS,
),
("/apps/files_antivirus", files_antivirus_5_5::STATEMENTS),
])
}
pub struct StatementList {
statements: Vec<(&'static str, &'static [LoggingStatement])>,
}
impl StatementList {
pub fn new(statements: Vec<(&'static str, &'static [LoggingStatement])>) -> StatementList {
StatementList { statements }
}
pub fn iter(&self) -> impl Iterator<Item = &'static LoggingStatement> + '_ {
self.statements
.iter()
.copied()
.flat_map(|(_, list)| list.iter())
}
pub fn get(&self, mut index: usize) -> Option<LoggingStatementWithPathPrefix> {
for (prefix, list) in &self.statements {
if index < list.len() {
return list
.get(index)
.map(|statement| statement.with_path_prefix(prefix));
}
index -= list.len()
}
None
}
}

7
data/src/data/files_accesscontrol_1_19.rs Normal file
View file

@ -0,0 +1,7 @@
pub const STATEMENTS: &[crate::LoggingStatement] = &[
crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/Operation.php" , line : 104usize , placeholders : & ["$e->getMessage()"] , exception : None , regex : "^(.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Operation.php" , line : 106usize , placeholders : & [] , exception : Some ("OCP\\Files\\ForbiddenException") , regex : "^Access denied$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Operation.php" , line : 203usize , placeholders : & ["$this->l->t('No rule given')"] , exception : Some ("UnexpectedValueException") , regex : "^(.*)$" , },
];

45
data/src/data/files_antivirus_5_5.rs Normal file
View file

@ -0,0 +1,45 @@
pub const STATEMENTS: &[crate::LoggingStatement] = &[
crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 77usize , placeholders : & [] , exception : None , regex : "^Antivirus background scan disabled, skipping$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 90usize , placeholders : & ["$e->getMessage()"] , exception : None , regex : "^(.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 95usize , placeholders : & [] , exception : None , regex : "^Start background scan$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 109usize , placeholders : & ["$e->getMessage()"] , exception : None , regex : "^(.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 125usize , placeholders : & ["$e->getMessage()"] , exception : None , regex : "^(.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Info , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 149usize , placeholders : & [] , exception : None , regex : "^Tried to scan non file$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 153usize , placeholders : & ["__METHOD__" , "$e->getMessage()"] , exception : None , regex : "^(.*), exception: (.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 180usize , placeholders : & ["$batchSize"] , exception : None , regex : "^Batch size is: (.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 288usize , placeholders : & ["$file->getId()"] , exception : None , regex : "^Scanning file with fileid: (.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Db/ItemMapper.php" , line : 40usize , placeholders : & [] , exception : Some ("\\InvalidArgumentException") , regex : "^$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/Status.php" , line : 86usize , placeholders : & ["__METHOD__" , "$e->getMessage()"] , exception : None , regex : "^(.*), exception: (.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/AppConfig.php" , line : 173usize , placeholders : & ["$key"] , exception : Some ("\\BadFunctionCallException") , regex : "^(.*) is not a valid key$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/AppConfig.php" , line : 189usize , placeholders : & ["$key"] , exception : Some ("\\BadFunctionCallException") , regex : "^(.*) is not a valid key$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/AppConfig.php" , line : 242usize , placeholders : & ["$methodName"] , exception : Some ("\\BadFunctionCallException") , regex : "^(.*) does not exist$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ExternalKaspersky.php" , line : 58usize , placeholders : & [] , exception : Some ("\\RuntimeException") , regex : "^The Kaspersky port and host are not set up\\.$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/Scanner/ExternalKaspersky.php" , line : 90usize , placeholders : & ["$response"] , exception : None , regex : "^Response :: (.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ICAP.php" , line : 63usize , placeholders : & [] , exception : Some ("\\RuntimeException") , regex : "^The ICAP port and host are not set up\\.$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ICAP.php" , line : 139usize , placeholders : & [] , exception : Some ("\\RuntimeException") , regex : "^Invalid response from ICAP server$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/LocalClam.php" , line : 41usize , placeholders : & ["$this->avPath"] , exception : Some ("\\RuntimeException") , regex : "^The antivirus executable could not be found at (.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/LocalClam.php" , line : 60usize , placeholders : & [] , exception : Some ("\\RuntimeException") , regex : "^Error starting process$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/Scanner/LocalClam.php" , line : 74usize , placeholders : & ["$result" , "$output"] , exception : None , regex : "^Exit code :: (.*) Response :: (.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/Scanner/ScannerBase.php" , line : 178usize , placeholders : & [] , exception : None , regex : "^reinit scanner$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Warn , path : "/lib/Scanner/ScannerBase.php" , line : 191usize , placeholders : & [] , exception : None , regex : "^Failed to write a chunk\\. Check if Stream Length matches StreamMaxLength in anti virus daemon settings$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ScannerFactory.php" , line : 52usize , placeholders : & ["$avMode"] , exception : Some ("\\InvalidArgumentException") , regex : "^Application is misconfigured\\. Please check the settings at the admin page\\. Invalid mode: (.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ExternalClam.php" , line : 41usize , placeholders : & ["$avSocket" , "$errstr" , "$errno"] , exception : Some ("\\RuntimeException") , regex : "^Cannot connect to \"(.*)\": (.*) \\(code (.*)\\)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ExternalClam.php" , line : 47usize , placeholders : & [] , exception : Some ("\\RuntimeException") , regex : "^The ClamAV port and host are not set up\\.$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ExternalClam.php" , line : 51usize , placeholders : & ["$avHost" , "$avPort"] , exception : Some ("\\RuntimeException") , regex : "^Could not connect to ClamAV via (.*):(.*)\\. Please check that ClamAV is running and reachable\\.$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/Scanner/ExternalClam.php" , line : 65usize , placeholders : & ["$response"] , exception : None , regex : "^Response :: (.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/AvirWrapper.php" , line : 114usize , placeholders : & ["$e->getMessage()"] , exception : None , regex : "^(.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Warn , path : "/lib/AvirWrapper.php" , line : 173usize , placeholders : & ["$status->getDetails()" , "$owner" , "$path"] , exception : None , regex : "^Infected file deleted\\. (.*) Account: (.*) Path: (.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/AvirWrapper.php" , line : 188usize , placeholders : & ["$status->getDetails()" , "$path" , "$owner"] , exception : None , regex : "^Infected file deleted\\. (.*) File: (.*) Account: (.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/AvirWrapper.php" , line : 191usize , placeholders : & ["$this->l10n->t(\n\t\t\t\t'Virus %s is detected in the file. Upload cannot be completed.',\n\t\t\t\t$status->getDetails()\n\t\t\t)"] , exception : Some ("OCP\\Files\\InvalidContentException") , regex : "^(.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/ICAP/ResponseParser.php" , line : 58usize , placeholders : & [] , exception : Some ("RuntimeException") , regex : "^Empty ICAP response$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/ICAP/ResponseParser.php" , line : 63usize , placeholders : & ["$icapHeader"] , exception : Some ("RuntimeException") , regex : "^Unknown ICAP response: \"(.*)\"$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/ICAP/ICAPClient.php" , line : 73usize , placeholders : & ["$this->host" , "$this->port" , "$errorMessage" , "$errorCode"] , exception : Some ("RuntimeException") , regex : "^Cannot connect to \"tcp://(.*):(.*)\": (.*) \\(code (.*)\\)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/ICAP/ICAPTlsClient.php" , line : 63usize , placeholders : & ["$this->host" , "$this->port" , "$errorMessage" , "$errorCode"] , exception : Some ("RuntimeException") , regex : "^Cannot connect to \"tls://(.*):(.*)\": (.*) \\(code (.*)\\)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Activity/Provider.php" , line : 53usize , placeholders : & [] , exception : Some ("\\InvalidArgumentException") , regex : "^$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/Item.php" , line : 160usize , placeholders : & ["__METHOD__" , "$e->getMessage()"] , exception : None , regex : "^(.*), exception: (.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Item.php" , line : 200usize , placeholders : & [] , exception : Some ("\\RuntimeException") , regex : "^$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/Item.php" , line : 245usize , placeholders : & ["$message" , "$this->generateExtraInfo()"] , exception : None , regex : "^(.*)(.*)$" , },
crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/Item.php" , line : 252usize , placeholders : & ["$message" , "$this->generateExtraInfo()"] , exception : None , regex : "^(.*)(.*)$" , },
];

2
data/src/lib.rs
View file

@ -1,5 +1,5 @@
mod data;
mod types;
pub use data::{get_statements, MAX_VERSION, MIN_VERSION};
pub use data::*;
pub use types::*;

92
data/src/types.rs
View file

@ -46,34 +46,88 @@ pub struct LoggingStatement {
pub regex: &'static str,
}
impl Display for LoggingStatement {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
if let Some(exception) = self.exception {
write!(
f,
"{}({}): {} line {}",
exception,
self.message(),
self.path,
self.line
)
} else {
write!(f, "{}: {} line {}", self.message(), self.path, self.line)
impl LoggingStatement {
pub fn with_path_prefix(&self, path_prefix: &'static str) -> LoggingStatementWithPathPrefix {
LoggingStatementWithPathPrefix {
level: self.level,
path_prefix,
path: self.path,
line: self.line,
placeholders: self.placeholders,
exception: self.exception,
regex: self.regex,
}
}
pub fn message(&self) -> impl Display + '_ {
LoggingMessage {
message: self.clone(),
}
}
}
impl LoggingStatement {
pub fn message(&self) -> impl Display + '_ {
LoggingMessage { message: &self }
#[derive(Debug, PartialEq, Clone)]
pub struct LoggingStatementWithPathPrefix {
pub level: LogLevel,
pub path_prefix: &'static str,
pub path: &'static str,
pub line: usize,
pub placeholders: &'static [&'static str],
pub exception: Option<&'static str>,
pub regex: &'static str,
}
impl From<&LoggingStatementWithPathPrefix> for LoggingStatement {
fn from(value: &LoggingStatementWithPathPrefix) -> Self {
LoggingStatement {
level: value.level,
path: value.path,
line: value.line,
placeholders: value.placeholders,
exception: value.exception,
regex: value.regex,
}
}
}
struct LoggingMessage<'a> {
message: &'a LoggingStatement,
impl Display for LoggingStatementWithPathPrefix {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
if let Some(exception) = self.exception {
write!(
f,
"{}({}): {}{} line {}",
exception,
self.message(),
self.path_prefix,
self.path,
self.line
)
} else {
write!(
f,
"{}: {}{} line {}",
self.message(),
self.path_prefix,
self.path,
self.line
)
}
}
}
impl<'a> Display for LoggingMessage<'a> {
impl LoggingStatementWithPathPrefix {
pub fn message(&self) -> impl Display + '_ {
LoggingMessage {
message: self.into(),
}
}
}
struct LoggingMessage {
message: LoggingStatement,
}
impl Display for LoggingMessage {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
if self.message.regex.is_empty() {
return Ok(());

5
nix/extracted-logs.nix
View file

@ -10,8 +10,9 @@
mode ? "json",
}: let
ext = if mode == "rust" then "rs" else "json";
cleanedMajor = builtins.replaceStrings ["."] ["_"] major;
in stdenvNoCC.mkDerivation rec {
pname = "extractor-logs-${name}-${major}";
pname = "extractor-logs-${name}-${cleanedMajor}";
inherit version;
src = fetchzip {
@ -26,6 +27,6 @@ in stdenvNoCC.mkDerivation rec {
installPhase = ''
mkdir -p $out
cp logs.* $out/${name}_${major}.${ext}
cp logs.* $out/${name}_${cleanedMajor}.${ext}
'';
}

5
nix/overlay.nix
View file

@ -4,7 +4,10 @@ final: prev: let
inherit (prev.lib) importJSON;
inherit (prev.lib.lists) flatten;
allPackages = importJSON ./versions.json;
packages = {server = {"29" = allPackages.server."29";};};
packages = {
server = {"29" = allPackages.server."29";};
inherit (allPackages) files_accesscontrol files_antivirus;
};
loggingFor = mode: name:
mapAttrs (major: data: (final.callPackage ./extracted-logs.nix {

14
nix/versions.json
View file

@ -30,5 +30,19 @@
"url": "https://download.nextcloud.com/server/releases/nextcloud-24.0.12.zip",
"sha256": "sha256-+VlQVSTXtEsuoJ3Ic9o4DmFH4CQhMK8KtwiS+SqhiYE="
}
},
"files_accesscontrol": {
"1.19": {
"version": "1.19.1",
"url": "https://github.com/nextcloud-releases/files_accesscontrol/releases/download/v1.19.1/files_accesscontrol-v1.19.1.tar.gz",
"sha256": "sha256-e9387k5yn3lAZ7uN0YU8zNVkX8sCoNxyc5M3E3azo80="
}
},
"files_antivirus": {
"5.5": {
"version": "5.5.7",
"url": "https://github.com/nextcloud-releases/files_antivirus/releases/download/v5.5.7/files_antivirus-v5.5.7.tar.gz",
"sha256": "sha256-Sg6AeqopaXrXrZ6loq+B5ZBjh2SHIflXaf6u/cIBqUo="
}
}
}

9
src/main.rs
View file

@ -32,11 +32,8 @@ fn main() -> MainResult {
let first = lines.next().unwrap();
let first_parsed: LogLine = serde_json::from_str(&first).unwrap();
let statements = get_statements(
"server",
first_parsed.major_version().unwrap_or(MAX_VERSION),
);
let matcher = Matcher::new(statements);
let statements = get_statements(first_parsed.major_version().unwrap_or(MAX_VERSION));
let matcher = Matcher::new(&statements);
let lines = once(first).chain(lines);
let mut error_count = 0;
@ -67,7 +64,7 @@ fn main() -> MainResult {
counts.sort_by_key(|(_, count)| *count);
counts.reverse();
for (match_result, count) in counts {
println!("{}: {}", match_result.display(statements), count);
println!("{}: {}", match_result.display(&statements), count);
}
if unmatched_total > 0 {
eprintln!("\n{unmatched_total} lines couldn't be matched:");

30
src/matcher.rs
View file

@ -1,5 +1,5 @@
use crate::logline::LogLine;
use cloud_log_analyser_data::{LogLevel, LoggingStatement};
use cloud_log_analyser_data::{LogLevel, LoggingStatement, StatementList};
use regex::Regex;
use std::fmt::{Display, Formatter};
@ -34,7 +34,7 @@ pub struct Matcher {
}
impl Matcher {
pub fn new(statements: &[LoggingStatement]) -> Matcher {
pub fn new(statements: &StatementList) -> Matcher {
Matcher {
matches: statements.iter().map(LogMatch::new).collect(),
}
@ -90,7 +90,7 @@ pub enum MatchResult {
}
impl MatchResult {
pub fn display<'a>(&'a self, log_statements: &'a [LoggingStatement]) -> impl Display + 'a {
pub fn display<'a>(&'a self, log_statements: &'a StatementList) -> impl Display + 'a {
MatchResultDisplay {
log_statements,
result: &self,
@ -99,7 +99,7 @@ impl MatchResult {
}
struct MatchResultDisplay<'a> {
log_statements: &'a [LoggingStatement],
log_statements: &'a StatementList,
result: &'a MatchResult,
}
@ -130,7 +130,7 @@ impl Display for MatchResultDisplay<'_> {
fn test_matcher() {
use crate::logline::Exception;
let statements = &[
const STATEMENTS: &[LoggingStatement] = &[
LoggingStatement {
line: 68,
level: LogLevel::Exception,
@ -169,32 +169,35 @@ fn test_matcher() {
path: "short",
placeholders: &["$path"],
regex: "^Not allowed to rename (.*)$",
exception: "Bar\\FooException".into(),
exception: Some("Bar\\FooException"),
},
];
let matcher = Matcher::new(statements);
let matcher = Matcher::new(&StatementList::new(vec![("", STATEMENTS)]));
assert_eq!(
Some(0),
Some(MatchResult::Single(0)),
matcher.match_log(&LogLine {
version: "29",
app: "core".into(),
level: LogLevel::Error,
message: "Not allowed to rename a shared album".into(),
exception: None,
})
);
assert_eq!(
Some(3),
Some(MatchResult::List(vec![3, 4])),
matcher.match_log(&LogLine {
version: "29",
app: "core".into(),
level: LogLevel::Error,
message: "Not allowed to rename an album".into(),
exception: None,
})
);
assert_eq!(
Some(1),
Some(MatchResult::Single(1)),
matcher.match_log(&LogLine {
version: "29",
app: "core".into(),
level: LogLevel::Error,
message: "You are not allowed to edit link shares that you don't own".into(),
exception: None,
@ -204,16 +207,18 @@ fn test_matcher() {
None,
matcher.match_log(&LogLine {
version: "29",
app: "core".into(),
level: LogLevel::Info,
message: "You are not allowed to edit link shares that you don't own".into(),
exception: None,
})
);
assert_eq!(
Some(2),
Some(MatchResult::Single(2)),
matcher.match_log(
&LogLine {
version: "29",
app: "core".into(),
level: LogLevel::Error,
message: "Unsupported query value for mimetype: %/text, only values in the format \"mime/type\" or \"mime/%\" are supported".into(),
exception: None,
@ -221,10 +226,11 @@ fn test_matcher() {
)
);
assert_eq!(
Some(4),
Some(MatchResult::Single(4)),
matcher.match_log(
&LogLine {
version: "29",
app: "core".into(),
level: LogLevel::Error,
message: "Unsupported query value for mimetype: %/text, only values in the format \"mime/type\" or \"mime/%\" are supported".into(),
exception: Some(Exception {