add app data

data/src/data.rs

@@ -1,21 +1,62 @@
+use crate::{LoggingStatement, LoggingStatementWithPathPrefix};
+
 // mod server_24;
 // mod server_25;
 // mod server_26;
 // mod server_27;
 // mod server_28;
+mod files_accesscontrol_1_19;
+mod files_antivirus_5_5;
 mod server_29;
 
 pub const MIN_VERSION: u32 = 24;
 pub const MAX_VERSION: u32 = 29;
 
-pub fn get_statements(name: &str, version: u32) -> &[crate::LoggingStatement] {
+pub fn get_statements(version: u32) -> StatementList {
-    match (name, version) {
+    let server = match version {
         // ("server", 24) => server_24::STATEMENTS,
         // ("server", 25) => server_25::STATEMENTS,
         // ("server", 26) => server_26::STATEMENTS,
         // ("server", 27) => server_27::STATEMENTS,
         // ("server", 28) => server_28::STATEMENTS,
-        ("server", 29) => server_29::STATEMENTS,
+        29 => server_29::STATEMENTS,
         _ => server_29::STATEMENTS,
+    };
+    StatementList::new(vec![
+        ("", server),
+        (
+            "/apps/files_accesscontrol",
+            files_accesscontrol_1_19::STATEMENTS,
+        ),
+        ("/apps/files_antivirus", files_antivirus_5_5::STATEMENTS),
+    ])
+}
+
+pub struct StatementList {
+    statements: Vec<(&'static str, &'static [LoggingStatement])>,
+}
+
+impl StatementList {
+    pub fn new(statements: Vec<(&'static str, &'static [LoggingStatement])>) -> StatementList {
+        StatementList { statements }
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &'static LoggingStatement> + '_ {
+        self.statements
+            .iter()
+            .copied()
+            .flat_map(|(_, list)| list.iter())
+    }
+
+    pub fn get(&self, mut index: usize) -> Option<LoggingStatementWithPathPrefix> {
+        for (prefix, list) in &self.statements {
+            if index < list.len() {
+                return list
+                    .get(index)
+                    .map(|statement| statement.with_path_prefix(prefix));
+            }
+            index -= list.len()
+        }
+        None
     }
 }

data/src/data/files_accesscontrol_1_19.rs

@@ -0,0 +1,7 @@
+pub const STATEMENTS: &[crate::LoggingStatement] = &[
+
+	crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/Operation.php" , line : 104usize , placeholders : & ["$e->getMessage()"] , exception : None , regex : "^(.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Operation.php" , line : 106usize , placeholders : & [] , exception : Some ("OCP\\Files\\ForbiddenException") , regex : "^Access denied$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Operation.php" , line : 203usize , placeholders : & ["$this->l->t('No rule given')"] , exception : Some ("UnexpectedValueException") , regex : "^(.*)$" , },
+];
+

data/src/data/files_antivirus_5_5.rs

@@ -0,0 +1,45 @@
+pub const STATEMENTS: &[crate::LoggingStatement] = &[
+
+	crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 77usize , placeholders : & [] , exception : None , regex : "^Antivirus background scan disabled, skipping$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 90usize , placeholders : & ["$e->getMessage()"] , exception : None , regex : "^(.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 95usize , placeholders : & [] , exception : None , regex : "^Start background scan$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 109usize , placeholders : & ["$e->getMessage()"] , exception : None , regex : "^(.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 125usize , placeholders : & ["$e->getMessage()"] , exception : None , regex : "^(.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Info , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 149usize , placeholders : & [] , exception : None , regex : "^Tried to scan non file$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 153usize , placeholders : & ["__METHOD__" , "$e->getMessage()"] , exception : None , regex : "^(.*), exception: (.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 180usize , placeholders : & ["$batchSize"] , exception : None , regex : "^Batch size is: (.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/BackgroundJob/BackgroundScanner.php" , line : 288usize , placeholders : & ["$file->getId()"] , exception : None , regex : "^Scanning file with fileid: (.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Db/ItemMapper.php" , line : 40usize , placeholders : & [] , exception : Some ("\\InvalidArgumentException") , regex : "^$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/Status.php" , line : 86usize , placeholders : & ["__METHOD__" , "$e->getMessage()"] , exception : None , regex : "^(.*), exception: (.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/AppConfig.php" , line : 173usize , placeholders : & ["$key"] , exception : Some ("\\BadFunctionCallException") , regex : "^(.*) is not a valid key$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/AppConfig.php" , line : 189usize , placeholders : & ["$key"] , exception : Some ("\\BadFunctionCallException") , regex : "^(.*) is not a valid key$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/AppConfig.php" , line : 242usize , placeholders : & ["$methodName"] , exception : Some ("\\BadFunctionCallException") , regex : "^(.*) does not exist$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ExternalKaspersky.php" , line : 58usize , placeholders : & [] , exception : Some ("\\RuntimeException") , regex : "^The Kaspersky port and host are not set up\\.$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/Scanner/ExternalKaspersky.php" , line : 90usize , placeholders : & ["$response"] , exception : None , regex : "^Response :: (.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ICAP.php" , line : 63usize , placeholders : & [] , exception : Some ("\\RuntimeException") , regex : "^The ICAP port and host are not set up\\.$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ICAP.php" , line : 139usize , placeholders : & [] , exception : Some ("\\RuntimeException") , regex : "^Invalid response from ICAP server$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/LocalClam.php" , line : 41usize , placeholders : & ["$this->avPath"] , exception : Some ("\\RuntimeException") , regex : "^The antivirus executable could not be found at (.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/LocalClam.php" , line : 60usize , placeholders : & [] , exception : Some ("\\RuntimeException") , regex : "^Error starting process$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/Scanner/LocalClam.php" , line : 74usize , placeholders : & ["$result" , "$output"] , exception : None , regex : "^Exit code :: (.*) Response :: (.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/Scanner/ScannerBase.php" , line : 178usize , placeholders : & [] , exception : None , regex : "^reinit scanner$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Warn , path : "/lib/Scanner/ScannerBase.php" , line : 191usize , placeholders : & [] , exception : None , regex : "^Failed to write a chunk\\. Check if Stream Length matches StreamMaxLength in anti virus daemon settings$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ScannerFactory.php" , line : 52usize , placeholders : & ["$avMode"] , exception : Some ("\\InvalidArgumentException") , regex : "^Application is misconfigured\\. Please check the settings at the admin page\\. Invalid mode: (.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ExternalClam.php" , line : 41usize , placeholders : & ["$avSocket" , "$errstr" , "$errno"] , exception : Some ("\\RuntimeException") , regex : "^Cannot connect to \"(.*)\": (.*) \\(code (.*)\\)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ExternalClam.php" , line : 47usize , placeholders : & [] , exception : Some ("\\RuntimeException") , regex : "^The ClamAV port and host are not set up\\.$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Scanner/ExternalClam.php" , line : 51usize , placeholders : & ["$avHost" , "$avPort"] , exception : Some ("\\RuntimeException") , regex : "^Could not connect to ClamAV via (.*):(.*)\\. Please check that ClamAV is running and reachable\\.$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/Scanner/ExternalClam.php" , line : 65usize , placeholders : & ["$response"] , exception : None , regex : "^Response :: (.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/AvirWrapper.php" , line : 114usize , placeholders : & ["$e->getMessage()"] , exception : None , regex : "^(.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Warn , path : "/lib/AvirWrapper.php" , line : 173usize , placeholders : & ["$status->getDetails()" , "$owner" , "$path"] , exception : None , regex : "^Infected file deleted\\. (.*) Account: (.*) Path: (.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/AvirWrapper.php" , line : 188usize , placeholders : & ["$status->getDetails()" , "$path" , "$owner"] , exception : None , regex : "^Infected file deleted\\. (.*) File: (.*) Account: (.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/AvirWrapper.php" , line : 191usize , placeholders : & ["$this->l10n->t(\n\t\t\t\t'Virus %s is detected in the file. Upload cannot be completed.',\n\t\t\t\t$status->getDetails()\n\t\t\t)"] , exception : Some ("OCP\\Files\\InvalidContentException") , regex : "^(.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/ICAP/ResponseParser.php" , line : 58usize , placeholders : & [] , exception : Some ("RuntimeException") , regex : "^Empty ICAP response$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/ICAP/ResponseParser.php" , line : 63usize , placeholders : & ["$icapHeader"] , exception : Some ("RuntimeException") , regex : "^Unknown ICAP response: \"(.*)\"$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/ICAP/ICAPClient.php" , line : 73usize , placeholders : & ["$this->host" , "$this->port" , "$errorMessage" , "$errorCode"] , exception : Some ("RuntimeException") , regex : "^Cannot connect to \"tcp://(.*):(.*)\": (.*) \\(code (.*)\\)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/ICAP/ICAPTlsClient.php" , line : 63usize , placeholders : & ["$this->host" , "$this->port" , "$errorMessage" , "$errorCode"] , exception : Some ("RuntimeException") , regex : "^Cannot connect to \"tls://(.*):(.*)\": (.*) \\(code (.*)\\)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Activity/Provider.php" , line : 53usize , placeholders : & [] , exception : Some ("\\InvalidArgumentException") , regex : "^$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/Item.php" , line : 160usize , placeholders : & ["__METHOD__" , "$e->getMessage()"] , exception : None , regex : "^(.*), exception: (.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Exception , path : "/lib/Item.php" , line : 200usize , placeholders : & [] , exception : Some ("\\RuntimeException") , regex : "^$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Debug , path : "/lib/Item.php" , line : 245usize , placeholders : & ["$message" , "$this->generateExtraInfo()"] , exception : None , regex : "^(.*)(.*)$" , },
+	crate :: LoggingStatement { level : crate :: LogLevel :: Error , path : "/lib/Item.php" , line : 252usize , placeholders : & ["$message" , "$this->generateExtraInfo()"] , exception : None , regex : "^(.*)(.*)$" , },
+];
+

data/src/lib.rs

@@ -1,5 +1,5 @@
 mod data;
 mod types;
 
-pub use data::{get_statements, MAX_VERSION, MIN_VERSION};
+pub use data::*;
 pub use types::*;

data/src/types.rs

@@ -46,34 +46,88 @@ pub struct LoggingStatement {
     pub regex: &'static str,
 }
 
-impl Display for LoggingStatement {
+impl LoggingStatement {
+    pub fn with_path_prefix(&self, path_prefix: &'static str) -> LoggingStatementWithPathPrefix {
+        LoggingStatementWithPathPrefix {
+            level: self.level,
+            path_prefix,
+            path: self.path,
+            line: self.line,
+            placeholders: self.placeholders,
+            exception: self.exception,
+            regex: self.regex,
+        }
+    }
+
+    pub fn message(&self) -> impl Display + '_ {
+        LoggingMessage {
+            message: self.clone(),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct LoggingStatementWithPathPrefix {
+    pub level: LogLevel,
+    pub path_prefix: &'static str,
+    pub path: &'static str,
+    pub line: usize,
+    pub placeholders: &'static [&'static str],
+    pub exception: Option<&'static str>,
+    pub regex: &'static str,
+}
+
+impl From<&LoggingStatementWithPathPrefix> for LoggingStatement {
+    fn from(value: &LoggingStatementWithPathPrefix) -> Self {
+        LoggingStatement {
+            level: value.level,
+            path: value.path,
+            line: value.line,
+            placeholders: value.placeholders,
+            exception: value.exception,
+            regex: value.regex,
+        }
+    }
+}
+
+impl Display for LoggingStatementWithPathPrefix {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         if let Some(exception) = self.exception {
             write!(
                 f,
-                "{}({}): {} line {}",
+                "{}({}): {}{} line {}",
                 exception,
                 self.message(),
+                self.path_prefix,
                 self.path,
                 self.line
             )
         } else {
-            write!(f, "{}: {} line {}", self.message(), self.path, self.line)
+            write!(
+                f,
+                "{}: {}{} line {}",
+                self.message(),
+                self.path_prefix,
+                self.path,
+                self.line
+            )
         }
     }
 }
 
-impl LoggingStatement {
+impl LoggingStatementWithPathPrefix {
     pub fn message(&self) -> impl Display + '_ {
-        LoggingMessage { message: &self }
+        LoggingMessage {
+            message: self.into(),
+        }
     }
 }
 
-struct LoggingMessage<'a> {
+struct LoggingMessage {
-    message: &'a LoggingStatement,
+    message: LoggingStatement,
 }
 
-impl<'a> Display for LoggingMessage<'a> {
+impl Display for LoggingMessage {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         if self.message.regex.is_empty() {
             return Ok(());

nix/extracted-logs.nix

@@ -10,8 +10,9 @@
   mode ? "json",
 }: let
   ext = if mode == "rust" then "rs" else "json";
+  cleanedMajor = builtins.replaceStrings ["."] ["_"] major;
 in stdenvNoCC.mkDerivation rec {
-  pname = "extractor-logs-${name}-${major}";
+  pname = "extractor-logs-${name}-${cleanedMajor}";
   inherit version;
 
   src = fetchzip {
@@ -26,6 +27,6 @@ in stdenvNoCC.mkDerivation rec {
 
   installPhase = ''
     mkdir -p $out
-    cp logs.* $out/${name}_${major}.${ext}
+    cp logs.* $out/${name}_${cleanedMajor}.${ext}
   '';
 }

nix/overlay.nix

@@ -4,7 +4,10 @@ final: prev: let
   inherit (prev.lib) importJSON;
   inherit (prev.lib.lists) flatten;
   allPackages = importJSON ./versions.json;
-  packages = {server = {"29" = allPackages.server."29";};};
+  packages = {
+    server = {"29" = allPackages.server."29";};
+    inherit (allPackages) files_accesscontrol files_antivirus;
+  };
 
   loggingFor = mode: name:
     mapAttrs (major: data: (final.callPackage ./extracted-logs.nix {

nix/versions.json

@@ -30,5 +30,19 @@
       "url": "https://download.nextcloud.com/server/releases/nextcloud-24.0.12.zip",
       "sha256": "sha256-+VlQVSTXtEsuoJ3Ic9o4DmFH4CQhMK8KtwiS+SqhiYE="
     }
+  },
+  "files_accesscontrol": {
+    "1.19": {
+      "version": "1.19.1",
+      "url": "https://github.com/nextcloud-releases/files_accesscontrol/releases/download/v1.19.1/files_accesscontrol-v1.19.1.tar.gz",
+      "sha256": "sha256-e9387k5yn3lAZ7uN0YU8zNVkX8sCoNxyc5M3E3azo80="
+    }
+  },
+  "files_antivirus": {
+    "5.5": {
+      "version": "5.5.7",
+      "url": "https://github.com/nextcloud-releases/files_antivirus/releases/download/v5.5.7/files_antivirus-v5.5.7.tar.gz",
+      "sha256": "sha256-Sg6AeqopaXrXrZ6loq+B5ZBjh2SHIflXaf6u/cIBqUo="
+    }
   }
 }

src/main.rs

@@ -32,11 +32,8 @@ fn main() -> MainResult {
     let first = lines.next().unwrap();
     let first_parsed: LogLine = serde_json::from_str(&first).unwrap();
 
-    let statements = get_statements(
+    let statements = get_statements(first_parsed.major_version().unwrap_or(MAX_VERSION));
-        "server",
+    let matcher = Matcher::new(&statements);
-        first_parsed.major_version().unwrap_or(MAX_VERSION),
-    );
-    let matcher = Matcher::new(statements);
 
     let lines = once(first).chain(lines);
     let mut error_count = 0;
@@ -67,7 +64,7 @@ fn main() -> MainResult {
     counts.sort_by_key(|(_, count)| *count);
     counts.reverse();
     for (match_result, count) in counts {
-        println!("{}: {}", match_result.display(statements), count);
+        println!("{}: {}", match_result.display(&statements), count);
     }
     if unmatched_total > 0 {
         eprintln!("\n{unmatched_total} lines couldn't be matched:");

src/matcher.rs

@@ -1,5 +1,5 @@
 use crate::logline::LogLine;
-use cloud_log_analyser_data::{LogLevel, LoggingStatement};
+use cloud_log_analyser_data::{LogLevel, LoggingStatement, StatementList};
 use regex::Regex;
 use std::fmt::{Display, Formatter};
 
@@ -34,7 +34,7 @@ pub struct Matcher {
 }
 
 impl Matcher {
-    pub fn new(statements: &[LoggingStatement]) -> Matcher {
+    pub fn new(statements: &StatementList) -> Matcher {
         Matcher {
             matches: statements.iter().map(LogMatch::new).collect(),
         }
@@ -90,7 +90,7 @@ pub enum MatchResult {
 }
 
 impl MatchResult {
-    pub fn display<'a>(&'a self, log_statements: &'a [LoggingStatement]) -> impl Display + 'a {
+    pub fn display<'a>(&'a self, log_statements: &'a StatementList) -> impl Display + 'a {
         MatchResultDisplay {
             log_statements,
             result: &self,
@@ -99,7 +99,7 @@ impl MatchResult {
 }
 
 struct MatchResultDisplay<'a> {
-    log_statements: &'a [LoggingStatement],
+    log_statements: &'a StatementList,
     result: &'a MatchResult,
 }
 
@@ -130,7 +130,7 @@ impl Display for MatchResultDisplay<'_> {
 fn test_matcher() {
     use crate::logline::Exception;
 
-    let statements = &[
+    const STATEMENTS: &[LoggingStatement] = &[
         LoggingStatement {
             line: 68,
             level: LogLevel::Exception,
@@ -169,32 +169,35 @@ fn test_matcher() {
             path: "short",
             placeholders: &["$path"],
             regex: "^Not allowed to rename (.*)$",
-            exception: "Bar\\FooException".into(),
+            exception: Some("Bar\\FooException"),
         },
     ];
-    let matcher = Matcher::new(statements);
+    let matcher = Matcher::new(&StatementList::new(vec![("", STATEMENTS)]));
     assert_eq!(
-        Some(0),
+        Some(MatchResult::Single(0)),
         matcher.match_log(&LogLine {
             version: "29",
+            app: "core".into(),
             level: LogLevel::Error,
             message: "Not allowed to rename a shared album".into(),
             exception: None,
         })
     );
     assert_eq!(
-        Some(3),
+        Some(MatchResult::List(vec![3, 4])),
         matcher.match_log(&LogLine {
             version: "29",
+            app: "core".into(),
             level: LogLevel::Error,
             message: "Not allowed to rename an album".into(),
             exception: None,
         })
     );
     assert_eq!(
-        Some(1),
+        Some(MatchResult::Single(1)),
         matcher.match_log(&LogLine {
             version: "29",
+            app: "core".into(),
             level: LogLevel::Error,
             message: "You are not allowed to edit link shares that you don't own".into(),
             exception: None,
@@ -204,16 +207,18 @@ fn test_matcher() {
         None,
         matcher.match_log(&LogLine {
             version: "29",
+            app: "core".into(),
             level: LogLevel::Info,
             message: "You are not allowed to edit link shares that you don't own".into(),
             exception: None,
         })
     );
     assert_eq!(
-        Some(2),
+        Some(MatchResult::Single(2)),
         matcher.match_log(
             &LogLine {
                 version: "29",
+                app: "core".into(),
                 level:  LogLevel::Error,
                 message: "Unsupported query value for mimetype: %/text, only values in the format \"mime/type\" or \"mime/%\" are supported".into(),
                 exception: None,
@@ -221,10 +226,11 @@ fn test_matcher() {
         )
     );
     assert_eq!(
-        Some(4),
+        Some(MatchResult::Single(4)),
         matcher.match_log(
             &LogLine {
                 version: "29",
+                app: "core".into(),
                 level:  LogLevel::Error,
                 message: "Unsupported query value for mimetype: %/text, only values in the format \"mime/type\" or \"mime/%\" are supported".into(),
                 exception: Some(Exception {