handle more logs

schema.sql

@@ -1,8 +1,8 @@
 CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
 
-CREATE TYPE team AS ENUM ('blue', 'red');
+CREATE TYPE team AS ENUM ('blue', 'red', 'other');
 
-CREATE TYPE class_type AS ENUM ('scout', 'soldier', 'pyro', 'demoman', 'heavyweapons', 'engineer', 'medic', 'sniper', 'spy');
+CREATE TYPE class_type AS ENUM ('scout', 'soldier', 'pyro', 'demoman', 'heavyweapons', 'engineer', 'medic', 'sniper', 'spy', 'unknown');
 
 CREATE TYPE game_mode AS ENUM ('ultiduo', '4v4', '6v6', '7v7', '9v9', 'other');
 

src/data.rs

@@ -6,6 +6,14 @@ use serde::Deserialize;
 pub enum TeamId {
     Blue,
     Red,
+    #[serde(other)]
+    Other,
+}
+
+impl Default for TeamId {
+    fn default() -> Self {
+        TeamId::Other
+    }
 }
 
 #[derive(Debug, Clone, Copy, sqlx::Type, Deserialize, Eq, PartialEq)]
@@ -22,6 +30,8 @@ pub enum Class {
     Medic,
     Sniper,
     Spy,
+    #[serde(other)]
+    Unknown,
 }
 
 #[derive(Debug, Clone, Copy, sqlx::Type, Eq, PartialEq)]
@@ -47,6 +57,8 @@ pub enum EventType {
     PointCap,
     MedicDeath,
     RoundWin,
+    #[serde(other)]
+    Other,
 }
 
 #[derive(Debug, Clone, Copy, sqlx::Type, Deserialize, Hash, Eq, PartialEq)]

src/database.rs

@@ -6,11 +6,12 @@ use sqlx::PgPool;
 use std::collections::HashMap;
 use steamid_ng::SteamID;
 
-pub async fn store_log(pool: &PgPool, id: u32, log: &NormalizedLog) -> Result<(), sqlx::Error> {
+pub async fn store_log(pool: &PgPool, id: i32, log: &NormalizedLog) -> Result<(), sqlx::Error> {
+    let mut tx = pool.begin().await?;
     sqlx::query!(
         "INSERT INTO logs(id, red_score, blue_score, length, game_mode, map, type, date)\
             VALUES($1, $2, $3, $4, $5, $6, $7, $8)",
-        id as i32,
+        id,
         log.teams.red.score as i32,
         log.teams.blue.score as i32,
         log.info.total_length as i32,
@@ -19,7 +20,7 @@ pub async fn store_log(pool: &PgPool, id: u32, log: &NormalizedLog) -> Result<()
         log.info.map_type() as MapType,
         log.info.date() as DateTime<Utc>
     )
-    .execute(pool)
+    .execute(&mut tx)
     .await?;
 
     for (num, round) in log.rounds.iter().enumerate() {
@@ -31,9 +32,9 @@ pub async fn store_log(pool: &PgPool, id: u32, log: &NormalizedLog) -> Result<()
             VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
             RETURNING id"#,
             num as i32,
-            id as i32,
+            id,
             round.length as i32,
-            round.winner as TeamId,
+            round.winner.unwrap_or_default() as TeamId,
             round.first_cap as TeamId,
             round.team.red.score as i32,
             round.team.blue.score as i32,
@@ -44,7 +45,7 @@ pub async fn store_log(pool: &PgPool, id: u32, log: &NormalizedLog) -> Result<()
             round.team.red.charges as i32,
             round.team.blue.charges as i32,
         )
-        .fetch_one(pool)
+        .fetch_one(&mut tx)
         .await?
         .id;
 
@@ -59,7 +60,7 @@ pub async fn store_log(pool: &PgPool, id: u32, log: &NormalizedLog) -> Result<()
                         *team as TeamId,
                         *point as i32,
                     )
-                    .execute(pool)
+                    .execute(&mut tx)
                     .await?;
                 }
                 Event::RoundWin { time, team } => {
@@ -68,9 +69,9 @@ pub async fn store_log(pool: &PgPool, id: u32, log: &NormalizedLog) -> Result<()
                             VALUES($1, $2, $3)",
                         round_id,
                         *time as i32,
-                        *team as TeamId,
+                        team.unwrap_or_default() as TeamId,
                     )
-                    .execute(pool)
+                    .execute(&mut tx)
                     .await?;
                 }
                 Event::MedicDeath {
@@ -88,7 +89,7 @@ pub async fn store_log(pool: &PgPool, id: u32, log: &NormalizedLog) -> Result<()
                         u64::from(*steamid) as i64,
                         u64::from(*killer) as i64,
                     )
-                    .execute(pool)
+                    .execute(&mut tx)
                     .await?;
                 }
                 Event::Drop {
@@ -104,7 +105,7 @@ pub async fn store_log(pool: &PgPool, id: u32, log: &NormalizedLog) -> Result<()
                         *team as TeamId,
                         u64::from(*steamid) as i64,
                     )
-                    .execute(pool)
+                    .execute(&mut tx)
                     .await?;
                 }
                 Event::Charge {
@@ -122,9 +123,10 @@ pub async fn store_log(pool: &PgPool, id: u32, log: &NormalizedLog) -> Result<()
                         *medigun as Medigun,
                         u64::from(*steamid) as i64,
                     )
-                    .execute(pool)
+                    .execute(&mut tx)
                     .await?;
                 }
+                _ => {}
             }
         }
     }
@@ -204,7 +206,7 @@ pub async fn store_log(pool: &PgPool, id: u32, log: &NormalizedLog) -> Result<()
             kills.sniper as i32,
             kills.spy as i32
         )
-        .fetch_one(pool)
+        .fetch_one(&mut tx)
         .await?
         .id;
 
@@ -221,7 +223,7 @@ pub async fn store_log(pool: &PgPool, id: u32, log: &NormalizedLog) -> Result<()
                 class.assists as i32,
                 class.dmg as i32,
             )
-            .fetch_one(pool)
+            .fetch_one(&mut tx)
             .await?
             .id;
 
@@ -236,12 +238,14 @@ pub async fn store_log(pool: &PgPool, id: u32, log: &NormalizedLog) -> Result<()
                         stats.hits as i32,
                         stats.dmg as i32,
                     )
-                    .execute(pool)
+                    .execute(&mut tx)
                     .await?;
             }
         }
     }
 
+    tx.commit().await?;
+
     Ok(())
 }
 

src/main.rs

@@ -6,18 +6,82 @@ mod raw;
 use crate::database::store_log;
 use crate::normalized::NormalizedLog;
 use main_error::MainError;
-use sqlx::PgPool;
+use sqlx::{postgres::PgQueryAs, PgPool};
-use std::fs;
 
 #[tokio::main]
 async fn main() -> Result<(), MainError> {
     let database_url = dotenv::var("DATABASE_URL")?;
-
+    let raw_database_url = dotenv::var("RAW_DATABASE_URL")?;
-    let content = fs::read_to_string("tests/data/2522305.json").unwrap();
-    let parsed: NormalizedLog = serde_json::from_str(&content).unwrap();
 
     let pool = PgPool::builder().max_size(2).build(&database_url).await?;
-    dbg!(store_log(&pool, 2522305, &parsed).await)?;
+    let raw_pool = PgPool::builder()
+        .max_size(2)
+        .build(&raw_database_url)
+        .await?;
+
+    let max = get_max_log(&raw_pool).await?;
+    let from = get_max_stored_log(&pool).await?;
+
+    for id in (from + 1)..=max {
+        println!("{}", id);
+        if let Some(log) = get_log(&raw_pool, id).await? {
+            println!("{}", log.info.map);
+            store_log(&pool, id, &log).await?;
+        } else {
+            println!("invalid");
+        }
+    }
 
     Ok(())
 }
+
+async fn get_max_stored_log(pool: &PgPool) -> Result<i32, MainError> {
+    Ok(sqlx::query!(r#"SELECT MAX(id) as id from logs"#)
+        .fetch_one(pool)
+        .await?
+        .id
+        .unwrap_or_default())
+}
+
+async fn get_max_log(pool: &PgPool) -> Result<i32, MainError> {
+    let row: (i32,) = sqlx::query_as(r#"SELECT MAX(id) as id from logs_raw"#)
+        .fetch_one(pool)
+        .await?;
+    Ok(row.0)
+}
+
+async fn get_log(pool: &PgPool, id: i32) -> Result<Option<NormalizedLog>, MainError> {
+    let row: (serde_json::Value,) =
+        sqlx::query_as(r#"SELECT json as id from logs_raw where id = $1"#)
+            .bind(id)
+            .fetch_one(pool)
+            .await?;
+
+    if is_valid(&row.0) {
+        Ok(serde_json::from_value(row.0).ok())
+    } else {
+        Ok(None)
+    }
+}
+
+fn is_valid(value: &serde_json::Value) -> bool {
+    if value.get("success").is_none() {
+        return false;
+    }
+    if value.get("success").unwrap().as_bool().unwrap_or_default() == false {
+        return false;
+    }
+
+    let rounds = value
+        .get("rounds")
+        .or_else(|| value.get("info").map(|info| info.get("rounds")).unwrap())
+        .unwrap();
+
+    for round in rounds.as_array().unwrap() {
+        if round.get("length").unwrap().as_i64().unwrap() < 0 {
+            return false;
+        }
+    }
+
+    true
+}

src/normalized.rs

@@ -94,7 +94,7 @@ impl Info {
 #[derive(Debug, Clone)]
 pub struct Round {
     pub start_time: u64,
-    pub winner: TeamId,
+    pub winner: Option<TeamId>,
     pub first_cap: TeamId,
     pub length: u32,
     pub team: Teams,
@@ -218,7 +218,7 @@ pub fn map_is_stopwatch(map: &str) -> bool {
 fn normalize_stopwatch_events(log: &mut NormalizedLog) {
     if map_is_stopwatch(&log.info.map)
         && log.rounds.len() >= 2
-        && log.rounds[1].winner == TeamId::Blue
+        && log.rounds[1].winner == Some(TeamId::Blue)
     {
         let first_half_rounds = get_round_point_capped(&log.rounds[0]);
         let second_half_rounds = get_round_point_capped(&log.rounds[1]);
@@ -284,6 +284,7 @@ fn normalize_event_times(log: &mut NormalizedLog) {
                 Event::Drop { time, .. } => *time += prev_round_end_time,
                 Event::MedicDeath { time, .. } => *time += prev_round_end_time,
                 Event::RoundWin { time, .. } => *time += prev_round_end_time,
+                Event::Other => {}
             });
         }
         prev_round_end_time = get_round_end_time(round);
@@ -317,8 +318,8 @@ fn normalize_stopwatch_score(log: &mut NormalizedLog) {
             log.teams.red.score = 1;
         } else {
             let first_half_cap_time = get_last_cap_time(&log.rounds[0]);
-            let second_half_cap_time =
+            let second_half_cap_time = get_last_cap_time(&log.rounds[1])
-                get_last_cap_time(&log.rounds[1]) - get_round_end_time(&log.rounds[0]);
+                .saturating_sub(get_round_end_time(&log.rounds[0]));
 
             if first_half_cap_time < second_half_cap_time {
                 log.teams.blue.score = 1;

src/raw.rs

@@ -35,7 +35,7 @@ pub struct Teams {
 
 #[derive(Debug, Clone, Deserialize, Default)]
 pub struct Team {
-    pub score: u8,
+    pub score: u32,
     #[serde(default)]
     pub kills: u32,
     #[serde(default)]
@@ -73,29 +73,29 @@ pub struct Player {
     pub hr: u16,
     #[serde(default)]
     pub lks: u16,
-    pub ubers: u8,
+    pub ubers: u32,
     #[serde(default)]
-    pub ubertypes: HashMap<Medigun, u8>,
+    pub ubertypes: HashMap<Medigun, u32>,
-    pub drops: u8,
+    pub drops: u32,
-    pub medkits: u8,
+    pub medkits: u32,
     #[serde(default)]
     pub medkits_hp: u16,
-    pub backstabs: u8,
+    pub backstabs: u32,
-    pub headshots: u8,
+    pub headshots: u32,
     #[serde(default)]
-    pub headshots_hit: u8,
+    pub headshots_hit: u32,
     pub heal: u32,
-    pub cpc: u8,
+    pub cpc: u32,
-    pub ic: u8,
+    pub ic: u32,
     pub medicstat: Option<MedicStats>,
 }
 
 #[derive(Debug, Clone, Deserialize)]
 pub struct MedicStats {
-    pub advantages_lost: u8,
+    pub advantages_lost: u32,
     pub biggest_advantage_list: u16,
-    pub deaths_within_20s_after_uber: u8,
+    pub deaths_within_20s_after_uber: u32,
-    pub deaths_with_95_99_uber: u8,
+    pub deaths_with_95_99_uber: u32,
     pub avg_time_before_healing: f32,
     pub avg_time_to_build: f32,
     pub avg_time_before_using: f32,
@@ -169,7 +169,7 @@ impl From<RawWeaponStats> for WeaponStat {
 pub struct Round {
     #[serde(default)]
     pub start_time: u64,
-    pub winner: TeamId,
+    pub winner: Option<TeamId>,
     #[serde(rename = "firstcap")]
     pub first_cap: Option<TeamId>,
     pub length: u32,
@@ -211,13 +211,15 @@ pub enum Event {
     },
     RoundWin {
         time: u32,
-        team: TeamId,
+        team: Option<TeamId>,
     },
     Drop {
         time: u32,
         steamid: SteamID,
         team: TeamId,
     },
+    #[serde(other)]
+    Other,
 }
 
 impl Event {
@@ -228,6 +230,7 @@ impl Event {
             Event::Drop { time, .. } => *time,
             Event::MedicDeath { time, .. } => *time,
             Event::PointCap { time, .. } => *time,
+            Event::Other => 0,
         }
     }
 }
@@ -235,23 +238,23 @@ impl Event {
 #[derive(Debug, Clone, Deserialize, Default)]
 pub struct ClassNumbers {
     #[serde(default)]
-    pub scout: u8,
+    pub scout: u32,
     #[serde(default)]
-    pub soldier: u8,
+    pub soldier: u32,
     #[serde(default)]
-    pub pyro: u8,
+    pub pyro: u32,
     #[serde(default)]
-    pub demoman: u8,
+    pub demoman: u32,
     #[serde(default)]
-    pub heavyweapons: u8,
+    pub heavyweapons: u32,
     #[serde(default)]
-    pub engineer: u8,
+    pub engineer: u32,
     #[serde(default)]
-    pub medic: u8,
+    pub medic: u32,
     #[serde(default)]
-    pub sniper: u8,
+    pub sniper: u32,
     #[serde(default)]
-    pub spy: u8,
+    pub spy: u32,
 }
 
 #[derive(Debug, Clone, Deserialize)]