archive more match data

archiver/.sqlx/query-037b9db0c5f69bce8930f7515a793e18b0018587dc476dedd4c756e88a2fa877.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE matches SET map = $2, week = $3, format = $4, default_data = $5 WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Varchar",
+        "Int4",
+        {
+          "Custom": {
+            "name": "game_mode",
+            "kind": {
+              "Enum": ["highlander", "eights", "sixes", "fours", "ultiduo"]
+            }
+          }
+        },
+        "Date"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "037b9db0c5f69bce8930f7515a793e18b0018587dc476dedd4c756e88a2fa877"
+}

archiver/.sqlx/query-41dfbd9d622c42e42346499e44d2d537a277f5a5b24b98d46b0645c218270adc.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE matches SET map = $2, week = $3, format = $4 WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Varchar",
+        "Int4",
+        {
+          "Custom": {
+            "name": "game_mode",
+            "kind": {
+              "Enum": ["highlander", "eights", "sixes", "fours", "ultiduo"]
+            }
+          }
+        }
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "41dfbd9d622c42e42346499e44d2d537a277f5a5b24b98d46b0645c218270adc"
+}

archiver/.sqlx/query-5fff8da677117dfd5d246a552a87cac254d6dee623c0e8497a28c2ce7a2d342b.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO maps (\n                    format, season, week, date, map\n                  ) VALUES ($1, $2, $3, $4, $5)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        {
+          "Custom": {
+            "name": "game_mode",
+            "kind": {
+              "Enum": ["highlander", "eights", "sixes", "fours", "ultiduo"]
+            }
+          }
+        },
+        "Int4",
+        "Int4",
+        "Date",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "5fff8da677117dfd5d246a552a87cac254d6dee623c0e8497a28c2ce7a2d342b"
+}

archiver/.sqlx/query-b679f54e09d9d2f7e5640725bfe2dbf553a6fcd68947e247e0d68a5c8eb43eab.json

@@ -0,0 +1,29 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT date FROM maps WHERE format = $1 AND week = $2 AND map = $3",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "date",
+        "type_info": "Date"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        {
+          "Custom": {
+            "name": "game_mode",
+            "kind": {
+              "Enum": ["highlander", "eights", "sixes", "fours", "ultiduo"]
+            }
+          }
+        },
+        "Int4",
+        "Text"
+      ]
+    },
+    "nullable": [false]
+  },
+  "hash": "b679f54e09d9d2f7e5640725bfe2dbf553a6fcd68947e247e0d68a5c8eb43eab"
+}

archiver/.sqlx/query-dbe1b2fc74c5390ebe7371c26bd1e9e6b1ed97f170f53e0f52bbb828e756eaae.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "select id from matches where map IS NULL ORDER BY id ASC",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [false]
+  },
+  "hash": "dbe1b2fc74c5390ebe7371c26bd1e9e6b1ed97f170f53e0f52bbb828e756eaae"
+}

archiver/Cargo.lock

@@ -94,6 +94,7 @@ dependencies = [
  "serde",
  "sqlx",
  "thiserror 2.0.12",
+ "time",
  "tokio",
  "tokio-stream",
  "toml",

archiver/Cargo.toml

@@ -19,3 +19,4 @@ thiserror = "2.0.12"
 main_error = "0.1.2"
 tokio-stream = "0.1.17"
 futures-util = "0.3.31"
+time = "0.3.41"

archiver/migrations/20250418172836_match_data.sql

@@ -0,0 +1,5 @@
+ALTER TABLE matches
+    ADD COLUMN IF NOT EXISTS map          VARCHAR,
+    ADD COLUMN IF NOT EXISTS week         INT,
+    ADD COLUMN IF NOT EXISTS format       game_mode,
+    ADD COLUMN IF NOT EXISTS default_data DATE;

archiver/src/archive.rs

@@ -4,6 +4,10 @@ use sqlx::{query, Error, Executor, PgPool, Postgres};
 use std::ops::Range;
 use std::str::FromStr;
 use thiserror::Error;
+use time::format_description::FormatItem;
+use time::macros::format_description;
+use time::parsing::Parsed;
+use time::Date;
 use tokio_stream::Stream;
 use ugc_scraper_types::{
     Class, GameMode, MapHistory, MatchInfo, Membership, MembershipRole, NameChange, Player, Record,
@@ -484,4 +488,126 @@ impl Archive {
 
         Ok(())
     }
+
+    pub fn get_match_ids_without_map(
+        &self,
+    ) -> impl Stream<Item = Result<u32, ArchiveError>> + use<'_> {
+        query!("select id from matches where map IS NULL ORDER BY id ASC")
+            .fetch(&self.pool)
+            .map_err(|error| ArchiveError::Query {
+                description: "getting match ids",
+                error,
+            })
+            .map_ok(|map| map.id as u32)
+    }
+
+    pub async fn get_match_date(
+        &self,
+        match_info: &MatchInfo,
+    ) -> Result<Option<Date>, ArchiveError> {
+        const MATCH_DATE_FORMAT: &[FormatItem<'static>] = format_description!("[weekday case_sensitive:false repr:short], [month repr:short] [day padding:none] [year]");
+        const MATCH_DATE_FORMAT2: &[FormatItem<'static>] = format_description!("[weekday case_sensitive:false repr:short] [month repr:short] [day padding:none] [year]");
+        if let Ok(match_date) = parse_old_match_date(&match_info.default_date) {
+            return Ok(Some(match_date));
+        } else if !match_info.format.is_tf2() {
+            return Ok(None);
+        }
+
+        let options = query!(
+            "SELECT date FROM maps WHERE format = $1 AND week = $2 AND map = $3",
+            match_info.format as GameMode,
+            match_info.week as i32,
+            match_info.map,
+        )
+        .fetch_all(&self.pool)
+        .await
+        .map_err(|error| ArchiveError::Query {
+            description: "searching map history",
+            error,
+        })?;
+
+        for row in options {
+            let possible_date: Date = row.date;
+            let Some(match_date) = try_date_formats(
+                match_info.default_date.as_str(),
+                possible_date.year(),
+                &[MATCH_DATE_FORMAT, MATCH_DATE_FORMAT2],
+            ) else {
+                break;
+            };
+
+            if match_date == possible_date {
+                return Ok(Some(possible_date));
+            }
+        }
+
+        Ok(None)
+    }
+
+    pub async fn update_match_details(
+        &self,
+        id: u32,
+        match_info: &MatchInfo,
+        date: Option<Date>,
+    ) -> Result<(), ArchiveError> {
+        let format = match_info.format.is_tf2().then_some(match_info.format);
+        if let Some(date) = date {
+            query!(
+                "UPDATE matches SET map = $2, week = $3, format = $4, default_data = $5 WHERE id = $1",
+                id as i32,
+                match_info.map,
+                match_info.week as i32,
+                format as Option<GameMode>,
+                date
+            )
+                .execute(&self.pool)
+                .await
+                .map_err(|error| ArchiveError::Query {
+                    description: "updating match",
+                    error,
+                })?;
+        } else {
+            query!(
+                "UPDATE matches SET map = $2, week = $3, format = $4 WHERE id = $1",
+                id as i32,
+                match_info.map,
+                match_info.week as i32,
+                format as Option<GameMode>,
+            )
+            .execute(&self.pool)
+            .await
+            .map_err(|error| ArchiveError::Query {
+                description: "updating match",
+                error,
+            })?;
+        }
+        Ok(())
+    }
+}
+
+fn parse_old_match_date(date: &str) -> Result<Date, time::Error> {
+    const MATCH_DATE_FORMAT_OLD: &[FormatItem<'static>] = format_description!("[weekday case_sensitive:false repr:short], [month padding:none]/[day padding:none]/[year repr:last_two]");
+    let mut parsed = Parsed::new();
+    parsed.parse_items(date.as_bytes(), MATCH_DATE_FORMAT_OLD)?;
+
+    let year = parsed.year_last_two().unwrap() as i32 + 2000;
+    parsed.set_year(year);
+    Ok(Date::try_from(parsed)?)
+}
+
+#[test]
+fn test_parse_old_match_date() {
+    assert_eq!(
+        Date::from_calendar_date(2009, time::Month::May, 13).unwrap(),
+        parse_old_match_date("Wed, 5/13/09").unwrap()
+    );
+}
+
+fn try_date_formats(date: &str, year: i32, formats: &[&[FormatItem<'static>]]) -> Option<Date> {
+    for format in formats {
+        if let Ok(match_date) = Date::parse(&format!("{} {}", date, year), format) {
+            return Some(match_date);
+        };
+    }
+    None
 }

archiver/src/main.rs

@@ -30,6 +30,7 @@ enum Command {
     Players,
     Teams,
     FixupTeams,
+    FixupMatches,
     MembershipHistory,
     MapHistory { format: String },
 }
@@ -55,6 +56,9 @@ async fn main() -> MainResult {
         Command::FixupTeams => {
             fixup_teams(&client, &archive).await?;
         }
+        Command::FixupMatches => {
+            fixup_matches(&client, &archive).await?;
+        }
         Command::MembershipHistory => {
             archive_team_roster_history(&client, &archive).await?;
         }
@@ -210,6 +214,29 @@ async fn archive_map_history(client: &UgcClient, archive: &Archive, mode: GameMo
     Ok(())
 }
 
+async fn fixup_matches(client: &UgcClient, archive: &Archive) -> MainResult {
+    let mut match_ids = pin!(archive.get_match_ids_without_map());
+
+    while let Some(Ok(id)) = match_ids.next().await {
+        let _span = span!(Level::INFO, "fixup_match", id = id).entered();
+        let match_info = client.get_match(id).await?;
+        let date = archive.get_match_date(&match_info).await?;
+        if date.is_none()
+            && (match_info.format == GameMode::Highlander
+                || match_info.format == GameMode::Sixes
+                || match_info.format == GameMode::Fours
+                || match_info.format == GameMode::Ultiduo)
+        {
+            error!("failed to parse match date");
+            panic!();
+        }
+        info!(date = ?date, format = %match_info.format, "updating match");
+        archive.update_match_details(id, &match_info, date).await?;
+        sleep(Duration::from_millis(500)).await;
+    }
+    Ok(())
+}
+
 trait NotFoundResultExt<T>: Sized {
     fn check_not_found(self) -> Result<Option<T>, UgcClientError>;
 }