match date

archiver/.sqlx/query-26e463ea8a7316befaee41794d230c0e78c956ccd345c38f58a049cb6b6bfdc2.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "select id from teams where id > $1 and format in ('highlander', 'sixes', 'fours', 'ultiduo') order by id asc",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": ["Int4"]
+    },
+    "nullable": [false]
+  },
+  "hash": "26e463ea8a7316befaee41794d230c0e78c956ccd345c38f58a049cb6b6bfdc2"
+}

archiver/.sqlx/query-2d58774b5365ac8ba3996246c26fb6b7ca63a8a6f8c086b85979f6ea7b5b8ebe.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE matches SET default_date = $2 WHERE id = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": ["Int4", "Date"]
+    },
+    "nullable": []
+  },
+  "hash": "2d58774b5365ac8ba3996246c26fb6b7ca63a8a6f8c086b85979f6ea7b5b8ebe"
+}

archiver/.sqlx/query-41dfbd9d622c42e42346499e44d2d537a277f5a5b24b98d46b0645c218270adc.json

@@ -1,24 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE matches SET map = $2, week = $3, format = $4 WHERE id = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Varchar",
-        "Int4",
-        {
-          "Custom": {
-            "name": "game_mode",
-            "kind": {
-              "Enum": ["highlander", "eights", "sixes", "fours", "ultiduo"]
-            }
-          }
-        }
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "41dfbd9d622c42e42346499e44d2d537a277f5a5b24b98d46b0645c218270adc"
-}

archiver/.sqlx/query-626c573368c57c8bf544ba40e68ca80e034e2c918e0fe7d0e2319354f80b7de2.json

@@ -1,18 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT MIN(id) as id FROM matches WHERE id < 0",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [null]
-  },
-  "hash": "626c573368c57c8bf544ba40e68ca80e034e2c918e0fe7d0e2319354f80b7de2"
-}

archiver/.sqlx/query-7abd1664e5a53abd08cf3b437032218cac18e0dc2ca67aefa2d1cde6db125108.json

@@ -1,25 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE matches SET map = $2, week = $3, format = $4, default_date = $5 WHERE id = $1",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int4",
-        "Varchar",
-        "Int4",
-        {
-          "Custom": {
-            "name": "game_mode",
-            "kind": {
-              "Enum": ["highlander", "eights", "sixes", "fours", "ultiduo"]
-            }
-          }
-        },
-        "Date"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "7abd1664e5a53abd08cf3b437032218cac18e0dc2ca67aefa2d1cde6db125108"
-}

archiver/.sqlx/query-a3a5e50cf1dd433faf196b9cccae1417316e064be45e2ee7ba0840c0931e51f4.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "select LEAST(MIN(team_home), MIN(team_away)) as team_id from matches INNER JOIN teams ON (team_home = teams.id OR team_away = teams.id) WHERE matches.default_date IS NULL AND matches.format in ('highlander', 'sixes', 'fours', 'ultiduo')",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "team_id",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [null]
+  },
+  "hash": "a3a5e50cf1dd433faf196b9cccae1417316e064be45e2ee7ba0840c0931e51f4"
+}

archiver/.sqlx/query-ccb1cfabaf4c593cfbd8206e5c2bb2be59105f965cdcbbe04ca394d2c3751613.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT date FROM maps WHERE format = $1 AND week = $2 AND map = $3",
+  "query": "SELECT date FROM maps WHERE format = $1 AND week = $2 AND season = $3",
   "describe": {
     "columns": [
       {
@@ -20,10 +20,10 @@
           }
         },
         "Int4",
-        "Text"
+        "Int4"
       ]
     },
     "nullable": [false]
   },
-  "hash": "b679f54e09d9d2f7e5640725bfe2dbf553a6fcd68947e247e0d68a5c8eb43eab"
+  "hash": "ccb1cfabaf4c593cfbd8206e5c2bb2be59105f965cdcbbe04ca394d2c3751613"
 }

archiver/src/archive.rs

@@ -7,9 +7,9 @@ use thiserror::Error;
 use time::format_description::FormatItem;
 use time::macros::format_description;
 use time::parsing::Parsed;
-use time::{Date, Duration};
+use time::Date;
 use tokio_stream::Stream;
-use tracing::{debug, info, warn};
+use tracing::{debug, error, warn};
 use ugc_scraper_types::{
     Class, GameMode, MapHistory, MatchInfo, Membership, MembershipRole, NameChange, Player, Record,
     Region, RosterHistory, SteamID, Team, TeamRef, TeamSeason,
@@ -21,6 +21,8 @@ const MATCH_DATE_FORMAT: &[FormatItem<'static>] = format_description!(
 const MATCH_DATE_FORMAT2: &[FormatItem<'static>] = format_description!(
     "[weekday case_sensitive:false repr:short] [month repr:short] [day padding:none] [year]"
 );
+
+#[allow(dead_code)]
 const MATCH_DATE_FORMATS: &[&[FormatItem<'static>]] = &[MATCH_DATE_FORMAT, MATCH_DATE_FORMAT2];
 
 #[derive(Debug, Error)]
@@ -202,6 +204,22 @@ impl Archive {
         .map_ok(|map| map.id as u32)
     }
 
+    pub fn get_team_ids_in(
+        &self,
+        min: u32,
+    ) -> impl Stream<Item = Result<u32, ArchiveError>> + use<'_> {
+        query!(
+            "select id from teams where id > $1 and format in ('highlander', 'sixes', 'fours', 'ultiduo') order by id asc",
+            min as i32,
+        )
+        .fetch(&self.pool)
+        .map_err(|error| ArchiveError::Query {
+            description: "getting team ids",
+            error,
+        })
+        .map_ok(|map| map.id as u32)
+    }
+
     pub async fn get_max_roster_history(&self) -> Result<u32, ArchiveError> {
         if let Some(row) =
             query!("select team_id as max from membership_history order by team_id desc limit 1;")
@@ -514,6 +532,20 @@ impl Archive {
             .map_ok(|map| map.id as u32)
     }
 
+    pub async fn get_min_team_id_without_default_date(&self) -> Result<Option<u32>, ArchiveError> {
+        Ok(query!(r#"select LEAST(MIN(team_home), MIN(team_away)) as team_id from matches
+                INNER JOIN teams ON (team_home = teams.id OR team_away = teams.id)
+                WHERE matches.default_date IS NULL AND matches.format in ('highlander', 'sixes', 'fours', 'ultiduo')
+                    AND region in ('europe', 'north-america', 'south-america', 'australia')
+            "#)
+            .fetch_one(&self.pool)
+            .await
+            .map_err(|error| ArchiveError::Query {
+                description: "getting team ids",
+                error,
+            })?.team_id.map(|id| id as u32))
+    }
+
     pub async fn get_min_team_id_without_match_seasons(&self) -> Result<u32, ArchiveError> {
         Ok(query!("select LEAST(MIN(team_home), MIN(team_away)) as team_id from matches INNER JOIN teams ON (team_home = teams.id OR team_away = teams.id) WHERE season IS NULL")
             .fetch_one(&self.pool)
@@ -535,46 +567,26 @@ impl Archive {
             .is_some())
     }
 
-    #[allow(dead_code)]
+    pub async fn get_match_year(
-    pub async fn get_match_date(
         &self,
-        match_info: &MatchInfo,
+        format: GameMode,
-    ) -> Result<Option<Date>, ArchiveError> {
+        season: u32,
-        if let Ok(match_date) = parse_old_match_date(&match_info.default_date) {
+        week: u8,
-            return Ok(Some(match_date));
+    ) -> Result<Option<u32>, ArchiveError> {
-        } else if !match_info.format.is_tf2() {
+        let option = query!(
-            return Ok(None);
+            "SELECT date FROM maps WHERE format = $1 AND week = $2 AND season = $3",
-        }
+            format as GameMode,
-
+            week as i32,
-        let options = query!(
+            season as i32,
-            "SELECT date FROM maps WHERE format = $1 AND week = $2 AND map = $3",
-            match_info.format as GameMode,
-            match_info.week as i32,
-            match_info.map,
         )
-        .fetch_all(&self.pool)
+        .fetch_optional(&self.pool)
         .await
         .map_err(|error| ArchiveError::Query {
             description: "searching map history",
             error,
         })?;
 
-        for row in options {
+        Ok(option.map(|row| row.date.year() as u32))
-            let possible_date: Date = row.date;
-            let Some(match_date) = try_date_formats(
-                match_info.default_date.as_str(),
-                possible_date.year(),
-                MATCH_DATE_FORMATS,
-            ) else {
-                break;
-            };
-
-            if (match_date - possible_date).abs() < Duration::days(7) {
-                return Ok(Some(possible_date));
-            }
-        }
-
-        Ok(None)
     }
 
     pub async fn get_team_format(&self, id: u32) -> Result<GameMode, ArchiveError> {
@@ -592,46 +604,6 @@ impl Archive {
     }
 
     #[allow(dead_code)]
-    pub async fn update_match_details(
-        &self,
-        id: u32,
-        match_info: &MatchInfo,
-        date: Option<Date>,
-    ) -> Result<(), ArchiveError> {
-        let format = match_info.format.is_tf2().then_some(match_info.format);
-        if let Some(date) = date {
-            query!(
-                "UPDATE matches SET map = $2, week = $3, format = $4, default_date = $5 WHERE id = $1",
-                id as i32,
-                match_info.map,
-                match_info.week as i32,
-                format as Option<GameMode>,
-                date
-            )
-                .execute(&self.pool)
-                .await
-                .map_err(|error| ArchiveError::Query {
-                    description: "updating match",
-                    error,
-                })?;
-        } else {
-            query!(
-                "UPDATE matches SET map = $2, week = $3, format = $4 WHERE id = $1",
-                id as i32,
-                match_info.map,
-                match_info.week as i32,
-                format as Option<GameMode>,
-            )
-            .execute(&self.pool)
-            .await
-            .map_err(|error| ArchiveError::Query {
-                description: "updating match",
-                error,
-            })?;
-        }
-        Ok(())
-    }
-
     pub async fn update_match_details_from_team_matches(
         &self,
         team: &TeamRef,
@@ -662,15 +634,7 @@ impl Archive {
                 if options.len() == 1 {
                     options[0] as i32
                 } else if options.is_empty() {
-                    let fake_id = Self::find_next_negative_match_id(&mut *transaction).await?;
+                    panic!("failed to find match");
-                    assert!(fake_id < 0);
-                    info!(id = fake_id, "inserting synthetic match");
-                    panic!("?");
-                    let fake_match = match_info
-                        .match_info(team, format)
-                        .expect("no match info but we do have opponent");
-                    self.store_match(fake_id, fake_match).await?;
-                    fake_id
                 } else {
                     warn!(
                         possible_options = options.len(),
@@ -690,7 +654,7 @@ impl Archive {
                 format as GameMode,
                 season.season as i32
             )
-            .execute(&self.pool)
+            .execute(&mut *transaction)
             .await
             .map_err(|error| ArchiveError::Query {
                 description: "updating match with team match data",
@@ -709,6 +673,63 @@ impl Archive {
         Ok(())
     }
 
+    pub async fn update_match_date_from_team_matches(
+        &self,
+        format: GameMode,
+        season: &TeamSeason,
+    ) -> Result<(), ArchiveError> {
+        let mut transaction = self
+            .pool
+            .begin()
+            .await
+            .map_err(|error| ArchiveError::Query {
+                description: "beginning team matches transaction",
+                error,
+            })?;
+
+        for match_info in season.matches.iter() {
+            if let Some(id) = match_info.result.match_id() {
+                let Some(year) = self
+                    .get_match_year(format, season.season, match_info.week)
+                    .await?
+                else {
+                    error!(
+                        r#match = id,
+                        ?format,
+                        season = season.season,
+                        week = match_info.week,
+                        "Can't find year'"
+                    );
+                    panic!("Can't find year for match");
+                };
+
+                let date = parse_match_date(&match_info.date, year as i32);
+
+                query!(
+                    "UPDATE matches SET default_date = $2 WHERE id = $1",
+                    id as i32,
+                    date,
+                )
+                .execute(&mut *transaction)
+                .await
+                .map_err(|error| ArchiveError::Query {
+                    description: "updating match date with team match data",
+                    error,
+                })?;
+            }
+        }
+
+        transaction
+            .commit()
+            .await
+            .map_err(|error| ArchiveError::Query {
+                description: "commiting team matches transaction",
+                error,
+            })?;
+
+        Ok(())
+    }
+
     async fn find_match_id(
         db: impl Executor<'_, Database = Postgres>,
         week: u8,
@@ -735,21 +756,14 @@ impl Archive {
                 .collect(),
         )
     }
+}
 
-    async fn find_next_negative_match_id(
+#[allow(dead_code)]
-        db: impl Executor<'_, Database = Postgres>,
+fn parse_match_date(date: &str, year: i32) -> Date {
-    ) -> Result<i32, ArchiveError> {
+    if let Ok(date) = parse_old_match_date(date) {
-        let min = query!("SELECT MIN(id) as id FROM matches WHERE id < 0")
+        return date;
-            .fetch_optional(db)
-            .await
-            .map_err(|error| ArchiveError::Query {
-                description: "getting next negative match",
-                error,
-            })?
-            .map(|row| row.id.unwrap_or_default())
-            .unwrap_or_default();
-        Ok(min - 1)
     }
+    try_date_formats(date, year, MATCH_DATE_FORMATS).expect("failed to parse date")
 }
 
 fn parse_old_match_date(date: &str) -> Result<Date, time::Error> {
@@ -770,6 +784,7 @@ fn test_parse_old_match_date() {
     );
 }
 
+#[allow(dead_code)]
 fn try_date_formats(date: &str, year: i32, formats: &[&[FormatItem<'static>]]) -> Option<Date> {
     for format in formats {
         match Date::parse(&format!("{} {}", date, year), format) {

archiver/src/main.rs

@@ -35,7 +35,6 @@ enum Command {
     MapHistory { format: String },
 }
 
-const LAST_MATCH: u32 = 117047;
 const MAYBE_FIRST_MATCH: u32 = 14486;
 
 #[tokio::main]
@@ -79,7 +78,7 @@ async fn archive_matches(client: &UgcClient, archive: &Archive) -> MainResult {
         .await?
         .unwrap_or(MAYBE_FIRST_MATCH - 1)
         + 1;
-    for id in 200..=MAYBE_FIRST_MATCH {
+    for id in next_match..=MAYBE_FIRST_MATCH {
         archive_match(client, archive, id).await.ok();
         sleep(Duration::from_millis(500)).await;
     }
@@ -228,44 +227,71 @@ async fn archive_map_history(client: &UgcClient, archive: &Archive, mode: GameMo
 
 async fn fixup_matches(client: &UgcClient, archive: &Archive) -> MainResult {
     let min_team = archive.get_min_team_id_without_match_seasons().await?;
-    dbg!(min_team);
+    if min_team > 0 {
-    let mut team_ids = pin!(archive.get_team_ids(min_team - 1));
+        let mut team_ids = pin!(archive.get_team_ids(min_team - 1));
+
+        while let Some(Ok(team_id)) = team_ids.next().await {
+            let _span = span!(Level::INFO, "fixup_matches", team_id).entered();
+            let format = archive.get_team_format(team_id).await?;
+            let matches = client.get_team_matches(team_id).await?;
+            info!(
+                seasons = matches.seasons.len(),
+                ?format,
+                "updating matches for team"
+            );
+
+            for season in matches.seasons.iter() {
+                for season_match in season.matches.iter() {
+                    if let Some(match_id) = season_match.result.match_id() {
+                        if !archive.has_match(match_id).await? {
+                            warn!(match_id, "match not archived yet");
+                            sleep(Duration::from_millis(500)).await;
+                            if let Err(_) = archive_match(client, archive, match_id).await {
+                                let match_info = season_match
+                                    .match_info(&matches.team, season.format)
+                                    .expect("failed to build match info");
+                                assert_eq!(format, match_info.format);
+                                info!("reconstructed match");
+                                archive.store_match(match_id as i32, match_info).await?;
+                            }
+                        }
+                    }
+                }
+
+                archive
+                    .update_match_details_from_team_matches(&matches.team, format, season)
+                    .await?;
+            }
+
+            sleep(Duration::from_millis(500)).await;
+        }
+    }
+
+    let mut last_team_id = 0;
+    while let Some(team_id) = archive.get_min_team_id_without_default_date().await? {
+        if team_id == last_team_id {
+            panic!("team didn't get fixed up");
+        }
+        last_team_id = team_id;
 
-    while let Some(Ok(team_id)) = team_ids.next().await {
         let _span = span!(Level::INFO, "fixup_matches", team_id).entered();
         let format = archive.get_team_format(team_id).await?;
         let matches = client.get_team_matches(team_id).await?;
         info!(
             seasons = matches.seasons.len(),
             ?format,
-            "updating matches for team"
+            "updating match date for team"
         );
 
         for season in matches.seasons.iter() {
-            for season_match in season.matches.iter() {
-                if let Some(match_id) = season_match.result.match_id() {
-                    if !archive.has_match(match_id).await? {
-                        warn!(match_id, "match not archived yet");
-                        sleep(Duration::from_millis(500)).await;
-                        if let Err(_) = archive_match(client, archive, match_id).await {
-                            let match_info = season_match
-                                .match_info(&matches.team, season.format)
-                                .expect("failed to build match info");
-                            assert_eq!(format, match_info.format);
-                            info!("reconstructed match");
-                            archive.store_match(match_id as i32, match_info).await?;
-                        }
-                    }
-                }
-            }
-
             archive
-                .update_match_details_from_team_matches(&matches.team, format, season)
+                .update_match_date_from_team_matches(format, season)
                 .await?;
         }
 
         sleep(Duration::from_millis(500)).await;
     }
+
     Ok(())
 }