map history

archiver/.sqlx/query-104cec686e47c560a136e7ac479fc6a301ae08c7b339e18dda446d3a0990f655.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO player_honors (\n                steam_id, team_id, season, division, format\n              ) VALUES ($1, $2, $3, $4, $5)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int4",
+        "Int2",
+        "Varchar",
+        {
+          "Custom": {
+            "name": "game_mode",
+            "kind": {
+              "Enum": ["highlander", "eights", "sixes", "fours", "ultiduo"]
+            }
+          }
+        }
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "104cec686e47c560a136e7ac479fc6a301ae08c7b339e18dda446d3a0990f655"
+}

archiver/.sqlx/query-133b2606adb67687107f717ec04114683e8ad6d6e9bb3177561971f0f12335f3.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "select steam_id as max from players order by steam_id desc limit 1;",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "max",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [false]
+  },
+  "hash": "133b2606adb67687107f717ec04114683e8ad6d6e9bb3177561971f0f12335f3"
+}

archiver/.sqlx/query-2df08faad317dd3f037d2fde18794a1a3b1c7b154118f3fca1f26ae2e3f9381c.json

@@ -0,0 +1,42 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO players (\n                steam_id, name, avatar, favorite_classes, country\n              ) VALUES ($1, $2, $3, $4, $5)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Varchar",
+        "Varchar",
+        {
+          "Custom": {
+            "name": "player_class[]",
+            "kind": {
+              "Array": {
+                "Custom": {
+                  "name": "player_class",
+                  "kind": {
+                    "Enum": [
+                      "scout",
+                      "soldier",
+                      "pyro",
+                      "demoman",
+                      "engineer",
+                      "heavy",
+                      "medic",
+                      "sniper",
+                      "spy"
+                    ]
+                  }
+                }
+              }
+            }
+          }
+        },
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "2df08faad317dd3f037d2fde18794a1a3b1c7b154118f3fca1f26ae2e3f9381c"
+}

archiver/.sqlx/query-d06507f78e0ac005f9e978816ffea6441d19232d08b3205993ee76b86c3e558b.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "select distinct steam_id from membership_history where steam_id > $1 order by steam_id asc",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "steam_id",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": ["Int8"]
+    },
+    "nullable": [false]
+  },
+  "hash": "d06507f78e0ac005f9e978816ffea6441d19232d08b3205993ee76b86c3e558b"
+}

archiver/migrations/20250416070625_player.sql

@@ -5,7 +5,8 @@ CREATE TABLE players
     steam_id         BIGINT         NOT NULL,
     name             VARCHAR        NOT NULL,
     avatar           VARCHAR,
-    favorite_classes player_class[] NOT NULL
+    favorite_classes player_class[] NOT NULL,
+    country          VARCHAR
 );
 
 CREATE UNIQUE INDEX players_steam_id_idx

archiver/migrations/20250418154032_maps.sql

@@ -0,0 +1,14 @@
+CREATE TABLE maps
+(
+    format game_mode NOT NULL,
+    season INT       NOT NULL,
+    week   INT       NOT NULL,
+    date   DATE      NOT NULL,
+    map    VARCHAR
+);
+
+CREATE UNIQUE INDEX maps_format_season_week_idx
+    ON maps USING BTREE (format, season, week);
+
+CREATE INDEX maps_map_idx
+    ON maps USING BTREE (map);

archiver/src/archive.rs

@@ -6,8 +6,8 @@ use std::str::FromStr;
 use thiserror::Error;
 use tokio_stream::Stream;
 use ugc_scraper_types::{
-    GameMode, MatchInfo, Membership, MembershipRole, NameChange, Record, Region, RosterHistory,
-    Team,
+    Class, GameMode, MapHistory, MatchInfo, Membership, MembershipRole, NameChange, Player, Record,
+    Region, RosterHistory, SteamID, Team,
 };
 
 #[derive(Debug, Error)]
@@ -21,6 +21,8 @@ pub enum ArchiveError {
         description: &'static str,
         error: sqlx::Error,
     },
+    #[error("Error while parsing dates for {format}")]
+    DateFormat { format: GameMode },
 }
 
 pub struct Archive {
@@ -200,6 +202,38 @@ impl Archive {
         }
     }
 
+    pub fn get_players_ids(
+        &self,
+        min: SteamID,
+    ) -> impl Stream<Item = Result<SteamID, ArchiveError>> + use<'_> {
+        query!(
+            "select distinct steam_id from membership_history where steam_id > $1 order by steam_id asc",
+            u64::from(min) as i64,
+        )
+            .fetch(&self.pool)
+            .map_err(|error| ArchiveError::Query {
+                description: "getting player steam ids",
+                error,
+            })
+            .map_ok(|map| (map.steam_id as u64).into())
+    }
+
+    pub async fn get_max_player(&self) -> Result<SteamID, ArchiveError> {
+        if let Some(row) =
+            query!("select steam_id as max from players order by steam_id desc limit 1;")
+                .fetch_optional(&self.pool)
+                .await
+                .map_err(|error| ArchiveError::Query {
+                    description: "getting latest team membership history",
+                    error,
+                })?
+        {
+            Ok((row.max as u64).into())
+        } else {
+            Ok(0.into())
+        }
+    }
+
     pub fn get_no_region_teams(&self) -> impl Stream<Item = Result<u32, ArchiveError>> + use<'_> {
         query!("select id from teams where region IS NULL and format != 'eights' order by id desc")
             .fetch(&self.pool)
@@ -345,4 +379,109 @@ impl Archive {
 
         Ok(())
     }
+
+    pub async fn store_player(&self, player: Player) -> Result<(), ArchiveError> {
+        let mut transaction = self
+            .pool
+            .begin()
+            .await
+            .map_err(|error| ArchiveError::Query {
+                description: "beginning player transaction",
+                error,
+            })?;
+
+        query!(
+            "INSERT INTO players (
+                steam_id, name, avatar, favorite_classes, country
+              ) VALUES ($1, $2, $3, $4, $5)",
+            u64::from(player.steam_id) as i64,
+            player.name,
+            player.avatar,
+            player.favorite_classes as Vec<Class>,
+            player.country,
+        )
+        .execute(&mut *transaction)
+        .await
+        .map_err(|error| ArchiveError::Query {
+            description: "inserting player",
+            error,
+        })?;
+
+        for honors in player.honors.iter() {
+            query!(
+                "INSERT INTO player_honors (
+                steam_id, team_id, season, division, format
+              ) VALUES ($1, $2, $3, $4, $5)",
+                u64::from(player.steam_id) as i64,
+                honors.team.id as i32,
+                honors.season as i16,
+                honors.division,
+                honors.format as GameMode,
+            )
+            .execute(&mut *transaction)
+            .await
+            .map_err(|error| ArchiveError::Query {
+                description: "inserting player honors",
+                error,
+            })?;
+        }
+
+        transaction
+            .commit()
+            .await
+            .map_err(|error| ArchiveError::Query {
+                description: "commiting player transaction",
+                error,
+            })?;
+
+        Ok(())
+    }
+
+    pub async fn store_map_history(
+        &self,
+        format: GameMode,
+        maps: &MapHistory,
+    ) -> Result<(), ArchiveError> {
+        let mut transaction = self
+            .pool
+            .begin()
+            .await
+            .map_err(|error| ArchiveError::Query {
+                description: "beginning map history transaction",
+                error,
+            })?;
+
+        // who knows, the website doesn't say
+        let current_season_year = 2024;
+
+        for week in maps.weeks(current_season_year) {
+            let week = week.map_err(|_| ArchiveError::DateFormat { format })?;
+            query!(
+                "INSERT INTO maps (
+                    format, season, week, date, map
+                  ) VALUES ($1, $2, $3, $4, $5)",
+                format as GameMode,
+                week.season as i32,
+                week.week as i32,
+                week.date,
+                week.map,
+            )
+            .execute(&mut *transaction)
+            .await
+            .map_err(|error| ArchiveError::Query {
+                description: "inserting map history",
+                error,
+            })?;
+        }
+
+        transaction
+            .commit()
+            .await
+            .map_err(|error| ArchiveError::Query {
+                description: "commiting map history transaction",
+                error,
+            })?;
+
+        Ok(())
+    }
 }

archiver/src/client.rs

@@ -2,7 +2,7 @@ use reqwest::{Client, ClientBuilder, Error, Response, StatusCode};
 use serde::de::DeserializeOwned;
 use thiserror::Error;
 use ugc_scraper_types::{
-    GameMode, MapHistory, MatchInfo, MembershipHistory, Player, RosterHistory, Team,
+    GameMode, MapHistory, MatchInfo, MembershipHistory, Player, RosterHistory, SteamID, Team,
     TeamRosterData, TeamSeasonMatch, Transaction,
 };
 
@@ -65,11 +65,14 @@ impl UgcClient {
         self.send_request(Endpoint::TeamMatches { id }).await
     }
 
-    pub async fn get_player(&self, id: u32) -> Result<Player, UgcClientError> {
+    pub async fn get_player(&self, id: SteamID) -> Result<Player, UgcClientError> {
         self.send_request(Endpoint::Player { id }).await
     }
 
-    pub async fn get_player_history(&self, id: u32) -> Result<MembershipHistory, UgcClientError> {
+    pub async fn get_player_history(
+        &self,
+        id: SteamID,
+    ) -> Result<MembershipHistory, UgcClientError> {
         self.send_request(Endpoint::PlayerHistory { id }).await
     }
 
@@ -88,8 +91,8 @@ impl UgcClient {
 #[derive(Debug, Copy, Clone)]
 pub enum Endpoint {
     Match { id: u32 },
-    Player { id: u32 },
-    PlayerHistory { id: u32 },
+    Player { id: SteamID },
+    PlayerHistory { id: SteamID },
     Transactions { format: GameMode },
     Team { id: u32 },
     TeamRoster { id: u32 },
@@ -101,8 +104,10 @@ impl Endpoint {
     pub fn build_url(&self, api_url: &str) -> String {
         match self {
             Endpoint::Match { id } => format!("{}/match/{id}", api_url),
-            Endpoint::Player { id } => format!("{}/player/{id}", api_url),
-            Endpoint::PlayerHistory { id } => format!("{}/player/{id}/history", api_url),
+            Endpoint::Player { id } => format!("{}/player/{}", api_url, u64::from(*id)),
+            Endpoint::PlayerHistory { id } => {
+                format!("{}/player/{}/history", api_url, u64::from(*id))
+            }
             Endpoint::Transactions { format } => format!("{}/transactions/{format}", api_url),
             Endpoint::Team { id } => format!("{}/team/{id}", api_url),
             Endpoint::TeamRoster { id } => format!("{}/team/{id}/roster", api_url),

archiver/src/main.rs

@@ -9,10 +9,12 @@ use clap::{Parser, Subcommand};
 use main_error::MainResult;
 use std::path::PathBuf;
 use std::pin::pin;
+use std::str::FromStr;
 use std::time::Duration;
 use tokio::time::sleep;
 use tokio_stream::StreamExt;
 use tracing::{error, info, span, warn, Level};
+use ugc_scraper_types::GameMode;
 
 #[derive(Debug, Parser)]
 struct Args {
@@ -25,9 +27,11 @@ struct Args {
 #[derive(Debug, Subcommand)]
 enum Command {
     Matches,
+    Players,
     Teams,
     FixupTeams,
     MembershipHistory,
+    MapHistory { format: String },
 }
 
 const LAST_MATCH: u32 = 117047;
@@ -54,6 +58,13 @@ async fn main() -> MainResult {
         Command::MembershipHistory => {
             archive_team_roster_history(&client, &archive).await?;
         }
+        Command::Players => {
+            archive_players(&client, &archive).await?;
+        }
+        Command::MapHistory { format } => {
+            let format = GameMode::from_str(&format)?;
+            archive_map_history(&client, &archive, format).await?;
+        }
     }
     Ok(())
 }
@@ -162,6 +173,43 @@ async fn fixup_teams(client: &UgcClient, archive: &Archive) -> MainResult {
     Ok(())
 }
 
+async fn archive_players(client: &UgcClient, archive: &Archive) -> MainResult {
+    let last = archive.get_max_player().await?;
+    let mut ids = pin!(archive.get_players_ids(last));
+
+    while let Some(Ok(steam_id)) = ids.next().await {
+        let _span = span!(
+            Level::INFO,
+            "archive_player",
+            steam_id = u64::from(steam_id)
+        )
+        .entered();
+        match client.get_player(steam_id).await.check_not_found() {
+            Ok(Some(player)) => {
+                info!("storing player");
+                archive.store_player(player).await?;
+                // panic!();
+            }
+            Ok(None) => {
+                warn!("player not found");
+            }
+            Err(e) => {
+                error!("error fetching player: {:?}", e);
+                panic!();
+            }
+        }
+        sleep(Duration::from_millis(500)).await;
+    }
+    Ok(())
+}
+
+async fn archive_map_history(client: &UgcClient, archive: &Archive, mode: GameMode) -> MainResult {
+    let history = client.get_maps(mode).await?;
+    archive.store_map_history(mode, &history).await?;
+
+    Ok(())
+}
+
 trait NotFoundResultExt<T>: Sized {
     fn check_not_found(self) -> Result<Option<T>, UgcClientError>;
 }