robin/ugc-scaper
1
0
Fork 0
mirror of https://codeberg.org/icewind/ugc-scaper.git synced 2026-06-03 10:14:11 +02:00
Code Issues Projects Releases Packages Wiki Activity

map history

Browse source
This commit is contained in:
Robin Appelman 2025-04-18 19:06:39 +02:00
commit d4311dc9df
10 changed files with 377 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

25
archiver/.sqlx/query-104cec686e47c560a136e7ac479fc6a301ae08c7b339e18dda446d3a0990f655.json generated Normal file
View file

@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO player_honors (\n steam_id, team_id, season, division, format\n ) VALUES ($1, $2, $3, $4, $5)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Int4",
"Int2",
"Varchar",
{
"Custom": {
"name": "game_mode",
"kind": {
"Enum": ["highlander", "eights", "sixes", "fours", "ultiduo"]
}
}
}
]
},
"nullable": []
},
"hash": "104cec686e47c560a136e7ac479fc6a301ae08c7b339e18dda446d3a0990f655"
}

18
archiver/.sqlx/query-133b2606adb67687107f717ec04114683e8ad6d6e9bb3177561971f0f12335f3.json generated Normal file
View file

@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "select steam_id as max from players order by steam_id desc limit 1;",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "max",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [false]
},
"hash": "133b2606adb67687107f717ec04114683e8ad6d6e9bb3177561971f0f12335f3"
}

42
archiver/.sqlx/query-2df08faad317dd3f037d2fde18794a1a3b1c7b154118f3fca1f26ae2e3f9381c.json generated Normal file
View file

@ -0,0 +1,42 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO players (\n steam_id, name, avatar, favorite_classes, country\n ) VALUES ($1, $2, $3, $4, $5)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Varchar",
"Varchar",
{
"Custom": {
"name": "player_class[]",
"kind": {
"Array": {
"Custom": {
"name": "player_class",
"kind": {
"Enum": [
"scout",
"soldier",
"pyro",
"demoman",
"engineer",
"heavy",
"medic",
"sniper",
"spy"
]
}
}
}
}
}
},
"Varchar"
]
},
"nullable": []
},
"hash": "2df08faad317dd3f037d2fde18794a1a3b1c7b154118f3fca1f26ae2e3f9381c"
}

18
archiver/.sqlx/query-d06507f78e0ac005f9e978816ffea6441d19232d08b3205993ee76b86c3e558b.json generated Normal file
View file

@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "select distinct steam_id from membership_history where steam_id > $1 order by steam_id asc",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "steam_id",
"type_info": "Int8"
}
],
"parameters": {
"Left": ["Int8"]
},
"nullable": [false]
},
"hash": "d06507f78e0ac005f9e978816ffea6441d19232d08b3205993ee76b86c3e558b"
}

3
archiver/migrations/20250416070625_player.sql
View file

@ -5,7 +5,8 @@ CREATE TABLE players
steam_id BIGINT NOT NULL,
name VARCHAR NOT NULL,
avatar VARCHAR,
favorite_classes player_class[] NOT NULL
favorite_classes player_class[] NOT NULL,
country VARCHAR
);
CREATE UNIQUE INDEX players_steam_id_idx

14
archiver/migrations/20250418154032_maps.sql Normal file
View file

@ -0,0 +1,14 @@
CREATE TABLE maps
(
format game_mode NOT NULL,
season INT NOT NULL,
week INT NOT NULL,
date DATE NOT NULL,
map VARCHAR
);
CREATE UNIQUE INDEX maps_format_season_week_idx
ON maps USING BTREE (format, season, week);
CREATE INDEX maps_map_idx
ON maps USING BTREE (map);

143
archiver/src/archive.rs
View file

@ -6,8 +6,8 @@ use std::str::FromStr;
use thiserror::Error;
use tokio_stream::Stream;
use ugc_scraper_types::{
GameMode, MatchInfo, Membership, MembershipRole, NameChange, Record, Region, RosterHistory,
Team,
Class, GameMode, MapHistory, MatchInfo, Membership, MembershipRole, NameChange, Player, Record,
Region, RosterHistory, SteamID, Team,
};
#[derive(Debug, Error)]
@ -21,6 +21,8 @@ pub enum ArchiveError {
description: &'static str,
error: sqlx::Error,
},
#[error("Error while parsing dates for {format}")]
DateFormat { format: GameMode },
}
pub struct Archive {
@ -200,6 +202,38 @@ impl Archive {
}
}
pub fn get_players_ids(
&self,
min: SteamID,
) -> impl Stream<Item = Result<SteamID, ArchiveError>> + use<'_> {
query!(
"select distinct steam_id from membership_history where steam_id > $1 order by steam_id asc",
u64::from(min) as i64,
)
.fetch(&self.pool)
.map_err(|error| ArchiveError::Query {
description: "getting player steam ids",
error,
})
.map_ok(|map| (map.steam_id as u64).into())
}
pub async fn get_max_player(&self) -> Result<SteamID, ArchiveError> {
if let Some(row) =
query!("select steam_id as max from players order by steam_id desc limit 1;")
.fetch_optional(&self.pool)
.await
.map_err(|error| ArchiveError::Query {
description: "getting latest team membership history",
error,
})?
{
Ok((row.max as u64).into())
} else {
Ok(0.into())
}
}
pub fn get_no_region_teams(&self) -> impl Stream<Item = Result<u32, ArchiveError>> + use<'_> {
query!("select id from teams where region IS NULL and format != 'eights' order by id desc")
.fetch(&self.pool)
@ -345,4 +379,109 @@ impl Archive {
Ok(())
}
pub async fn store_player(&self, player: Player) -> Result<(), ArchiveError> {
let mut transaction = self
.pool
.begin()
.await
.map_err(|error| ArchiveError::Query {
description: "beginning player transaction",
error,
})?;
query!(
"INSERT INTO players (
steam_id, name, avatar, favorite_classes, country
) VALUES ($1, $2, $3, $4, $5)",
u64::from(player.steam_id) as i64,
player.name,
player.avatar,
player.favorite_classes as Vec<Class>,
player.country,
)
.execute(&mut *transaction)
.await
.map_err(|error| ArchiveError::Query {
description: "inserting player",
error,
})?;
for honors in player.honors.iter() {
query!(
"INSERT INTO player_honors (
steam_id, team_id, season, division, format
) VALUES ($1, $2, $3, $4, $5)",
u64::from(player.steam_id) as i64,
honors.team.id as i32,
honors.season as i16,
honors.division,
honors.format as GameMode,
)
.execute(&mut *transaction)
.await
.map_err(|error| ArchiveError::Query {
description: "inserting player honors",
error,
})?;
}
transaction
.commit()
.await
.map_err(|error| ArchiveError::Query {
description: "commiting player transaction",
error,
})?;
Ok(())
}
pub async fn store_map_history(
&self,
format: GameMode,
maps: &MapHistory,
) -> Result<(), ArchiveError> {
let mut transaction = self
.pool
.begin()
.await
.map_err(|error| ArchiveError::Query {
description: "beginning map history transaction",
error,
})?;
// who knows, the website doesn't say
let current_season_year = 2024;
for week in maps.weeks(current_season_year) {
let week = week.map_err(|_| ArchiveError::DateFormat { format })?;
query!(
"INSERT INTO maps (
format, season, week, date, map
) VALUES ($1, $2, $3, $4, $5)",
format as GameMode,
week.season as i32,
week.week as i32,
week.date,
week.map,
)
.execute(&mut *transaction)
.await
.map_err(|error| ArchiveError::Query {
description: "inserting map history",
error,
})?;
}
transaction
.commit()
.await
.map_err(|error| ArchiveError::Query {
description: "commiting map history transaction",
error,
})?;
Ok(())
}
}

19
archiver/src/client.rs
View file

@ -2,7 +2,7 @@ use reqwest::{Client, ClientBuilder, Error, Response, StatusCode};
use serde::de::DeserializeOwned;
use thiserror::Error;
use ugc_scraper_types::{
GameMode, MapHistory, MatchInfo, MembershipHistory, Player, RosterHistory, Team,
GameMode, MapHistory, MatchInfo, MembershipHistory, Player, RosterHistory, SteamID, Team,
TeamRosterData, TeamSeasonMatch, Transaction,
};
@ -65,11 +65,14 @@ impl UgcClient {
self.send_request(Endpoint::TeamMatches { id }).await
}
pub async fn get_player(&self, id: u32) -> Result<Player, UgcClientError> {
pub async fn get_player(&self, id: SteamID) -> Result<Player, UgcClientError> {
self.send_request(Endpoint::Player { id }).await
}
pub async fn get_player_history(&self, id: u32) -> Result<MembershipHistory, UgcClientError> {
pub async fn get_player_history(
&self,
id: SteamID,
) -> Result<MembershipHistory, UgcClientError> {
self.send_request(Endpoint::PlayerHistory { id }).await
}
@ -88,8 +91,8 @@ impl UgcClient {
#[derive(Debug, Copy, Clone)]
pub enum Endpoint {
Match { id: u32 },
Player { id: u32 },
PlayerHistory { id: u32 },
Player { id: SteamID },
PlayerHistory { id: SteamID },
Transactions { format: GameMode },
Team { id: u32 },
TeamRoster { id: u32 },
@ -101,8 +104,10 @@ impl Endpoint {
pub fn build_url(&self, api_url: &str) -> String {
match self {
Endpoint::Match { id } => format!("{}/match/{id}", api_url),
Endpoint::Player { id } => format!("{}/player/{id}", api_url),
Endpoint::PlayerHistory { id } => format!("{}/player/{id}/history", api_url),
Endpoint::Player { id } => format!("{}/player/{}", api_url, u64::from(*id)),
Endpoint::PlayerHistory { id } => {
format!("{}/player/{}/history", api_url, u64::from(*id))
}
Endpoint::Transactions { format } => format!("{}/transactions/{format}", api_url),
Endpoint::Team { id } => format!("{}/team/{id}", api_url),
Endpoint::TeamRoster { id } => format!("{}/team/{id}/roster", api_url),

48
archiver/src/main.rs
View file

@ -9,10 +9,12 @@ use clap::{Parser, Subcommand};
use main_error::MainResult;
use std::path::PathBuf;
use std::pin::pin;
use std::str::FromStr;
use std::time::Duration;
use tokio::time::sleep;
use tokio_stream::StreamExt;
use tracing::{error, info, span, warn, Level};
use ugc_scraper_types::GameMode;
#[derive(Debug, Parser)]
struct Args {
@ -25,9 +27,11 @@ struct Args {
#[derive(Debug, Subcommand)]
enum Command {
Matches,
Players,
Teams,
FixupTeams,
MembershipHistory,
MapHistory { format: String },
}
const LAST_MATCH: u32 = 117047;
@ -54,6 +58,13 @@ async fn main() -> MainResult {
Command::MembershipHistory => {
archive_team_roster_history(&client, &archive).await?;
}
Command::Players => {
archive_players(&client, &archive).await?;
}
Command::MapHistory { format } => {
let format = GameMode::from_str(&format)?;
archive_map_history(&client, &archive, format).await?;
}
}
Ok(())
}
@ -162,6 +173,43 @@ async fn fixup_teams(client: &UgcClient, archive: &Archive) -> MainResult {
Ok(())
}
async fn archive_players(client: &UgcClient, archive: &Archive) -> MainResult {
let last = archive.get_max_player().await?;
let mut ids = pin!(archive.get_players_ids(last));
while let Some(Ok(steam_id)) = ids.next().await {
let _span = span!(
Level::INFO,
"archive_player",
steam_id = u64::from(steam_id)
)
.entered();
match client.get_player(steam_id).await.check_not_found() {
Ok(Some(player)) => {
info!("storing player");
archive.store_player(player).await?;
// panic!();
}
Ok(None) => {
warn!("player not found");
}
Err(e) => {
error!("error fetching player: {:?}", e);
panic!();
}
}
sleep(Duration::from_millis(500)).await;
}
Ok(())
}
async fn archive_map_history(client: &UgcClient, archive: &Archive, mode: GameMode) -> MainResult {
let history = client.get_maps(mode).await?;
archive.store_map_history(mode, &history).await?;
Ok(())
}
trait NotFoundResultExt<T>: Sized {
fn check_not_found(self) -> Result<Option<T>, UgcClientError>;
}

66
types/src/lib.rs
View file

@ -3,6 +3,9 @@ use std::fmt::Display;
use std::str::FromStr;
pub use steamid_ng::SteamID;
use thiserror::Error;
use time::error::Parse;
use time::format_description::FormatItem;
use time::macros::format_description;
use time::{Date, OffsetDateTime};
#[cfg(feature = "serde")]
@ -319,6 +322,10 @@ pub struct MatchInfo {
pub team_away: TeamRef,
pub score_home: u8,
pub score_away: u8,
pub map: String,
pub week: u8,
pub format: GameMode,
pub default_date: Date,
}
#[derive(Debug, Clone, Error)]
@ -396,14 +403,14 @@ impl GameMode {
}
pub fn is_tf2(&self) -> bool {
match self {
GameMode::Highlander => true,
GameMode::Eights => true,
GameMode::Sixes => true,
GameMode::Fours => true,
GameMode::Ultiduo => true,
_ => false,
}
matches!(
self,
GameMode::Highlander
| GameMode::Eights
| GameMode::Sixes
| GameMode::Fours
| GameMode::Ultiduo
)
}
}
@ -458,7 +465,7 @@ impl FromStr for Region {
type Err = InvalidRegion;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let s = s.trim_matches(&['*', '(', ')']);
let s = s.trim_matches(['*', '(', ')']);
match s {
"Euro" => Ok(Region::Europe),
"Europe" => Ok(Region::Europe),
@ -533,6 +540,47 @@ pub struct MapHistory {
pub previous: Vec<PreviousSeasonMapList>,
}
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct Week<'a> {
pub season: u8,
pub week: u8,
pub map: &'a str,
#[cfg_attr(feature = "serde", serde(with = "serde_date"))]
pub date: Date,
}
impl MapHistory {
pub fn weeks(&self, current_season_year: u16) -> impl Iterator<Item = Result<Week, Parse>> {
const CURRENT_DATE_FORMAT: &[FormatItem<'static>] = format_description!("[weekday case_sensitive:false repr:short], [month repr:short] [day padding:none] [year]");
let current_season = self.current.maps.iter().map(move |map| {
Ok(Week {
season: self.current.season,
week: map.week,
map: map.map.as_str(),
date: Date::parse(
&format!("{} {current_season_year}", map.date),
CURRENT_DATE_FORMAT,
)?,
})
});
let past_seasons = self
.previous
.iter()
.flat_map(|season| season.maps.iter().map(|map| (season.season, map)))
.map(|(season, map)| {
Ok(Week {
season,
week: map.week,
map: map.map.as_str(),
date: map.date,
})
});
current_season.chain(past_seasons)
}
}
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct CurrentSeasonMapList {