robin/ugc-scaper
1
0
Fork 0
mirror of https://codeberg.org/icewind/ugc-scaper.git synced 2026-06-03 10:14:11 +02:00
Code Issues Projects Releases Packages Wiki Activity

updates

Browse source
This commit is contained in:
Robin Appelman 2026-03-26 23:44:23 +01:00
commit fb383652c8
22 changed files with 3161 additions and 3071 deletions
Download patch file Download diff file Expand all files Collapse all files

1761
archiver/Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

10
archiver/Cargo.toml
View file

@ -1,18 +1,18 @@
[package]
name = "archiver"
version = "0.1.0"
edition = "2021"
edition = "2024"
rust-version = "1.88.0"
[dependencies]
# ugc-scraper-types = "0.1.2"
ugc-scraper-types = { version = "0.2.0", path = "../types", features = ["sqlx"] }
reqwest = { version = "0.12.15", features = ["json"] }
ugc-scraper-types = { version = "0.3.0", path = "../types", features = ["sqlx"] }
reqwest = { version = "0.13.2", features = ["json"] }
clap = { version = "4.5.35", features = ["derive"] }
tracing = "0.1.41"
tracing-subscriber = "0.3.19"
serde = { version = "1.0.219", features = ["derive"] }
serde_json = "1.0.140"
toml = "0.8.20"
toml = "1.1.0"
secretfile = "0.1.0"
tokio = { version = "1.44.2", features = ["macros", "rt-multi-thread"] }
sqlx = { version = "0.8.3", features = ["postgres", "runtime-tokio", "time"] }

23
archiver/src/archive.rs
View file

@ -1,19 +1,20 @@
use crate::steam_id_from_u64;
use futures_util::stream::TryStreamExt;
use serde::{Serialize, Serializer};
use sqlx::postgres::PgConnectOptions;
use sqlx::{query, query_as, Error, Executor, PgPool, Postgres};
use sqlx::{Error, Executor, PgPool, Postgres, query, query_as};
use std::ops::Range;
use std::str::FromStr;
use thiserror::Error;
use time::Date;
use time::format_description::FormatItem;
use time::macros::format_description;
use time::parsing::Parsed;
use time::Date;
use tokio_stream::Stream;
use tracing::{debug, error, warn};
use ugc_scraper_types::{
serde_steam_id_as_string, Class, GameMode, MapHistory, MatchInfo, Membership, MembershipRole,
NameChange, Player, Record, Region, RosterHistory, SteamID, Team, TeamRef, TeamSeason,
Class, GameMode, MapHistory, MatchInfo, Membership, MembershipRole, NameChange, Player, Record,
Region, RosterHistory, SteamID, Team, TeamRef, TeamSeason, serde_steam_id_as_string,
};
const MATCH_DATE_FORMAT: &[FormatItem<'static>] = format_description!(
@ -251,7 +252,7 @@ impl Archive {
description: "getting player steam ids",
error,
})
.map_ok(|map| (map.steam_id as u64).into())
.map_ok(|map| steam_id_from_u64(map.steam_id as u64))
}
pub async fn get_max_player(&self) -> Result<SteamID, ArchiveError> {
@ -264,9 +265,9 @@ impl Archive {
error,
})?
{
Ok((row.max as u64).into())
Ok(steam_id_from_u64(row.max as u64))
} else {
Ok(0.into())
Ok(steam_id_from_u64(0))
}
}
@ -850,7 +851,7 @@ pub struct PlayerData {
impl From<PlayerDataRaw> for PlayerData {
fn from(player: PlayerDataRaw) -> Self {
PlayerData {
steam_id: (player.steam_id as u64).into(),
steam_id: steam_id_from_u64(player.steam_id as u64),
name: player.name,
avatar: player.avatar,
country: player.country,
@ -897,7 +898,7 @@ impl From<MembershipDataRaw> for MembershipData {
fn from(membership: MembershipDataRaw) -> Self {
MembershipData {
team_id: membership.team_id,
steam_id: (membership.steam_id as u64).into(),
steam_id: steam_id_from_u64(membership.steam_id as u64),
role: membership.role,
joined: membership.joined,
left: membership.left,
@ -930,7 +931,9 @@ fn parse_match_date(date: &str, year: i32) -> Date {
}
fn parse_old_match_date(date: &str) -> Result<Date, time::Error> {
const MATCH_DATE_FORMAT_OLD: &[FormatItem<'static>] = format_description!("[weekday case_sensitive:false repr:short], [month padding:none]/[day padding:none]/[year repr:last_two]");
const MATCH_DATE_FORMAT_OLD: &[FormatItem<'static>] = format_description!(
"[weekday case_sensitive:false repr:short], [month padding:none]/[day padding:none]/[year repr:last_two]"
);
let mut parsed = Parsed::new();
parsed.parse_items(date.as_bytes(), MATCH_DATE_FORMAT_OLD)?;

32
archiver/src/main.rs
View file

@ -17,8 +17,8 @@ use std::str::FromStr;
use std::time::Duration;
use tokio::time::sleep;
use tokio_stream::StreamExt;
use tracing::{error, info, instrument, span, warn, Level};
use ugc_scraper_types::GameMode;
use tracing::{Level, error, info, instrument, span, warn};
use ugc_scraper_types::{GameMode, SteamID};
#[derive(Debug, Parser)]
struct Args {
@ -258,18 +258,18 @@ async fn fixup_matches(client: &UgcClient, archive: &Archive) -> MainResult {
for season in matches.seasons.iter() {
for season_match in season.matches.iter() {
if let Some(match_id) = season_match.result.match_id() {
if !archive.has_match(match_id).await? {
warn!(match_id, "match not archived yet");
sleep(Duration::from_millis(500)).await;
if archive_match(client, archive, match_id).await.is_err() {
let match_info = season_match
.match_info(&matches.team, season.format)
.expect("failed to build match info");
assert_eq!(format, match_info.format);
info!("reconstructed match");
archive.store_match(match_id as i32, match_info).await?;
}
if let Some(match_id) = season_match.result.match_id()
&& !archive.has_match(match_id).await?
{
warn!(match_id, "match not archived yet");
sleep(Duration::from_millis(500)).await;
if archive_match(client, archive, match_id).await.is_err() {
let match_info = season_match
.match_info(&matches.team, season.format)
.expect("failed to build match info");
assert_eq!(format, match_info.format);
info!("reconstructed match");
archive.store_match(match_id as i32, match_info).await?;
}
}
}
@ -388,3 +388,7 @@ async fn dump_data(archive: &Archive, data: Data, output: &str) -> MainResult {
writeln!(&mut output, "\n]")?;
Ok(())
}
pub fn steam_id_from_u64(id: u64) -> SteamID {
SteamID::from_steam64(id).unwrap_or_else(|_| SteamID::from_steam64(0).unwrap())
}