robin/ugc-scaper
1
0
Fork 0
mirror of https://codeberg.org/icewind/ugc-scaper.git synced 2026-06-03 10:14:11 +02:00
Code Issues Projects Releases Packages Wiki Activity

more player honors data

Browse source
This commit is contained in:
Robin Appelman 2025-04-16 09:41:39 +02:00
commit b70f2ee336
7 changed files with 491 additions and 213 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/parser/mod.rs
View file

@ -1,5 +1,7 @@
use crate::{ParseError, Result};
use regex::Regex;
use scraper::{ElementRef, Selector};
use std::sync::OnceLock;
use steamid_ng::SteamID;
use time::format_description::FormatItem;
use time::macros::format_description;
@ -103,3 +105,8 @@ fn steam_id_from_link(link: &str) -> Result<SteamID, ParseError> {
})
.map(SteamID::from)
}
static WHITESPACE_REGEX: OnceLock<Regex> = OnceLock::new();
fn whitespace_regex() -> &'static Regex {
WHITESPACE_REGEX.get_or_init(|| Regex::new("[\n\t ]+").unwrap())
}

72
src/parser/player.rs
View file

@ -1,5 +1,5 @@
use super::{ElementExt, Parser};
use crate::data::{Class, Honors, Player, TeamMemberShip, TeamRef};
use crate::data::{Class, GameMode, Honors, Player, TeamMemberShip, TeamRef};
use crate::parser::{select_last_text, select_text, team_id_from_link, DATE_FORMAT};
use crate::{ParseError, Result};
use scraper::{Html, Selector};
@ -10,12 +10,13 @@ use time::Date;
const SELECTOR_PLAYER_NAME: &str = ".container .col-md-4 > h3 > b";
const SELECTOR_PLAYER_ID: &str = r#"a[href*="steam://friends/add"]"#;
const SELECTOR_PLAYER_FLAG: &str = r#"img[data-cfsrc*="/images/flags/"]"#;
const SELECTOR_PLAYER_HONORS_GROUP: &str =
".container .col-md-6:nth-child(2) .white-row-small .row-fluid";
const SELECTOR_PLAYER_HONORS_HEADER: &str = "h5";
const SELECTOR_PLAYER_HONORS_LEAGUE: &str = "li div";
const SELECTOR_PLAYER_HONORS_TEAM: &str = "li small";
const SELECTOR_PLAYER_HONORS_TEAM: &str = "li small a";
const SELECTOR_PLAYER_TEAM_GROUP: &str =
".container .col-md-6:nth-child(1) .white-row-small .row-fluid";
@ -32,6 +33,7 @@ const SELECTOR_CLASS: &str =
pub struct PlayerParser {
selector_name: Selector,
selector_id: Selector,
selector_flag: Selector,
selector_honors_header: Selector,
selector_honors_group: Selector,
@ -59,6 +61,7 @@ impl PlayerParser {
PlayerParser {
selector_name: Selector::parse(SELECTOR_PLAYER_NAME).unwrap(),
selector_id: Selector::parse(SELECTOR_PLAYER_ID).unwrap(),
selector_flag: Selector::parse(SELECTOR_PLAYER_FLAG).unwrap(),
selector_honors_header: Selector::parse(SELECTOR_PLAYER_HONORS_HEADER).unwrap(),
selector_honors_group: Selector::parse(SELECTOR_PLAYER_HONORS_GROUP).unwrap(),
@ -106,6 +109,12 @@ impl Parser for PlayerParser {
.unwrap_or_default()
.to_string();
let country = document
.select(&self.selector_flag)
.next()
.and_then(|e| e.attr("title"))
.map(String::from);
let avatar_element =
document
.select(&self.selector_avatar)
@ -135,22 +144,44 @@ impl Parser for PlayerParser {
})
.map(|((format_res, season), team)| {
let format = format_res?;
Ok(Honors {
format: format.to_string(),
season: season
.first_text()
.ok_or(ParseError::EmptyText {
let game_mode =
GameMode::from_str(format).map_err(|_| ParseError::InvalidText {
text: format.into(),
role: "player honors format",
})?;
let division = season.first_text().ok_or(ParseError::EmptyText {
selector: SELECTOR_PLAYER_HONORS_LEAGUE,
role: "player honors division",
})?;
let season = division
.split(' ')
.nth(1)
.unwrap_or_default()
.parse()
.map_err(|_| ParseError::InvalidText {
text: division.into(),
role: "player honors season",
})?
.to_string(),
team: team
.first_text()
.ok_or(ParseError::EmptyText {
selector: SELECTOR_PLAYER_HONORS_TEAM,
})?;
let team_name = team.first_text().unwrap_or_default().to_string();
let team_link = team.attr("href").unwrap_or_default();
let team_id: u32 = team_link
.rsplit('=')
.next()
.unwrap_or_default()
.parse()
.map_err(|_| ParseError::InvalidLink {
link: team_link.into(),
role: "player honors team",
})?
.to_string(),
})?;
Ok(Honors {
format: game_mode,
division: division.splitn(3, ' ').last().unwrap_or_default().into(),
season,
team: TeamRef {
id: team_id,
name: team_name,
},
})
})
.collect::<Result<Vec<_>>>()?;
@ -178,12 +209,7 @@ impl Parser for PlayerParser {
})?
.attr("href")
.unwrap_or_default();
let name = select_text(item, &self.selector_team_name).ok_or(
ParseError::ElementNotFound {
selector: SELECTOR_PLAYER_TEAM_NAME,
role: "players team name",
},
)?;
let name = select_text(item, &self.selector_team_name).unwrap_or_default();
let league = select_text(item, &self.selector_team_league).ok_or(
ParseError::ElementNotFound {
selector: SELECTOR_PLAYER_TEAM_LEAGUE,
@ -198,8 +224,9 @@ impl Parser for PlayerParser {
)?;
let id = team_id_from_link(link)?;
let since = match since.rsplit_once('\n') {
let since = match since.rsplit_once(['\n', ' ', '\t']) {
Some((_, since)) => {
let since = since.trim();
Date::parse(since, DATE_FORMAT).map_err(|_| ParseError::InvalidDate {
role: "team join date",
date: since.to_string(),
@ -232,6 +259,7 @@ impl Parser for PlayerParser {
honors,
teams,
favorite_classes,
country,
})
}
}

8
src/parser/team.rs
View file

@ -1,13 +1,11 @@
use super::{select_text_empty, ElementExt, Parser};
use super::{select_text_empty, whitespace_regex, ElementExt, Parser};
use crate::data::{Membership, NameChange, Record, Team};
use crate::parser::{
select_text, steam_id_from_link, DATE_FORMAT, MEMBER_DATE_ALT_FORMAT, MEMBER_DATE_FORMAT,
};
use crate::{ParseError, Result, ScrapeError};
use regex::Regex;
use scraper::{Html, Selector};
use std::str::FromStr;
use std::sync::OnceLock;
use time::{Date, PrimitiveDateTime, Time, UtcOffset};
use ugc_scraper_types::{GameMode, MembershipRole, Region};
@ -118,13 +116,11 @@ impl TeamParser {
}
}
static WHITESPACE_REGEX: OnceLock<Regex> = OnceLock::new();
impl Parser for TeamParser {
type Output = Team;
fn parse(&self, document: &str) -> Result<Self::Output> {
let whitespace_regex = WHITESPACE_REGEX.get_or_init(|| Regex::new("[\n\t ]+").unwrap());
let whitespace_regex = whitespace_regex();
let document = Html::parse_document(document);
let root = document.root_element();

190
tests/snapshots/snapshot__parse_player_classes_html.snap
View file

@ -8,99 +8,175 @@ expression: parsed
"steam_id": "76561198049312442",
"honors": [
{
"format": "TF2 Highlander",
"season": "Season 32 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 32,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 31 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 31,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 30 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 30,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 29 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 29,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 28 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 28,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 27 Premium EU",
"team": "Xenon"
"format": "9v9",
"division": "Premium EU",
"season": 27,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 26 Premium EU",
"team": "Xenon"
"format": "9v9",
"division": "Premium EU",
"season": 26,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 25 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 25,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 24 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 24,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 23 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 23,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 22 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 22,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 21 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 21,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 20 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 20,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 19 Euro Gold",
"team": "Xenon"
"format": "9v9",
"division": "Euro Gold",
"season": 19,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 18 Euro Silver",
"team": "Xenon"
"format": "9v9",
"division": "Euro Silver",
"season": 18,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 17 Euro Silver",
"team": "Xenon"
"format": "9v9",
"division": "Euro Silver",
"season": 17,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 6vs6",
"season": "Season 32 Europe",
"team": "Ulku Ocaklari"
"format": "6v6",
"division": "Europe",
"season": 32,
"team": {
"name": "Ulku Ocaklari",
"id": 29445
}
},
{
"format": "TF2 6vs6",
"season": "Season 31 Europe",
"team": "Ulku Ocaklari"
"format": "6v6",
"division": "Europe",
"season": 31,
"team": {
"name": "Ulku Ocaklari",
"id": 29445
}
},
{
"format": "TF2 6vs6",
"season": "Season 30 Europe",
"team": "Ulku Ocaklari"
"format": "6v6",
"division": "Europe",
"season": 30,
"team": {
"name": "Ulku Ocaklari",
"id": 29445
}
}
],
"teams": [

10
tests/snapshots/snapshot__parse_player_empty_html.snap
View file

@ -8,9 +8,13 @@ expression: parsed
"steam_id": "76561197967332647",
"honors": [
{
"format": "TF2 Highlander",
"season": "Season 6 NA Steel",
"team": "Penguin Doom Squad"
"format": "9v9",
"division": "NA Steel",
"season": 6,
"team": {
"name": "Penguin Doom Squad",
"id": 3975
}
}
],
"teams": [],

400
tests/snapshots/snapshot__parse_player_html.snap
View file

@ -8,204 +8,364 @@ expression: parsed
"steam_id": "76561198024494988",
"honors": [
{
"format": "TF2 Highlander",
"season": "Season 32 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 32,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 31 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 31,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 30 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 30,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 29 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 29,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 28 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 28,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 27 Premium EU",
"team": "Xenon"
"format": "9v9",
"division": "Premium EU",
"season": 27,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 26 Premium EU",
"team": "Xenon"
"format": "9v9",
"division": "Premium EU",
"season": 26,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 25 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 25,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 24 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 24,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 23 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 23,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 22 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 22,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 21 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 21,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 20 Euro Platinum",
"team": "Xenon"
"format": "9v9",
"division": "Euro Platinum",
"season": 20,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 19 Euro Gold",
"team": "Xenon"
"format": "9v9",
"division": "Euro Gold",
"season": 19,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 18 Euro Silver",
"team": "Xenon"
"format": "9v9",
"division": "Euro Silver",
"season": 18,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 17 Euro Silver",
"team": "Xenon"
"format": "9v9",
"division": "Euro Silver",
"season": 17,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 16 Euro Silver",
"team": "Xenon"
"format": "9v9",
"division": "Euro Silver",
"season": 16,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 15 Euro Silver",
"team": "Xenon"
"format": "9v9",
"division": "Euro Silver",
"season": 15,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 14 Euro Silver",
"team": "Xenon"
"format": "9v9",
"division": "Euro Silver",
"season": 14,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 13 Euro Silver",
"team": "Xenon"
"format": "9v9",
"division": "Euro Silver",
"season": 13,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 12 Euro Silver",
"team": "Xenon"
"format": "9v9",
"division": "Euro Silver",
"season": 12,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 Highlander",
"season": "Season 11 Euro Steel",
"team": "Xenon"
"format": "9v9",
"division": "Euro Steel",
"season": 11,
"team": {
"name": "Xenon",
"id": 7861
}
},
{
"format": "TF2 6vs6",
"season": "Season 35 Europe",
"team": "Controller Gamers"
"format": "6v6",
"division": "Europe",
"season": 35,
"team": {
"name": "Controller Gamers",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 34 Europe",
"team": "Bye week"
"format": "6v6",
"division": "Europe",
"season": 34,
"team": {
"name": "Bye week",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 33 Europe",
"team": "meta.tf"
"format": "6v6",
"division": "Europe",
"season": 33,
"team": {
"name": "meta.tf",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 32 Europe",
"team": "meta.tf"
"format": "6v6",
"division": "Europe",
"season": 32,
"team": {
"name": "meta.tf",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 31 Europe",
"team": "meta.tf"
"format": "6v6",
"division": "Europe",
"season": 31,
"team": {
"name": "meta.tf",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 30 Europe",
"team": "meta.tf"
"format": "6v6",
"division": "Europe",
"season": 30,
"team": {
"name": "meta.tf",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 29 Europe",
"team": "meta.tf"
"format": "6v6",
"division": "Europe",
"season": 29,
"team": {
"name": "meta.tf",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 28 Europe",
"team": "meta.tf"
"format": "6v6",
"division": "Europe",
"season": 28,
"team": {
"name": "meta.tf",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 27 Europe",
"team": "meta.tf"
"format": "6v6",
"division": "Europe",
"season": 27,
"team": {
"name": "meta.tf",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 26 Europe",
"team": "meta.tf"
"format": "6v6",
"division": "Europe",
"season": 26,
"team": {
"name": "meta.tf",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 25 Europe",
"team": "meta.tf"
"format": "6v6",
"division": "Europe",
"season": 25,
"team": {
"name": "meta.tf",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 24 Europe",
"team": "meta.tf"
"format": "6v6",
"division": "Europe",
"season": 24,
"team": {
"name": "meta.tf",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 23 Europe",
"team": "Giel and the 9wiels"
"format": "6v6",
"division": "Europe",
"season": 23,
"team": {
"name": "Giel and the 9wiels",
"id": 6929
}
},
{
"format": "TF2 6vs6",
"season": "Season 19 Euro Platinum",
"team": "sExy eSports"
"format": "6v6",
"division": "Euro Platinum",
"season": 19,
"team": {
"name": "sExy eSports",
"id": 17736
}
},
{
"format": "TF2 6vs6",
"season": "Season 18 Euro Steel",
"team": "BigHorseDong"
"format": "6v6",
"division": "Euro Steel",
"season": 18,
"team": {
"name": "BigHorseDong",
"id": 16277
}
},
{
"format": "TF2 6vs6",
"season": "Season 13 Euro Steel",
"team": "Necronoms"
"format": "6v6",
"division": "Euro Steel",
"season": 13,
"team": {
"name": "Necronoms",
"id": 8622
}
},
{
"format": "TF2 4vs4",
"season": "Season 6 Silver Euro",
"team": "sExy eSports"
"format": "4v4",
"division": "Silver Euro",
"season": 6,
"team": {
"name": "sExy eSports",
"id": 17790
}
},
{
"format": "TF2 4vs4",
"season": "Season 5 Silver Euro",
"team": "sExy eSports"
"format": "4v4",
"division": "Silver Euro",
"season": 5,
"team": {
"name": "sExy eSports",
"id": 17790
}
}
],
"teams": [

13
types/src/lib.rs
View file

@ -69,14 +69,16 @@ pub struct Player {
pub honors: Vec<Honors>,
pub teams: Vec<TeamMemberShip>,
pub favorite_classes: Vec<Class>,
pub country: Option<String>,
}
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct Honors {
pub format: String,
pub season: String,
pub team: String,
pub format: GameMode,
pub division: String,
pub season: u8,
pub team: TeamRef,
}
#[derive(Debug, Clone)]
@ -128,11 +130,15 @@ pub struct Team {
#[derive(Debug, Clone, Copy)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[cfg_attr(feature = "serde", serde(rename_all = "lowercase"))]
#[cfg_attr(feature = "sqlx", derive(sqlx::Type))]
#[cfg_attr(feature = "sqlx", sqlx(type_name = "player_class"))]
#[cfg_attr(feature = "sqlx", sqlx(rename_all = "lowercase"))]
pub enum Class {
Scout,
Soldier,
Pyro,
Demoman,
Engineer,
Heavy,
Medic,
Sniper,
@ -148,6 +154,7 @@ impl FromStr for Class {
"soldier" => Ok(Class::Soldier),
"pyro" => Ok(Class::Pyro),
"demoman" => Ok(Class::Demoman),
"engineer" => Ok(Class::Engineer),
"heavy" => Ok(Class::Heavy),
"medic" => Ok(Class::Medic),
"sniper" => Ok(Class::Sniper),