robin/ugc-scaper
1
0
Fork 0
mirror of https://codeberg.org/icewind/ugc-scaper.git synced 2026-06-03 10:14:11 +02:00
Code Issues Projects Releases Packages Wiki Activity

improve not found team detection

Browse source
This commit is contained in:
Robin Appelman 2025-04-21 16:39:01 +02:00
commit cf8de44bbb
7 changed files with 42 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

1
archiver/src/main.rs
View file

@ -228,6 +228,7 @@ async fn archive_map_history(client: &UgcClient, archive: &Archive, mode: GameMo
async fn fixup_matches(client: &UgcClient, archive: &Archive) -> MainResult {
let min_team = archive.get_min_team_id_without_match_seasons().await?;
dbg!(min_team);
let mut team_ids = pin!(archive.get_team_ids(min_team - 1));
while let Some(Ok(team_id)) = team_ids.next().await {

12
src/parser/match_page.rs
View file

@ -6,7 +6,7 @@ use scraper::{Html, Selector};
const SELECTOR_MATCH_FORMAT: &str = "h3.page-header > strong.styleColor";
const SELECTOR_MATCH_COMMENT_AUTHOR: &str = ".row-fluid .col-md-12 span.text-success";
const SELECTOR_MATCH_COMMENT: &str = ".row-fluid .col-md-12 > .white-row-light-small > p";
const SELECTOR_MATCH_COMMENT: &str = ".row-fluid .col-md-12 > .text-center > p";
const SELECTOR_MATCH_TEAM_LINK: &str = "a[href^=\"team_page\"]:not(.btn-large)";
const SELECTOR_MATCH_RESULT_TEAM: &str =
".table.table-condensed.table-bordered tr:nth-child(2) td:nth-child(1)";
@ -119,10 +119,7 @@ impl Parser for MatchPageParser {
role: "home team link",
})?
.first_text()
.ok_or(ParseError::EmptyText {
role: "home team name",
selector: SELECTOR_MATCH_RESULT_TEAM,
})?
.unwrap_or_default()
.to_string();
let team_name_away = team_names
.next()
@ -131,10 +128,7 @@ impl Parser for MatchPageParser {
role: "away team link",
})?
.first_text()
.ok_or(ParseError::EmptyText {
role: "away team name",
selector: SELECTOR_MATCH_RESULT_TEAM,
})?
.unwrap_or_default()
.to_string();
let mut team_scores = document.select(&self.selector_result_score);

9
src/parser/team.rs
View file

@ -132,13 +132,14 @@ impl Parser for TeamParser {
.unwrap_or_default()
.to_string();
match (tag.as_str(), name.as_str()) {
("", "") => return Err(ScrapeError::NotFound),
(_, "") => name = tag.clone(),
let image = document.select(&self.selector_image).next();
match (tag.as_str(), name.as_str(), image.is_some()) {
("", "", false) => return Err(ScrapeError::NotFound),
(_, "", true) => name = tag.clone(),
_ => {}
};
let image = document.select(&self.selector_image).next();
let image = image.and_then(|image| {
image
.attr("data-cfsrc")

8
src/parser/team_matches.rs
View file

@ -267,12 +267,8 @@ impl Parser for TeamMatchesParser {
role: "match team link",
})?;
let team_name = select_text(document.root_element(), &self.selector_team_name).ok_or(
ParseError::ElementNotFound {
selector: SELECTOR_TEAM_NAME,
role: "match team name",
},
)?;
let team_name =
select_text(document.root_element(), &self.selector_team_name).unwrap_or_default();
let team = TeamRef {
id: team_id,
name: team_name.into(),

2
tests/snapshots/snapshot__parse_match_html.snap
View file

@ -3,7 +3,7 @@ source: tests/snapshot.rs
expression: parsed
---
{
"comment": "https://logs.tf/3509421#76561198288857894\r\nhttps://logs.tf/3509435#76561198288857894",
"comment": null,
"comment_author": "Vkid E-sports:",
"team_home": {
"name": "Vkid E-Sports",

36
tests/snapshots/snapshot__parse_team_matches_old_html.snap
View file

@ -22,7 +22,9 @@ expression: parsed
"opponent": {
"name": "[$0.00]",
"id": 2137
}
},
"score": 0,
"score_opponent": 0
},
"map": "cp_gravelpit"
},
@ -36,7 +38,9 @@ expression: parsed
"opponent": {
"name": "-DeX-",
"id": 2130
}
},
"score": 0,
"score_opponent": 0
},
"map": "cp_gravelpit"
},
@ -50,7 +54,9 @@ expression: parsed
"opponent": {
"name": "[UGS]",
"id": 2127
}
},
"score": 3,
"score_opponent": 0
},
"map": "cp_dustbowl"
},
@ -74,7 +80,9 @@ expression: parsed
"opponent": {
"name": ".tKd",
"id": 2169
}
},
"score": 0,
"score_opponent": 0
},
"map": "ctf_impact"
},
@ -88,7 +96,9 @@ expression: parsed
"opponent": {
"name": "[FOoM]",
"id": 2172
}
},
"score": 0,
"score_opponent": 0
},
"map": "cp_science2"
},
@ -102,7 +112,9 @@ expression: parsed
"opponent": {
"name": "[SWAT]",
"id": 2188
}
},
"score": 1,
"score_opponent": 0
},
"map": "cp_junction"
},
@ -116,7 +128,9 @@ expression: parsed
"opponent": {
"name": "[FOoM]",
"id": 2172
}
},
"score": 0,
"score_opponent": 0
},
"map": "cp_castle3"
},
@ -130,7 +144,9 @@ expression: parsed
"opponent": {
"name": "[=1=]",
"id": 2197
}
},
"score": 0,
"score_opponent": 0
},
"map": "cp_dustbowl"
},
@ -144,7 +160,9 @@ expression: parsed
"opponent": {
"name": "[FOoM]",
"id": 2172
}
},
"score": 0,
"score_opponent": 0
},
"map": "cp_science"
}

3
types/src/lib.rs
View file

@ -466,14 +466,17 @@ impl FromStr for GameMode {
"1v1" => Ok(GameMode::Ones),
"Highlander" => Ok(GameMode::Highlander),
"TF2 Highlander" => Ok(GameMode::Highlander),
"TF2-H" => Ok(GameMode::Highlander),
"ASIA TF2-H" => Ok(GameMode::Highlander),
"ASIA TF2-6" => Ok(GameMode::Sixes),
"ASIA TF2-4" => Ok(GameMode::Fours),
"TF2 8vs8" => Ok(GameMode::Eights),
"8vs8" => Ok(GameMode::Eights),
"TF2 6vs6" => Ok(GameMode::Sixes),
"TF2-6" => Ok(GameMode::Sixes),
"6vs6" => Ok(GameMode::Sixes),
"TF2 4vs4" => Ok(GameMode::Fours),
"TF2-4" => Ok(GameMode::Fours),
"4vs4" => Ok(GameMode::Fours),
"TF2 2vs2" => Ok(GameMode::Ultiduo),
"2vs2" => Ok(GameMode::Ultiduo),