robin/ugc-scaper
1
0
Fork 0
mirror of https://codeberg.org/icewind/ugc-scaper.git synced 2026-06-03 10:14:11 +02:00
Code Issues Projects Releases Packages Wiki Activity

previous season map history

Browse source
This commit is contained in:
Robin Appelman 2023-11-21 20:20:30 +01:00
commit 62b8ff9b5c
6 changed files with 7974 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/data.rs
View file

@ -283,7 +283,7 @@ pub struct CurrentSeasonMapList {
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
pub struct PreviousSeasonMapList {
pub season: u8,
pub maps: Vec<PreviousSeasonMapList>,
pub maps: Vec<PreviousSeasonMap>,
}
#[derive(Debug, Clone)]

100
src/parser/map_history.rs
View file

@ -1,8 +1,11 @@
use super::Parser;
use crate::data::{CurrentSeasonMap, CurrentSeasonMapList, MapHistory};
use crate::parser::select_text;
use crate::data::{
CurrentSeasonMap, CurrentSeasonMapList, MapHistory, PreviousSeasonMap, PreviousSeasonMapList,
};
use crate::parser::{select_text, ElementExt};
use crate::{ParseError, Result};
use scraper::{Html, Selector};
use time::{Date, Month};
const SELECTOR_CURRENT_ROW: &str = "table.table.table-condensed.table-responsive tbody tr";
const SELECTOR_CURRENT_SEASON: &str = "div.row > div > div.white-row-small > h5:nth-child(2), div.row-fluid > div > div.white-row-small > h4:first-child+h5";
@ -11,8 +14,11 @@ const SELECTOR_CURRENT_MAP: &str = "td:nth-child(2)";
const SELECTOR_CURRENT_DATE: &str = "td:nth-child(4) small";
const SELECTOR_CURRENT_DATE_ALT: &str = "td:nth-child(5) small";
const SELECTOR_PREVIOUS: &str = "table.table.table-condensed.table-bordered";
const SELECTOR_PREVIOUS_SEASON: &str = "tr.top-bar td h3.text-info";
const SELECTOR_PREVIOUS: &str =
"table.table.table-condensed.table-bordered tbody tr:not(:first-child)";
const SELECTOR_PREVIOUS_WEEK: &str = "td:nth-child(1)";
const SELECTOR_PREVIOUS_DATE: &str = "td:nth-child(2)";
const SELECTOR_PREVIOUS_MAP: &str = "td:nth-child(3)";
pub struct MapHistoryParser {
selector_current_row: Selector,
@ -21,6 +27,11 @@ pub struct MapHistoryParser {
selector_current_map: Selector,
selector_current_date: Selector,
selector_current_date_alt: Selector,
selector_previous: Selector,
selector_previous_week: Selector,
selector_previous_date: Selector,
selector_previous_map: Selector,
}
impl Default for MapHistoryParser {
@ -38,6 +49,11 @@ impl MapHistoryParser {
selector_current_map: Selector::parse(SELECTOR_CURRENT_MAP).unwrap(),
selector_current_date: Selector::parse(SELECTOR_CURRENT_DATE).unwrap(),
selector_current_date_alt: Selector::parse(SELECTOR_CURRENT_DATE_ALT).unwrap(),
selector_previous: Selector::parse(SELECTOR_PREVIOUS).unwrap(),
selector_previous_week: Selector::parse(SELECTOR_PREVIOUS_WEEK).unwrap(),
selector_previous_date: Selector::parse(SELECTOR_PREVIOUS_DATE).unwrap(),
selector_previous_map: Selector::parse(SELECTOR_PREVIOUS_MAP).unwrap(),
}
}
}
@ -101,12 +117,86 @@ impl Parser for MapHistoryParser {
})
.collect::<Result<_>>()?;
let mut previous = Vec::with_capacity(8);
let mut prev_season = None;
for row in document.select(&self.selector_previous) {
if row.attr("class") == Some("top-bar") {
if let Some(season) = prev_season.take() {
previous.push(season);
}
let season = row
.first_text()
.unwrap_or_default()
.trim_start_matches("Season ");
dbg!(season);
let season = season.parse().map_err(|_| ParseError::InvalidText {
role: "previous season number",
text: season.to_string(),
})?;
prev_season = Some(PreviousSeasonMapList {
season,
maps: Vec::with_capacity(8),
});
} else if row
.children()
.filter(|child| child.value().is_element())
.count()
== 3
{
if let Some(season) = prev_season.as_mut() {
let week = select_text(row, &self.selector_previous_week).ok_or(
ParseError::ElementNotFound {
selector: SELECTOR_PREVIOUS_WEEK,
role: "previous season week number",
},
)?;
if week != "Week" {
let week = week.parse().map_err(|_| ParseError::InvalidText {
role: "previous season week number",
text: week.to_string(),
})?;
let date = select_text(row, &self.selector_previous_date).ok_or(
ParseError::ElementNotFound {
selector: SELECTOR_PREVIOUS_DATE,
role: "previous season week number",
},
)?;
let date = parse_date(date)?;
let map = select_text(row, &self.selector_previous_map)
.ok_or(ParseError::ElementNotFound {
selector: SELECTOR_PREVIOUS_MAP,
role: "previous season map",
})?
.to_string();
season.maps.push(PreviousSeasonMap { week, date, map })
}
}
}
}
if let Some(season) = prev_season {
previous.push(season);
}
Ok(MapHistory {
current: CurrentSeasonMapList {
season,
maps: current_weeks,
},
previous: Vec::new(),
previous,
})
}
}
fn parse_date(date: &str) -> Result<Date> {
let err = || ParseError::InvalidDate {
date: date.to_string(),
role: "previous season date",
};
let mut parts = date.split('/');
let month: u8 = parts.next().ok_or_else(err)?.parse().map_err(|_| err())?;
let month = Month::try_from(month).map_err(|_| err())?;
let date: u8 = parts.next().ok_or_else(err)?.parse().map_err(|_| err())?;
let year: i32 = parts.next().ok_or_else(err)?.parse().map_err(|_| err())?;
let year = 2000 + year;
Ok(Date::from_calendar_date(year, month, date).map_err(|_| err())?)
}

428
tests/snapshots/snapshot__parse_maps_2_html.snap
View file

@ -74,5 +74,431 @@ expression: parsed
}
]
},
"previous": []
"previous": [
{
"season": 7,
"maps": [
{
"week": 1,
"map": "ultiduo_baloo_v2",
"date": "2022-02-01"
},
{
"week": 2,
"map": "koth_ultiduo_r_b7",
"date": "2022-02-08"
},
{
"week": 3,
"map": "ultiduo_grove_b4",
"date": "2022-02-15"
},
{
"week": 4,
"map": "ultiduo_gullywash_b2",
"date": "2022-02-22"
},
{
"week": 5,
"map": "ultiduo_noodle",
"date": "2022-03-01"
},
{
"week": 6,
"map": "ultiduo_seclusion_b3",
"date": "2022-03-08"
},
{
"week": 7,
"map": "ultiduo_obsidiian_a10",
"date": "2022-03-15"
},
{
"week": 8,
"map": "ultiduo_lookout_b1",
"date": "2022-03-22"
},
{
"week": 9,
"map": "Best of 3 Maps",
"date": "2022-03-29"
},
{
"week": 10,
"map": "Best of 3 Maps",
"date": "2022-04-05"
},
{
"week": 11,
"map": "Best of 3 Maps",
"date": "2022-04-12"
}
]
},
{
"season": 6,
"maps": [
{
"week": 1,
"map": "ultiduo_grove_b4",
"date": "2021-10-05"
},
{
"week": 2,
"map": "ultiduo_obsidiian_a10",
"date": "2021-10-12"
},
{
"week": 3,
"map": "koth_ultiduo_r_b7",
"date": "2021-10-19"
},
{
"week": 4,
"map": "ultiduo_seclusion_b3",
"date": "2021-10-26"
},
{
"week": 5,
"map": "ultiduo_baloo_v2",
"date": "2021-11-02"
},
{
"week": 6,
"map": "ultiduo_furnace_b1",
"date": "2021-11-09"
},
{
"week": 7,
"map": "ultiduo_gullywash_b2",
"date": "2021-11-16"
},
{
"week": 8,
"map": "ultiduo_thaw_b3",
"date": "2021-11-23"
},
{
"week": 9,
"map": "Best of 3 Maps",
"date": "2021-11-30"
},
{
"week": 10,
"map": "Best of 3 Maps",
"date": "2021-12-07"
},
{
"week": 11,
"map": "Best of 3 Maps",
"date": "2021-12-14"
}
]
},
{
"season": 5,
"maps": [
{
"week": 1,
"map": "ultiduo_noodle",
"date": "2021-06-15"
},
{
"week": 2,
"map": "ultiduo_gullywash_b2",
"date": "2021-06-22"
},
{
"week": 3,
"map": "ultiduo_lookout_b1",
"date": "2021-06-29"
},
{
"week": 4,
"map": "ultiduo_baloo_v2",
"date": "2021-07-06"
},
{
"week": 5,
"map": "ultiduo_grove_b4",
"date": "2021-07-13"
},
{
"week": 6,
"map": "ultiduo_coaltown_v8",
"date": "2021-07-20"
},
{
"week": 7,
"map": "koth_ultiduo_r_b7",
"date": "2021-07-27"
},
{
"week": 8,
"map": "ultiduo_raksha_beta7",
"date": "2021-08-03"
},
{
"week": 9,
"map": "Best of 3 Maps",
"date": "2021-08-10"
},
{
"week": 10,
"map": "Best of 3 Maps",
"date": "2021-08-17"
},
{
"week": 11,
"map": "Best of 3 Maps",
"date": "2021-08-24"
}
]
},
{
"season": 4,
"maps": [
{
"week": 1,
"map": "ultiduo_baloo_v2",
"date": "2021-02-16"
},
{
"week": 2,
"map": "ultiduo_badlands_b1",
"date": "2021-02-23"
},
{
"week": 3,
"map": "ultiduo_grove_b4",
"date": "2021-03-02"
},
{
"week": 4,
"map": "ultiduo_noodle",
"date": "2021-03-09"
},
{
"week": 5,
"map": "koth_ultiduo_r_b7",
"date": "2021-03-16"
},
{
"week": 6,
"map": "ultiduo_gullywash_b2",
"date": "2021-03-23"
},
{
"week": 7,
"map": "ultiduo_seclusion_b3",
"date": "2021-03-30"
},
{
"week": 8,
"map": "ultiduo_lookout_b1",
"date": "2021-04-06"
},
{
"week": 9,
"map": "Best of 3 Maps",
"date": "2021-04-13"
},
{
"week": 10,
"map": "Best of 3 Maps",
"date": "2021-04-20"
},
{
"week": 11,
"map": "Best of 3 Maps",
"date": "2021-04-27"
}
]
},
{
"season": 3,
"maps": [
{
"week": 1,
"map": "ultiduo_baloo_v2",
"date": "2020-10-06"
},
{
"week": 2,
"map": "ultiduo_gullywash_b2",
"date": "2020-10-13"
},
{
"week": 3,
"map": "ultiduo_grove_b4",
"date": "2020-10-20"
},
{
"week": 4,
"map": "ultiduo_raksha_beta7",
"date": "2020-10-27"
},
{
"week": 5,
"map": "ultiduo_noodle",
"date": "2020-11-03"
},
{
"week": 6,
"map": "koth_ultiduo_r_b7",
"date": "2020-11-10"
},
{
"week": 7,
"map": "ultiduo_lookout_b1",
"date": "2020-11-17"
},
{
"week": 8,
"map": "ultiduo_obsidiian_a10",
"date": "2020-11-24"
},
{
"week": 9,
"map": "Best of 3 Maps",
"date": "2020-12-01"
},
{
"week": 10,
"map": "Best of 3 Maps",
"date": "2020-12-08"
},
{
"week": 11,
"map": "Finals 3 Maps",
"date": "2020-12-15"
}
]
},
{
"season": 2,
"maps": [
{
"week": 1,
"map": "ultiduo_baloo_v2",
"date": "2020-06-09"
},
{
"week": 2,
"map": "ultiduo_champions_a4b",
"date": "2020-06-16"
},
{
"week": 3,
"map": "ultiduo_thaw_b3",
"date": "2020-06-23"
},
{
"week": 4,
"map": "ultiduo_lookout_b1",
"date": "2020-06-30"
},
{
"week": 5,
"map": "ultiduo_nicecicles_v2",
"date": "2020-07-07"
},
{
"week": 6,
"map": "ultiduo_gullywash_b2",
"date": "2020-07-14"
},
{
"week": 7,
"map": "koth_ultiduo_r_b7",
"date": "2020-07-21"
},
{
"week": 8,
"map": "ultiduo_noodle",
"date": "2020-07-28"
},
{
"week": 9,
"map": "Best of 3 Maps",
"date": "2020-08-04"
},
{
"week": 10,
"map": "Best of 3 Maps",
"date": "2020-08-11"
},
{
"week": 11,
"map": "Finals 3 Maps",
"date": "2020-08-18"
}
]
},
{
"season": 1,
"maps": [
{
"week": 1,
"map": "ultiduo_gullywash_b2 (old)",
"date": "2020-02-25"
},
{
"week": 2,
"map": "koth_ultiduo_r_b7 (old)",
"date": "2020-03-03"
},
{
"week": 3,
"map": "ultiduo_baloo_v2 (old)",
"date": "2020-03-10"
},
{
"week": 4,
"map": "ultiduo_badlands_b1 (old)",
"date": "2020-03-17"
},
{
"week": 5,
"map": "ultiduo_gullywash_b2 (old)",
"date": "2020-03-24"
},
{
"week": 6,
"map": "koth_ultiduo_r_b7 (old)",
"date": "2020-03-31"
},
{
"week": 7,
"map": "ultiduo_baloo_v2 (old)",
"date": "2020-04-07"
},
{
"week": 8,
"map": "ultiduo_badlands_b1 (old)",
"date": "2020-04-14"
},
{
"week": 9,
"map": "Best of 3 Maps",
"date": "2020-04-21"
},
{
"week": 10,
"map": "Best of 3 Maps",
"date": "2020-04-28"
},
{
"week": 11,
"map": "Best of 3 Maps",
"date": "2020-05-05"
},
{
"week": 12,
"map": "Finals 3 Maps",
"date": "2020-05-12"
}
]
}
]
}

1798
tests/snapshots/snapshot__parse_maps_4_html.snap
View file

File diff suppressed because it is too large Load diff

3003
tests/snapshots/snapshot__parse_maps_6_html.snap
View file

File diff suppressed because it is too large Load diff

2653
tests/snapshots/snapshot__parse_maps_9_html.snap
View file

File diff suppressed because it is too large Load diff