mirror of
https://codeberg.org/icewind/ugc-scaper.git
synced 2026-06-03 18:24:10 +02:00
fix teams with no timezone
This commit is contained in:
parent
14faa36a12
commit
1a997aef36
9 changed files with 3043 additions and 13 deletions
2
Cargo.lock
generated
2
Cargo.lock
generated
|
|
@ -1540,7 +1540,7 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ugc-scraper"
|
name = "ugc-scraper"
|
||||||
version = "0.2.2"
|
version = "0.2.3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"insta",
|
"insta",
|
||||||
"main_error",
|
"main_error",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "ugc-scraper"
|
name = "ugc-scraper"
|
||||||
version = "0.2.2"
|
version = "0.2.3"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.67.0"
|
rust-version = "1.67.0"
|
||||||
description = "Scraper for ugcleague.com"
|
description = "Scraper for ugcleague.com"
|
||||||
|
|
|
||||||
4
api-server/Cargo.lock
generated
4
api-server/Cargo.lock
generated
|
|
@ -1742,9 +1742,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ugc-scraper"
|
name = "ugc-scraper"
|
||||||
version = "0.2.2"
|
version = "0.2.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1128dc71e95e4888137a818c8f0bb091f42fe5a024e0ee0d8ea651185415e403"
|
checksum = "910dd93cead2cdb81f3a4f7c8e728353c01a22101f10e717f56dc2a8c1ef81f3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"scraper",
|
"scraper",
|
||||||
|
|
|
||||||
|
|
@ -6,8 +6,8 @@ edition = "2021"
|
||||||
[dependencies]
|
[dependencies]
|
||||||
tokio = { version = "1.34.0", features = ["macros", "rt-multi-thread", "rt"] }
|
tokio = { version = "1.34.0", features = ["macros", "rt-multi-thread", "rt"] }
|
||||||
main_error = "0.1.2"
|
main_error = "0.1.2"
|
||||||
#ugc-scraper = { path = "../", version = "0.2.2" }
|
#ugc-scraper = { path = "../", version = "0.2.3" }
|
||||||
ugc-scraper = "0.2.2"
|
ugc-scraper = "0.2.3"
|
||||||
axum = "0.6.20"
|
axum = "0.6.20"
|
||||||
steamid-ng = "1.0.0"
|
steamid-ng = "1.0.0"
|
||||||
thiserror = "1.0.50"
|
thiserror = "1.0.50"
|
||||||
|
|
|
||||||
|
|
@ -90,7 +90,7 @@ pub struct Team {
|
||||||
pub tag: String,
|
pub tag: String,
|
||||||
pub image: String,
|
pub image: String,
|
||||||
pub format: String,
|
pub format: String,
|
||||||
pub timezone: String,
|
pub timezone: Option<String>,
|
||||||
pub division: String,
|
pub division: String,
|
||||||
pub description: String,
|
pub description: String,
|
||||||
pub titles: Vec<String>,
|
pub titles: Vec<String>,
|
||||||
|
|
|
||||||
|
|
@ -154,12 +154,7 @@ impl Parser for TeamParser {
|
||||||
})?
|
})?
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
let timezone = select_text(root, &self.selector_team_timezone)
|
let timezone = select_text(root, &self.selector_team_timezone).map(String::from);
|
||||||
.ok_or(ParseError::ElementNotFound {
|
|
||||||
selector: SELECTOR_TEAM_TIMEZONE,
|
|
||||||
role: "team timzone",
|
|
||||||
})?
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
let description = select_text(root, &self.selector_team_description)
|
let description = select_text(root, &self.selector_team_description)
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
|
|
|
||||||
2894
tests/data/team_8157.html
Normal file
2894
tests/data/team_8157.html
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -37,6 +37,14 @@ fn test_parse_older_team_html() {
|
||||||
assert_json_snapshot!(parsed);
|
assert_json_snapshot!(parsed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_parse_team_no_tz_html() {
|
||||||
|
let body = read_to_string("tests/data/team_8157.html").unwrap();
|
||||||
|
let parser = TeamParser::new();
|
||||||
|
let parsed = parser.parse(&body).unwrap();
|
||||||
|
assert_json_snapshot!(parsed);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_parse_team_changed_name_html() {
|
fn test_parse_team_changed_name_html() {
|
||||||
let body = read_to_string("tests/data/team_6929.html").unwrap();
|
let body = read_to_string("tests/data/team_6929.html").unwrap();
|
||||||
|
|
|
||||||
133
tests/snapshots/snapshot__parse_team_no_tz_html.snap
Normal file
133
tests/snapshots/snapshot__parse_team_no_tz_html.snap
Normal file
|
|
@ -0,0 +1,133 @@
|
||||||
|
---
|
||||||
|
source: tests/snapshot.rs
|
||||||
|
expression: parsed
|
||||||
|
---
|
||||||
|
{
|
||||||
|
"name": "Barney and the Ganks",
|
||||||
|
"tag": "Grand Meister",
|
||||||
|
"image": "https://steamcdn-a.akamaihd.net/steamcommunity/public/images/avatars/d7/d71ff4e1e2635c3d7c1be8d96bc40b499c665aad_full.jpg",
|
||||||
|
"format": "TF2 Highlander",
|
||||||
|
"timezone": null,
|
||||||
|
"division": "NA Platinum",
|
||||||
|
"description": "",
|
||||||
|
"titles": [],
|
||||||
|
"members": [
|
||||||
|
{
|
||||||
|
"name": "rchl",
|
||||||
|
"steam_id": 76561198027925375,
|
||||||
|
"role": "Leader",
|
||||||
|
"since": "+002013-08-13T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Yollide",
|
||||||
|
"steam_id": 76561197964088385,
|
||||||
|
"role": "Leader",
|
||||||
|
"since": "+002013-08-13T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Gankmeister",
|
||||||
|
"steam_id": 76561198017648752,
|
||||||
|
"role": "Leader",
|
||||||
|
"since": "+002013-08-14T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Hng",
|
||||||
|
"steam_id": 76561198042955655,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-08-13T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "snake kryptonite",
|
||||||
|
"steam_id": 76561197998673784,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-08-13T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Hugh",
|
||||||
|
"steam_id": 76561197968399860,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-08-13T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Joy",
|
||||||
|
"steam_id": 76561197997012073,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-08-13T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "BIG RAT",
|
||||||
|
"steam_id": 76561197975762186,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-08-13T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "GoldenBrownBear",
|
||||||
|
"steam_id": 76561197991937066,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-08-22T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "bfl Satan",
|
||||||
|
"steam_id": 76561197996097401,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-08-27T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Murow",
|
||||||
|
"steam_id": 76561198046399702,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-08-31T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Ibby",
|
||||||
|
"steam_id": 76561198001095962,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-09-05T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "mpb",
|
||||||
|
"steam_id": 76561197995307557,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-09-09T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Hei",
|
||||||
|
"steam_id": 76561197987102243,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-09-15T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "to go to bed forev",
|
||||||
|
"steam_id": 76561198025368981,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-09-15T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stanplsStochast1c",
|
||||||
|
"steam_id": 76561197995209440,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-09-28T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ChinGoo",
|
||||||
|
"steam_id": 76561197990116981,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-10-03T00:00:00.000000000Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Bruce Leeroy",
|
||||||
|
"steam_id": 76561198010708247,
|
||||||
|
"role": "Member",
|
||||||
|
"since": "+002013-10-07T00:00:00.000000000Z"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"season": 11,
|
||||||
|
"division": "NA Platinum",
|
||||||
|
"wins": 5,
|
||||||
|
"losses": 5
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"name_changes": []
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue