steam group on roster page

Cargo.lock

@@ -1550,7 +1550,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
 [[package]]
 name = "ugc-scraper"
-version = "0.3.3"
+version = "0.4.0"
 dependencies = [
  "insta",
  "main_error",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ugc-scraper"
-version = "0.3.3"
+version = "0.4.0"
 edition = "2021"
 rust-version = "1.67.0"
 description = "Scraper for ugcleague.com"

api-server/Cargo.lock

@@ -1851,9 +1851,9 @@ dependencies = [
 
 [[package]]
 name = "ugc-scraper"
-version = "0.3.3"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "082110c325d2a7b6282d643f5753860ae04ae9890df4f145d10db25ad449c06f"
+checksum = "5866c81814a2fc1495630ac7fd74c5564cceff96305ea10c7ee2dd85caadc45e"
 dependencies = [
  "reqwest",
  "scraper",

api-server/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 tokio = { version = "1.34.0", features = ["macros", "rt-multi-thread", "rt", "signal"] }
 main_error = "0.1.2"
 #ugc-scraper = { path = "../", version = "*" }
-ugc-scraper = "0.3.3"
+ugc-scraper = "0.4.0"
 axum = "0.7.4"
 steamid-ng = "1.0.0"
 thiserror = "1.0.50"

examples/team.rs

@@ -18,7 +18,7 @@ async fn main() -> MainResult {
 
     println!();
     println!("previous players ");
-    let roster_history = client.team_roster_history(id).await?;
+    let roster_history = client.team_roster_history(id).await?.history;
     for roster_item in roster_history {
         if let Some(left) = roster_item.left {
             println!(

src/data.rs

@@ -168,6 +168,14 @@ pub struct Record {
     pub losses: u8,
 }
 
+#[derive(Debug, Clone)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize))]
+#[non_exhaustive]
+pub struct TeamRosterData {
+    pub steam_group: String,
+    pub history: Vec<RosterHistory>,
+}
+
 #[derive(Debug, Clone)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize))]
 pub struct RosterHistory {

src/lib.rs

@@ -4,8 +4,8 @@ mod error;
 pub mod parser;
 
 use crate::data::{
-    GameMode, MapHistory, MatchInfo, MembershipHistory, Player, RosterHistory, Seasons, Team,
-    TeamRef, TeamSeason, Transaction,
+    GameMode, MapHistory, MatchInfo, MembershipHistory, Player, Seasons, Team, TeamRef,
+    TeamRosterData, TeamSeason, Transaction,
 };
 use crate::parser::{
     MapHistoryParser, MatchPageParser, Parser, PlayerDetailsParser, PlayerParser, SeasonsParser,
@@ -101,7 +101,7 @@ impl UgcClient {
     }
 
     /// Retrieve team roster history
-    pub async fn team_roster_history(&self, id: u32) -> Result<Vec<RosterHistory>> {
+    pub async fn team_roster_history(&self, id: u32) -> Result<TeamRosterData> {
         let body = self
             .client
             .get(&format!(

src/parser/team.rs

@@ -154,7 +154,10 @@ impl Parser for TeamParser {
         let steam_group = root
             .select(&self.selector_steam_group)
             .next()
-            .and_then(|link| link.attr("href").map(String::from));
+            .and_then(|link| {
+                link.attr("href")
+                    .map(|group| group.replace("http://http", "http"))
+            });
 
         let division = select_text(root, &self.selector_team_division)
             .ok_or(ParseError::ElementNotFound {

src/parser/team_roster_history.rs

@@ -1,5 +1,5 @@
 use super::Parser;
-use crate::data::RosterHistory;
+use crate::data::{RosterHistory, TeamRosterData};
 use crate::parser::{select_text, ROSTER_HISTORY_DATE_FORMAT};
 use crate::{ParseError, Result};
 use scraper::{Html, Selector};
@@ -13,12 +13,15 @@ const SELECTOR_ROSTER_ID: &str = "h5 small";
 const SELECTOR_ROSTER_JOINED: &str = "span.text-success small";
 const SELECTOR_ROSTER_LEFT: &str = "span.text-danger small";
 
+const SELECTOR_STEAM: &str = r#"p.muted a[href*="//steamcommunity.com/groups"]"#;
+
 pub struct TeamRosterHistoryParser {
     selector_item: Selector,
     selector_name: Selector,
     selector_id: Selector,
     selector_joined: Selector,
     selector_left: Selector,
+    selector_steam_group: Selector,
 }
 
 impl Default for TeamRosterHistoryParser {
@@ -35,17 +38,29 @@ impl TeamRosterHistoryParser {
             selector_id: Selector::parse(SELECTOR_ROSTER_ID).unwrap(),
             selector_joined: Selector::parse(SELECTOR_ROSTER_JOINED).unwrap(),
             selector_left: Selector::parse(SELECTOR_ROSTER_LEFT).unwrap(),
+            selector_steam_group: Selector::parse(SELECTOR_STEAM).unwrap(),
         }
     }
 }
 
 impl Parser for TeamRosterHistoryParser {
-    type Output = Vec<RosterHistory>;
+    type Output = TeamRosterData;
 
     fn parse(&self, document: &str) -> Result<Self::Output> {
         let document = Html::parse_document(document);
 
-        document
+        let steam_group = document.select(&self.selector_steam_group).next().ok_or(
+            ParseError::ElementNotFound {
+                selector: SELECTOR_STEAM,
+                role: "team steam group",
+            },
+        )?;
+        let steam_group = steam_group
+            .attr("href")
+            .unwrap_or_default()
+            .replace("http://http", "http");
+
+        let history = document
             .select(&self.selector_item)
             .map(|item| {
                 let name =
@@ -92,6 +107,10 @@ impl Parser for TeamRosterHistoryParser {
                         .transpose()?,
                 })
             })
-            .collect::<Result<Vec<_>>>()
+            .collect::<Result<Vec<_>>>()?;
+        Ok(TeamRosterData {
+            history,
+            steam_group,
+        })
     }
 }

tests/snapshots/snapshot__parse_team_empty_name_change_html.snap

@@ -8,7 +8,7 @@ expression: parsed
   "image": "clan_avatars/32437_thumbnail.jpg",
   "format": "TF2 Highlander",
   "timezone": null,
-  "steam_group": "http://https://steamcommunity.com/groups/xyxxx-",
+  "steam_group": "https://steamcommunity.com/groups/xyxxx-",
   "division": "North America",
   "description": "",
   "titles": [

tests/snapshots/snapshot__parse_team_html.snap

@@ -8,7 +8,7 @@ expression: parsed
   "image": "https://steamcdn-a.akamaihd.net/steamcommunity/public/images/avatars/db/dbabbd8bab7ccf6d27a9d4ca2e73a76e085bb201_full.jpg",
   "format": "TF2 Highlander",
   "timezone": "West-Euro",
-  "steam_group": "http://https://steamcommunity.com/groups/XenonxTF2",
+  "steam_group": "https://steamcommunity.com/groups/XenonxTF2",
   "division": "Euro Platinum",
   "description": "Contact Kimo or Icewind for everything.",
   "titles": [

tests/snapshots/snapshot__parse_team_roster_history_html.snap

@@ -2,7 +2,9 @@
 source: tests/snapshot.rs
 expression: parsed
 ---
-[
+{
+  "steam_group": "https://steamcommunity.com/groups/XenonxTF2",
+  "history": [
     {
       "name": "marko",
       "steam_id": "76561198274165935",
@@ -946,3 +948,4 @@ expression: parsed
       "left": "2016-05-27"
     }
   ]
+}