add download convenience method

.github/workflows/rust.yml

@@ -33,7 +33,7 @@ jobs:
       api:
         image: demostf/api
         env:
-          DEMO_ROOT: /tmp
+          DEMO_ROOT: /demos
           DEMO_HOST: localhost
           DB_TYPE: pgsql
           DB_HOST: api-test-db
@@ -43,12 +43,16 @@ jobs:
           DB_PASSWORD: test
           APP_ROOT: https://api.localhost
           EDIT_SECRET: edit
+        volumes:
+          - /tmp:/demos
       api-test:
         image: demostf/api-nginx-test
         env:
           POSTGRES_PASSWORD: test
         ports:
           - 80:80
+        volumes:
+          - /tmp:/demos
 
     steps:
       - uses: actions/checkout@v2

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "demostf-client"
-version = "0.3.2"
+version = "0.3.3"
 authors = ["Robin Appelman <robin@icewind.nl>"]
 edition = "2018"
 description = "Api client for demos.tf"
@@ -22,6 +22,7 @@ bytes = "1.1.0"
 futures-util = "0.3.21"
 tracing = "0.1.33"
 tinyvec = "1.5.1"
+md5 = "0.7.0"
 
 [dev-dependencies]
 tokio = { version = "1", features = ["macros"] }

src/lib.rs

@@ -1,15 +1,17 @@
 use bytes::Bytes;
 use futures_util::{Stream, StreamExt};
+use md5::Context;
 use reqwest::{multipart, Client, IntoUrl, StatusCode, Url};
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use std::borrow::Cow;
 use std::fmt::{self, Debug, Display, Formatter};
+use std::io::Write;
 use std::str::FromStr;
 pub use steamid_ng::SteamID;
 use thiserror::Error;
 use time::OffsetDateTime;
 use tinyvec::TinyVec;
-use tracing::{debug, instrument};
+use tracing::{debug, error, instrument};
 
 #[derive(Debug, Error)]
 pub enum Error {
@@ -29,6 +31,8 @@ pub enum Error {
     InvalidResponse(String),
     #[error("Demo {0} not found")]
     DemoNotFound(u32),
+    #[error("Error while writing demo data")]
+    Write(#[source] std::io::Error),
 }
 
 impl From<reqwest::Error> for Error {
@@ -85,14 +89,46 @@ impl Demo {
         &self,
         client: &ApiClient,
     ) -> Result<impl Stream<Item = Result<Bytes, Error>>, Error> {
-        debug!(id = self.id, "starting download");
+        debug!(id = self.id, url = display(&self.url), "starting download");
         Ok(client
             .client
             .get(&self.url)
             .send()
             .await?
+            .error_for_status()?
             .bytes_stream()
-            .map(|chunk| Ok(chunk?)))
+            .map(|chunk| chunk.map_err(Error::from)))
+    }
+
+    /// Download a demo and save it to a writer, verifying the md5 hash in the process
+    #[instrument(skip(target))]
+    pub async fn save<W: Write>(&self, client: &ApiClient, mut target: W) -> Result<(), Error> {
+        debug!(id = self.id, url = display(&self.url), "starting download");
+        let mut response = client
+            .client
+            .get(&self.url)
+            .send()
+            .await?
+            .error_for_status()?;
+
+        let mut context = Context::new();
+
+        while let Some(chunk) = response.chunk().await? {
+            context.consume(&chunk);
+            target.write_all(&chunk).map_err(Error::Write)?;
+        }
+
+        let calculated = context.compute().0;
+
+        if calculated != self.hash {
+            error!(
+                calculated = display(hex::encode(calculated)),
+                expected = display(hex::encode(self.hash)),
+                "hash mismatch"
+            );
+            return Err(Error::HashMisMatch);
+        }
+        Ok(())
     }
 }
 
@@ -586,7 +622,14 @@ impl ApiClient {
     pub async fn get_user(&self, user_id: u32) -> Result<User, Error> {
         let mut url = self.base_url.clone();
         url.set_path(&format!("/users/{}", user_id));
-        Ok(self.client.get(url).send().await?.json().await?)
+        Ok(self
+            .client
+            .get(url)
+            .send()
+            .await?
+            .error_for_status()?
+            .json()
+            .await?)
     }
 
     /// List demos with the provided options
@@ -612,7 +655,14 @@ impl ApiClient {
     pub async fn get_chat(&self, demo_id: u32) -> Result<Vec<ChatMessage>, Error> {
         let mut url = self.base_url.clone();
         url.set_path(&format!("/demos/{}/chat", demo_id));
-        Ok(self.client.get(url).send().await?.json().await?)
+        Ok(self
+            .client
+            .get(url)
+            .send()
+            .await?
+            .error_for_status()?
+            .json()
+            .await?)
     }
 
     #[instrument]

start_test_server.sh

@@ -5,12 +5,12 @@ docker rm -f api-test-fpm
 docker rm -f api-test
 
 mkdir -p /tmp/api-test-data
-chmod 0777 /tmp/api-test-data
+chmod -R 0777 /tmp/api-test-data
 
 docker run -d --name api-test-db -e POSTGRES_PASSWORD=test -p 15432:5432 demostf/db
 docker run -d --name api-test-fpm --link api-test-db:db -v /tmp/api-test-data:/demos \
   -e DEMO_ROOT=/demos -e DEMO_HOST=localhost -e DB_TYPE=pgsql \
   -e DB_HOST=db -e DB_PORT=5432 -e DB_DATABASE=postgres -e DB_USERNAME=postgres \
-  -e DB_PASSWORD=test -e APP_ROOT=http://api.localhost -e EDIT_SECRET=edit \
+  -e DB_PASSWORD=test -e APP_ROOT=http://localhost:8888 -e EDIT_SECRET=edit \
   demostf/api
-docker run -d --name api-test --link api-test-fpm:api -p 8888:80 demostf/api-nginx-test
+docker run -d --name api-test --link api-test-fpm:api -v /tmp/api-test-data:/demos -p 8888:80 demostf/api-nginx-test

tests/tests.rs

@@ -1,5 +1,6 @@
 use demostf_client::{ApiClient, Error, ListOrder, ListParams};
 use sqlx::postgres::PgPoolOptions;
+use std::fs::read;
 use std::sync::atomic::{AtomicBool, Ordering};
 use steamid_ng::SteamID;
 use tracing_subscriber::EnvFilter;
@@ -205,6 +206,47 @@ async fn test_set_url_invalid_key() {
     assert!(matches!(res.unwrap_err(), Error::InvalidApiKey));
 }
 
+#[tokio::test]
+async fn test_set_url_invalid_hash() {
+    let client = test_client().await;
+
+    let res = client
+        .set_url(
+            1,
+            "tests",
+            "tests",
+            "http://example.com/tests",
+            [0; 16],
+            "edit",
+        )
+        .await;
+    assert!(matches!(res.unwrap_err(), Error::HashMisMatch));
+}
+
+#[tokio::test]
+async fn test_set_url() {
+    let client = test_client().await;
+    let demo = client.get(1).await.unwrap();
+
+    client
+        .set_url(
+            1,
+            "example",
+            "tests",
+            "http://example.com/tests",
+            demo.hash,
+            "edit",
+        )
+        .await
+        .unwrap();
+
+    let moved = client.get(1).await.unwrap();
+
+    assert_eq!(moved.backend, "example");
+    assert_eq!(moved.path, "tests");
+    assert_eq!(moved.url, "http://example.com/tests");
+}
+
 #[tokio::test]
 async fn test_get_demo_not_found() {
     let client = test_client().await;
@@ -267,3 +309,24 @@ async fn test_list_players() {
         .unwrap();
     assert_eq!(demos.len(), 0);
 }
+
+#[tokio::test]
+async fn test_download_demo() {
+    let client = test_client().await;
+
+    let mut demo = client.get(1).await.unwrap();
+
+    let demos_url =
+        std::env::var("API_ROOT").unwrap_or_else(|_| "http://localhost:8888/".to_string());
+
+    // fixup the url to one that is actually usable
+    demo.url = format!(
+        "{}static/01/b2/01b2265d875026b91d59a2785abfd50d_test.dem",
+        demos_url
+    );
+
+    let mut data: Vec<u8> = Vec::new();
+    demo.save(&client, &mut data).await.unwrap();
+
+    assert_eq!(data.len(), read("./tests/data/gully.dem").unwrap().len());
+}