initial version

src/api.rs

@@ -0,0 +1,103 @@
+use crate::Error;
+use chrono::{DateTime, Utc};
+use md5::Digest;
+use serde::{Deserialize, Deserializer};
+use smol_str::SmolStr;
+use std::fmt;
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Demo {
+    pub id: u32,
+    pub url: String,
+    pub name: String,
+    pub server: SmolStr,
+    pub duration: u16,
+    pub nick: SmolStr,
+    pub map: SmolStr,
+    #[serde(with = "chrono::serde::ts_seconds")]
+    pub time: DateTime<Utc>,
+    pub red: SmolStr,
+    pub blue: SmolStr,
+    pub red_score: u8,
+    pub blue_score: u8,
+    pub player_count: u8,
+    pub uploader: u32,
+    #[serde(deserialize_with = "hex_to_digest")]
+    pub hash: Digest,
+    pub backend: SmolStr,
+    pub path: String,
+}
+
+/// Deserializes a lowercase hex string to a `Vec<u8>`.
+pub fn hex_to_digest<'de, D>(deserializer: D) -> Result<Digest, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    use hex::FromHex;
+    use serde::de::Error;
+
+    let string = String::deserialize(deserializer)?;
+
+    if string.len() == 0 {
+        return Ok(Digest([0; 16]));
+    }
+
+    <[u8; 16]>::from_hex(&string)
+        .map_err(|err| Error::custom(err.to_string()))
+        .map(Digest)
+}
+
+#[derive(Debug)]
+pub enum ListOrder {
+    Ascending,
+    Descending,
+}
+
+impl Default for ListOrder {
+    fn default() -> Self {
+        ListOrder::Descending
+    }
+}
+
+impl fmt::Display for ListOrder {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            ListOrder::Ascending => "ASC".fmt(f),
+            ListOrder::Descending => "DESC".fmt(f),
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct ListParams {
+    order: ListOrder,
+    backend: Option<String>,
+}
+
+impl ListParams {
+    pub fn with_backend(self, backend: impl ToString) -> Self {
+        ListParams {
+            backend: Some(backend.to_string()),
+            ..self
+        }
+    }
+
+    pub fn with_order(self, order: ListOrder) -> Self {
+        ListParams { order, ..self }
+    }
+}
+
+pub fn list_demos(params: ListParams, page: u32) -> Result<Vec<Demo>, Error> {
+    let mut req = ureq::get("https://api.demos.tf/demos");
+    req.query("page", &format!("{}", page))
+        .query("order", &format!("{}", params.order));
+
+    if let Some(backend) = params.backend.as_ref() {
+        req.query("backend", backend);
+    }
+
+    let resp = req.call();
+
+    Ok(resp.into_json_deserialize()?)
+}

src/backup.rs

@@ -0,0 +1,54 @@
+use crate::api::{list_demos, ListOrder, ListParams};
+use crate::store::Store;
+use crate::Error;
+use md5::Digest;
+
+pub struct Backup {
+    store: Store,
+}
+
+impl Backup {
+    pub fn new(store: Store) -> Self {
+        Backup { store }
+    }
+
+    fn backup_demo(&self, name: &str, url: &str, hash: Digest) -> Result<(), Error> {
+        let resp = ureq::get(url).call();
+
+        let digest = self.store.store(name, &mut resp.into_reader())?;
+
+        if digest == hash || digest == Digest([0; 16]) {
+            Ok(())
+        } else {
+            let _ = self.store.remove(name);
+            Err(Error::DigestMismatch {
+                expected: hash,
+                got: digest,
+            })
+        }
+    }
+
+    fn backup_page(&self, page: u32) -> Result<usize, Error> {
+        let demos = list_demos(ListParams::default().with_order(ListOrder::Ascending), page)?;
+
+        for demo in demos.iter() {
+            if demo.url != "" {
+                let name = demo.url.rsplit('/').next().unwrap();
+                println!("{} {}", demo.id, name);
+                if !self.store.exists(name) {
+                    self.backup_demo(name, &demo.url, demo.hash)?;
+                }
+            }
+        }
+
+        Ok(demos.len())
+    }
+
+    pub fn backup_from(&self, mut page: u32) -> Result<u32, Error> {
+        while self.backup_page(page)? > 0 {
+            page += 1;
+        }
+
+        Ok(page)
+    }
+}

src/main.rs

@@ -1,3 +1,46 @@
-fn main() {
-    println!("Hello, world!");
+mod backup;
+mod store;
+
+use crate::backup::Backup;
+use crate::store::Store;
+use main_error::MainError;
+use md5::Digest;
+use std::cmp::max;
+use std::collections::HashMap;
+use std::path::PathBuf;
+use thiserror::Error;
+
+mod api;
+
+#[derive(Debug, Error)]
+pub enum Error {
+    #[error("Request failed: {0}")]
+    Request(#[from] std::io::Error),
+    #[error("MD5 digest mismatch for downloaded demo, expected {expected:?}, received {got:?}")]
+    DigestMismatch { expected: Digest, got: Digest },
+}
+
+fn main() -> Result<(), MainError> {
+    let mut args: HashMap<_, _> = dotenv::vars().collect();
+    let store = Store::new(args.get("STORAGE_ROOT").expect("no STORAGE_ROOT set"));
+    let state_path = PathBuf::from(args.remove("STATE_FILE").expect("no STATE_FILE set"));
+    let backup = Backup::new(store);
+
+    let last_page = if state_path.is_file() {
+        max(
+            std::fs::read_to_string(&state_path)?
+                .trim()
+                .parse::<u32>()?
+                - 1,
+            1,
+        )
+    } else {
+        1u32
+    };
+
+    let current_page = backup.backup_from(last_page)?;
+
+    std::fs::write(&state_path, format!("{}", current_page))?;
+
+    Ok(())
 }

src/store.rs

@@ -0,0 +1,58 @@
+use md5::{Context, Digest};
+use std::fs;
+use std::fs::File;
+use std::io::{ErrorKind, Read, Write};
+use std::path::{Path, PathBuf};
+
+pub struct Store {
+    basedir: PathBuf,
+}
+
+impl Store {
+    pub fn new(basedir: impl AsRef<Path>) -> Self {
+        Store {
+            basedir: basedir.as_ref().to_path_buf(),
+        }
+    }
+
+    pub fn store(&self, name: &str, data: &mut impl Read) -> std::io::Result<Digest> {
+        let path = self.generate_path(name);
+        fs::create_dir_all(path.parent().unwrap())?;
+
+        let mut file = File::create(&path)?;
+
+        let mut context = Context::new();
+        let mut buf = [0u8; 8 * 1024];
+
+        // copy the file and compute the digest was we go
+        loop {
+            let len = match data.read(&mut buf) {
+                Ok(0) => return Ok(context.compute()),
+                Ok(len) => len,
+                Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
+                Err(e) => return Err(e),
+            };
+
+            let data = &buf[..len];
+            context.consume(data);
+
+            file.write_all(data)?;
+        }
+    }
+
+    pub fn exists(&self, name: &str) -> bool {
+        self.generate_path(name).is_file()
+    }
+
+    pub fn remove(&self, name: &str) -> std::io::Result<()> {
+        fs::remove_file(self.generate_path(name))
+    }
+
+    fn generate_path(&self, name: &str) -> PathBuf {
+        let mut path = self.basedir.clone();
+        path.push(&name[0..2]);
+        path.push(&name[2..4]);
+        path.push(name);
+        path
+    }
+}