From 71f1ba6f3db2085ad1fd269f98baf863fd876e65 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Sat, 30 Apr 2022 19:53:18 +0200 Subject: [PATCH] move to api client crate --- Cargo.lock | 770 +++++++++++++++++++++++++++++++++++++++++++++----- Cargo.toml | 10 +- Dockerfile | 2 +- src/api.rs | 101 ------- src/backup.rs | 38 ++- src/main.rs | 19 +- src/store.rs | 34 +-- 7 files changed, 764 insertions(+), 210 deletions(-) delete mode 100644 src/api.rs diff --git a/Cargo.lock b/Cargo.lock index 68be56b..10ea3a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,10 +3,22 @@ version = 3 [[package]] -name = "adler" -version = "1.0.2" +name = "aho-corasick" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] [[package]] name = "autocfg" @@ -18,14 +30,18 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" name = "backup" version = "0.1.0" dependencies = [ - "chrono", + "bytes", + "demostf-client", "dotenv", + "futures-util", "hex", "main_error", "md5", "serde", "thiserror", - "ureq", + "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -40,6 +56,12 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + [[package]] name = "cc" version = "1.0.73" @@ -53,32 +75,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "chrono" -version = "0.4.19" +name = "demostf-client" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "7ba6215021299e2f440f91c6c32bb5dd8f19ae7e94c9c692124f52daa9ec5b57" dependencies = [ - "libc", - "num-integer", - "num-traits", + "bytes", + "futures-util", + "hex", + "reqwest", "serde", + "steamid-ng", + "thiserror", "time", - "winapi", -] - -[[package]] -name = "chunked_transfer" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" - -[[package]] -name = "crc32fast" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" -dependencies = [ - "cfg-if", ] [[package]] @@ -88,17 +97,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" [[package]] -name = "flate2" -version = "1.0.23" +name = "encoding_rs" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" dependencies = [ "cfg-if", - "crc32fast", - "libc", - "miniz_oxide", ] +[[package]] +name = "enum_primitive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4551092f4d519593039259a9ed8daedf0da12e5109c5280338073eaeb81180" +dependencies = [ + "num-traits 0.1.43", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -109,12 +130,169 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-channel" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" + +[[package]] +name = "futures-macro" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" + +[[package]] +name = "futures-task" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" + +[[package]] +name = "futures-util" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "h2" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util 0.7.1", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "http" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" +dependencies = [ + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "idna" version = "0.2.3" @@ -126,6 +304,22 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" + [[package]] name = "itoa" version = "1.0.1" @@ -181,12 +375,91 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] -name = "miniz_oxide" -version = "0.5.1" +name = "memchr" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" dependencies = [ - "adler", + "mime", + "unicase", +] + +[[package]] +name = "mio" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "wasi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "ntapi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" +dependencies = [ + "winapi", +] + +[[package]] +name = "num" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7a8e9be5e039e2ff869df49155f1c06bd01ade2117ec783e56ab0932b67a8f" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits 0.2.14", +] + +[[package]] +name = "num-bigint" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" +dependencies = [ + "autocfg", + "num-integer", + "num-traits 0.2.14", +] + +[[package]] +name = "num-complex" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747d632c0c558b87dbabbe6a82f3b4ae03720d0646ac5b7b4dae89394be5f2c5" +dependencies = [ + "num-traits 0.2.14", ] [[package]] @@ -196,7 +469,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", - "num-traits", + "num-traits 0.2.14", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits 0.2.14", +] + +[[package]] +name = "num-rational" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits 0.2.14", +] + +[[package]] +name = "num-traits" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" +dependencies = [ + "num-traits 0.2.14", ] [[package]] @@ -208,6 +513,25 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "num_threads" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aba1801fb138d8e85e11d0fc70baf4fe1cdfffda7c6cd34a854905df588e5ed0" +dependencies = [ + "libc", +] + [[package]] name = "once_cell" version = "1.10.0" @@ -220,6 +544,18 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "proc-macro2" version = "1.0.37" @@ -238,6 +574,63 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "reqwest" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-rustls", + "tokio-util 0.6.9", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + [[package]] name = "ring" version = "0.16.20" @@ -265,6 +658,15 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls-pemfile" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360" +dependencies = [ + "base64", +] + [[package]] name = "ryu" version = "1.0.9" @@ -312,12 +714,70 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "slab" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" + +[[package]] +name = "smallvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" + +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "spin" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "steamid-ng" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb049f8faa2cba570c5366dbaf88ee5849725b16edb771848639fac92e33673" +dependencies = [ + "enum_primitive", + "lazy_static", + "num", + "regex", + "serde", + "serde_derive", + "thiserror", +] + [[package]] name = "syn" version = "1.0.92" @@ -350,14 +810,23 @@ dependencies = [ ] [[package]] -name = "time" -version = "0.1.44" +name = "thread_local" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" dependencies = [ "libc", - "wasi", - "winapi", + "num_threads", + "serde", ] [[package]] @@ -375,6 +844,153 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +[[package]] +name = "tokio" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f48b6d60512a392e34dbf7fd456249fd2de3c83669ab642e021903f4015185b" +dependencies = [ + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "pin-project-lite", + "socket2", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-macros" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4151fda0cf2798550ad0b34bcfc9b9dcc2a9d2471c895c68f3a8818e54f2389e" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-util" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +dependencies = [ + "lazy_static", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" +dependencies = [ + "ansi_term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.8" @@ -402,25 +1018,6 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" -[[package]] -name = "ureq" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9399fa2f927a3d327187cbd201480cee55bee6ac5d3c77dd27f0c6814cff16d5" -dependencies = [ - "base64", - "chunked_transfer", - "flate2", - "log", - "once_cell", - "rustls", - "serde", - "serde_json", - "url", - "webpki", - "webpki-roots", -] - [[package]] name = "url" version = "2.2.2" @@ -434,10 +1031,32 @@ dependencies = [ ] [[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +name = "valuable" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" @@ -464,6 +1083,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.80" @@ -543,3 +1174,12 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] diff --git a/Cargo.toml b/Cargo.toml index 8e468bd..3bc378e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,10 +6,14 @@ edition = "2018" [dependencies] serde = { version = "1.0.136", features = ["derive"] } -ureq = { version = "2.4.0", features = ["json"] } thiserror = "1.0.30" -chrono = { version = "0.4", features = ["serde"] } md5 = "0.7.0" hex = "0.4.3" dotenv = "0.15.0" -main_error = "0.1.2" \ No newline at end of file +main_error = "0.1.2" +demostf-client = { version = "0.3.1", default-features = false, features = ["rustls-tls"] } +tokio = { version = "1.17.0", features = ["rt-multi-thread", "macros"] } +tracing = "0.1.33" +tracing-subscriber = "0.3.11" +bytes = "1.1.0" +futures-util = "0.3.21" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7941076..5c8f5be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,5 +8,5 @@ RUN cargo build --release FROM alpine:latest COPY --from=build /home/rust/src/target/x86_64-unknown-linux-musl/release/backup / - +ENV RUST_LOG=info CMD ["/backup"] \ No newline at end of file diff --git a/src/api.rs b/src/api.rs deleted file mode 100644 index ececd46..0000000 --- a/src/api.rs +++ /dev/null @@ -1,101 +0,0 @@ -use crate::Error; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Deserializer}; -use std::borrow::Cow; -use std::fmt; - -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Demo { - pub id: u32, - pub url: String, - pub name: String, - pub server: String, - pub duration: u16, - pub nick: String, - pub map: String, - #[serde(with = "chrono::serde::ts_seconds")] - pub time: DateTime, - pub red: String, - pub blue: String, - pub red_score: u8, - pub blue_score: u8, - pub player_count: u8, - pub uploader: u32, - #[serde(deserialize_with = "hex_to_digest")] - pub hash: [u8; 16], - pub backend: String, - pub path: String, -} - -/// Deserializes a lowercase hex string to a `[u8; 16]`. -pub fn hex_to_digest<'de, D>(deserializer: D) -> Result<[u8; 16], D::Error> -where - D: Deserializer<'de>, -{ - use hex::FromHex; - use serde::de::Error; - - let string = >::deserialize(deserializer)?; - - if string.len() == 0 { - return Ok([0; 16]); - } - - <[u8; 16]>::from_hex(string.as_ref()).map_err(|err| Error::custom(err.to_string())) -} - -#[derive(Debug)] -pub enum ListOrder { - Ascending, - Descending, -} - -impl Default for ListOrder { - fn default() -> Self { - ListOrder::Descending - } -} - -impl fmt::Display for ListOrder { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - ListOrder::Ascending => "ASC".fmt(f), - ListOrder::Descending => "DESC".fmt(f), - } - } -} - -#[derive(Debug, Default)] -pub struct ListParams { - order: ListOrder, - backend: Option, -} - -impl ListParams { - #[allow(dead_code)] - pub fn with_backend(self, backend: impl ToString) -> Self { - ListParams { - backend: Some(backend.to_string()), - ..self - } - } - - pub fn with_order(self, order: ListOrder) -> Self { - ListParams { order, ..self } - } -} - -pub fn list_demos(params: ListParams, page: u32) -> Result, Error> { - let mut req = ureq::get("https://api.demos.tf/demos") - .query("page", &format!("{}", page)) - .query("order", &format!("{}", params.order)); - - if let Some(backend) = params.backend.as_ref() { - req = req.query("backend", backend); - } - - let resp = req.call()?; - - Ok(resp.into_json()?) -} diff --git a/src/backup.rs b/src/backup.rs index 309f874..c9ddded 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -1,41 +1,53 @@ -use crate::api::{list_demos, ListOrder, ListParams}; use crate::store::Store; use crate::Error; +use demostf_client::{ApiClient, Demo, ListOrder, ListParams}; +use tracing::{info, instrument}; pub struct Backup { + client: ApiClient, store: Store, } impl Backup { pub fn new(store: Store) -> Self { - Backup { store } + Backup { + store, + client: ApiClient::new(), + } } - fn backup_demo(&self, name: &str, url: &str, hash: [u8; 16]) -> Result<(), Error> { - let resp = ureq::get(url).call()?; + #[instrument(skip_all, fields(demo.id = demo.id, demo.name = name))] + async fn backup_demo(&self, name: &str, demo: &Demo) -> Result<(), Error> { + info!("backing up"); + let chunks = demo.download(&self.client).await?; - let digest = self.store.store(name, &mut resp.into_reader())?; + let digest = self.store.store(name, chunks).await?; - if digest == hash || digest == [0; 16] { + if digest == demo.hash || digest == [0; 16] { Ok(()) } else { let _ = self.store.remove(name); Err(Error::DigestMismatch { - expected: hash, + expected: demo.hash, got: digest, }) } } - fn backup_page(&self, page: u32) -> Result { - let demos = list_demos(ListParams::default().with_order(ListOrder::Ascending), page)?; + #[instrument(skip(self))] + async fn backup_page(&self, page: u32) -> Result { + let demos = self + .client + .list(ListParams::default().with_order(ListOrder::Ascending), page) + .await?; for demo in demos.iter() { if !demo.url.is_empty() { let name = demo.url.rsplit('/').next().unwrap(); - println!("{} {}", demo.id, name); if !self.store.exists(name) { - self.backup_demo(name, &demo.url, demo.hash)?; + self.backup_demo(name, demo).await?; + } else { + info!(demo = demo.id, name, "already backed up"); } } } @@ -43,8 +55,8 @@ impl Backup { Ok(demos.len()) } - pub fn backup_from(&self, mut page: u32) -> Result { - while self.backup_page(page)? > 0 { + pub async fn backup_from(&self, mut page: u32) -> Result { + while self.backup_page(page).await? > 0 { page += 1; } diff --git a/src/main.rs b/src/main.rs index d91b5b4..aa84c19 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,26 +8,22 @@ use std::cmp::max; use std::collections::HashMap; use std::path::PathBuf; use thiserror::Error; - -mod api; +use tracing::info; #[derive(Debug, Error)] pub enum Error { #[error("Request failed: {0}")] Request(#[from] std::io::Error), - #[error("Request failed: {0}")] - UReq(Box), + #[error(transparent)] + Api(#[from] demostf_client::Error), #[error("MD5 digest mismatch for downloaded demo, expected {expected:?}, received {got:?}")] DigestMismatch { expected: [u8; 16], got: [u8; 16] }, } -impl From for Error { - fn from(e: ureq::Error) -> Self { - Error::UReq(Box::new(e)) - } -} +#[tokio::main] +async fn main() -> Result<(), MainError> { + tracing_subscriber::fmt::init(); -fn main() -> Result<(), MainError> { let mut args: HashMap<_, _> = dotenv::vars().collect(); let store = Store::new(args.get("STORAGE_ROOT").expect("no STORAGE_ROOT set")); let state_path = PathBuf::from(args.remove("STATE_FILE").expect("no STATE_FILE set")); @@ -44,8 +40,9 @@ fn main() -> Result<(), MainError> { } else { 1u32 }; + info!(last_page, "starting backup"); - let current_page = backup.backup_from(last_page)?; + let current_page = backup.backup_from(last_page).await?; std::fs::write(&state_path, format!("{}", current_page))?; diff --git a/src/store.rs b/src/store.rs index 71f45c6..9f77b0a 100644 --- a/src/store.rs +++ b/src/store.rs @@ -1,9 +1,13 @@ +use crate::Error; +use bytes::Bytes; +use futures_util::{Stream, StreamExt}; use md5::Context; use std::fs; use std::fs::{File, Permissions}; -use std::io::{ErrorKind, Read, Write}; +use std::io::Write; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; +use tokio::pin; pub struct Store { basedir: PathBuf, @@ -16,30 +20,28 @@ impl Store { } } - pub fn store(&self, name: &str, data: &mut impl Read) -> std::io::Result<[u8; 16]> { + pub async fn store( + &self, + name: &str, + data: impl Stream>, + ) -> Result<[u8; 16], Error> { let path = self.generate_path(name); fs::create_dir_all(path.parent().unwrap())?; let mut file = File::create(&path)?; let mut context = Context::new(); - let mut buf = [0u8; 8 * 1024]; + pin!(data); // copy the file and compute the digest was we go - loop { - let len = match data.read(&mut buf) { - Ok(0) => return Ok(context.compute().0), - Ok(len) => len, - Err(ref e) if e.kind() == ErrorKind::Interrupted => continue, - Err(e) => return Err(e), - }; - - let data = &buf[..len]; - context.consume(data); - - file.write_all(data)?; - file.set_permissions(Permissions::from_mode(0o644))?; + while let Some(chunk) = data.next().await { + let chunk = chunk?; + context.consume(&chunk); + file.write_all(&chunk)?; } + file.set_permissions(Permissions::from_mode(0o644))?; + + Ok(context.compute().0) } pub fn exists(&self, name: &str) -> bool {