proper config file

This commit is contained in:
Robin Appelman 2024-03-27 13:33:29 +01:00
commit b361d49340
7 changed files with 389 additions and 65 deletions

1
.gitignore vendored
View file

@ -2,3 +2,4 @@
.direnv
.env
result
config.toml

294
Cargo.lock generated
View file

@ -26,6 +26,54 @@ dependencies = [
"memchr",
]
[[package]]
name = "anstream"
version = "0.6.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb"
dependencies = [
"anstyle",
"anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
"utf8parse",
]
[[package]]
name = "anstyle"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc"
[[package]]
name = "anstyle-parse"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648"
dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "anstyle-wincon"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
dependencies = [
"anstyle",
"windows-sys 0.52.0",
]
[[package]]
name = "autocfg"
version = "1.1.0"
@ -86,6 +134,52 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clap"
version = "4.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c"
dependencies = [
"clap_builder",
"clap_derive",
]
[[package]]
name = "clap_builder"
version = "4.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7"
dependencies = [
"anstream",
"anstyle",
"clap_lex",
"strsim",
]
[[package]]
name = "clap_derive"
version = "4.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "clap_lex"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
[[package]]
name = "colorchoice"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
[[package]]
name = "core-foundation"
version = "0.9.4"
@ -125,14 +219,17 @@ dependencies = [
name = "demostf-mover"
version = "0.1.0"
dependencies = [
"clap",
"demostf-client",
"dotenvy",
"futures-util",
"main_error",
"md5",
"secretfile",
"serde",
"thiserror",
"time",
"tokio",
"toml",
"tracing",
"tracing-subscriber",
]
@ -147,12 +244,6 @@ dependencies = [
"serde",
]
[[package]]
name = "dotenvy"
version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
[[package]]
name = "encoding_rs"
version = "0.8.33"
@ -295,6 +386,12 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
[[package]]
name = "heck"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
[[package]]
name = "hermit-abi"
version = "0.3.3"
@ -489,7 +586,7 @@ checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09"
dependencies = [
"libc",
"wasi",
"windows-sys",
"windows-sys 0.48.0",
]
[[package]]
@ -743,7 +840,7 @@ dependencies = [
"libc",
"spin",
"untrusted",
"windows-sys",
"windows-sys 0.48.0",
]
[[package]]
@ -800,19 +897,28 @@ dependencies = [
]
[[package]]
name = "serde"
version = "1.0.195"
name = "secretfile"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02"
checksum = "746c54b939ab8d393b536765393c0bd7634fca94eed62321ec3e3559293f6c21"
dependencies = [
"thiserror",
]
[[package]]
name = "serde"
version = "1.0.197"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.195"
version = "1.0.197"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c"
checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
dependencies = [
"proc-macro2",
"quote",
@ -830,6 +936,15 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_spanned"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1"
dependencies = [
"serde",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@ -873,7 +988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9"
dependencies = [
"libc",
"windows-sys",
"windows-sys 0.48.0",
]
[[package]]
@ -897,6 +1012,12 @@ dependencies = [
"thiserror",
]
[[package]]
name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "syn"
version = "2.0.48"
@ -931,18 +1052,18 @@ dependencies = [
[[package]]
name = "thiserror"
version = "1.0.56"
version = "1.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad"
checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.56"
version = "1.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471"
checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
dependencies = [
"proc-macro2",
"quote",
@ -1016,7 +1137,7 @@ dependencies = [
"pin-project-lite",
"socket2",
"tokio-macros",
"windows-sys",
"windows-sys 0.48.0",
]
[[package]]
@ -1054,6 +1175,40 @@ dependencies = [
"tracing",
]
[[package]]
name = "toml"
version = "0.8.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"toml_edit",
]
[[package]]
name = "toml_datetime"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1"
dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
version = "0.22.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4"
dependencies = [
"indexmap",
"serde",
"serde_spanned",
"toml_datetime",
"winnow",
]
[[package]]
name = "tower-service"
version = "0.3.2"
@ -1170,6 +1325,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "utf8parse"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
[[package]]
name = "valuable"
version = "0.1.0"
@ -1320,7 +1481,16 @@ version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
dependencies = [
"windows-targets",
"windows-targets 0.48.5",
]
[[package]]
name = "windows-sys"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets 0.52.4",
]
[[package]]
@ -1329,13 +1499,28 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
"windows_aarch64_gnullvm 0.48.5",
"windows_aarch64_msvc 0.48.5",
"windows_i686_gnu 0.48.5",
"windows_i686_msvc 0.48.5",
"windows_x86_64_gnu 0.48.5",
"windows_x86_64_gnullvm 0.48.5",
"windows_x86_64_msvc 0.48.5",
]
[[package]]
name = "windows-targets"
version = "0.52.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b"
dependencies = [
"windows_aarch64_gnullvm 0.52.4",
"windows_aarch64_msvc 0.52.4",
"windows_i686_gnu 0.52.4",
"windows_i686_msvc 0.52.4",
"windows_x86_64_gnu 0.52.4",
"windows_x86_64_gnullvm 0.52.4",
"windows_x86_64_msvc 0.52.4",
]
[[package]]
@ -1344,42 +1529,93 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9"
[[package]]
name = "windows_aarch64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675"
[[package]]
name = "windows_i686_gnu"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
[[package]]
name = "windows_i686_gnu"
version = "0.52.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3"
[[package]]
name = "windows_i686_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
[[package]]
name = "windows_i686_msvc"
version = "0.52.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02"
[[package]]
name = "windows_x86_64_gnu"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177"
[[package]]
name = "windows_x86_64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8"
[[package]]
name = "winnow"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8"
dependencies = [
"memchr",
]
[[package]]
name = "winreg"
version = "0.50.0"
@ -1387,5 +1623,5 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
dependencies = [
"cfg-if",
"windows-sys",
"windows-sys 0.48.0",
]

View file

@ -11,6 +11,9 @@ tracing = "0.1.40"
tracing-subscriber = "0.3.18"
futures-util = "0.3.30"
main_error = "0.1.2"
dotenvy = "0.15.7"
time = "0.3.31"
md5 = "0.7.0"
serde = { version = "1.0.197", features = ["derive"] }
toml = "0.8.12"
secretfile = "0.1.0"
clap = { version = "=4.4.18", features = ["derive"] }

14
config.samle.toml Normal file
View file

@ -0,0 +1,14 @@
[source]
root = "/data/demos" # root directory of the source demo storage
backend = "freezer" # name of the source demo storage
[target]
root = "/data2/demos" # root directory of the target demo storage
backend = "freezer2" # name of the target demo storage
[api]
url = "https://api.demos.tf" # optional, base url of the api
key_file = "" # file containing the api key
[migrate]
age = 172800 # age of demos in secconds to migrate

View file

@ -6,10 +6,33 @@
}:
with lib; let
cfg = config.services.demostf-mover;
format = pkgs.formats.toml {};
configFile = format.generate "demostf-mover.toml" {
api = {
url = cfg.api;
key_file = "$CREDENTIALS_DIRECTORY/api_key";
};
source = {
backend = cfg.sourceBackend;
root = cfg.source;
};
target = {
backend = cfg.targetBackend;
root = cfg.target;
};
migrate = {
age = cfg.age;
};
};
in {
options.services.demostf-mover = {
enable = mkEnableOption "Enables the demos mover service";
api = mkOption {
type = types.str;
default = "https://api.demos.tf";
description = "Api endpoint to migrate demos for";
};
source = mkOption {
type = types.str;
description = "source directory";
@ -61,19 +84,12 @@ in {
systemd.services.demostf-mover = {
description = "Move demos for demos.tf";
environment = {
SOURCE_ROOT = cfg.source;
TARGET_ROOT = cfg.target;
SOURCE_BACKEND = cfg.sourceBackend;
TARGET_BACKEND = cfg.targetBackend;
AGE = toString cfg.age;
RUST_LOG = cfg.logLevel;
};
serviceConfig = {
ExecStart = "${cfg.package}/bin/demostf-mover";
EnvironmentFile = cfg.keyFile;
ExecStart = "${cfg.package}/bin/demostf-mover ${configFile}";
ReadWritePaths = [cfg.source cfg.target];
LoadCredential = [
"api_key:${cfg.keyFile}"
];
Restart = "on-failure";
User = cfg.user;
PrivateTmp = true;

55
src/config.rs Normal file
View file

@ -0,0 +1,55 @@
use demostf_client::ApiClient;
use serde::Deserialize;
use std::fs::read_to_string;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum ConfigError {
#[error("Failed to read config from {path}: {error}")]
Read { error: std::io::Error, path: String },
#[error("Failed to parse config from {path}: {error}")]
Parse {
error: toml::de::Error,
path: String,
},
}
#[derive(Debug, Deserialize)]
pub struct Config {
pub api: ApiConfig,
pub source: StorageConfig,
pub target: StorageConfig,
pub migrate: MigrateConfig,
}
impl Config {
pub fn load(path: String) -> Result<Self, ConfigError> {
let content = read_to_string(&path).map_err(|error| ConfigError::Read {
error,
path: path.clone(),
})?;
toml::from_str(&content).map_err(|error| ConfigError::Parse { error, path })
}
}
#[derive(Debug, Deserialize)]
pub struct ApiConfig {
#[serde(default = "default_api_base")]
pub url: String,
pub key_file: String,
}
fn default_api_base() -> String {
ApiClient::DEMOS_TF_BASE_URL.into()
}
#[derive(Debug, Deserialize)]
pub struct MigrateConfig {
pub age: u64,
}
#[derive(Debug, Deserialize)]
pub struct StorageConfig {
pub root: String,
pub backend: String,
}

View file

@ -1,7 +1,9 @@
use crate::config::{Config, ConfigError};
use clap::Parser;
use demostf_client::{ApiClient, Demo, ListOrder, ListParams};
use main_error::{MainError, MainResult};
use md5::Context;
use std::collections::HashMap;
use secretfile::{load, SecretError};
use std::fs::{copy, create_dir_all, remove_file, write, File};
use std::io::Read;
use std::path::{Path, PathBuf};
@ -11,6 +13,8 @@ use time::OffsetDateTime;
use tokio::time::timeout;
use tracing::{error, info, instrument, warn};
mod config;
#[derive(Debug, Error)]
pub enum Error {
#[error("Request failed: {0}")]
@ -19,36 +23,31 @@ pub enum Error {
Api(#[from] demostf_client::Error),
#[error("Backup timed out")]
Timeout,
#[error(transparent)]
Config(#[from] ConfigError),
#[error(transparent)]
Secret(#[from] SecretError),
}
#[derive(Debug, Parser)]
struct Args {
/// Config file
config: String,
}
#[tokio::main]
async fn main() -> Result<(), MainError> {
tracing_subscriber::fmt::init();
let mut args: HashMap<_, _> = dotenvy::vars().collect();
let args = Args::parse();
let config = Config::load(args.config)?;
let source_root: PathBuf = args
.remove("SOURCE_ROOT")
.expect("no SOURCE_ROOT set")
.trim_end_matches('/')
.into();
let target_root: PathBuf = args
.remove("TARGET_ROOT")
.expect("no TARGET_ROOT set")
.trim_end_matches('/')
.into();
let api_key = args.remove("KEY").expect("no KEY set");
let source_backend = args
.remove("SOURCE_BACKEND")
.expect("no SOURCE_BACKEND set");
let target_backend = args
.remove("TARGET_BACKEND")
.expect("no TARGET_BACKEND set");
let age: u64 = args
.get("AGE")
.expect("no AGE set")
.parse()
.expect("invalid AGE");
let source_root: PathBuf = config.source.root.trim_end_matches('/').into();
let target_root: PathBuf = config.target.root.trim_end_matches('/').into();
let api_key = load(&config.api.key_file)?;
let source_backend = config.source.backend;
let target_backend = config.target.backend;
let age = config.migrate.age;
let cutoff = OffsetDateTime::now_utc() - Duration::from_secs(age);
info!(cutoff = display(cutoff), "starting move");