flake reorg, nix integration testing

This commit is contained in:
Robin Appelman 2024-12-24 15:52:24 +01:00
commit 5cfd70d583
14 changed files with 3221 additions and 145 deletions

32
.github/workflows/ci.yml vendored Normal file
View file

@ -0,0 +1,32 @@
name: "CI"
on:
pull_request:
push:
jobs:
matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v4
- uses: cachix/install-nix-action@v27
- id: set-matrix
run: echo "matrix={\"check\":$(nix eval --json '.#checks.x86_64-linux' --apply 'builtins.attrNames')}" | tee -a $GITHUB_OUTPUT
checks:
runs-on: ubuntu-latest
needs: [matrix]
strategy:
fail-fast: false
matrix: ${{fromJson(needs.matrix.outputs.matrix)}}
name: ${{ matrix.check }}
steps:
- uses: actions/checkout@v4
- uses: cachix/install-nix-action@v26
- uses: icewind1991/attic-action@v1
with:
name: ci
instance: https://cache.icewind.me
authToken: "${{ secrets.ATTIC_TOKEN }}"
- run: nix build .#checks.x86_64-linux.${{ matrix.check }}

View file

@ -1,72 +0,0 @@
on: [push, pull_request]
name: CI
jobs:
check:
name: Check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- uses: Swatinem/rust-cache@v1
- uses: actions-rs/cargo@v1
with:
command: check
test:
runs-on: ubuntu-latest
name: Tests
services:
api-test-db:
image: demostf/db
env:
POSTGRES_PASSWORD: test
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
ports:
- 5432:5432
api:
image: demostf/api
env:
DEMO_ROOT: /demos
DEMO_HOST: localhost
DB_TYPE: pgsql
DB_HOST: api-test-db
DB_PORT: 5432
DB_DATABASE: postgres
DB_USERNAME: postgres
DB_PASSWORD: test
APP_ROOT: https://api.localhost
EDIT_SECRET: edit
volumes:
- /tmp:/demos
api-test:
image: demostf/api-nginx-test
env:
POSTGRES_PASSWORD: test
ports:
- 80:80
volumes:
- /tmp:/demos
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
toolchain: stable
- uses: Swatinem/rust-cache@v1
- uses: actions-rs/cargo@v1
env:
DB_URL: postgres://postgres:test@localhost/postgres
API_ROOT: http://localhost/
with:
command: test
args: -- --test-threads 1
- name: logs
uses: jwalton/gh-docker-logs@v1
if: failure()

1
.gitignore vendored
View file

@ -1,4 +1,3 @@
/target
Cargo.lock
result
.direnv

2848
Cargo.lock generated Normal file

File diff suppressed because it is too large Load diff

View file

@ -1,33 +1,38 @@
[package]
name = "demostf-client"
version = "0.4.3"
version = "0.4.4"
authors = ["Robin Appelman <robin@icewind.nl>"]
edition = "2018"
edition = "2021"
description = "Api client for demos.tf"
license = "MIT OR Apache-2.0"
repository = "https://github.com/demostf/api-client"
readme = "README.md"
exclude = [ "tests/data/gully.dem" ]
exclude = ["tests/data/gully.dem"]
keywords = ["tf2", "demo"]
categories = ["api-bindings"]
rust-version = "1.81.0"
[dependencies]
serde = { version = "1.0.144", features = ["derive"] }
time = { version = "0.3.13", features = ["serde"] }
reqwest = { version = "0.11.11", default-features = false, features = ["json", "multipart", "stream"] }
thiserror = "1.0.32"
serde = { version = "1.0.216", features = ["derive"] }
time = { version = "0.3.37", features = ["serde"] }
reqwest = { version = "0.12.9", default-features = false, features = [
"json",
"multipart",
"stream",
] }
thiserror = "2.0.9"
hex = "0.4.3"
steamid-ng = "1.0.0"
bytes = "1.2.1"
futures-util = "0.3.23"
tracing = "0.1.36"
tinyvec = "1.6.0"
bytes = "1.9.0"
futures-util = "0.3.31"
tracing = "0.1.41"
tinyvec = { version = "1.8.1", features = ["alloc"] }
md5 = "0.7.0"
[dev-dependencies]
tokio = { version = "1.20.1", features = ["macros"] }
sqlx = { version = "0.6.1", features = ["postgres", "runtime-tokio-rustls"] }
tracing-subscriber = { version = "0.3.15", features = ["env-filter"] }
tokio = { version = "1.42.0", features = ["macros"] }
sqlx = { version = "0.8.2", features = ["postgres", "runtime-tokio-rustls"] }
tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
[features]
default = ["default-tls"]

102
flake.lock generated
View file

@ -1,66 +1,100 @@
{
"nodes": {
"naersk": {
"crane": {
"locked": {
"lastModified": 1733688869,
"narHash": "sha256-KrhxxFj1CjESDrL5+u/zsVH0K+Ik9tvoac/oFPoxSB8=",
"owner": "ipetkov",
"repo": "crane",
"rev": "604637106e420ad99907cae401e13ab6b452e7d9",
"type": "github"
},
"original": {
"owner": "ipetkov",
"repo": "crane",
"type": "github"
}
},
"flakelight": {
"inputs": {
"nixpkgs": "nixpkgs"
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1659610603,
"narHash": "sha256-LYgASYSPYo7O71WfeUOaEUzYfzuXm8c8eavJcel+pfI=",
"lastModified": 1734957624,
"narHash": "sha256-RbvX9lf9lWQwG9vTXkscOiWTrKf8lzjyeOvW/v8IuBY=",
"owner": "nix-community",
"repo": "naersk",
"rev": "c6a45e4277fa58abd524681466d3450f896dc094",
"repo": "flakelight",
"rev": "8c226ea0166784b02d4a58fbb015f9c01250221e",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "naersk",
"repo": "flakelight",
"type": "github"
}
},
"mill-scale": {
"inputs": {
"crane": "crane",
"flakelight": [
"flakelight"
],
"rust-overlay": "rust-overlay"
},
"locked": {
"lastModified": 1734998048,
"narHash": "sha256-EzIE9/jrgvUdNxvZrjD+qqkH5AHaaEbJDsLDzSuizYw=",
"path": "/home/robin/Projects/mill-scale",
"type": "path"
},
"original": {
"path": "/home/robin/Projects/mill-scale",
"type": "path"
}
},
"nixpkgs": {
"locked": {
"lastModified": 0,
"narHash": "sha256-ogcrJszrCg23/mIcLEOUCMKgdWlqMJ4QqezvX0V2ZQk=",
"path": "/nix/store/f6y01zll9swq7rvf82ya4r3hjs9j93py-source",
"type": "path"
},
"original": {
"id": "nixpkgs",
"type": "indirect"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 0,
"narHash": "sha256-ogcrJszrCg23/mIcLEOUCMKgdWlqMJ4QqezvX0V2ZQk=",
"path": "/nix/store/f6y01zll9swq7rvf82ya4r3hjs9j93py-source",
"type": "path"
"lastModified": 1734875076,
"narHash": "sha256-Pzyb+YNG5u3zP79zoi8HXYMs15Q5dfjDgwCdUI5B0nY=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "1807c2b91223227ad5599d7067a61665c52d1295",
"type": "github"
},
"original": {
"id": "nixpkgs",
"ref": "nixos-24.11",
"type": "indirect"
}
},
"root": {
"inputs": {
"naersk": "naersk",
"nixpkgs": "nixpkgs_2",
"utils": "utils"
"flakelight": "flakelight",
"mill-scale": "mill-scale",
"nixpkgs": "nixpkgs"
}
},
"utils": {
"rust-overlay": {
"inputs": {
"nixpkgs": [
"mill-scale",
"flakelight",
"nixpkgs"
]
},
"locked": {
"lastModified": 1659877975,
"narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
"lastModified": 1733884434,
"narHash": "sha256-8GXR9kC07dyOIshAyfZhG11xfvBRSZzYghnZ2weOKJU=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "d0483df44ddf0fd1985f564abccbe568e020ddf2",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"owner": "oxalica",
"repo": "rust-overlay",
"type": "github"
}
}

View file

@ -1,22 +1,25 @@
{
inputs = {
utils.url = "github:numtide/flake-utils";
naersk.url = "github:nix-community/naersk";
nixpkgs.url = "nixpkgs/nixos-24.11";
flakelight = {
url = "github:nix-community/flakelight";
inputs.nixpkgs.follows = "nixpkgs";
};
mill-scale = {
# url = "github:icewind1991/mill-scale";
url = "path:/home/robin/Projects/mill-scale";
inputs.flakelight.follows = "flakelight";
};
};
outputs = {
self,
nixpkgs,
utils,
naersk,
}:
utils.lib.eachDefaultSystem (system: let
pkgs = nixpkgs.legacyPackages."${system}";
naersk-lib = naersk.lib."${system}";
in rec {
devShell = pkgs.mkShell {
nativeBuildInputs = with pkgs; [rustc cargo bacon pkg-config cargo-edit cargo-outdated];
buildInputs = with pkgs; [ openssl ];
outputs = {mill-scale, ...}:
mill-scale ./. {
cargoTest = false;
withOverlays = [(import ./nix/overlay.nix)];
packages = {
test-runner = pkgs: pkgs.test-runner;
};
});
checks = {
test = pkgs: pkgs.nixosTest (import ./nix/test.nix);
};
};
}

27
nix/demostf-api.nix Normal file
View file

@ -0,0 +1,27 @@
{
php,
fetchFromGitHub,
}: let
phpWithExtensions = php.withExtensions ({
enabled,
all,
}:
enabled ++ (with all; [pdo apcu]));
in
phpWithExtensions.buildComposerProject (finalAttrs: {
pname = "demostf-api";
version = "0.1.0";
src = fetchFromGitHub {
owner = "demostf";
repo = "api";
rev = "1a8380360b993226ae1c6fcc226011e03a6c1467";
hash = "sha256-JcBRU1N44tt0QDLnj6z9MCT3V2s2dkf+JbpWb1rmXnY=";
};
vendorHash = "sha256-EYWCR2aJAoyWvEX+SML4Fb3F3KGcUtwCgqhAGT6ZjZ4=";
composerStrictValidation = false;
doCheck = false;
})

23
nix/demostf-parser.nix Normal file
View file

@ -0,0 +1,23 @@
{
rustPlatform,
fetchFromGitHub,
}:
rustPlatform.buildRustPackage {
pname = "demostf-parser";
version = "0.5.1";
src = fetchFromGitHub {
owner = "demostf";
repo = "parser";
rev = "0cd87a8a40e2a6af637d831b272c2758cebd2f9c";
hash = "sha256-bKcc0hWTkdYUDMI/DjUh45abuBeQEvkn6TsuAz02H5Y=";
};
cargoBuildFlags = ''
--bin parse_demo
'';
doCheck = false;
cargoHash = "sha256-/Fnw6l2fznrBK780E4q1PKFOkT0eiL+dE+UuhFA+V9M=";
}

5
nix/overlay.nix Normal file
View file

@ -0,0 +1,5 @@
final: prev: {
demostf-api = final.callPackage ./demostf-api.nix {};
demostf-parser = final.callPackage ./demostf-parser.nix {};
test-runner = final.callPackage ./test-runner.nix {};
}

42
nix/test-runner.nix Normal file
View file

@ -0,0 +1,42 @@
{
rustPlatform,
openssl,
pkg-config,
lib,
}: let
inherit (lib.sources) sourceByRegex;
inherit (builtins) fromTOML readFile;
src = sourceByRegex ../. ["Cargo.*" "(src|tests)(/.*)?"];
cargoPackage = (fromTOML (readFile ../Cargo.toml)).package;
in
rustPlatform.buildRustPackage {
pname = cargoPackage.name;
inherit (cargoPackage) version;
inherit src;
buildInputs = [
openssl
];
nativeBuildInputs = [
pkg-config
];
doCheck = false;
buildPhase = ''
cargo build --tests
'';
installPhase = ''
mkdir -p $out/bin
cp target/debug/deps/tests-???????????????? $out/bin/api-test
'';
cargoLock = {
lockFile = ../Cargo.lock;
};
meta.mainProgram = "api-test";
}

126
nix/test.nix Normal file
View file

@ -0,0 +1,126 @@
{
pkgs,
lib,
...
}: {
name = "demostf-api-client-test";
nodes.machine = {config, ...}: let
fpmCfg = config.services.phpfpm.pools.demostf-api;
in {
config = {
users.groups.demostf = {};
users.users.demostf = {
group = "demostf";
isSystemUser = true;
};
services.postgresql = {
enable = true;
ensureDatabases = ["demostf"];
ensureUsers = [
{
name = "demostf";
ensureDBOwnership = true;
}
];
initialScript = pkgs.writeText "init-sql-script" ''
CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
'';
};
services.nginx = {
enable = true;
virtualHosts."localhost" = {
root = "/var/empty";
extraConfig = ''
try_files $uri /index.php?$query_string /index.php;
'';
locations = {
"~ ^(.+?\\.php)(/.*)?$" = {
extraConfig = ''
fastcgi_param PATH_INFO $2;
fastcgi_pass unix:${fpmCfg.socket};
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME ${pkgs.demostf-api}/share/php/demostf-api/src/public/index.php;
include ${pkgs.nginx}/conf/fastcgi_params;
client_max_body_size 250m;
'';
};
"= /upload" = {
extraConfig = ''
fastcgi_pass unix:${fpmCfg.socket};
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME ${pkgs.demostf-api}/share/php/demostf-api/src/public/upload.php;
include ${pkgs.nginx}/conf/fastcgi_params;
client_max_body_size 250m;
'';
};
"/static/" = {
alias = "/demos/";
};
};
};
};
services.phpfpm.pools.demostf-api = {
phpPackage = pkgs.php.buildEnv {
extensions = {
enabled,
all,
}:
enabled ++ (with all; [pdo apcu]);
extraConfig = ''
post_max_size = 150M
upload_max_filesize = 150M
'';
};
settings = {
"clear_env" = "no";
"pm" = "dynamic";
"pm.max_children" = "25";
"pm.start_servers" = "5";
"pm.min_spare_servers" = "5";
"pm.max_spare_servers" = "15";
"catch_workers_output" = "yes";
"listen.owner" = "nginx";
"listen.group" = "nginx";
};
phpEnv = {
BASE_HOST = "demos.tf";
DEMO_ROOT = "/demos";
DEMO_HOST = "localhost";
DB_TYPE = "pgsql";
DB_HOST = "/run/postgresql";
DB_PORT = "5432";
DB_DATABASE = "demostf";
DB_USERNAME = "demostf";
APP_ROOT = "http://localhost";
EDIT_SECRET = "edit";
PARSER_PATH = "${pkgs.demostf-parser}/bin/parse_demo";
};
user = "demostf";
group = "demostf";
};
};
};
testScript = let
testBinary = lib.getExe pkgs.test-runner;
initSql = pkgs.fetchurl {
url = "https://github.com/demostf/db/raw/refs/heads/master/schema.sql";
hash = "sha256-AwXN9mh9CRk6HWdvyUR+YdBkpmExNIDOIeDMz6XqjEQ=";
};
in ''
machine.succeed("mkdir /demos && chmod 0777 /demos");
machine.wait_for_unit("postgresql")
machine.succeed("sudo -u demostf psql demostf demostf < ${initSql}");
machine.succeed("sudo -u postgres psql postgres postgres -c \"alter user demostf with password 'test';\"");
machine.wait_for_unit("phpfpm-demostf-api")
machine.wait_for_unit("nginx")
machine.wait_until_succeeds("curl http://127.0.0.1", timeout=45)
machine.succeed("DB_URL='postgres://demostf:test@localhost/demostf'\
TEST_DEMO='${../tests/data/gully.dem}'\
API_ROOT='http://localhost/'\
${testBinary} --test-threads 1", timeout=180)
'';
}

View file

@ -7,6 +7,10 @@ use tracing_subscriber::EnvFilter;
static SETUP_DONE: AtomicBool = AtomicBool::new(false);
fn test_demo_path() -> String {
std::env::var("TEST_DEMO").unwrap_or_else(|_| "./tests/data/gully.dem".to_string())
}
async fn setup() {
let _ = tracing_subscriber::fmt()
.with_env_filter(EnvFilter::from_default_env())
@ -38,18 +42,18 @@ async fn setup() {
for table in &tables {
sqlx::query(&format!("TRUNCATE TABLE {}", table))
.execute(&mut transaction)
.execute(&mut *transaction)
.await
.unwrap();
sqlx::query(&format!("ALTER SEQUENCE {}_id_seq RESTART with 1", table))
.execute(&mut transaction)
.execute(&mut *transaction)
.await
.unwrap();
}
sqlx::query("INSERT INTO users(steamid, name, avatar, token)\
VALUES(76561198024494988, 'Icewind', 'http://cdn.akamai.steamstatic.com/steamcommunity/public/images/avatars/75/75b84075b70535c5cfb3499af03b3e4e7a7b556f_medium.jpg', 'test_token')")
.execute(&mut transaction).await.unwrap();
.execute(&mut *transaction).await.unwrap();
transaction.commit().await.unwrap();
@ -58,14 +62,14 @@ async fn setup() {
let client = ApiClient::with_base_url(&api_root).unwrap();
upload(&client, "./tests/data/gully.dem", "test.dem", "R", "B").await;
upload(&client, &test_demo_path(), "test.dem", "R", "B").await;
let mut transaction = pool.begin().await.unwrap();
let views = ["map_list", "name_list", "users_named"];
for view in views {
sqlx::query(&format!("REFRESH MATERIALIZED VIEW {}", view))
.execute(&mut transaction)
.execute(&mut *transaction)
.await
.unwrap();
}
@ -108,7 +112,7 @@ async fn upload(client: &ApiClient, source: &str, name: &str, red: &str, blue: &
async fn test_upload_invalid_key() {
let client = test_client().await;
let data = std::fs::read("./tests/data/gully.dem").unwrap();
let data = std::fs::read(test_demo_path()).unwrap();
let err = client
.upload_demo(
@ -181,7 +185,7 @@ async fn test_get_chat() {
let chat = client.get_chat(1).await.unwrap();
assert_eq!(chat.len(), 134);
assert_eq!(chat.len(), 199);
assert_eq!(chat[0].user, "distraughtduck4");
assert_eq!(chat[0].time, 0);
@ -367,5 +371,5 @@ async fn test_download_demo() {
let mut data: Vec<u8> = Vec::new();
demo.save(&client, &mut data).await.unwrap();
assert_eq!(data.len(), read("./tests/data/gully.dem").unwrap().len());
assert_eq!(data.len(), read(test_demo_path()).unwrap().len());
}