flake rework

This commit is contained in:
Robin Appelman 2024-11-08 23:14:56 +01:00
commit 2e3ef9dfd3
11 changed files with 718 additions and 566 deletions

View file

@ -1,55 +1,17 @@
on: [push, pull_request]
name: CI
name: "CI"
on:
pull_request:
push:
jobs:
check:
checks:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v20
- uses: actions/checkout@v4
- uses: cachix/install-nix-action@v26
- uses: icewind1991/attic-action@v1
with:
name: ci
instance: https://cache.icewind.me
authToken: '${{ secrets.ATTIC_TOKEN }}'
- run: nix build .#check
clippy:
runs-on: ubuntu-latest
needs: check
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v20
- uses: icewind1991/attic-action@v1
with:
name: ci
instance: https://cache.icewind.me
authToken: '${{ secrets.ATTIC_TOKEN }}'
- run: nix build .#clippy
msrv:
runs-on: ubuntu-latest
needs: check
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v20
- uses: icewind1991/attic-action@v1
with:
name: ci
instance: https://cache.icewind.me
authToken: '${{ secrets.ATTIC_TOKEN }}'
- run: nix build .#msrv
build:
runs-on: ubuntu-latest
needs: check
steps:
- uses: actions/checkout@v3
- uses: cachix/install-nix-action@v20
- uses: icewind1991/attic-action@v1
with:
name: ci
instance: https://cache.icewind.me
authToken: '${{ secrets.ATTIC_TOKEN }}'
- run: nix build .#ugc-api-server
authToken: "${{ secrets.ATTIC_TOKEN }}"
- run: nix flake check --keep-going

866
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -2,24 +2,24 @@
name = "ugc-scraper"
version = "0.4.1"
edition = "2021"
rust-version = "1.67.0"
rust-version = "1.70.0"
description = "Scraper for ugcleague.com"
license = "MIT OR Apache-2.0"
homepage = "https://github.com/icewind1991/ugc-scaper"
[dependencies]
tokio = "1.36.0"
reqwest = "0.11.24"
tokio = "1.38.1"
reqwest = "0.11.27"
scraper = "0.18.1"
thiserror = "1.0.57"
time = { version = "0.3.34", features = ["parsing", "macros"] }
thiserror = "2.0.1"
time = { version = "0.3.36", features = ["parsing", "macros"] }
steamid-ng = "1.0.0"
serde = { version = "1.0.197", features = ["derive"], optional = true }
serde = { version = "1.0.214", features = ["derive"], optional = true }
[dev-dependencies]
tokio = { version = "1.36.0", features = ["macros", "rt-multi-thread", "rt"] }
tokio = { version = "1.38.1", features = ["macros", "rt-multi-thread", "rt"] }
main_error = "0.1.2"
insta = { version = "1.35.1", features = ["json"] }
insta = { version = "1.41.1", features = ["json"] }
test-case = "3.3.1"
[profile.dev.package]

View file

@ -1,6 +1,6 @@
# ugc-scraper
*We have ugc api at home*
_We have ugc api at home_
## Usage
@ -22,4 +22,4 @@ async fn main() -> Result<()> {
Ok(())
}
```
```

View file

@ -38,4 +38,4 @@ Get match information
### `/maps/:format`
Get map history by format (`9v9`, `6v6`, `4v4`, `2v2`)
Get map history by format (`9v9`, `6v6`, `4v4`, `2v2`)

108
flake.lock generated
View file

@ -1,63 +1,98 @@
{
"nodes": {
"naersk": {
"crane": {
"locked": {
"lastModified": 1730060262,
"narHash": "sha256-RMgSVkZ9H03sxC+Vh4jxtLTCzSjPq18UWpiM0gq6shQ=",
"owner": "ipetkov",
"repo": "crane",
"rev": "498d9f122c413ee1154e8131ace5a35a80d8fa76",
"type": "github"
},
"original": {
"owner": "ipetkov",
"repo": "crane",
"type": "github"
}
},
"flakelight": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1698420672,
"narHash": "sha256-/TdeHMPRjjdJub7p7+w55vyABrsJlt5QkznPYy55vKA=",
"lastModified": 1730724080,
"narHash": "sha256-0dNZpQzIsN+KxbsBGCwRQUJPHrvEuu/x0sJq0Z7xSCg=",
"owner": "nix-community",
"repo": "naersk",
"rev": "aeb58d5e8faead8980a807c840232697982d47b9",
"repo": "flakelight",
"rev": "aa5d58bd458e846e8dc7d160d9167b74d6976a41",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "naersk",
"repo": "flakelight",
"type": "github"
}
},
"mill-scale": {
"inputs": {
"crane": "crane",
"flakelight": [
"flakelight"
],
"rust-overlay": "rust-overlay"
},
"locked": {
"lastModified": 1731105211,
"narHash": "sha256-NzpaQuPvMS2TcW3Mbhe6AEhtCzk/vQO523SFCz/sbes=",
"owner": "icewind1991",
"repo": "mill-scale",
"rev": "c38c7c897b81928b0548f4a9d8958af7bea02934",
"type": "github"
},
"original": {
"owner": "icewind1991",
"repo": "mill-scale",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1708161998,
"narHash": "sha256-6KnemmUorCvlcAvGziFosAVkrlWZGIc6UNT9GUYr0jQ=",
"lastModified": 1730883749,
"narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "84d981bae8b5e783b3b548de505b22880559515f",
"rev": "dba414932936fde69f0606b4f1d87c5bc0003ede",
"type": "github"
},
"original": {
"id": "nixpkgs",
"ref": "nixos-23.11",
"ref": "nixos-24.05",
"type": "indirect"
}
},
"root": {
"inputs": {
"naersk": "naersk",
"nixpkgs": "nixpkgs",
"rust-overlay": "rust-overlay",
"utils": "utils"
"flakelight": "flakelight",
"mill-scale": "mill-scale",
"nixpkgs": "nixpkgs"
}
},
"rust-overlay": {
"inputs": {
"flake-utils": [
"utils"
],
"nixpkgs": [
"mill-scale",
"flakelight",
"nixpkgs"
]
},
"locked": {
"lastModified": 1708241671,
"narHash": "sha256-zSulX9tP4R35Y8A842dGSzaHMVP91W2Ry0SXvQKD2BQ=",
"lastModified": 1730255392,
"narHash": "sha256-9pydem8OVxa0TwjUai1PJe0yHAJw556CWCEwyoAq8Ik=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "d500e370b26f9b14303cb39bf1509df0a920c8b0",
"rev": "7509d76ce2b3d22b40bd25368b45c0a9f7f36c89",
"type": "github"
},
"original": {
@ -65,39 +100,6 @@
"repo": "rust-overlay",
"type": "github"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1705309234,
"narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
}
},
"root": "root",

137
flake.nix
View file

@ -1,115 +1,30 @@
{
inputs = {
nixpkgs.url = "nixpkgs/nixos-23.11";
utils.url = "github:numtide/flake-utils";
naersk.url = "github:nix-community/naersk";
naersk.inputs.nixpkgs.follows = "nixpkgs";
rust-overlay.url = "github:oxalica/rust-overlay";
rust-overlay.inputs.nixpkgs.follows = "nixpkgs";
rust-overlay.inputs.flake-utils.follows = "utils";
};
outputs = {
self,
nixpkgs,
utils,
naersk,
rust-overlay,
}:
utils.lib.eachDefaultSystem (system: let
overlays = [
(import rust-overlay)
(import ./overlay.nix)
];
pkgs = (import nixpkgs) {
inherit system overlays;
};
inherit (pkgs) lib callPackage rust-bin mkShell;
inherit (lib.sources) sourceByRegex;
inherit (builtins) fromTOML readFile;
msrv = (fromTOML (readFile ./Cargo.toml)).package.rust-version;
toolchain = rust-bin.stable.latest.default;
msrvToolchain = rust-bin.stable."${msrv}".default;
naersk' = callPackage naersk {
rustc = toolchain;
cargo = toolchain;
};
msrvNaersk = callPackage naersk {
rustc = msrvToolchain;
cargo = msrvToolchain;
};
src = sourceByRegex ./. ["Cargo.*" "(src|derive|benches|tests|examples)(/.*)?"];
deps = with pkgs; [
pkg-config
openssl
];
nearskOpt = {
pname = "ugc-scraper";
root = src;
nativeBuildInputs = deps;
};
in rec {
packages = rec {
check = naersk'.buildPackage (nearskOpt
// {
mode = "check";
});
clippy = naersk'.buildPackage (nearskOpt
// {
mode = "clippy";
});
test = naersk'.buildPackage (nearskOpt
// {
release = false;
mode = "test";
});
msrv = msrvNaersk.buildPackage (nearskOpt
// {
mode = "check";
});
inherit (pkgs) ugc-api-server;
default = ugc-api-server;
};
devShells = let
tools = with pkgs; [
bacon
cargo-insta
cargo-edit
cargo-outdated
cargo-audit
cargo-msrv
cargo-semver-checks
];
in {
default = mkShell {
OPENSSL_NO_VENDOR = 1;
nativeBuildInputs = [toolchain] ++ tools ++ deps;
};
msrv = mkShell {
OPENSSL_NO_VENDOR = 1;
nativeBuildInputs = [msrvToolchain] ++ tools ++ deps;
};
};
})
// {
overlays.default = import ./overlay.nix;
nixosModules.default = {
pkgs,
config,
lib,
...
}: {
imports = [./module.nix];
config = lib.mkIf config.services.ugc-api-server.enable {
nixpkgs.overlays = [self.overlays.default];
services.ugc-api-server.package = lib.mkDefault pkgs.ugc-api-server;
};
};
nixpkgs.url = "nixpkgs/nixos-24.05";
flakelight = {
url = "github:nix-community/flakelight";
inputs.nixpkgs.follows = "nixpkgs";
};
mill-scale = {
url = "github:icewind1991/mill-scale";
inputs.flakelight.follows = "flakelight";
};
};
outputs = { mill-scale, ... }: mill-scale ./. {
extraFilesRegex = [ ".*\.html" ];
nixosModules = { outputs, ... }: {
default =
{ pkgs
, config
, lib
, ...
}: {
imports = [ ./module.nix ];
config = lib.mkIf config.services.ugc-api-server.enable {
nixpkgs.overlays = [ (import ./overlay.nix) ];
services.ugc-api-server.package = lib.mkDefault pkgs.ugc-api-server;
};
};
};
};
}

View file

@ -1,12 +1,12 @@
{
config,
lib,
pkgs,
...
{ config
, lib
, pkgs
, ...
}:
with lib; let
cfg = config.services.ugc-api-server;
in {
in
{
options.services.ugc-api-server = {
enable = mkEnableOption "ugc api server";
@ -30,12 +30,12 @@ in {
config = mkIf cfg.enable {
systemd.services."ugc-api-server" = {
wantedBy = ["multi-user.target"];
after = ["network-online.target"];
wants = ["network-online.target"];
wantedBy = [ "multi-user.target" ];
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
environment = {
RUST_LOG = cfg.logLevel;
PORT = toString cfg.port;
PORT = toString cfg.port;
};
serviceConfig = {
@ -61,7 +61,7 @@ in {
RestrictAddressFamilies = "AF_INET AF_INET6";
RestrictRealtime = true;
ProtectProc = "noaccess";
SystemCallFilter = ["@system-service" "~@resources" "~@privileged"];
SystemCallFilter = [ "@system-service" "~@resources" "~@privileged" ];
PrivateUsers = true;
ProcSubset = "pid";
};

View file

@ -1,3 +1,3 @@
final: prev: {
ugc-api-server = final.callPackage ./package.nix {};
ugc-api-server = final.callPackage ./package.nix { };
}

View file

@ -1,32 +1,33 @@
{
rustPlatform,
openssl,
pkg-config,
lib,
}: let
{ rustPlatform
, openssl
, pkg-config
, lib
,
}:
let
inherit (lib.sources) sourceByRegex;
inherit (builtins) fromTOML readFile;
src = sourceByRegex ./api-server ["Cargo.*" "(src)(/.*)?" "README.md"];
src = sourceByRegex ./api-server [ "Cargo.*" "(src)(/.*)?" "README.md" ];
version = (fromTOML (readFile api-server/Cargo.toml)).package.version;
in
rustPlatform.buildRustPackage rec {
pname = "ugc-api-server";
rustPlatform.buildRustPackage rec {
pname = "ugc-api-server";
inherit src version;
inherit src version;
buildInputs = [
openssl
];
buildInputs = [
openssl
];
nativeBuildInputs = [
pkg-config
];
nativeBuildInputs = [
pkg-config
];
OPENSSL_NO_VENDOR = 1;
OPENSSL_NO_VENDOR = 1;
doCheck = false;
doCheck = false;
cargoLock = {
lockFile = ./api-server/Cargo.lock;
};
}
cargoLock = {
lockFile = ./api-server/Cargo.lock;
};
}

View file

@ -1,6 +1,10 @@
#[cfg(feature = "serde")]
use insta::assert_json_snapshot;
#[cfg(feature = "serde")]
use std::fs::read_to_string;
#[cfg(feature = "serde")]
use test_case::test_case;
#[cfg(feature = "serde")]
use ugc_scraper::parser::{
MapHistoryParser, MatchPageParser, Parser, PlayerDetailsParser, PlayerParser, SeasonsParser,
TeamLookupParser, TeamMatchesParser, TeamParser, TeamRosterHistoryParser, TransactionParser,
@ -8,16 +12,18 @@ use ugc_scraper::parser::{
#[test_case("player_76561198024494988.html", "player")]
#[test_case("player_76561198049312442.html", "player_classes")]
#[cfg(feature = "serde")]
fn test_parse_player_html(input: &str, name: &str) {
let body = read_to_string(&format!("tests/data/{input}")).unwrap();
let body = read_to_string(format!("tests/data/{input}")).unwrap();
let parser = PlayerParser::new();
let parsed = parser.parse(&body).unwrap();
assert_json_snapshot!(format!("parse_{name}_html"), parsed);
}
#[test_case("player_details_76561198024494988.html", "player_details")]
#[cfg(feature = "serde")]
fn test_parse_player_details_html(input: &str, name: &str) {
let body = read_to_string(&format!("tests/data/{input}")).unwrap();
let body = read_to_string(format!("tests/data/{input}")).unwrap();
let parser = PlayerDetailsParser::new();
let parsed = parser.parse(&body).unwrap();
assert_json_snapshot!(format!("parse_{name}_html"), parsed);
@ -28,8 +34,9 @@ fn test_parse_player_details_html(input: &str, name: &str) {
#[test_case("team_8157.html", "team_no_tz")]
#[test_case("team_6929.html", "team_changed_name")]
#[test_case("team_32437.html", "team_empty_name_change")]
#[cfg(feature = "serde")]
fn test_parse_team_html(input: &str, name: &str) {
let body = read_to_string(&format!("tests/data/{input}")).unwrap();
let body = read_to_string(format!("tests/data/{input}")).unwrap();
let parser = TeamParser::new();
let parsed = parser.parse(&body).unwrap();
assert_json_snapshot!(format!("parse_{name}_html"), parsed);
@ -37,24 +44,27 @@ fn test_parse_team_html(input: &str, name: &str) {
#[test_case("team_roster_history_7861.html", "team_roster_history")]
#[test_case("team_roster_history_2133.html", "team_roster_history_no_group")]
#[cfg(feature = "serde")]
fn test_parse_team_roster_history_html(input: &str, name: &str) {
let body = read_to_string(&format!("tests/data/{input}")).unwrap();
let body = read_to_string(format!("tests/data/{input}")).unwrap();
let parser = TeamRosterHistoryParser::new();
let parsed = parser.parse(&body).unwrap();
assert_json_snapshot!(format!("parse_{name}_html"), parsed);
}
#[test_case("team_matches_7861.html", "team_matches")]
#[cfg(feature = "serde")]
fn test_parse_team_matches_html(input: &str, name: &str) {
let body = read_to_string(&format!("tests/data/{input}")).unwrap();
let body = read_to_string(format!("tests/data/{input}")).unwrap();
let parser = TeamMatchesParser::new();
let parsed = parser.parse(&body).unwrap();
assert_json_snapshot!(format!("parse_{name}_html"), parsed);
}
#[test_case("index.html", "seasons")]
#[cfg(feature = "serde")]
fn test_parse_seasons_html(input: &str, name: &str) {
let body = read_to_string(&format!("tests/data/{input}")).unwrap();
let body = read_to_string(format!("tests/data/{input}")).unwrap();
let parser = SeasonsParser::new();
let parsed = parser.parse(&body).unwrap();
assert_json_snapshot!(format!("parse_{name}_html"), parsed);
@ -64,24 +74,27 @@ fn test_parse_seasons_html(input: &str, name: &str) {
#[test_case("lookup_6v6.html", "seasons_6")]
#[test_case("lookup_4v4.html", "seasons_4")]
#[test_case("lookup_2v2.html", "seasons_2")]
#[cfg(feature = "serde")]
fn test_parse_seasons_mode_html(input: &str, name: &str) {
let body = read_to_string(&format!("tests/data/{input}")).unwrap();
let body = read_to_string(format!("tests/data/{input}")).unwrap();
let parser = TeamLookupParser::new();
let parsed = parser.parse(&body).unwrap();
assert_json_snapshot!(format!("parse_{name}_html"), parsed);
}
#[test_case("match_116246.html", "match")]
#[cfg(feature = "serde")]
fn test_parse_match_html(input: &str, name: &str) {
let body = read_to_string(&format!("tests/data/{input}")).unwrap();
let body = read_to_string(format!("tests/data/{input}")).unwrap();
let parser = MatchPageParser::new();
let parsed = parser.parse(&body).unwrap();
assert_json_snapshot!(format!("parse_{name}_html"), parsed);
}
#[test_case("transactions_4v4.html", "transaction")]
#[cfg(feature = "serde")]
fn test_parse_transaction_html(input: &str, name: &str) {
let body = read_to_string(&format!("tests/data/{input}")).unwrap();
let body = read_to_string(format!("tests/data/{input}")).unwrap();
let parser = TransactionParser::new();
let parsed = parser.parse(&body).unwrap();
assert_json_snapshot!(format!("parse_{name}_html"), parsed);
@ -91,8 +104,9 @@ fn test_parse_transaction_html(input: &str, name: &str) {
#[test_case("map_6v6.html", "maps_6")]
#[test_case("map_4v4.html", "maps_4")]
#[test_case("map_2v2.html", "maps_2")]
#[cfg(feature = "serde")]
fn test_parse_maps_html(input: &str, name: &str) {
let body = read_to_string(&format!("tests/data/{input}")).unwrap();
let body = read_to_string(format!("tests/data/{input}")).unwrap();
let parser = MapHistoryParser::new();
let parsed = parser.parse(&body).unwrap();
assert_json_snapshot!(format!("parse_{name}_html"), parsed);