flake reorg, nix integration testing

.github/workflows/ci.yaml

@@ -13,70 +13,5 @@ jobs:
         with:
           name: ci
           instance: https://cache.icewind.me
-          authToken: '${{ secrets.ATTIC_TOKEN }}'
+          authToken: "${{ secrets.ATTIC_TOKEN }}"
-      - run: nix build
+      - run: nix flake check --keep-going
-
-  run:
-    runs-on: ubuntu-latest
-    name: Run
-    needs: build
-
-    services:
-      api-test-db:
-        image: demostf/db
-        env:
-          POSTGRES_PASSWORD: test
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-        ports:
-          - 5432:5432
-      api:
-        image: demostf/api
-        env:
-          DEMO_ROOT: /tmp
-          DEMO_HOST: localhost
-          DB_TYPE: pgsql
-          DB_HOST: api-test-db
-          DB_PORT: 5432
-          DB_DATABASE: postgres
-          DB_USERNAME: postgres
-          DB_PASSWORD: test
-          APP_ROOT: https://api.localhost
-          EDIT_SECRET: edit
-      api-test:
-        image: demostf/api-nginx-test
-        env:
-          POSTGRES_PASSWORD: test
-        ports:
-          - 80:80
-
-    steps:
-      - uses: nschloe/action-cached-lfs-checkout@v1
-      - uses: cachix/install-nix-action@v26
-      - uses: icewind1991/attic-action@v1
-        with:
-          name: ci
-          instance: https://cache.icewind.me
-          authToken: '${{ secrets.ATTIC_TOKEN }}'
-      - run: nix run
-        env:
-          DB_URL: postgres://postgres:test@localhost/postgres
-          BASE_URL: http://localhost/
-          EDIT_KEY: edit
-
-  docker:
-    runs-on: ubuntu-latest
-    needs: [build, run]
-    steps:
-      - name: Checkout code
-        uses: nschloe/action-cached-lfs-checkout@v1
-      - uses: cachix/install-nix-action@v26
-      - uses: icewind1991/attic-action@v1
-        with:
-          name: ci
-          instance: https://cache.icewind.me
-          authToken: '${{ secrets.ATTIC_TOKEN }}'
-      - run: nix build .#docker
-      - name: Push image
-        if: github.ref == 'refs/heads/main'
-        run: |
-          skopeo copy --dest-creds="${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }}" "docker-archive:$(nix build .#docker --print-out-paths)" "docker://demostf/api-test"

Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Robin Appelman <robin@icewind.nl>"]
 edition = "2018"
 
 [dependencies]
-demostf-client = { version = "0.4.2", default-features = false, features = ["rustls-tls"] }
+demostf-client = { version = "0.4.4" }
 sqlx = { version = "0.6.0", features = ["postgres", "runtime-tokio-rustls"] }
 dotenv = "0.15.0"
 color-eyre = "0.6.1"

README.md

@@ -8,8 +8,8 @@ Test suite for demos.tf api
 
 Start the test with the following environment variables
 
- - `DB_URL` - [sqlx](https://github.com/launchbadge/sqlx) database url for the database used by the api
+- `DB_URL` - [sqlx](https://github.com/launchbadge/sqlx) database url for the database used by the api
- - `BASE_URL` - base url for the api
+- `BASE_URL` - base url for the api
- - `EDIT_KEY` - edit key for the api
+- `EDIT_KEY` - edit key for the api
 
 Note that the test suite is destructive, all data saved in the database will be wiped for each test run.

docker.nix

@@ -1,15 +0,0 @@
-{
-  dockerTools,
-  demostf-api-test,
-}:
-dockerTools.buildLayeredImage {
-  name = "demostf/api-test";
-  tag = "latest";
-  maxLayers = 5;
-  contents = [
-    demostf-api-test
-  ];
-  config = {
-    Cmd = ["api-test"];
-  };
-}

flake.lock

@@ -1,56 +1,103 @@
 {
   "nodes": {
-    "flake-utils": {
+    "crane": {
-      "inputs": {
-        "systems": "systems"
-      },
       "locked": {
-        "lastModified": 1705309234,
+        "lastModified": 1733688869,
-        "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
+        "narHash": "sha256-KrhxxFj1CjESDrL5+u/zsVH0K+Ik9tvoac/oFPoxSB8=",
-        "owner": "numtide",
+        "owner": "ipetkov",
-        "repo": "flake-utils",
+        "repo": "crane",
-        "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
+        "rev": "604637106e420ad99907cae401e13ab6b452e7d9",
         "type": "github"
       },
       "original": {
-        "owner": "numtide",
+        "owner": "ipetkov",
-        "repo": "flake-utils",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "flakelight": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1734957624,
+        "narHash": "sha256-RbvX9lf9lWQwG9vTXkscOiWTrKf8lzjyeOvW/v8IuBY=",
+        "owner": "nix-community",
+        "repo": "flakelight",
+        "rev": "8c226ea0166784b02d4a58fbb015f9c01250221e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "flakelight",
+        "type": "github"
+      }
+    },
+    "mill-scale": {
+      "inputs": {
+        "crane": "crane",
+        "flakelight": [
+          "flakelight"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1735052218,
+        "narHash": "sha256-I30wh6G8fSUO4EseexxiDXcxyUhXR6C8BvEeKn6xyfE=",
+        "owner": "icewind1991",
+        "repo": "mill-scale",
+        "rev": "7e45bb598ff63a8416ee3c26743b20644563bd93",
+        "type": "github"
+      },
+      "original": {
+        "owner": "icewind1991",
+        "repo": "mill-scale",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1708529342,
+        "lastModified": 1734875076,
-        "narHash": "sha256-gTzSstZWvAJgVz5U6coxVtAlY7dldPKgljjR9+cGr/I=",
+        "narHash": "sha256-Pzyb+YNG5u3zP79zoi8HXYMs15Q5dfjDgwCdUI5B0nY=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "700804df18b73e2fe360d950f371aaec1691dea2",
+        "rev": "1807c2b91223227ad5599d7067a61665c52d1295",
         "type": "github"
       },
       "original": {
         "id": "nixpkgs",
-        "ref": "release-23.11",
+        "ref": "nixos-24.11",
         "type": "indirect"
       }
     },
     "root": {
       "inputs": {
-        "flake-utils": "flake-utils",
+        "flakelight": "flakelight",
+        "mill-scale": "mill-scale",
         "nixpkgs": "nixpkgs"
       }
     },
-    "systems": {
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "mill-scale",
+          "flakelight",
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1681028828,
+        "lastModified": 1733884434,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "narHash": "sha256-8GXR9kC07dyOIshAyfZhG11xfvBRSZzYghnZ2weOKJU=",
-        "owner": "nix-systems",
+        "owner": "oxalica",
-        "repo": "default",
+        "repo": "rust-overlay",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "rev": "d0483df44ddf0fd1985f564abccbe568e020ddf2",
         "type": "github"
       },
       "original": {
-        "owner": "nix-systems",
+        "owner": "oxalica",
-        "repo": "default",
+        "repo": "rust-overlay",
         "type": "github"
       }
     }

flake.nix

@@ -1,50 +1,25 @@
 {
   inputs = {
-    nixpkgs.url = "nixpkgs/release-23.11";
+    nixpkgs.url = "nixpkgs/nixos-24.11";
-    flake-utils.url = "github:numtide/flake-utils";
+    flakelight = {
+      url = "github:nix-community/flakelight";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+    mill-scale = {
+      url = "github:icewind1991/mill-scale";
+      inputs.flakelight.follows = "flakelight";
+    };
   };
-
+  outputs = {mill-scale, ...}:
-  outputs = {
+    mill-scale ./. {
-    self,
+      cargoTest = false;
-    nixpkgs,
+      extraPaths = [./data];
-    flake-utils,
+      withOverlays = [(import ./nix/overlay.nix)];
-  }:
+      packages = {
-  flake-utils.lib.eachDefaultSystem (system: let
+        test-runner = pkgs: pkgs.test-runner;
-      overlays = [
-        (import ./overlay.nix)
-      ];
-      pkgs = (import nixpkgs) {
-        inherit system overlays;
       };
-      tools = with pkgs; [
+      checks = {
-        rustc
+        test = pkgs: pkgs.nixosTest (import ./nix/test.nix);
-        cargo
-        bacon
-        cargo-edit
-        cargo-outdated
-        clippy
-        cargo-audit
-        cargo-msrv
-      ];
-      dependencies = with pkgs; [
-        openssl
-        pkg-config
-      ];
-    in rec {
-      packages = rec {
-        inherit (pkgs) demostf-api-test demostf-api-test-docker;
-        docker = demostf-api-test-docker;
-        default = demostf-api-test;
       };
-      apps = rec {
+    };
-        api-test = {
-          type = "app";
-          program = "${pkgs.demostf-api-test}/bin/api-test";
-        };
-        default = api-test;
-      };
-      devShells.default = pkgs.mkShell {
-        nativeBuildInputs = tools ++ dependencies;
-      };
-    });
 }

nix/demostf-api.nix

@@ -0,0 +1,27 @@
+{
+  php,
+  fetchFromGitHub,
+}: let
+  phpWithExtensions = php.withExtensions ({
+    enabled,
+    all,
+  }:
+    enabled ++ (with all; [pdo apcu]));
+in
+  phpWithExtensions.buildComposerProject (finalAttrs: {
+    pname = "demostf-api";
+    version = "0.1.0";
+
+    src = fetchFromGitHub {
+      owner = "demostf";
+      repo = "api";
+      rev = "1a8380360b993226ae1c6fcc226011e03a6c1467";
+      hash = "sha256-JcBRU1N44tt0QDLnj6z9MCT3V2s2dkf+JbpWb1rmXnY=";
+    };
+
+    vendorHash = "sha256-EYWCR2aJAoyWvEX+SML4Fb3F3KGcUtwCgqhAGT6ZjZ4=";
+
+    composerStrictValidation = false;
+
+    doCheck = false;
+  })

nix/demostf-parser.nix

@@ -0,0 +1,23 @@
+{
+  rustPlatform,
+  fetchFromGitHub,
+}:
+rustPlatform.buildRustPackage {
+  pname = "demostf-parser";
+  version = "0.5.1";
+
+  src = fetchFromGitHub {
+    owner = "demostf";
+    repo = "parser";
+    rev = "0cd87a8a40e2a6af637d831b272c2758cebd2f9c";
+    hash = "sha256-bKcc0hWTkdYUDMI/DjUh45abuBeQEvkn6TsuAz02H5Y=";
+  };
+
+  cargoBuildFlags = ''
+    --bin parse_demo
+  '';
+
+  doCheck = false;
+
+  cargoHash = "sha256-/Fnw6l2fznrBK780E4q1PKFOkT0eiL+dE+UuhFA+V9M=";
+}

nix/overlay.nix

@@ -0,0 +1,5 @@
+prev: final: {
+  demostf-api = final.callPackage ./demostf-api.nix {};
+  demostf-parser = final.callPackage ./demostf-parser.nix {};
+  demostf-api-test = final.callPackage ./package.nix {};
+}

nix/package.nix

@@ -0,0 +1,28 @@
+{
+  rustPlatform,
+  lib,
+  pkg-config,
+  openssl,
+}: let
+  inherit (lib.sources) sourceByRegex;
+  inherit (builtins) fromTOML readFile;
+  src = sourceByRegex ../. ["Cargo.*" "(src|data)(/.*)?"];
+  cargoPackage = (fromTOML (readFile ../Cargo.toml)).package;
+in
+  rustPlatform.buildRustPackage {
+    pname = cargoPackage.name;
+    inherit (cargoPackage) version;
+    inherit src;
+
+    buildInputs = [openssl];
+
+    nativeBuildInputs = [pkg-config];
+
+    doCheck = false;
+
+    cargoLock = {
+      lockFile = ../Cargo.lock;
+    };
+
+    meta.mainProgram = "api-test";
+  }

nix/test.nix

@@ -0,0 +1,126 @@
+{
+  pkgs,
+  lib,
+  ...
+}: {
+  name = "demostf-api-client-test";
+  nodes.machine = {config, ...}: let
+    fpmCfg = config.services.phpfpm.pools.demostf-api;
+  in {
+    config = {
+      users.groups.demostf = {};
+      users.users.demostf = {
+        group = "demostf";
+        isSystemUser = true;
+      };
+
+      services.postgresql = {
+        enable = true;
+        ensureDatabases = ["demostf"];
+        ensureUsers = [
+          {
+            name = "demostf";
+            ensureDBOwnership = true;
+          }
+        ];
+        initialScript = pkgs.writeText "init-sql-script" ''
+          CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
+        '';
+      };
+
+      services.nginx = {
+        enable = true;
+        virtualHosts."localhost" = {
+          root = "/var/empty";
+          extraConfig = ''
+            try_files $uri /index.php?$query_string /index.php;
+          '';
+          locations = {
+            "~ ^(.+?\\.php)(/.*)?$" = {
+              extraConfig = ''
+                fastcgi_param  PATH_INFO $2;
+                fastcgi_pass   unix:${fpmCfg.socket};
+                fastcgi_index  index.php;
+                fastcgi_param  SCRIPT_FILENAME ${pkgs.demostf-api}/share/php/demostf-api/src/public/index.php;
+                include ${pkgs.nginx}/conf/fastcgi_params;
+                client_max_body_size 250m;
+              '';
+            };
+            "= /upload" = {
+              extraConfig = ''
+                fastcgi_pass   unix:${fpmCfg.socket};
+                fastcgi_index  index.php;
+                fastcgi_param  SCRIPT_FILENAME ${pkgs.demostf-api}/share/php/demostf-api/src/public/upload.php;
+                include ${pkgs.nginx}/conf/fastcgi_params;
+                client_max_body_size 250m;
+              '';
+            };
+            "/static/" = {
+              alias = "/demos/";
+            };
+          };
+        };
+      };
+
+      services.phpfpm.pools.demostf-api = {
+        phpPackage = pkgs.php.buildEnv {
+          extensions = {
+            enabled,
+            all,
+          }:
+            enabled ++ (with all; [pdo apcu]);
+          extraConfig = ''
+            post_max_size = 150M
+            upload_max_filesize = 150M
+          '';
+        };
+        settings = {
+          "clear_env" = "no";
+          "pm" = "dynamic";
+          "pm.max_children" = "25";
+          "pm.start_servers" = "5";
+          "pm.min_spare_servers" = "5";
+          "pm.max_spare_servers" = "15";
+          "catch_workers_output" = "yes";
+          "listen.owner" = "nginx";
+          "listen.group" = "nginx";
+        };
+        phpEnv = {
+          BASE_HOST = "demos.tf";
+          DEMO_ROOT = "/demos";
+          DEMO_HOST = "localhost";
+          DB_TYPE = "pgsql";
+          DB_HOST = "/run/postgresql";
+          DB_PORT = "5432";
+          DB_DATABASE = "demostf";
+          DB_USERNAME = "demostf";
+          APP_ROOT = "http://localhost";
+          EDIT_SECRET = "edit";
+          PARSER_PATH = "${pkgs.demostf-parser}/bin/parse_demo";
+        };
+        user = "demostf";
+        group = "demostf";
+      };
+    };
+  };
+
+  testScript = let
+    testBinary = lib.getExe pkgs.demostf-api-test;
+    initSql = pkgs.fetchurl {
+      url = "https://github.com/demostf/db/raw/refs/heads/master/schema.sql";
+      hash = "sha256-AwXN9mh9CRk6HWdvyUR+YdBkpmExNIDOIeDMz6XqjEQ=";
+    };
+  in ''
+    machine.succeed("mkdir /demos && chmod 0777 /demos");
+    machine.wait_for_unit("postgresql")
+    machine.succeed("sudo -u demostf psql demostf demostf < ${initSql}");
+    machine.succeed("sudo -u postgres psql postgres postgres -c \"alter user demostf with password 'test';\"");
+    machine.wait_for_unit("phpfpm-demostf-api")
+    machine.wait_for_unit("nginx")
+    machine.wait_until_succeeds("curl http://127.0.0.1", timeout=45)
+    machine.succeed("DB_URL='postgres://demostf:test@localhost/demostf'\
+        BASE_URL='http://localhost/'\
+        EDIT_KEY='edit'\
+        ${testBinary}", timeout=180)
+  '';
+}

overlay.nix

@@ -1,4 +0,0 @@
-prev: final: {
-  demostf-api-test = final.callPackage ./package.nix {};
-  demostf-api-test-docker = final.callPackage ./docker.nix {};
-}

package.nix

@@ -1,26 +0,0 @@
-{
-  rustPlatform,
-  lib,
-  pkg-config,
-  openssl,
-}: let
-  inherit (lib.sources) sourceByRegex;
-  inherit (builtins) fromTOML readFile;
-  version = (fromTOML (readFile ./Cargo.toml)).package.version;
-  src = sourceByRegex ./. ["Cargo.*" "(src|data)(/.*)?"];
-in
-  rustPlatform.buildRustPackage rec {
-    pname = "demostf-api-test";
-
-    inherit src version;
-
-    buildInputs = [openssl];
-
-    nativeBuildInputs = [pkg-config];
-
-    doCheck = false;
-
-    cargoLock = {
-      lockFile = ./Cargo.lock;
-    };
-  }