Refactor tests to load flake inputs with flake-compat

This makes `nix fmt` just works and we no longer have to override flake
inputs.
This commit is contained in:
Jörg Thalheim
2025-08-17 11:20:09 +02:00
parent 2379bc4099
commit 4bafcc2454
10 changed files with 491 additions and 190 deletions

View File

@@ -11,11 +11,11 @@ jobs:
steps:
- uses: actions/checkout@v5
- uses: cachix/install-nix-action@v31
- run: cd tests && nix fmt .. -- --fail-on-change
- run: nix build .#checks.x86_64-linux.formatting
tests:
needs: nixfmt
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- uses: cachix/install-nix-action@v31
- run: nix run ./tests#run .
- run: nix run .#run-tests

View File

@@ -42,7 +42,7 @@ Link the profile in the table in README.md and in flake.nix.
## 3. Testing
Run `nix run ./tests#run .` to evaluate all hardware profiles.
Run `nix run .#run-tests` to evaluate all hardware profiles.
Because profiles can only be tested with the appropriate hardware, quality
assurance is up to *you*.

View File

@@ -2,7 +2,35 @@
description = "nixos-hardware";
outputs =
{ ... }:
{ self, ... }:
let
# Import private inputs (for development)
privateInputs =
(import ./tests/flake-compat.nix {
src = ./tests;
}).defaultNix;
systems = [
"aarch64-linux"
"riscv64-linux"
"x86_64-linux"
];
formatSystems = [ "aarch64-linux" "x86_64-linux" ];
# Helper to iterate over systems
eachSystem =
f:
privateInputs.nixos-unstable-small.lib.genAttrs systems (
system: f privateInputs.nixos-unstable-small.legacyPackages.${system} system
);
eachSystemFormat =
f:
privateInputs.nixos-unstable-small.lib.genAttrs formatSystems (
system: f privateInputs.nixos-unstable-small.legacyPackages.${system} system
);
in
{
nixosModules =
@@ -420,5 +448,40 @@
common-pc-laptop-ssd = import ./common/pc/ssd;
common-pc-ssd = import ./common/pc/ssd;
};
# Add formatter for `nix fmt`
formatter = eachSystemFormat (
pkgs: _system:
(privateInputs.treefmt-nix.lib.evalModule pkgs ./tests/treefmt.nix).config.build.wrapper
);
# Add packages
packages = eachSystem (
pkgs: _system: {
run-tests = pkgs.callPackage ./tests/run-tests.nix {
inherit self;
};
}
);
# Add checks for `nix run .#run-tests`
checks = eachSystem (
pkgs: system:
let
treefmtEval = privateInputs.treefmt-nix.lib.evalModule pkgs ./tests/treefmt.nix;
nixosTests = import ./tests/nixos-tests.nix {
inherit
self
privateInputs
system
pkgs
;
};
in
pkgs.lib.optionalAttrs (self.formatter ? ${system}) {
formatting = treefmtEval.config.build.check self;
}
// nixosTests
);
};
}

299
tests/flake-compat.nix Normal file
View File

@@ -0,0 +1,299 @@
# Compatibility function to allow flakes to be used by
# non-flake-enabled Nix versions. Given a source tree containing a
# 'flake.nix' and 'flake.lock' file, it fetches the flake inputs and
# calls the flake's 'outputs' function. It then returns an attrset
# containing 'defaultNix' (to be used in 'default.nix'), 'shellNix'
# (to be used in 'shell.nix').
{
src,
system ? builtins.currentSystem or "unknown-system",
}:
let
lockFilePath = src + "/flake.lock";
lockFile = builtins.fromJSON (builtins.readFile lockFilePath);
fetchTree =
builtins.fetchTree or (
info:
if info.type == "github" then
{
outPath = fetchTarball (
{
url = "https://api.${info.host or "github.com"}/repos/${info.owner}/${info.repo}/tarball/${info.rev}";
}
// (if info ? narHash then { sha256 = info.narHash; } else { })
);
rev = info.rev;
shortRev = builtins.substring 0 7 info.rev;
lastModified = info.lastModified;
lastModifiedDate = formatSecondsSinceEpoch info.lastModified;
narHash = info.narHash;
}
else if info.type == "git" then
{
outPath = builtins.fetchGit (
{
url = info.url;
}
// (if info ? rev then { inherit (info) rev; } else { })
// (if info ? ref then { inherit (info) ref; } else { })
// (if info ? submodules then { inherit (info) submodules; } else { })
);
lastModified = info.lastModified;
lastModifiedDate = formatSecondsSinceEpoch info.lastModified;
narHash = info.narHash;
revCount = info.revCount or 0;
}
// (
if info ? rev then
{
rev = info.rev;
shortRev = builtins.substring 0 7 info.rev;
}
else
{ }
)
else if info.type == "path" then
{
outPath = builtins.path {
path = info.path;
sha256 = info.narHash;
};
narHash = info.narHash;
}
else if info.type == "tarball" then
{
outPath = fetchTarball (
{ inherit (info) url; } // (if info ? narHash then { sha256 = info.narHash; } else { })
);
}
else if info.type == "gitlab" then
{
inherit (info) rev narHash lastModified;
outPath = fetchTarball (
{
url = "https://${info.host or "gitlab.com"}/api/v4/projects/${info.owner}%2F${info.repo}/repository/archive.tar.gz?sha=${info.rev}";
}
// (if info ? narHash then { sha256 = info.narHash; } else { })
);
shortRev = builtins.substring 0 7 info.rev;
}
else if info.type == "sourcehut" then
{
inherit (info) rev narHash lastModified;
outPath = fetchTarball (
{
url = "https://${info.host or "git.sr.ht"}/${info.owner}/${info.repo}/archive/${info.rev}.tar.gz";
}
// (if info ? narHash then { sha256 = info.narHash; } else { })
);
shortRev = builtins.substring 0 7 info.rev;
}
else
# FIXME: add Mercurial, tarball inputs.
throw "flake input has unsupported input type '${info.type}'"
);
callFlake4 =
flakeSrc: locks:
let
flake = import (flakeSrc + "/flake.nix");
inputs = builtins.mapAttrs (
_n: v:
if v.flake or true then
callFlake4 (fetchTree (v.locked // v.info)) v.inputs
else
fetchTree (v.locked // v.info)
) locks;
outputs = flakeSrc // (flake.outputs (inputs // { self = outputs; }));
in
assert flake.edition == 201909;
outputs;
callLocklessFlake =
flakeSrc:
let
flake = import (flakeSrc + "/flake.nix");
outputs = flakeSrc // (flake.outputs ({ self = outputs; }));
in
outputs;
rootSrc =
let
# Try to clean the source tree by using fetchGit, if this source
# tree is a valid git repository.
tryFetchGit =
src:
if isGit && !isShallow then
let
res = builtins.fetchGit src;
in
if res.rev == "0000000000000000000000000000000000000000" then
removeAttrs res [
"rev"
"shortRev"
]
else
res
else
{
outPath =
# Massage `src` into a store path.
if builtins.isPath src then
if
dirOf (toString src) == builtins.storeDir
# `builtins.storePath` is not available in pure-eval mode.
&& builtins ? currentSystem
then
# If it's already a store path, don't copy it again.
builtins.storePath src
else
"${src}"
else
src;
};
# NB git worktrees have a file for .git, so we don't check the type of .git
isGit = builtins.pathExists (src + "/.git");
isShallow = builtins.pathExists (src + "/.git/shallow");
in
{
lastModified = 0;
lastModifiedDate = formatSecondsSinceEpoch 0;
}
// (if src ? outPath then src else tryFetchGit src);
# Format number of seconds in the Unix epoch as %Y%m%d%H%M%S.
formatSecondsSinceEpoch =
t:
let
rem = x: y: x - x / y * y;
days = t / 86400;
secondsInDay = rem t 86400;
hours = secondsInDay / 3600;
minutes = (rem secondsInDay 3600) / 60;
seconds = rem t 60;
# Courtesy of https://stackoverflow.com/a/32158604.
z = days + 719468;
era = (if z >= 0 then z else z - 146096) / 146097;
doe = z - era * 146097;
yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
y = yoe + era * 400;
doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
mp = (5 * doy + 2) / 153;
d = doy - (153 * mp + 2) / 5 + 1;
m = mp + (if mp < 10 then 3 else -9);
y' = y + (if m <= 2 then 1 else 0);
pad = s: if builtins.stringLength s < 2 then "0" + s else s;
in
"${toString y'}${pad (toString m)}${pad (toString d)}${pad (toString hours)}${pad (toString minutes)}${pad (toString seconds)}";
allNodes = builtins.mapAttrs (
key: node:
let
sourceInfo =
if key == lockFile.root then
rootSrc
else
fetchTree (node.info or { } // removeAttrs node.locked [ "dir" ]);
subdir = if key == lockFile.root then "" else node.locked.dir or "";
outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir);
flake = import (outPath + "/flake.nix");
inputs = builtins.mapAttrs (_inputName: inputSpec: allNodes.${resolveInput inputSpec}) (
node.inputs or { }
);
# Resolve a input spec into a node name. An input spec is
# either a node name, or a 'follows' path from the root
# node.
resolveInput =
inputSpec: if builtins.isList inputSpec then getInputByPath lockFile.root inputSpec else inputSpec;
# Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
# root node, returning the final node.
getInputByPath =
nodeName: path:
if path == [ ] then
nodeName
else
getInputByPath
# Since this could be a 'follows' input, call resolveInput.
(resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
(builtins.tail path);
outputs = flake.outputs (inputs // { self = result; });
result =
outputs
# We add the sourceInfo attribute for its metadata, as they are
# relevant metadata for the flake. However, the outPath of the
# sourceInfo does not necessarily match the outPath of the flake,
# as the flake may be in a subdirectory of a source.
# This is shadowed in the next //
// sourceInfo
// {
# This shadows the sourceInfo.outPath
inherit outPath;
inherit inputs;
inherit outputs;
inherit sourceInfo;
_type = "flake";
};
in
if node.flake or true then
assert builtins.isFunction flake.outputs;
result
else
sourceInfo
) lockFile.nodes;
result =
if !(builtins.pathExists lockFilePath) then
callLocklessFlake rootSrc
else if lockFile.version == 4 then
callFlake4 rootSrc (lockFile.inputs)
else if lockFile.version >= 5 && lockFile.version <= 7 then
allNodes.${lockFile.root}
else
throw "lock file '${lockFilePath}' has unsupported version ${toString lockFile.version}";
in
rec {
outputs = result;
defaultNix =
builtins.removeAttrs result [ "__functor" ]
// (
if result ? defaultPackage.${system} then { default = result.defaultPackage.${system}; } else { }
)
// (
if result ? packages.${system}.default then
{ default = result.packages.${system}.default; }
else
{ }
);
shellNix =
defaultNix
// (if result ? devShell.${system} then { default = result.devShell.${system}; } else { })
// (
if result ? devShells.${system}.default then
{ default = result.devShells.${system}.default; }
else
{ }
);
}

75
tests/flake.lock generated
View File

@@ -1,78 +1,39 @@
{
"nodes": {
"flake-parts": {
"inputs": {
"nixpkgs-lib": [
"nixos-unstable-small"
]
},
"locked": {
"lastModified": 1743550720,
"narHash": "sha256-hIshGgKZCgWh6AYJpJmRgFdR3WUbkY04o82X05xqQiY=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "c621e8422220273271f52058f618c94e405bb0f5",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "flake-parts",
"type": "github"
}
},
"nixos-hardware": {
"locked": {
"lastModified": 1747083103,
"narHash": "sha256-dMx20S2molwqJxbmMB4pGjNfgp5H1IOHNa1Eby6xL+0=",
"owner": "NixOS",
"repo": "nixos-hardware",
"rev": "d1d68fe8b00248caaa5b3bbe4984c12b47e0867d",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "nixos-hardware",
"type": "github"
}
},
"nixos-stable": {
"locked": {
"lastModified": 1751211869,
"narHash": "sha256-1Cu92i1KSPbhPCKxoiVG5qnoRiKTgR5CcGSRyLpOd7Y=",
"ref": "nixos-25.05",
"rev": "b43c397f6c213918d6cfe6e3550abfe79b5d1c51",
"shallow": true,
"type": "git",
"url": "https://github.com/NixOS/nixpkgs"
"lastModified": 1755274400,
"narHash": "sha256-rTInmnp/xYrfcMZyFMH3kc8oko5zYfxsowaLv1LVobY=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "ad7196ae55c295f53a7d1ec39e4a06d922f3b899",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-25.05",
"shallow": true,
"type": "git",
"url": "https://github.com/NixOS/nixpkgs"
"repo": "nixpkgs",
"type": "github"
}
},
"nixos-unstable-small": {
"locked": {
"lastModified": 1747040834,
"narHash": "sha256-iKQKoNlZmxQq+O2WfImm/jn97g5GZBVW5EZEoCTXZ3I=",
"ref": "nixos-unstable-small",
"rev": "e4f52f3ea82ddd3754b467e3fdc0d709685c9a05",
"shallow": true,
"type": "git",
"url": "https://github.com/NixOS/nixpkgs"
"lastModified": 1755375481,
"narHash": "sha256-43PgCQFgFD1nM/7dncytV0c5heNHe/gXrEud18ZWcZU=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "35f1742e4f1470817ff8203185e2ce0359947f12",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable-small",
"shallow": true,
"type": "git",
"url": "https://github.com/NixOS/nixpkgs"
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"flake-parts": "flake-parts",
"nixos-hardware": "nixos-hardware",
"nixos-stable": "nixos-stable",
"nixos-unstable-small": "nixos-unstable-small",
"treefmt-nix": "treefmt-nix"

View File

@@ -1,127 +1,12 @@
{
description = "Test flake for nixos-hardware";
description = "Private dev inputs for nixos-hardware";
inputs = {
nixos-unstable-small.url = "git+https://github.com/NixOS/nixpkgs?shallow=1&ref=nixos-unstable-small";
nixos-stable.url = "git+https://github.com/NixOS/nixpkgs?shallow=1&ref=nixos-25.05";
# override in the test
nixos-hardware.url = "github:NixOS/nixos-hardware";
flake-parts.url = "github:hercules-ci/flake-parts";
flake-parts.inputs.nixpkgs-lib.follows = "nixos-unstable-small";
nixos-unstable-small.url = "github:NixOS/nixpkgs/nixos-unstable-small";
nixos-stable.url = "github:NixOS/nixpkgs/nixos-25.05";
treefmt-nix.url = "github:numtide/treefmt-nix";
treefmt-nix.inputs.nixpkgs.follows = "nixos-unstable-small";
};
outputs =
inputs@{ flake-parts, ... }:
flake-parts.lib.mkFlake { inherit inputs; } {
imports = [
inputs.treefmt-nix.flakeModule
];
systems = [
"aarch64-linux"
"x86_64-linux"
"riscv64-linux"
];
perSystem =
{
system,
lib,
pkgs,
...
}:
let
blackList = [
# does import-from-derivation
"toshiba-swanky"
# uses custom nixpkgs config
"raspberry-pi-2"
# deprecated profiles
"framework"
"asus-zephyrus-ga402x"
"lenovo-yoga-7-14ARH7"
];
# There are more, but for those we need to force it.
# In future we should probably already define it in our module.
aarch64Systems = [
"raspberry-pi-3"
"raspberry-pi-4"
"raspberry-pi-5"
];
matchArch =
moduleName:
if builtins.elem moduleName aarch64Systems then
pkgs.hostPlatform.system == "aarch64-linux"
else
# TODO also add riscv64
pkgs.hostPlatform.system == "x86_64-linux";
modules = lib.filterAttrs (
name: _: !(builtins.elem name blackList || lib.hasPrefix "common-" name) && matchArch name
) inputs.nixos-hardware.nixosModules;
buildProfile = import ./build-profile.nix;
unfreeNixpkgs =
importPath:
import importPath {
config = {
allowBroken = true;
allowUnfree = true;
nvidia.acceptLicense = true;
};
overlays = [ ];
inherit system;
};
nixpkgsUnstable = unfreeNixpkgs inputs.nixos-unstable-small;
nixpkgsStable = unfreeNixpkgs inputs.nixos-stable;
checksForNixpkgs =
channel: nixpkgs:
lib.mapAttrs' (
name: module:
lib.nameValuePair "${channel}-${name}" (buildProfile {
pkgs = nixpkgs;
profile = module;
})
) modules;
in
{
_module.args.pkgs = nixpkgsUnstable;
treefmt = {
flakeCheck = pkgs.hostPlatform.system != "riscv64-linux";
projectRootFile = "COPYING";
programs = {
deadnix = {
enable = true;
no-lambda-pattern-names = true;
};
nixfmt = {
enable = true;
package = pkgs.nixfmt-rfc-style;
};
};
settings = {
on-unmatched = "info";
};
};
checks =
checksForNixpkgs "nixos-unstable" nixpkgsUnstable
// checksForNixpkgs "nixos-stable" nixpkgsStable;
packages.run = pkgs.writeShellScriptBin "run.py" ''
#!${pkgs.bash}/bin/bash
export PATH=${
lib.makeBinPath [
pkgs.nix-eval-jobs
pkgs.nix-eval-jobs.nix
]
}
exec ${pkgs.python3.interpreter} ${./.}/run.py --nixos-hardware "$@"
'';
};
};
outputs = inputs: inputs;
}

70
tests/nixos-tests.nix Normal file
View File

@@ -0,0 +1,70 @@
{
self,
privateInputs,
system,
pkgs,
}:
let
# Hardware profile checks
blackList = [
# does import-from-derivation
"toshiba-swanky"
# uses custom nixpkgs config
"raspberry-pi-2"
# deprecated profiles
"framework"
"asus-zephyrus-ga402x"
"lenovo-yoga-7-14ARH7"
];
aarch64Systems = [
"raspberry-pi-3"
"raspberry-pi-4"
"raspberry-pi-5"
];
matchArch =
moduleName:
if builtins.elem moduleName aarch64Systems then
system == "aarch64-linux"
else
# TODO also add riscv64
system == "x86_64-linux";
modules = pkgs.lib.filterAttrs (
name: _: !(builtins.elem name blackList || pkgs.lib.hasPrefix "common-" name) && matchArch name
) self.nixosModules;
buildProfile = import ./build-profile.nix;
unfreeNixpkgs = {
config = {
allowBroken = true;
allowUnfree = true;
nvidia.acceptLicense = true;
};
overlays = [ ];
inherit system;
};
nixpkgsUnstable = import privateInputs.nixos-unstable-small unfreeNixpkgs;
nixpkgsStable = import privateInputs.nixos-stable unfreeNixpkgs;
# Build checks for both unstable and stable
unstableChecks = pkgs.lib.mapAttrs' (
name: module:
pkgs.lib.nameValuePair "unstable-${name}" (buildProfile {
pkgs = nixpkgsUnstable;
profile = module;
})
) modules;
stableChecks = pkgs.lib.mapAttrs' (
name: module:
pkgs.lib.nameValuePair "stable-${name}" (buildProfile {
pkgs = nixpkgsStable;
profile = module;
})
) modules;
in
unstableChecks // stableChecks

19
tests/run-tests.nix Normal file
View File

@@ -0,0 +1,19 @@
{
lib,
writeShellScriptBin,
bash,
python3,
nix-eval-jobs,
self,
}:
writeShellScriptBin "run-tests" ''
#!${bash}/bin/bash
export PATH=${
lib.makeBinPath [
nix-eval-jobs
nix-eval-jobs.nix
]
}
exec ${python3.interpreter} ${self}/tests/run.py
''

View File

@@ -30,22 +30,15 @@ def parse_args() -> argparse.Namespace:
action="store_true",
help="Print evaluation commands executed",
)
parser.add_argument(
"--nixos-hardware",
help="Print evaluation commands executed",
)
return parser.parse_args()
def run_eval_test(nixos_hardware: str, gcroot_dir: Path, jobs: int) -> list[str]:
def run_eval_test(gcroot_dir: Path, jobs: int) -> list[str]:
failed_profiles = []
cmd = [
"nix-eval-jobs",
"--extra-experimental-features",
"flakes",
"--override-input",
"nixos-hardware",
nixos_hardware,
"--gc-roots-dir",
str(gcroot_dir),
"--max-memory-size",
@@ -53,7 +46,7 @@ def run_eval_test(nixos_hardware: str, gcroot_dir: Path, jobs: int) -> list[str]
"--workers",
str(jobs),
"--flake",
str(TEST_ROOT) + "#checks",
str(ROOT) + "#checks",
"--force-recurse",
]
print(" ".join(map(shlex.quote, cmd)))
@@ -84,7 +77,7 @@ def main() -> None:
with TemporaryDirectory() as tmpdir:
gcroot_dir = Path(tmpdir) / "gcroot"
failed_profiles = run_eval_test(args.nixos_hardware, gcroot_dir, args.jobs)
failed_profiles = run_eval_test(gcroot_dir, args.jobs)
if len(failed_profiles) > 0:
print(f"\n{RED}The following {len(failed_profiles)} test(s) failed:{RESET}")

11
tests/treefmt.nix Normal file
View File

@@ -0,0 +1,11 @@
{
projectRootFile = "flake.nix";
programs = {
deadnix = {
enable = true;
no-lambda-pattern-names = true;
};
nixfmt.enable = true;
};
settings.on-unmatched = "info";
}