From fde70ab2bb3b199035d70b50e6e73650b6518346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20B=C3=B6hme?= Date: Sun, 20 Apr 2025 10:22:37 +0200 Subject: [PATCH 1/2] refactor: remote-builders to clan service --- clanServices/flake-module.nix | 9 ++ clanServices/remote-builders/default.nix | 171 ++++++++++++++++++++++ flake.lock | 40 ++--- flake.nix | 10 +- machines/moritz-desktop/configuration.nix | 2 - machines/moritz-server/configuration.nix | 2 - modules/remote_builders.nix | 123 ---------------- modules/shared.nix | 1 - 8 files changed, 208 insertions(+), 150 deletions(-) create mode 100644 clanServices/flake-module.nix create mode 100644 clanServices/remote-builders/default.nix delete mode 100644 modules/remote_builders.nix diff --git a/clanServices/flake-module.nix b/clanServices/flake-module.nix new file mode 100644 index 0000000..cf88ae7 --- /dev/null +++ b/clanServices/flake-module.nix @@ -0,0 +1,9 @@ +{ + self, + lib, + ... +}: { + clan.inventory.modules = { + remote-builders = lib.modules.importApply ./remote-builders/default.nix {inherit self;}; + }; +} diff --git a/clanServices/remote-builders/default.nix b/clanServices/remote-builders/default.nix new file mode 100644 index 0000000..3ddf99e --- /dev/null +++ b/clanServices/remote-builders/default.nix @@ -0,0 +1,171 @@ +{self}: { + _class = "clan.service"; + manifest.name = "remote-builders"; + + # Define what roles exist + roles.worker = { + interface = {lib, ...}: { + # These options can be set via 'roles.client.settings' + options.supportedFeatures = lib.mkOption { + type = with lib.types; + listOf ( + oneOf [ + "nixos-test" + "benchmark" + "big-parallel" + "kvm" + ] + ); + default = []; + description = '' + kvm | Everything which builds inside a vm, like NixOS tests + nixos-test | Machine can run NixOS tests + big-parallel | kernel config, libreoffice, evolution, llvm and chromium + benchmark | Machine can generate metrics (means the builds usually takes the same amount of time) + ''; + }; + }; + + # Maps over all instances and produces one result per instance. + perInstance = {roles, ...}: { + # Analog to 'perSystem' of flake-parts. + # For every instance of this service we will add a nixosModule to a client-machine + nixosModule = { + config, + lib, + ... + }: let + inherit (lib) filterAttrs hasAttr mapAttrsToList; + + clients = filterAttrs (name: _value: hasAttr name roles.client.machines) self.nixosConfigurations; + others = filterAttrs (_name: value: value.config.networking.hostName != config.networking.hostName) clients; + remotebuildKeys = + mapAttrsToList ( + _name: attrs: attrs.config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519.pub".value + ) + others; + in { + # Interaction examples what you could do here: + # - Get some settings of this machine + # settings.ipRanges + # + # - Get all controller names: + # allControllerNames = lib.attrNames roles.controller.machines + # + # - Get all roles of the machine: + # machine.roles + # + # - Get the settings that where applied to a specific controller machine: + # roles.controller.machines.jon.settings + # + # Add one systemd service for every instance + users.users.remotebuild = { + isNormalUser = true; + createHome = false; + group = "remotebuild"; + + openssh.authorizedKeys.keys = remotebuildKeys; + }; + + users.groups.remotebuild = {}; + }; + }; + }; + roles.client = { + interface = {}; + perInstance = {roles, ...}: { + nixosModule = { + config, + pkgs, + lib, + ... + }: let + inherit (lib) filterAttrs hasAttr mapAttrsToList concatLines; + + workers = filterAttrs (name: _value: hasAttr name roles.worker.machines) self.nixosConfigurations; + + mkBuilder = hostName: attrs: let + config' = attrs.config; + cfg' = roles.worker.machines.${hostName}.settings; + pkgs' = attrs.pkgs; + in { + # NOTE: https://github.com/NixOS/nix/issues/3177 + hostName = + if config'.networking.hostName == config.networking.hostName + then "local?root=/nix/store" + else hostName; + sshUser = + if config'.networking.hostName == config.networking.hostName + then null + else "remotebuild"; + # CPU architecture of the builder, and the operating system it runs. + # If your builder supports multiple architectures + # (e.g. search for "binfmt" for emulation), + systems = [pkgs'.system] ++ config'.boot.binfmt.emulatedSystems; + # Nix custom ssh-variant that avoids lots of "trusted-users" settings pain + protocol = + if config'.networking.hostName == config.networking.hostName + then null + else "ssh-ng"; + # default is 1 but may keep the builder idle in between builds + maxJobs = 3; + speedFactor = 1; + supportedFeatures = cfg'.supportedFeatures; + mandatoryFeatures = []; + }; + + buildMachines = mapAttrsToList mkBuilder workers; + + others = filterAttrs (_name: value: value.config.networking.hostName != config.networking.hostName) workers; + mkMatch = _name: value: '' + Match User remotebuild Host ${value.config.networking.hostName} + IdentityFile ${config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519".path} + ''; + sshConfig = concatLines (mapAttrsToList mkMatch others); + in { + programs.ssh.extraConfig = sshConfig; + + clan.core.vars.generators.remotebuild = { + files."ssh.id_ed25519" = {}; + files."ssh.id_ed25519.pub".secret = false; + runtimeInputs = [ + pkgs.coreutils + pkgs.openssh + ]; + script = '' + ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519 + ''; + }; + + nix = { + buildMachines = buildMachines; + # required, otherwise remote buildMachines above aren't used + distributedBuilds = true; + # optional, useful when the builder has a faster internet connection than yours + settings = { + builders-use-substitutes = true; + trusted-users = ["remotebuild"]; + }; + }; + }; + }; + }; + + # Maps over all machines and produces one result per machine. + perMachine = {...}: { + # Analog to 'perSystem' of flake-parts. + # For every machine of this service we will add exactly one nixosModule to a machine + nixosModule = {...}: { + # Interaction examples what you could do here: + # - Get the name of this machine + # machine.name + # + # - Get all roles of this machine across all instances: + # machine.roles + # + # - Get the settings of a specific instance of a specific machine + # instances.foo.roles.peer.machines.jon.settings + # + }; + }; +} diff --git a/flake.lock b/flake.lock index f671351..26058b1 100644 --- a/flake.lock +++ b/flake.lock @@ -50,11 +50,11 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1744472425, - "narHash": "sha256-zBx1nBO+vlaPgk1+prUjaREbJLMVW2omMtY2ppWtBPc=", + "lastModified": 1745001614, + "narHash": "sha256-NAADglNN3lh/3hgTbkynbYC2JuQCCGvv3Yizm9I64ns=", "ref": "refs/heads/main", - "rev": "d69ae6049a4bf325aaabe646c235d5777cbb56aa", - "revCount": 6272, + "rev": "779d28577de180c60ed4774c002cc5814162a0f9", + "revCount": 6386, "type": "git", "url": "https://git.clan.lol/clan/clan-core" }, @@ -98,11 +98,11 @@ ] }, "locked": { - "lastModified": 1744470659, - "narHash": "sha256-syTeDvkDg9eiFpn8gUOz3DLOHJoJFS1Md7DXJQBIQFg=", + "lastModified": 1744961442, + "narHash": "sha256-ymbmZjOaIoVwEuTpFDjlXPGufqmdlzdDEVpQAjNi8Ls=", "ref": "refs/heads/main", - "rev": "35cc96c4f952a93ecb1abc8caefddc52f69fdd98", - "revCount": 401, + "rev": "0328b8c5912db9562e1ab115f760ca7ba0044aff", + "revCount": 409, "type": "git", "url": "https://git.clan.lol/clan/data-mesher" }, @@ -119,11 +119,11 @@ ] }, "locked": { - "lastModified": 1744145203, - "narHash": "sha256-I2oILRiJ6G+BOSjY+0dGrTPe080L3pbKpc+gCV3Nmyk=", + "lastModified": 1744940522, + "narHash": "sha256-TNoetfICvd29DhxRPpmyKItQBDlqSvKcV+wGNkn14jk=", "owner": "nix-community", "repo": "disko", - "rev": "76c0a6dba345490508f36c1aa3c7ba5b6b460989", + "rev": "51d33bbb7f1e74ba5f9d9a77357735149da99081", "type": "github" }, "original": { @@ -712,11 +712,11 @@ ] }, "locked": { - "lastModified": 1744224272, - "narHash": "sha256-cqePj5nuC7flJWNncaVAFq1YZncU0PSyO0DEqGn+vYc=", + "lastModified": 1744478979, + "narHash": "sha256-dyN+teG9G82G+m+PX/aSAagkC+vUv0SgUw3XkPhQodQ=", "owner": "nix-darwin", "repo": "nix-darwin", - "rev": "113883e37d985d26ecb65282766e5719f2539103", + "rev": "43975d782b418ebf4969e9ccba82466728c2851b", "type": "github" }, "original": { @@ -1081,11 +1081,11 @@ ] }, "locked": { - "lastModified": 1744103455, - "narHash": "sha256-SR6+qjkPjGQG+8eM4dCcVtss8r9bre/LAxFMPJpaZeU=", + "lastModified": 1744669848, + "narHash": "sha256-pXyanHLUzLNd3MX9vsWG+6Z2hTU8niyphWstYEP3/GU=", "owner": "Mic92", "repo": "sops-nix", - "rev": "69d5a5a4635c27dae5a742f36108beccc506c1ba", + "rev": "61154300d945f0b147b30d24ddcafa159148026a", "type": "github" }, "original": { @@ -1198,11 +1198,11 @@ ] }, "locked": { - "lastModified": 1743748085, - "narHash": "sha256-uhjnlaVTWo5iD3LXics1rp9gaKgDRQj6660+gbUU3cE=", + "lastModified": 1744961264, + "narHash": "sha256-aRmUh0AMwcbdjJHnytg1e5h5ECcaWtIFQa6d9gI85AI=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "815e4121d6a5d504c0f96e5be2dd7f871e4fd99d", + "rev": "8d404a69efe76146368885110f29a2ca3700bee6", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 098bc21..3f9a5f6 100644 --- a/flake.nix +++ b/flake.nix @@ -66,6 +66,7 @@ imports = [ inputs.clan-core.flakeModules.default inputs.git-hooks-nix.flakeModule + ./clanServices/flake-module.nix ]; perSystem = { @@ -99,9 +100,14 @@ inventory = { machines = { - moritz-desktop.tags = ["peer"]; + moritz-desktop.tags = ["peer" "client" "worker"]; moritz-laptop.tags = ["peer"]; - moritz-server.tags = []; + moritz-server.tags = ["client" "worker"]; + }; + instances."remote-builders" = { + module.name = "remote-builders"; + roles.client.tags.client = {}; + roles.worker.tags.worker = {}; }; services.zerotier.default = { roles.peer.tags = [ diff --git a/machines/moritz-desktop/configuration.nix b/machines/moritz-desktop/configuration.nix index ba902e0..af41755 100644 --- a/machines/moritz-desktop/configuration.nix +++ b/machines/moritz-desktop/configuration.nix @@ -23,8 +23,6 @@ # This only works however if you have avahi running on your admin machine else use IP clan.core.networking.targetHost = "root@moritz-desktop"; - our.buildMachines.enable = true; - my = { ai.enable = true; profiles = { diff --git a/machines/moritz-server/configuration.nix b/machines/moritz-server/configuration.nix index e57c667..fb7a507 100644 --- a/machines/moritz-server/configuration.nix +++ b/machines/moritz-server/configuration.nix @@ -11,8 +11,6 @@ # This only works however if you have avahi running on your admin machine else use IP clan.core.networking.targetHost = "root@moritz-server"; - our.buildMachines.enable = true; - networking = { interfaces.enp2s0 = { ipv4.addresses = [ diff --git a/modules/remote_builders.nix b/modules/remote_builders.nix deleted file mode 100644 index 82f03d3..0000000 --- a/modules/remote_builders.nix +++ /dev/null @@ -1,123 +0,0 @@ -{ - lib, - config, - pkgs, - self, - ... -}: let - inherit - (lib) - filterAttrs - mkEnableOption - mkIf - mapAttrsToList - concatLines - mkOption - types - ; - cfg = config.our.buildMachines; - - builders = filterAttrs (_n: v: v.config.our.buildMachines.enable) self.nixosConfigurations; - others = filterAttrs (n: _v: n != config.networking.hostName) builders; - - mkBuilder = hostName: attrs: let - config' = attrs.config; - cfg' = config'.our.buildMachines; - pkgs' = attrs.pkgs; - in { - # NOTE: https://github.com/NixOS/nix/issues/3177 - hostName = - if hostName == config.networking.hostName - then "local?root=/nix/store" - else hostName; - sshUser = - if hostName == config.networking.hostName - then null - else "remotebuild"; - # CPU architecture of the builder, and the operating system it runs. - # If your builder supports multiple architectures - # (e.g. search for "binfmt" for emulation), - systems = [pkgs'.system] ++ config'.boot.binfmt.emulatedSystems; - # Nix custom ssh-variant that avoids lots of "trusted-users" settings pain - protocol = - if hostName == config.networking.hostName - then null - else "ssh-ng"; - # default is 1 but may keep the builder idle in between builds - maxJobs = 3; - speedFactor = 1; - supportedFeatures = cfg'.supportedFeatures; - mandatoryFeatures = []; - }; - - buildMachines = mapAttrsToList mkBuilder builders; - - remotebuildKeys = - mapAttrsToList ( - _name: attrs: attrs.config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519.pub".value - ) - others; - - mkMatch = hostName: _v: '' - Match User remotebuild Host ${hostName} - IdentityFile ${config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519".path} - ''; - sshConfig = concatLines (mapAttrsToList mkMatch others); -in { - options.our.buildMachines = { - enable = mkEnableOption "Use this machine as a remoteBuilder for others and vice versa."; - supportedFeatures = mkOption { - type = types.listOf ( - types.oneOf [ - "nixos-test" - "benchmark" - "big-parallel" - "kvm" - ] - ); - default = []; - description = '' - kvm | Everything which builds inside a vm, like NixOS tests - nixos-test | Machine can run NixOS tests - big-parallel | kernel config, libreoffice, evolution, llvm and chromium - benchmark | Machine can generate metrics (means the builds usually takes the same amount of time) - ''; - }; - }; - config = mkIf cfg.enable { - users.users.remotebuild = { - isNormalUser = true; - createHome = false; - group = "remotebuild"; - - openssh.authorizedKeys.keys = remotebuildKeys; - }; - - users.groups.remotebuild = {}; - - clan.core.vars.generators.remotebuild = { - files."ssh.id_ed25519" = {}; - files."ssh.id_ed25519.pub".secret = false; - runtimeInputs = [ - pkgs.coreutils - pkgs.openssh - ]; - script = '' - ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519 - ''; - }; - - programs.ssh.extraConfig = sshConfig; - - nix = { - buildMachines = buildMachines; - # required, otherwise remote buildMachines above aren't used - distributedBuilds = true; - # optional, useful when the builder has a faster internet connection than yours - settings = { - builders-use-substitutes = true; - trusted-users = ["remotebuild"]; - }; - }; - }; -} diff --git a/modules/shared.nix b/modules/shared.nix index c75006a..7ffaba8 100644 --- a/modules/shared.nix +++ b/modules/shared.nix @@ -10,7 +10,6 @@ clan-core.clanModules.state-version clan-core.clanModules.static-hosts clan-core.clanModules.machine-id - ./remote_builders.nix ]; networking.hosts."fd77:acc0:1d56:2265:499:9367:28e0:97d3" = ["moritz-remarkable"]; From 47855697be9ac1ff468e933fe80c352b19aeb3c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20B=C3=B6hme?= Date: Mon, 21 Apr 2025 10:45:25 +0200 Subject: [PATCH 2/2] refactor: remote-builders to be more readable --- clanServices/remote-builders/default.nix | 62 +++++++++++++++--------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/clanServices/remote-builders/default.nix b/clanServices/remote-builders/default.nix index 3ddf99e..5d38b7e 100644 --- a/clanServices/remote-builders/default.nix +++ b/clanServices/remote-builders/default.nix @@ -27,7 +27,11 @@ }; # Maps over all instances and produces one result per instance. - perInstance = {roles, ...}: { + perInstance = { + roles, + machine, + ... + }: { # Analog to 'perSystem' of flake-parts. # For every instance of this service we will add a nixosModule to a client-machine nixosModule = { @@ -38,7 +42,7 @@ inherit (lib) filterAttrs hasAttr mapAttrsToList; clients = filterAttrs (name: _value: hasAttr name roles.client.machines) self.nixosConfigurations; - others = filterAttrs (_name: value: value.config.networking.hostName != config.networking.hostName) clients; + others = filterAttrs (name: _value: name != machine.name) clients; remotebuildKeys = mapAttrsToList ( _name: attrs: attrs.config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519.pub".value @@ -73,40 +77,35 @@ }; roles.client = { interface = {}; - perInstance = {roles, ...}: { + perInstance = { + roles, + machine, + ... + }: { nixosModule = { config, pkgs, lib, ... }: let - inherit (lib) filterAttrs hasAttr mapAttrsToList concatLines; + inherit (lib) filterAttrs hasAttr mapAttrsToList concatLines optional; workers = filterAttrs (name: _value: hasAttr name roles.worker.machines) self.nixosConfigurations; + others = filterAttrs (name: _value: name != machine.name) workers; - mkBuilder = hostName: attrs: let + mkBuilder = name: attrs: let config' = attrs.config; - cfg' = roles.worker.machines.${hostName}.settings; + cfg' = roles.worker.machines.${name}.settings; pkgs' = attrs.pkgs; in { - # NOTE: https://github.com/NixOS/nix/issues/3177 - hostName = - if config'.networking.hostName == config.networking.hostName - then "local?root=/nix/store" - else hostName; - sshUser = - if config'.networking.hostName == config.networking.hostName - then null - else "remotebuild"; + hostName = name; + sshUser = "remotebuild"; # CPU architecture of the builder, and the operating system it runs. # If your builder supports multiple architectures # (e.g. search for "binfmt" for emulation), systems = [pkgs'.system] ++ config'.boot.binfmt.emulatedSystems; # Nix custom ssh-variant that avoids lots of "trusted-users" settings pain - protocol = - if config'.networking.hostName == config.networking.hostName - then null - else "ssh-ng"; + protocol = "ssh-ng"; # default is 1 but may keep the builder idle in between builds maxJobs = 3; speedFactor = 1; @@ -114,11 +113,28 @@ mandatoryFeatures = []; }; - buildMachines = mapAttrsToList mkBuilder workers; + otherBuildMachines = mapAttrsToList mkBuilder others; + buildMachines = + otherBuildMachines + ++ optional (hasAttr machine.name roles.worker.machines) + { + # NOTE: https://github.com/NixOS/nix/issues/3177 + hostName = "local?root=/nix/store"; + sshUser = null; + # CPU architecture of the builder, and the operating system it runs. + # If your builder supports multiple architectures + # (e.g. search for "binfmt" for emulation), + systems = [pkgs.system] ++ config.boot.binfmt.emulatedSystems; + protocol = null; + # default is 1 but may keep the builder idle in between builds + maxJobs = 3; + speedFactor = 1; + supportedFeatures = roles.worker.machines.${machine.name}.settings.supportedFeatures; + mandatoryFeatures = []; + }; - others = filterAttrs (_name: value: value.config.networking.hostName != config.networking.hostName) workers; - mkMatch = _name: value: '' - Match User remotebuild Host ${value.config.networking.hostName} + mkMatch = name: _value: '' + Match User remotebuild Host ${name} IdentityFile ${config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519".path} ''; sshConfig = concatLines (mapAttrsToList mkMatch others);