From 3f0eb9cc6f3ab1ddd9ee1e0603bfcb189c6607c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Moritz=20B=C3=B6hme?= <mail@moritzboeh.me>
Date: Sat, 12 Apr 2025 19:13:59 +0200
Subject: [PATCH] feat: add remote builders

---
 machines/moritz-desktop/configuration.nix |   6 ++
 machines/moritz-server/configuration.nix  |   2 +
 modules/remote_builders.nix               | 106 ++++++++++++++++++++++
 modules/shared.nix                        |   1 +
 4 files changed, 115 insertions(+)
 create mode 100644 modules/remote_builders.nix

diff --git a/machines/moritz-desktop/configuration.nix b/machines/moritz-desktop/configuration.nix
index 8a47f8c..0a5ac81 100644
--- a/machines/moritz-desktop/configuration.nix
+++ b/machines/moritz-desktop/configuration.nix
@@ -29,6 +29,12 @@
   # This only works however if you have avahi running on your admin machine else use IP
   clan.core.networking.targetHost = "root@moritz-desktop";
 
+  our.buildMachines = {
+    enable = true;
+    speedFactor = 2;
+    gpgAgentSupport = true;
+  };
+
   my = {
     ai.enable = true;
     profiles = {
diff --git a/machines/moritz-server/configuration.nix b/machines/moritz-server/configuration.nix
index 547e782..8bd9aa2 100644
--- a/machines/moritz-server/configuration.nix
+++ b/machines/moritz-server/configuration.nix
@@ -11,6 +11,8 @@
   # This only works however if you have avahi running on your admin machine else use IP
   clan.core.networking.targetHost = "root@moritz-server";
 
+  our.buildMachines.enable = true;
+
   networking = {
     interfaces.enp2s0 = {
       ipv4.addresses = [{
diff --git a/modules/remote_builders.nix b/modules/remote_builders.nix
new file mode 100644
index 0000000..a07ec02
--- /dev/null
+++ b/modules/remote_builders.nix
@@ -0,0 +1,106 @@
+{
+  config,
+  clan-core,
+  self,
+  lib,
+  ...
+}:
+
+let
+  inherit (lib)
+    filterAttrs
+    mkEnableOption
+    mkIf
+    mapAttrsToList
+    attrNames
+    map
+    concatLines
+    mkOption
+    types
+    ;
+  cfg = config.our.buildMachines;
+
+  others = filterAttrs (n: v: n != config.networking.hostName) self.nixosConfigurations;
+
+  mkBuilder =
+    hostName: attrs:
+    let
+      config' = attrs.config;
+      cfg' = config'.our.buildMachines;
+      pkgs' = attrs.pkgs;
+    in
+    mkIf cfg'.enable {
+      hostName = hostName;
+      sshUser = "remotebuild";
+      # CPU architecture of the builder, and the operating system it runs.
+      # If your builder supports multiple architectures
+      # (e.g. search for "binfmt" for emulation),
+      systems = [ pkgs'.system ] ++ config'.boot.binfmt.emulatedSystems;
+      # Nix custom ssh-variant that avoids lots of "trusted-users" settings pain
+      protocol = "ssh-ng";
+      # default is 1 but may keep the builder idle in between builds
+      maxJobs = 3;
+      speedFactor = cfg'.speedFactor - (cfg.speedFactor) + 1;
+      supportedFeatures = cfg'.supportedFeatures;
+      mandatoryFeatures = [ ];
+    };
+
+  buildMachines = mapAttrsToList mkBuilder others;
+
+  remotebuildKeys = mapAttrsToList (
+    _name: attrs: attrs.config.clan.core.vars.generators.openssh.files."ssh.id_ed25519.pub".value
+  ) others;
+in
+{
+  options.our.buildMachines = {
+    enable = mkEnableOption "Use this machine as a remoteBuilder for others and vice versa.";
+    supportedFeatures = mkOption {
+      type = types.listOf (
+        types.oneOf [
+          "nixos-test"
+          "benchmark"
+          "big-parallel"
+          "kvm"
+        ]
+      );
+      default = [ ];
+      description = ''
+        kvm          | Everything which builds inside a vm, like NixOS tests
+        nixos-test   | Machine can run NixOS tests
+        big-parallel | kernel config, libreoffice, evolution, llvm and chromium
+        benchmark    | Machine can generate metrics (means the builds usually takes the same amount of time)
+      '';
+    };
+    speedFactor = mkOption {
+      type = types.int;
+      default = 1;
+      description = "How fast is the builder compared to your local machine";
+    };
+  };
+  config = mkIf cfg.enable {
+    users.users.remotebuild = {
+      isNormalUser = true;
+      createHome = false;
+      group = "remotebuild";
+
+      openssh.authorizedKeys.keys = remotebuildKeys;
+    };
+
+    users.groups.remotebuild = { };
+
+    programs.ssh.extraConfig = ''
+      Match User remotebuild
+        IdentityFile  ${config.clan.core.vars.generators.openssh.files."ssh.id_ed25519".path}
+    '';
+    nix = {
+      buildMachines = buildMachines;
+      # required, otherwise remote buildMachines above aren't used
+      distributedBuilds = true;
+      # optional, useful when the builder has a faster internet connection than yours
+      settings = {
+        builders-use-substitutes = true;
+        trusted-users = [ "remotebuild" ];
+      };
+    };
+  };
+}
diff --git a/modules/shared.nix b/modules/shared.nix
index 75d24ad..12bc959 100644
--- a/modules/shared.nix
+++ b/modules/shared.nix
@@ -7,6 +7,7 @@
     clan-core.clanModules.state-version
     clan-core.clanModules.static-hosts
     clan-core.clanModules.machine-id
+    ./remote_builders.nix
   ];
 
   networking.hosts."fd77:acc0:1d56:2265:499:9367:28e0:97d3" = [ "moritz-remarkable" ];