clan/clanServices/remote-builders/default.nix

{self}: {
  _class = "clan.service";
  manifest.name = "remote-builders";

  # Define what roles exist
  roles.worker = {
    interface = {lib, ...}: {
      # These options can be set via 'roles.client.settings'
      options.supportedFeatures = lib.mkOption {
        type = with lib.types;
          listOf (enum [
            "nixos-test"
            "benchmark"
            "big-parallel"
            "kvm"
          ]);
        default = [];
        description = ''
          kvm          | Everything which builds inside a vm, like NixOS tests
          nixos-test   | Machine can run NixOS tests
          big-parallel | kernel config, libreoffice, evolution, llvm and chromium
          benchmark    | Machine can generate metrics (means the builds usually takes the same amount of time)
        '';
      };
    };

    # Maps over all instances and produces one result per instance.
    perInstance = {
      roles,
      machine,
      ...
    }: {
      # Analog to 'perSystem' of flake-parts.
      # For every instance of this service we will add a nixosModule to a client-machine
      nixosModule = {
        config,
        lib,
        ...
      }: let
        inherit (lib) filterAttrs hasAttr mapAttrsToList;

        clients = filterAttrs (name: _value: hasAttr name roles.client.machines) self.nixosConfigurations;
        others = filterAttrs (name: _value: name != machine.name) clients;
        remotebuildKeys =
          mapAttrsToList (
            _name: attrs: attrs.config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519.pub".value
          )
          others;
      in {
        # Interaction examples what you could do here:
        # - Get some settings of this machine
        # settings.ipRanges
        #
        # - Get all controller names:
        # allControllerNames = lib.attrNames roles.controller.machines
        #
        # - Get all roles of the machine:
        # machine.roles
        #
        # - Get the settings that where applied to a specific controller machine:
        # roles.controller.machines.jon.settings
        #
        # Add one systemd service for every instance
        users.users.remotebuild = {
          isNormalUser = true;
          createHome = false;
          group = "remotebuild";

          openssh.authorizedKeys.keys = remotebuildKeys;
        };

        users.groups.remotebuild = {};
      };
    };
  };
  roles.client = {
    interface = {};
    perInstance = {
      roles,
      machine,
      ...
    }: {
      nixosModule = {
        config,
        pkgs,
        lib,
        ...
      }: let
        inherit (lib) filterAttrs hasAttr mapAttrsToList concatLines optional;

        workers = filterAttrs (name: _value: hasAttr name roles.worker.machines) self.nixosConfigurations;
        others = filterAttrs (name: _value: name != machine.name) workers;

        mkBuilder = name: attrs: let
          config' = attrs.config;
          cfg' = roles.worker.machines.${name}.settings;
          pkgs' = attrs.pkgs;
        in {
          hostName = name;
          sshUser = "remotebuild";
          # CPU architecture of the builder, and the operating system it runs.
          # If your builder supports multiple architectures
          # (e.g. search for "binfmt" for emulation),
          systems = [pkgs'.system] ++ config'.boot.binfmt.emulatedSystems;
          # Nix custom ssh-variant that avoids lots of "trusted-users" settings pain
          protocol = "ssh-ng";
          # default is 1 but may keep the builder idle in between builds
          maxJobs = 3;
          speedFactor = 1;
          supportedFeatures = cfg'.supportedFeatures;
          mandatoryFeatures = [];
        };

        otherBuildMachines = mapAttrsToList mkBuilder others;
        buildMachines =
          otherBuildMachines
          ++ optional (hasAttr machine.name roles.worker.machines)
          {
            # NOTE: https://github.com/NixOS/nix/issues/3177
            hostName = "local?root=/nix/store";
            sshUser = null;
            # CPU architecture of the builder, and the operating system it runs.
            # If your builder supports multiple architectures
            # (e.g. search for "binfmt" for emulation),
            systems = [pkgs.system] ++ config.boot.binfmt.emulatedSystems;
            protocol = null;
            # default is 1 but may keep the builder idle in between builds
            maxJobs = 3;
            speedFactor = 1;
            supportedFeatures = roles.worker.machines.${machine.name}.settings.supportedFeatures;
            mandatoryFeatures = [];
          };

        mkMatch = name: _value: ''
          Match User remotebuild Host ${name}
            IdentityFile ${config.clan.core.vars.generators.remotebuild.files."ssh.id_ed25519".path}
            ConnectTimeout 1
        '';
        sshConfig = concatLines (mapAttrsToList mkMatch others);
      in {
        programs.ssh.extraConfig = sshConfig;

        clan.core.vars.generators.remotebuild = {
          files."ssh.id_ed25519" = {};
          files."ssh.id_ed25519.pub".secret = false;
          runtimeInputs = [
            pkgs.coreutils
            pkgs.openssh
          ];
          script = ''
            ssh-keygen -t ed25519 -N "" -f "$out"/ssh.id_ed25519
          '';
        };

        nix = {
          buildMachines = buildMachines;
          # required, otherwise remote buildMachines above aren't used
          distributedBuilds = true;
          # optional, useful when the builder has a faster internet connection than yours
          settings = {
            builders-use-substitutes = true;
            trusted-users = ["remotebuild"];
          };
        };
      };
    };
  };

  # Maps over all machines and produces one result per machine.
  perMachine = {...}: {
    # Analog to 'perSystem' of flake-parts.
    # For every machine of this service we will add exactly one nixosModule to a machine
    nixosModule = {...}: {
      # Interaction examples what you could do here:
      # - Get the name of this machine
      # machine.name
      #
      # - Get all roles of this machine across all instances:
      # machine.roles
      #
      # - Get the settings of a specific instance of a specific machine
      # instances.foo.roles.peer.machines.jon.settings
      #
    };
  };
}