Server configuration

{
  config,
  lib,
  pkgs,
  ...
}:
let
  cfg = config.mjm.server;
in
{
  options.mjm.server = {
    enable = lib.mkEnableOption "server setup";

    <<options>>
  };

  config = lib.mkIf cfg.enable (
    lib.mkMerge [
      <<config>>
    ]
  );

  _class = "nixos";
}

This module configures things that make sense specifically for my servers.

Nightly garbage collection

gc.enable = lib.mkEnableOption "automatic nightly garbage collection" // {
  default = true;
};

Automatic garbage collection is enabled by default on my servers.

(lib.mkIf cfg.gc.enable {
  <<gc-config>>
})

nix.gc = {
  automatic = true;
  randomizedDelaySec = "30min";
  options = "--delete-older-than 3d";
};

Garbage collection is enabled via the nix.gc options in NixOS. This sets up a systemd timer and service that runs daily. I introduce a randomized delay to avoid all machines running

this at the exact same time. And I choose to keep only the last three days of generations on the machine, as this has proven to preserve a reasonable amount of history in case things go wrong while not consuming excessive disk space.

nix.settings = {
  min-free = 100 * 1024 * 1024;
  max-free = 2 * 1024 * 1024 * 1024;
};

These settings control how Nix behaves when the disk gets full. If the disk has less space than min-free (100MB here), then Nix will try to garbage collect until the disk has at least max-free (2GB here) space.

Local-specific configuration

isLocal = lib.mkOption {
  type = lib.types.bool;
  default = true;
};

Most of my servers run in my homelab at my house, but I do have some VPSes that run elsewhere. Some of the configuration in this module only really makes sense for things that run locally, since the things they need to talk to aren't available from the wider internet. That's something I'd like to resolve at some point, but for now, I just don't set up those things on servers running outside the LAN.

(lib.mkIf cfg.isLocal {
  mjm.consul.enable = lib.mkDefault true;
  mjm.spire.agent.enable = lib.mkDefault true;
})

The Consul and SPIRE agents are universal across LAN servers, as many other things rely on them. This enables them by default to reduce boilerplate.

SSH host certificates

sshHostCert.enable = lib.mkEnableOption "SSH host certificate generation" // {
  default = cfg.isLocal;
};

(lib.mkIf cfg.sshHostCert.enable {
  <<ssh-cert-config>>
})

On local servers where Vault is reachable, I use it to generate a host certificate for SSH. This allows my workstations to automatically trust my servers without maintaining a known hosts file and doing trust-on-first-use. This is particularly useful for deploys via CI, since there's no machine-specific configuration for trust when SSHing into the machines being deployed to.

services.openssh.settings.HostCertificate = "/run/sshd-host-cert/cert";

The first step is to configure sshd to use the certificate that will be generated. A systemd service will be used to issue a certificate both on boot and regularly on a timer to keep it fresh.

systemd.services.sshd-host-cert = {
  description = "Issue SSH Server Host Certificate";
  wantedBy = [ "multi-user.target" ];
  before = [ "sshd.service" ];
  after = [
    "network-online.target"
    "spire-agent.service"
    "sshd-keygen.service"
  ];
  wants = [ "network-online.target" ];

  <<sshd-host-cert-unit>>
};

The sshd-host-cert service is responsible for connecting to Vault and having it sign the public key for the machine's host key. This should ideally happen before sshd starts, so that the host certificate file exists. If sshd starts without the host certificate file existing, it will simply not use it, requiring clients to trust the host's public key itself rather than the certificate's public key. It's very possible for this service to fail if Vault is unavailable, so the implementation accounts for possibly needing to restart sshd to get it to pick up the certificate once it's been issued.

This service depends on a working network connection to talk to Vault. It also needs the SPIRE agent to issue a JWT for logging in to Vault. And finally, it needs the host's SSH keys to be generated, since for some machines, these keys are not persisted and will be regenerated every boot.

startLimitIntervalSec = 0;
serviceConfig = {
  Type = "oneshot";
  RemainAfterExit = true;
  Restart = "on-failure";
  RestartSec = "5s";
  RuntimeDirectory = "sshd-host-cert";

  ExecStartPre = pkgs.execline.writeScript "sshd-host-cert-pre" "-P" ''
    <<sshd-host-cert-pre>>
  '';
  ExecStart = pkgs.execline.writeScript "sshd-host-cert-start" "-P" ''
    <<sshd-host-cert-start>>
  '';
};

The service itself is a pretty normal oneshot service, configured to restart indefinitely on failure. It's set to remain after exit for two reasons. First, it shouldn't re-run unnecessarily. Second, it needs to place the generated certificate in its RuntimeDirectory, which only exists as long as the unit does.

path = with pkgs; [
  vault
  glibc.getent
  spire-agent
  jq
  coreutils
  systemd
];
environment = {
  VAULT_ADDR = "https://vault.service.consul:8200";
  VAULT_CACERT = "/run/sshd-host-cert/bundle.0.pem";
};

The pre-start and start scripts need a handful of utilities in the PATH. The start script also needs some environment variables for talking to Vault.

Now let's look at the pre-start script:

foreground {
  loopwhilex -x 0
  foreground { sleep 2 }
  eltest -S ${config.mjm.spire.agent.socketPath}
}

This execline script starts by waiting for the SPIRE agent's socket to actually be present on disk. This is not necessarily true even once the spire-agent service is active: there's a small delay there, so this accounts for that. The script will continue once the socket is present.

spire-agent api fetch
  -socketPath ${config.mjm.spire.agent.socketPath}
  -write /run/sshd-host-cert

Once the SPIRE agent is ready, the script requests an X.509 certificate for this service. The certificate isn't actually needed, but it will also fetch the CA bundle, which is needed to validate Vault's TLS certificate.

That's all for the pre-start script. Now for the start script that will actually issue the SSH certificate:

importas -S VAULT_ADDR
backtick -E jwt {
  pipeline {
    spire-agent api fetch jwt
      -audience $VAULT_ADDR
      -output json
      -socketPath ${config.mjm.spire.agent.socketPath}
  }
  jq -r ".[0].svids[0].svid"
}

First, the VAULT_ADDR environment variable is imported to use with execline's substitution, since it needs to be passed as the audient when asking SPIRE to issue a JWT token. Then, the script runs a small pipeline to produce the JWT token and capture it as "jwt" to be substituted in later commands. The JSON output from the spire-agent CLI is used, with jq pulling the relevant field out of the output.

export VAULT_TOKEN placeholder
backtick VAULT_TOKEN {
  vault write
    -field=token
    auth/spiffe/login
    role=spiffe
    jwt=''${jwt}
}

Next, the script logins in to Vault, exchanging the JWT token from SPIRE for a Vault token. I believe the placeholder VAULT_TOKEN that is set first is to prevent Vault from trying to find a saved token somewhere else. We don't need a token yet to call this login endpoint, since the whole purpose is to get a Vault token from it.

if {
  redirfd -w 1 /run/sshd-host-cert/cert
  vault write
    -field=signed_key
    ssh-host-signer/sign/homelab-host
    cert_type=host
    public_key=@${"${(lib.findFirst (k: k.type == "ed25519") null config.services.openssh.hostKeys).path}.pub"}
    valid_principals=${config.networking.hostName}.home.mattmoriarity.com
}

Now that the script has a Vault token in the VAULT_TOKEN env var, it can attempt to sign the public SSH key to get a certificate. The output of the vault command is redirected to /run/sshd-host-cert/cert, which is where sshd has already been configured to look for the certificate. The ED25519 public key for the host is used for the sign request. And the certificate is issued specifically for the server's own FQDN.

The server is actually not allowed to request a host certificate for any other domain. The homelab-host role in Vault is configured to only allow a single domain based on the identity of the entity making the request. The SPIFFE ID that this systemd service is assigned is aliased in Vault to an entity which has a metadata field indicating its FQDN. This prevents servers from impersonating each other.

systemctl --no-block try-restart sshd.service

The last step for the script, now that the certificate has been issued, is to restart sshd if it is running. That is what the try-restart command in systemctl does. For some reason, it is important to not block on this, as it causes the command to hang indefinitely.

CapabilityBoundingSet = "";
DevicePolicy = "closed";
LockPersonality = true;
MemoryDenyWriteExecute = true;
PrivateDevices = true;
PrivateIPC = true;
PrivateUsers = "identity";
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = "read-only";
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectProc = "invisible";
RestrictAddressFamilies = [
  "AF_INET"
  "AF_INET6"
  "AF_UNIX"
];
RestrictNamespaces = true;
RestrictRealtime = true;
SystemCallArchitectures = "native";
SystemCallErrorNumber = "EPERM";
SystemCallFilter = [
  "@system-service"
  "~@resources @privileged"
];
UMask = "0027";

This is pretty normal systemd service hardening, and that's all for the sshd-host-cert service.

mjm.spire.entries =
  let
    inherit (config.networking) hostName;
  in
  {
    "sshd-host-cert-${hostName}" = {
      spiffe_id = "${hostName}/sshd";
      parent_id = hostName;
      selectors = [
        {
          type = "systemd";
          value = "id:sshd-host-cert.service";
        }
      ];
    };
  };

This creates the SPIRE registration entry that gives the sshd-host-cert service its identity. SSH certificates are issued to an identity of the form "spiffe://home.mattmoriarity.com/$hostname/sshd". Entity aliases for this ID are created for each machine in Vault, and the corresponding entity has metadata as describe above to indicate the FQDN it can request a certificate for.

mjm.consul.services.sshd = {
  port = 22;

  checks.host-cert = {
    name = "sshd host certificate is being used";
    script.args =
      let
        script = pkgs.execline.writeScript "ssh-host-cert-check" "-P" ''
          if -nt { ${pkgs.openssh}/bin/ssh-keyscan -c ${config.networking.hostName}.home.mattmoriarity.com }
          exit 2
        '';
      in
      [ "${script}" ];
    intervalSeconds = 60;
  };
};

The sshd service is registered on each host in Consul. Nothing needs to discover sshd this way, but it's a convenient way to surface a health check for whether the host certificate is actually being used. Arguably, it should probably be written as a node check rather than a service check, but I haven't created the infrastructure in Nix to define those nicely.

This check was more important when the sshd-host-cert service was less robust. It used to be a situation where it would fail and be stuck that way, requiring manual intervention, so knowing when that was needed was useful. Now, the service will generally succeed once it has what it needs, so it's less important to know when it's failing.

TODO: add the timer, and shorten the length that the certificates are valid for

Gathering telemetry with Alloy

alloy.enable = lib.mkEnableOption "Alloy for collecting telemetry" // {
  default = cfg.isLocal;
};

Alloy is Grafana's general-purpose collector of telemetry. It's an agent that is meant to be run locally on a server, and it can do things like scrape logs, run Prometheus collectors, or process traces before sending them on.

Alloy is enabled by default on LAN servers, because it depends on being able to connect to Loki, Prometheus, and Tempo. I'd like to find a way for the VPSes to use it, as they're currently a blind spot in my monitoring.

alloy.tracing.enable = lib.mkEnableOption "OpenTelemetry trace exporter";

This option enables configuring Alloy for the local tracing exporter. I don't enable this by default right now because it tends to cause weird bootstrapping cycles in the infra, where something Tempo needs to start is blocked from starting because of its own tunnel for connecting to Tempo. I'd like to figure out a way to handle this more gracefully.

(lib.mkIf cfg.alloy.enable {
  services.alloy.enable = true;

  <<alloy-config>>
})

mjm.services.alloy = {
  upstreams = {
    loki.port = 13101;
    prometheus.port = 13102;
    tempo = lib.mkIf cfg.alloy.tracing.enable { port = 15317; };
  };
};

Alloy needs a SPIFFE ID to be able to talk to the upstream services that collect the telemetry it gathers, and then it also needs to run local client tunnels to connect to them. The tunnel for Tempo is only configured if tracing is enabled.

mjm.spire.tunnels = {
  alloy-loki.onDemand = true;
  alloy-prometheus.onDemand = true;

  alloy-tempo = lib.mkIf cfg.alloy.tracing.enable {
    onDemand = true;
    # bit hacky, but tempo's service is not advertising the grpc port, so
    # we need to override it.
    target.port = lib.mkForce 14317;
  };
};

The tunnels defined by the upstream above need some extra configuration in this case. Usually my tunnels are configured to always be running, because they are registered as services in Consul and define a health check, so those requests require it to be running even when not being used. Marking the tunnels as on-demand disables those health checks, so the tunnel is only running when it's actually used.

This is relevant for Alloy in particular because it's also running on the host machine that runs all the upstream services those tunnels will connect to. Starting these tunnels before they are needed and before the service they connect has even had a chance to be started is pointless, and just creates unnecessary churn on the machine while it is trying to start VMs.

The Tempo upstream also needs to be told what port to target, because the Consul service advertising it is for a different port than the one for OpenTelemetry via gRPC.

Sending journal logs to Loki

environment.etc."alloy/journal.alloy".source = ./journal.alloy;

Alloy will load any config files found in /etc/alloy, so that's where the configuration will go.

loki.source.journal "read" {
  forward_to = [loki.write.endpoint.receiver]
  relabel_rules = loki.relabel.journal.rules
}

The first step is setting up a component to read logs from the journal and forward them to Loki. The component configures rules to relabel the logs before forwarding them.

loki.relabel "journal" {
  forward_to = []

  rule {
    source_labels = ["__journal__systemd_unit"]
    target_label = "systemd_unit"
  }
  rule {
    source_labels = ["__journal__systemd_unit"]
    regex = "(.*)\\.service"
    target_label = "service_name"
  }
  rule {
    source_labels = ["__journal__hostname"]
    target_label = "hostname"
  }
  rule {
    source_labels = ["__journal_syslog_identifier"]
    target_label = "syslog_identifier"
  }
}

These rules are used to relabel the logs. Most are just promoting exist labels so they are actually kept. The one exception is that .service units get a service_name label.

loki.write "endpoint" {
  endpoint {
    url = "http://localhost:13101/loki/api/v1/push"
  }
}

After relabeling, the logs are written to this Loki endpoint, which points to the local tunnel.

Gathering host metrics for Prometheus

environment.etc."alloy/prometheus.alloy".source = ./prometheus.alloy;

prometheus.remote_write "prod" {
  endpoint {
    url = "http://localhost:13102/api/v1/write"
  }
}

Usually Prometheus gathers metrics by scraping them, but Alloy instead uses the remote write API to push metrics to Prometheus. The benefit is that Prometheus doesn't have to be able to directly address the service producing the metrics. It also means that a temporary interruption in connectivity doesn't have to mean losing metrics, as they can be send with the next push.

The Prometheus endpoint configured here can be used by other Alloy config files that need to send metrics to Prometheus.

environment.etc."alloy/unix.alloy".source = ./unix.alloy;

prometheus.exporter.unix "local" {
  enable_collectors = ["processes", "systemd"]

  filesystem {
    mount_points_exclude = "^/(dev|proc|sys|nix/store|var/lib/docker/.+|run|run/.+)($|/)"
  }

  netclass {
    ignored_devices = "^(veth|docker)"
  }

  netdev {
    device_exclude = "^(veth|docker)"
  }
}

This sets up an embedded Prometheus node-exporter in the Alloy process. Alloy embeds many of the popular Prometheus exporters so that you don't have to run and manage them separately. I've configured mine to gather extra information on processes and systemd units that isn't enabled by default. I'm also excluding some filesystems and network interfaces to clean up dashboards.

prometheus.scrape "unix" {
  targets = prometheus.exporter.unix.local.targets
  forward_to = [prometheus.remote_write.prod.receiver]
}

The unix exporter above isn't doing anything useful until it is scraped, so this config does that. Those metrics could be relabeled if needed, but in my case, they are just forwarded directly to Prometheus.

Alloy's own metrics

environment.etc."alloy/self.alloy".source = ./self.alloy;

prometheus.exporter.self "default" {}

prometheus.scrape "self" {
  targets = prometheus.exporter.self.default.targets
  forward_to = [prometheus.remote_write.prod.receiver]
}

This is very similar to the unix exporter above, but requires less configuration. This forwards Alloy's own metrics to Prometheus, so they don't have to be scraped.

Sending traces to Tempo

environment.etc."alloy/otel.alloy".source = ./otel.alloy;
environment.etc."alloy/otel.alloy".enable = cfg.alloy.tracing.enable;

otelcol.receiver.otlp "otel" {
  grpc {
    endpoint = "127.0.0.1:4317"
  }
  http {
    endpoint = "127.0.0.1:4318"
  }

  output {
    traces = [otelcol.processor.memory_limiter.default.input]
  }
}

If tracing is enabled, an OpenTelemetry collector is set up to receive traces on the default OpenTelemetry endpoints. The collector is embedded in Alloy just like the Prometheus exporters.

Rather than outputting directly to Tempo, the traces will go through some processors first.

otelcol.processor.memory_limiter "default" {
  check_interval = "5s"
  limit = "400MiB"
  spike_limit = "100MiB"

  output {
    traces = [otelcol.processor.batch.default.input]
  }
}

The memory limiter keeps memory usage for the collector under control by dropping or garbage collecting data as needed when limits are hit. This is especially important when running the collector inside memory-constrained microVMs.

otelcol.processor.batch "default" {
  output {
    traces = [otelcol.exporter.otlp.tempo.input]
  }
}

The batch processor collects spans into batches to reduce the number of network requests needed and improve compression. I'm just using the default settings here.

otelcol.exporter.otlp "tempo" {
  client {
    endpoint = "localhost:15317"
    tls {
      insecure = true
    }
  }
}

Finally, after going through each processor, the traces are sent to the local Tempo tunnel.

Health checks

mjm.consul.services.alloy = {
  port = 12345;

  checks.up = {
    http.path = "/-/healthy";
    intervalSeconds = 30;
  };
};

Alloy is registered as a Consul service on each node where it runs so that health can be observed across the cluster.