Mutual TLS tunnels with Ghostunnel

{
  config,
  lib,
  pkgs,
  utils,
  ...
}:
let
  cfg = config.mjm.spire;
  enabledTunnels = lib.filterAttrs (_: t: t.enable) cfg.tunnels;
in
{
  options.mjm.spire.tunnels = lib.mkOption {
    default = { };
    type = lib.types.attrsOf (
      lib.types.submodule (
        { config, ... }:
        {
          options = {
            <<options>>
          };

          config = {
            <<tunnel-config>>
          };
        }
      )
    );
  };

  config = lib.mkIf (enabledTunnels != { }) {
    <<config>>
  };
}

One of the main reasons I use SPIRE throughout the lab is to enable mutual TLS for service-to-service communication. Usually this is done through some kind of sidecar proxy that takes care of the TLS so that individual services can just do plaintext communication and not have to worry about that complexity. Ghostunnel is a very simple yet flexible option for such a proxy, and it already supports the SPIFFE workload API, which makes it a good fit for me.

Tunnel options

These options are available within the scope of a particular entry of mjm.spire.tunnels.

enable = lib.mkOption {
  type = lib.types.bool;
  default = true;
};

Any tunnels that are defined are enabled by default, but this option exists to allow for disabling tunnels if necessary.

id = lib.mkOption {
  type = lib.types.str;
};

The name of the service whose identity the tunnel's certificate will use.

mode = lib.mkOption {
  type = lib.types.enum [
    "client"
    "server"
  ];
};

Tunnels can either be a server or client tunnel, based on which side of the connection will be protected with TLS and which will be plaintext.

Server tunnels protect the listen side of the tunnel. They should be used to expose a TLS-protected server for a service that is listening for plaintext on the same machine as the tunnel. The listener is remote while the target is local.

Client tunnels protect the target side of the tunnel. They should be used to give processes running on the same machine as the tunnel the ability to use plaintext to communicate with a remote service while trusting the tunnel to secure that communication with TLS when leaving the machine. The listener is local while the target is remote.

listen.port = lib.mkOption {
  type = with lib.types; nullOr port;
  default = null;
};

The TCP port the tunnel should listen on. This can be used for either server or client tunnels.

listen.socket = lib.mkOption {
  type = with lib.types; nullOr path;
  default = null;
};

The Unix domain socket path the tunnel should listen on. This only makes sense to use for client tunnels. Server tunnels are meant to listen for external traffic, which must be over a TCP port.

listen.address = lib.mkOption {
  type = lib.types.str;
};

listen.address = lib.mkMerge [
  (lib.mkIf (config.listen.port != null)
    "[::${lib.optionalString (config.mode == "client") "1"}]:${toString config.listen.port}"
  )
  (lib.mkIf (config.listen.socket != null) config.listen.socket)
];

The full address the tunnel should listen on. If listen.port is set, then the address will be for that port, and IP will either be "::" for server tunnels or "::1" for client tunnels. If listen.socket is set, then the address will just be the socket path.

It's also possible to set the address directly, in case these defaults are not sufficient. The address maps directly to the ListenStream= option in the systemd socket unit.

target.service = lib.mkOption {
  type = with lib.types; nullOr str;
  default = null;
};

The name of the service to connect to. This only makes sense for client tunnels. This is expected to match both the name of the Consul service to use to reach the remote service and the service name in the SPIFFE ID in the certificate the server presents.

target.tag = lib.mkOption {
  type = lib.types.str;
  default = "";
};

An optional tag to use with target.service. If set, the tunnel will only target service instances that have that tag set.

target.port = lib.mkOption {
  type = with lib.types; nullOr port;
  default = null;
};

The TCP port to use to connect to the target. If set, this will be combined with target.host to create the address of the target.

For server tunnels, the target.host should usually be left as the default "localhost" as the target is expected to be listening locally.

For client tunnels using target.service, this should usually not be set at all, since it requires all instances of the target to be listening on the same hard-coded port. Leaving the port unset will instead rely on SRV records which include the port for each instance.

target.host = lib.mkOption {
  type = with lib.types; nullOr str;
  default = "localhost";
};

target.host = lib.mkIf (config.target.service != null) (
  (lib.optionalString (config.target.tag != "") "${config.target.tag}.")
  + "${config.target.service}.service.consul"
);

The host to use to connect to the target. Defaults to localhost, which is appropriate for server tunnels. Client tunnels will usually set target.service instead, so this isn't usually set directly.

target.socket = lib.mkOption {
  type = with lib.types; nullOr path;
  default = null;
};

The Unix domain socket path to use to connect to the target. This only makes sense for server tunnels, since for client tunnels, the target should be on another machine.

target.address = lib.mkOption {
  type = with lib.types; nullOr str;
  default = null;
};

target.address = lib.mkMerge [
  (lib.mkIf (config.target.port != null) "${config.target.host}:${toString config.target.port}")
  (lib.mkIf (config.target.socket != null) "unix:${config.target.socket}")
];

The full address the tunnel should connect to. This doesn't usually need to be set directly, as it can be derived from target.port, target.host, and target.socket. Only one of target.address or target.srv should be set.

target.srv = lib.mkOption {
  type = with lib.types; nullOr str;
  default = null;
};

target.srv =
  lib.mkIf (config.target.service != null && config.target.port == null)
    "_${config.target.service}._${
      if config.target.tag == "" then "tcp" else config.target.tag
    }.service.consul";

The SRV record name to use to lookup which targets to connect to. This is the preferred way for client tunnels to connect to their targets, as the servers' ports can be assigned automatically independently of the clients.

It's not generally necessary to set this option directly: a tunnel with target.service set but target.port unset will determine the correct SRV record by default.

openFirewall = lib.mkOption {
  type = lib.types.bool;
  default = config.mode == "server";
};

Whether the tunnel's listen port should be allowed through the firewall. By default, this is just server tunnels, as they are intended for listening for outside traffic.

onDemand = lib.mkOption {
  type = lib.types.bool;
  default = false;
};

Whether the tunnel should always be running or if it should start only when needed. Most tunnels are not on-demand, since I want them to have their health continuously checked by Consul.

allowIngress = lib.mkOption {
  type = lib.types.bool;
  default = false;
};

Whether the ingress servers should be allowed to connect to the tunnel. Only makes sense for server tunnels, since this is done with TLS validation on the listen side. Any service which wants to be accessible outside the lab via ingress needs to enable this.

allowMetrics = lib.mkOption {
  type = lib.types.bool;
  default = false;
};

Whether Prometheus should be allowed to connect to the tunnel. Only makes sense for server tunnels, since this is done with TLS validation on the listen side. If the tunnel is exposing metrics that Prometheus will need to scrape, this option should be enabled.

allowedServices = lib.mkOption {
  type = with lib.types; listOf str;
  default = [ ];
};

allowedServices = lib.mkMerge [
  (lib.mkIf config.allowIngress [ "caddy" ])
  (lib.mkIf config.allowMetrics [ "prometheus" ])
];

The names of services that should be allowed to connect to the tunnel. Only makes sense for server tunnels, since this is done with TLS validation on the listen side. The allowIngress and allowMetrics options are implemented by adding to this list. Any other services that need to connect to the tunnel should be added as well.

extraArgs = lib.mkOption {
  type = with lib.types; listOf str;
  default = [ ];
};

An escape hatch to pass additional arguments to ghostunnel. They are added to the end of the command.

Config

systemd.sockets = lib.mapAttrs' (
  name: tunnel:
  lib.nameValuePair "${name}-tunnel" {
    description = "${if tunnel.mode == "server" then "Server" else "Client"} Tunnel '${name}' Socket";
    wantedBy = [ "sockets.target" ];
    partOf = [ "${name}-tunnel.service" ];
    startLimitIntervalSec = 0;
    socketConfig = {
      FileDescriptorName = "ghostunnel";
      ListenStream = tunnel.listen.address;
      Restart = "on-failure";
      RestartSec = "5s";
    };
  }
) enabledTunnels;

Every tunnel listens using a systemd socket unit. The file descriptor is always named "ghostunnel", which the corresponding systemd service will tell Ghostunnel to look for. The socket listens on the configured address, which might be either a TCP IP and port or a Unix domain socket path.

The socket unit is part of the corresponding service unit, so it will be stopped whenever systemd stops the service.

systemd.services = lib.mapAttrs' (
  name: tunnel:
  let
    inherit (tunnel) mode target onDemand;
    <<execStartArgs>>
  in
  lib.nameValuePair "${name}-tunnel" {
    description = "${if mode == "server" then "Server" else "Client"} Tunnel '${name}'";
    wantedBy = lib.mkIf (!onDemand) [ "multi-user.target" ];
    after = [
      "network.target"
      "${name}-tunnel.socket"
    ];
    requires = [ "${name}-tunnel.socket" ];

    environment.SPIFFE_ENDPOINT_SOCKET = "unix:${cfg.agent.socketPath}";

    startLimitIntervalSec = 0;

    serviceConfig = {
      Type = "notify-reload";
      ExecStart = utils.escapeSystemdExecArgs execStartArgs;
      DynamicUser = true;
      Restart = "always";
      WatchdogSec = 1;
      RuntimeDirectory = "${name}-tunnel";
      RuntimeDirectoryMode = "0755";

      <<tunnel-hardening>>
    };
  }
) enabledTunnels;

Every tunnel gets a systemd service to go along with its socket unit. The service requires the socket unit: without this, systemd could start the service without the socket, and the service would be expected to be able to handle this.

The tunnel will use the SPIFFE workload API to get its certificate, so it needs the environment configured to point at the SPIRE agent's socket path. The tunnel also gets a runtime directory so it expose a Unix socket there for health checks.

execStartArgs = [
  "${pkgs.ghostunnel}/bin/ghostunnel"
  tunnel.mode
  "--listen=systemd:ghostunnel"
  "--disable-landlock"
  "--use-workload-api"
  "--status=unix:/run/${name}-tunnel/status.sock"
  "--shutdown-timeout=1m"
]
++ lib.optional (target.address != null) "--target=${target.address}"
++ lib.optionals (target.srv != null) [
  "--target=srv:${target.srv}"
  "--override-server-name=${target.host}"
]
++ (
  if tunnel.mode == "server" then
    (
      lib.optional (tunnel.allowedServices == [ ]) "--disable-authentication"
      ++ (map (s: "--allow-uri=spiffe://home.mattmoriarity.com/svc/${s}") tunnel.allowedServices)
    )
  else
    [ "--verify-uri=spiffe://home.mattmoriarity.com/svc/${tunnel.target.service}" ]
)
++ lib.optional onDemand "--skip-resolve"
++ tunnel.extraArgs;

The arguments for ghostunnel are based on the values of the options described above. The --target option is set appropriately based on whether target.address or target.srv is set. Note that when using SRV records, the server name needs to be overridden, since what's in the certificates will not match either the SRV record name or the individual addresses in the SRV record entries. target.host is used for this, as it should be the normal Consul address for A/AAAA records.

If the tunnel is in server mode, then arguments are passed to control authentication based on SPIFFE IDs. If allowedServices is empty, all services are allowed to connect. Otherwise, an --allow-uri option is passed for each service name that is allowed.

If the tunnel is in client mode, then the tunnel will verify that the service on the other end has the identity that is expected based on target.service. This addresses a core trust problem with Consul: I'm not using Consul's ACLs and tokens since they are a pretty clumsy and awkward way to do things. This means that any machine can register an instance of any Consul service, and potentially steal traffic from the intended instances. The harm of this is mitigated by this verification on the client side: even if an impostor does register itself, clients will refuse to interact with it.

If a tunnel is on-demand, then the --skip-resolve flag is passed to delay trying to resolve the target until actually proxying a request. The on-demand tunnels are generally the ones that ideally should avoid failing in a big loop because of this.

CapabilityBoundingSet = "";
DevicePolicy = "closed";
LockPersonality = true;
MemoryDenyWriteExecute = true;
PrivateDevices = true;
PrivateIPC = true;
PrivateUsers = "identity";
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectProc = "invisible";
RestrictAddressFamilies = [
  "AF_INET"
  "AF_INET6"
  "AF_UNIX"
];
RestrictNamespaces = true;
RestrictRealtime = true;
SystemCallArchitectures = "native";
SystemCallErrorNumber = "EPERM";
SystemCallFilter = [
  "@system-service"
  "~@resources @privileged"
];
UMask = "0000";

This is pretty typical systemd hardening. The tunnel service needs network access but not much else. It could even be possible for server tunnels that target a Unix socket to have private networking too, since listening is handled by the socket unit, but I haven't tested that.

mjm.spire.entries = lib.mapAttrs' (
  name: tunnel:
  lib.nameValuePair "tunnel-${tunnel.id}-${name}" {
    spiffe_id = "svc/${tunnel.id}";
    selectors = [
      {
        type = "systemd";
        value = "id:${name}-tunnel.service";
      }
    ];
    dns_names = lib.mkIf (tunnel.mode == "server") [ "${tunnel.id}.service.consul" ];
  }
) enabledTunnels;

Each tunnel's systemd service gets a SPIRE entry to assign it the identity corresponding to its id. Server tunnels also get a DNS name set, so clients that are not tunnels can do ordinary server name verification.

mjm.consul.services = lib.mapAttrs' (
  name: tunnel:
  lib.nameValuePair "${name}-tunnel" {
    checks.up = lib.mkIf (!tunnel.onDemand) {
      http.path = "/_status";
      http.socket = "/run/${name}-tunnel/status.sock";
    };
  }
) enabledTunnels;

Each tunnel gets a Consul service to keep track of the tunnel's health independently of the service it targets. Tunnels marked as on-demand can't do this, since the health check would cause the tunnel to start anytime it was stopped, defeating the purpose. They get still get a service registered nonetheless.

networking.firewall.allowedTCPPorts =
  enabledTunnels
  |> lib.attrValues
  |> lib.filter (t: t.openFirewall)
  |> map (t: t.listen.port);

Any tunnels that are configured to open their port in the firewall are added to the list of allowed ports.