Retrieving secrets from Vault

{
  pkgs,
  config,
  lib,
  ...
}:
let
  cfg = config.mjm.services;
  vaultServices = lib.attrValues (lib.filterAttrs (_: s: s.secrets.enable) cfg);
in
{
  options.mjm.services =
    let
      serviceType = { name, ... }: {
        options = {
          <<options>>
        };
      };
    in
    lib.mkOption {
      type = with lib.types; attrsOf (submodule serviceType);
    };

  config = lib.mkIf (vaultServices != [ ]) {
    <<config>>
  };

  _class = "nixos";
}

I use Vault's key-value secrets engine to store static secret data that services need to be able to function. I've built tooling to integrate this with NixOS using systemd credentials.

=> spiffe-creds

Service options

secrets.enable = lib.mkEnableOption "Vault secrets";

Accessing secrets from Vault must be enabled for each service that needs it.

secrets.templates = lib.mkOption {
  type = lib.types.attrsOf (
    lib.types.submodule (
      { name, ... }:
      {
        options = {
          name = lib.mkOption {
            type = lib.types.str;
            default = name;
            description = "Basename of the file containing the rendered template content.";
          };
          text = lib.mkOption {
            type = lib.types.lines;
            description = "The template to use to generate the file.";
          };
          secrets = lib.mkOption {
            type = with lib.types; listOf str;
            default = [ ];
            description = "Names of credentials for the service that are used in the template.";
          };
        };
      }
    )
  );
  default = { };
};

Templates to use to generate files using the contents of secrets. Ideally, every service would be able to read secrets directly from individual files using systemd credentials. Since some can't, a basic templating mechanism is available for when there is no other option.

Each key in secrets.templates defines a single file that will be generated. The text option describes the contents of the file that will be generated. The secrets option must list the names of all secrets that will be used within the template.

secrets.user = lib.mkOption {
  type = with lib.types; nullOr str;
  default = null;
};

The user to use to generate any secret templates. If no templates are defined, this has no effect. Since all templated files for a service are generated by the same systemd service, this one setting applies to all templated files for the service.

secrets.group = lib.mkOption {
  type = with lib.types; nullOr str;
  default = null;
};

The group to use to generate any secret templates. If no templates are defined, this has no effect. Since all templated files for a service are generated by the same systemd service, this one setting applies to all templated files for the service.

secrets.socketPath = lib.mkOption {
  type = lib.types.path;
  default = "/run/${name}-creds.sock";
  readOnly = true;
};

The socket path to use with LoadCredential to access secrets for this service. This is exposed as an option to avoid needing to hardcode this path in modules.

Config

mjm.vault.services = lib.pipe vaultServices [
  (map (s: s.name))
  lib.uniqueStrings
];

Any services that have enabled Vault are added to mjm.vault.services. My deploy tooling will gather these from all machines to know which entities to set up in Vault. Each service is mapped to an entity with a generic "service" policy, which grants access to read secrets under the path "prod/services/$name".

systemd.sockets = lib.pipe vaultServices [
  (map (
    { name, ... }:
    lib.nameValuePair "spiffe-creds@${name}" {
      overrideStrategy = "asDropin";
      wantedBy = [ "sockets.target" ];
    }
  ))
  lib.listToAttrs
];

The spire service module sets up a "spiffe-creds@" template socket/service that can provide secrets on-demand over a Unix socket with systemd's LoadCredential property. For each service that has enabled Vault, the socket unit for that service needs to be enabled by making it wanted by sockets.target. The matching service will be started on-demand as needed by the socket unit.

mjm.spire.entries = lib.pipe vaultServices [
  (map (
    { name, ... }:
    lib.nameValuePair "spiffe-creds-${name}" {
      spiffe_id = "svc/${name}";
      selectors = [
        {
          type = "systemd";
          value = "id:spiffe-creds@${name}.service";
        }
      ];
    }
  ))
  lib.listToAttrs
];

Each spiffe-creds service also needs a SPIRE registration entry so that it can get a JWT token to authenticate with Vault. The registration entry ensures that each spiffe-creds instance accesses the correct secrets with the correct identity.

systemd.services = lib.pipe vaultServices [
  (lib.filter (s: s.secrets.templates != { }))
  (map (
    { name, secrets, ... }:
    let
      <<serviceDef>>
      <<mkRenderScript>>
      <<template-helpers>>
    in
    lib.nameValuePair "${name}-secrets" serviceDef
  ))
  lib.listToAttrs
];

Finally, each service that declares templates gets a systemd service generated called "$name-secrets" that loads the credentials and generates the templated files. Any service units that require the contents of the templated files should be configured to require and start after this service.

serviceDef = {
  description = "Generate Secrets Files for '${name}'";
  startLimitIntervalSec = 0;
  path = [
    pkgs.systemd
    pkgs.envsubst
  ];
  credentials.${name} = lib.genAttrs allSecretNames (_: { });
  serviceConfig = {
    Type = "oneshot";
    Restart = "on-failure";
    RestartSec = 10;
    RemainAfterExit = true;
    ExecStart = map mkRenderScript (lib.attrValues secrets.templates);

    DynamicUser = true;
    User = lib.mkIf (secrets.user != null) secrets.user;
    Group = lib.mkIf (secrets.group != null) secrets.group;
    PrivateNetwork = true;
    PrivateTmp = true;
    RuntimeDirectory = "${name}-secrets";
    RuntimeDirectoryMode = "0700";
  };
};

The service is a oneshot that will retry indefinitely on failure, in case it starts before Vault is available. All secrets used by any of the templates are loaded via the custom "credentials" option on the service. Each template becomes its own ExecStart script on the service.

allSecretNames = lib.pipe secrets.templates [
  lib.attrValues
  (lib.concatMap (t: t.secrets))
  lib.uniqueStrings
];

The list of secrets to load is just concatenating the list of secrets from each service and removing duplicates.

mkRenderScript =
  t:
  pkgs.execline.writeScript "render-${name}-secrets" "-P" ''
    ${lib.concatMapStringsSep "\n" (
      s: "backtick ${envVarName s} { systemd-creds cat ${secretName s} }"
    ) t.secrets}

    envsubst
      -i ${pkgs.writeText "${name}-secrets-template" t.text}
      -o /run/${name}-secrets/${t.name}
      -no-unset # fail if a secret is referenced but not declared (and therefore not exported)
      -no-empty # fail if reading the secret from vault fails, so it gets retried
  '';

To render a template, first the contents of each secret is loaded into an environment variable. Then envsubst is run to produce the desired output file.

secretName = s: "${name}.${lib.replaceStrings [ "/" ] [ "." ] s}";

envVarName =
  s:
  lib.pipe s [
    secretName
    (lib.replaceStrings [ "-" "." ] [ "_" "_" ])
    (s: "secret_${s}")
  ];

The secretName is the name for the systemd credential that contains the secret contents. The name is a list of period-separated components, starting with the service name and followed by the path components of the key for the secret. For instance, if the "grafana" service has a template that uses the "managed/oidc_client_secret" secret, the secretName for that is "grafana.managed.oidc_client_secret". This is what spiffe-creds expects and will translate into the correct path in Vault.

The envVarName is what should be used in the text of the template and will be replaced by the value of the secret. It starts with the secretName, replaces any dots or dashes with underscores, and prefixes that with "secret_". So the secret mentioned above would be referenced in the template with "${secret_grafana_managed_oidc_client_secret}".