Registering Consul services declaratively

#| file: services/consul/services.nix
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.mjm.consul;
  jsonFormat = pkgs.formats.json { };
  hostname = config.networking.hostName;
in
{
  options.mjm.consul.services = lib.mkOption {
    type =
      lib.types.attrsOf
      <| lib.types.submodule (
        { name, config, ... }:
        {
          options = {
            <<options>>
          };

          config = {
            <<serviceConfig>>
          };
        }
      );
    default = { };
  };

  config = lib.mkIf (cfg.services != { }) {
    <<config>>
  };

  _class = "nixos";
}

NixOS doesn't have a particularly convenient way to declare Consul services that should be registered, so I've added one of my own, tailored to my needs.

Consul service options

A new Consul service instance can be added by defining a key under mjm.consul.services. These are the options that can be used under that key:

enable = lib.mkOption {
  type = lib.types.bool;
  default = true;
};

Any services that are defined are enabled by default, but this option lets you disable them without removing them from the config.

id = lib.mkOption {
  type = lib.types.str;
  default = "${config.name}:${hostname}";
};

Each Consul service instance needs an ID. I'm not generally running multiple instances of a service on the same machine, so I like a convention of using the service name and the hostname separated by a colon.

name = lib.mkOption {
  type = lib.types.str;
  default = name;
};

The name of the service to register in Consul. This usually doesn't need to be set, as it defaults to the key that was used for the service definition.

port = lib.mkOption {
  type = with lib.types; nullOr port;
  default = null;
};

The port that clients can use to connect to the service.

metrics.enable = lib.mkEnableOption "scraping metrics with Prometheus";

My Prometheus setup is configured to automatically scrape metrics from services registered in Consul if they have certain metadata attributes. This option enables setting those attributes.

metrics.path = lib.mkOption {
  type = lib.types.str;
  default = "/metrics";
};

The path Prometheus should use to scrape metrics.

metrics.port = lib.mkOption {
  type = with lib.types; nullOr port;
  default = null;
};

The port Prometheus should use to scrape metrics, if it is different from the port already declared for the service.

metrics.tls = lib.mkOption {
  type = lib.types.bool;
  default = false;
};

If enabled, tells Prometheus to use HTTPS to scrape this service's metrics.

blockDeploy = lib.mkOption {
  type = lib.types.bool;
  default = false;
};

If enabled, then after deploying this machine, my deploy tooling will wait for this service to reported healthy in Consul before advancing to the next machine.

checks =
  let
    svcConfig = config;
  in
  lib.mkOption {
    default = { };
    type =
      lib.types.attrsOf
      <| lib.types.submodule (
        { name, config, ... }:
        {
          options = {
            <<check-options>>
          };

          config = {
            <<checkConfig>>
          };
        }
      );
  };

A Consul service can define one or more checks that determine the health of the service. These are defined in their own attribute set of submodules similar to the service itself. The options for checks will be shown further below.

serviceConfig = lib.mkOption {
  type = lib.types.submodule { freeformType = jsonFormat.type; };
};

serviceConfig = {
  id = lib.mkDefault config.id;
  name = lib.mkDefault config.name;
  port = lib.mkIf (config.port != null) (lib.mkDefault config.port);

  meta = lib.mkIf config.metrics.enable {
    metrics_path = config.metrics.path;
    metrics_port = lib.mkIf (config.metrics.port != null) (toString config.metrics.port);
    metrics_scheme = lib.mkIf config.metrics.tls "https";
  };
  checks = lib.mkIf (config.checks != { }) (
    config.checks
    |> lib.attrValues
    |> lib.filter (chk: chk.enable)
    |> map (chk: chk.checkConfig)
  );
};

This is the actual JSON that will be used in the Consul config file. It can be used to set things that don't have options defined above.

Check options

enable = lib.mkOption {
  type = lib.types.bool;
  default = true;
};

Like services, any check that is defined is automatically enabled, but it can be disabled without removing it from the config using this option.

id = lib.mkOption {
  type = lib.types.str;
  default = "${svcConfig.id}:${name}";
};

Checks also have IDs which should be unique, at least in the scope of the agent. By default, I generate these by combining the service ID with the key used to define the check. This seems to copy what Vault does for the Consul service it registers.

name = lib.mkOption {
  type = lib.types.str;
  default = "${svcConfig.name} is ready";
};

Checks can also have user-friendly names that are shown in the web UI. I generate one of these by default based on the service name.

intervalSeconds = lib.mkOption {
  type = lib.types.int;
  default = 15;
};

How often Consul will run the check. More frequent means more requests made to the service, but it also means a change in health is noticed more quickly.

timeoutSeconds = lib.mkOption {
  type = lib.types.int;
  default = 10;
};

A timeout after which a check will be considered failing if it hasn't completed yet. Should be kept shorter than the interval to avoid the same check overlapping itself.

http.url = lib.mkOption {
  type = with lib.types; nullOr str;
  default = null;
};

An absolute HTTP or HTTPS URL to check using a GET request. Usually, it's preferred to use the other HTTP options below, which will construct a full URL if needed.

http.port = lib.mkOption {
  type = with lib.types; nullOr port;
  default = svcConfig.port;
};

The TCP port to use for an HTTP health check. Defining a port on its own will not cause an HTTP check to be performed. It defaults to the port of the service, so usually it is not necessary to set this unless the service exposes its health checks on a separate listener.

http.path = lib.mkOption {
  type = with lib.types; nullOr str;
  default = null;
};

The path to check using an HTTP GET request. Setting this will cause an HTTP check to be performed, and is preferred over http.url. The path will be combined with the port to construct a full localhost URL for the check.

http.socket = lib.mkOption {
  type = with lib.types; nullOr path;
  default = null;
};

If set, then the HTTP path to check can be reached over this Unix socket path instead of via TCP. Since Consul doesn't support making HTTP requests this way natively, this is implemented as a script check using curl. http.path must be set as well.

script.args = lib.mkOption {
  type = with lib.types; nullOr (listOf str);
  default = null;
};

Arguments for a command to run as a script check.

checkConfig = lib.mkOption {
  type = lib.types.submodule { freeformType = jsonFormat.type; };
};

checkConfig = {
  id = lib.mkDefault config.id;
  name = lib.mkDefault config.name;
  http = lib.mkIf (config.http.socket == null) (
    lib.mkMerge [
      (lib.mkIf (
        config.http.path != null
      ) "http://localhost:${toString config.http.port}${config.http.path}")
      (lib.mkIf (config.http.url != null) config.http.url)
    ]
  );
  args = lib.mkMerge [
    (lib.mkIf (config.script.args != null) config.script.args)
    (lib.mkIf (config.http.socket != null) [
      (lib.getExe pkgs.curl)
      "--no-progress-meter"
      "--fail-with-body"
      "--unix-socket"
      config.http.socket
      "http://localhost${config.http.path}"
    ])
  ];
  interval = lib.mkDefault "${toString config.intervalSeconds}s";
  timeout = lib.mkDefault "${toString config.timeoutSeconds}s";
};

This is the actual JSON that will be used for the check in the Consul config file. It can be used to set things that don't have options defined above. It is also how the behavior of the options above are implemented.

Config

environment.etc."consul-services.json".source = jsonFormat.generate "consul-services.json" {
  services =
    cfg.services
    |> lib.attrValues
    |> lib.filter (svc: svc.enable)
    |> map (svc: svc.serviceConfig);
};
services.consul.extraConfigFiles = [ "/etc/consul-services.json" ];
systemd.services.consul.reloadTriggers = [ config.environment.etc."consul-services.json".source ];

The configuration for all of the services on the machine is written to /etc/consul-services.json, and Consul is configured to read that as well. The systemd service for Consul will be reloaded whenever this file is changed.

mjm.deploy.consulChecks =
  cfg.services |> lib.filterAttrs (_: svc: svc.blockDeploy) |> lib.attrNames;

Each service that was configured to block deploys is added to the list of Consul services to check for the machine. That list is what the deploy tooling actually uses.