{
config,
lib,
pkgs,
nodes,
...
}:
let
cfg = config.mjm.services;
in
{
options.mjm.services =
let
serviceType =
{ name, config, ... }:
let
yamlFormat = pkgs.formats.yaml { };
in
{
options = {
<<options>>
};
config = {
<<default-allowed-clients>>
<<client-config>>
};
};
in
lib.mkOption {
type = with lib.types; attrsOf (submodule serviceType);
};
config =
let
<<services-and-ports>>
in
{
<<config>>
};
_class = "nixos";
}A particularly common need for services in the lab is to expose an HTTP server that people or other services can connect to. This module provides options for easily configuring that and setting up all the necessary infrastructure to support it. At a minimum, this will set up a Consul service to allow the service to be discovered by others and a server tunnel that exposes the service via mutual TLS.
There is quite a bit of complexity in this module. I'm always looking for opportunities to simplify it, but the nature of running software you don't control is needing to be tolerant of its quirks. The complexity here helps keep the modules that define each service simple and straightforward, and it ensures that they all follow as close to the same patterns as is possible.
These options are available within the scope of a particular entry of mjm.services.
http.enable = lib.mkOption {
type = lib.types.bool;
default = config.http.port != null || config.http.socket != null;
};Support for serving HTTP requests is enabled automatically if the service configures either a TCP port or a Unix socket to listen on via one of the next two options.
http.socket = lib.mkOption {
type = with lib.types; nullOr path;
default = null;
};Listening on a Unix socket is preferred if the service supports it. Ideally this is via a systemd socket unit, but that's even less commonly supported.
http.port = lib.mkOption {
type = with lib.types; nullOr port;
default = null;
};If a Unix socket is not an option, listening locally on a TCP port is fine too. Note that services should not listen on an external IP. Clients should not be able to connect to the service for plaintext communication.
http.clients = lib.mkOption {
type = with lib.types; listOf str;
default = [ ];
};The names of other services that are allowed to connect to this one. This generally doesn't need to be set manually: other services that declare an upstream for this one will be automatically added as clients:
http.clients = lib.pipe nodes [ lib.attrValues (map (n: n.config.mjm.services)) lib.mergeAttrsList lib.attrValues (lib.filter (s: lib.hasAttr name s.upstreams)) (map (s: s.name)) ];
This looks at the definitions of all services across every machine, and gathers the names of each one that declares an upstream matching this service's name.
http.health.path = lib.mkOption {
type = with lib.types; nullOr str;
default = null;
};An HTTP path to use for checking the health of the service. If a GET request to the path succeeds, the service is considered healthy and able to receive traffic. This should almost always be set: ideally to a dedicated health endpoint if the service has one, but if not, any basic GET route will do to at least confirm the service is listening.
http.health.port = lib.mkOption {
type = with lib.types; nullOr port;
default = config.http.port;
};An alternative TCP port to use for the health check. Some services expose health checks on a separate listener from normal requests. This only needs to be set if it is different from the normal port.
http.health.blockDeploy = lib.mkOption {
type = lib.types.bool;
default = false;
};My deploy tooling can pause after deploying a host and wait for some of the Consul services running on it are healthy before advancing the deploy. This option enables that behavior for this service. When deploying a VM host, all of the services that block deploys from all microVMs on that host will be checked.
http.metrics.enable = lib.mkOption {
type = lib.types.bool;
default = false;
};Enables Prometheus scraping metrics from the service. This sets metadata on the Consul service which Prometheus will read to discover the service and scrape it. Metrics, like normal requests, are exposed via mutual TLS, so requests for metrics will come in through a tunnel.
http.metrics.path = lib.mkOption {
type = lib.types.str;
default = "/metrics";
};By default, metrics will be scraped from /metrics, but this path can be changed if the service exposes metrics from a different path.
http.metrics.port = lib.mkOption {
type = with lib.types; nullOr port;
default = null;
};If metrics are exposed over a separate port from normal requests, then that port can be set with this option. Since metrics must be exposed via mTLS over a tunnel, setting a separate port for them will create a second tunnel to serve them to Prometheus.
http.ingress.subdomain = lib.mkOption {
type = with lib.types; nullOr str;
default = null;
};An HTTP service can be exposed to people, rather than just other services, by setting an ingress subdomain. If set, the service will be accessible through the ingress Caddy server at https://$subdomain.midna.dev.
http.ingress.ipv4 = lib.mkOption {
type = lib.types.bool;
default = true;
};Whether the service should be accessible via IPv4 through the ingress. This affects the DNS records that are set up for the subdomain. All services exposed through the ingress are always exposed via IPv6, which connects directly to the currently active ingress server. Since my homelab does not have a static IPv4 address, I tunnel IPv4 requests through a VPS which then translates them to IPv6 and forwards them to the ingress server.
For some services, I would rather not have the traffic go through the VPS this way, so for those I can disable IPv4 traffic for them. Those services will not have any A records, only AAAA.
http.ingress.authMode = lib.mkOption {
type = lib.types.enum [
"none"
"oidc"
"proxy"
];
};How authentication for the service should be handled when coming through the ingress. This intentionally does not have a default, as the specific properties of the service need to be considered. The options in order of preference are:
Both "oidc" and "proxy" offer protection via integration with Authelia. Using OpenID Connect is preferred because it doesn't require Authelia to be online and healthy for every request to the service, only during authentication.
The "oidc" and "none" modes are functionally the same from the perspective of the ingress: traffic is sent directly to the service rather than first being forwarded to Authelia. The difference is "oidc" services will get added as clients in Authelia's configuration, using configuration set in options below.
http.ingress.proxy.rules = lib.mkOption {
type = lib.types.listOf (lib.types.submodule { freeformType = yamlFormat.type; });
default = [ ];
};For services using the "proxy" auth mode, this option allows defining custom rules for Authelia. This can be used to, among other things, exempt certain paths from requiring authentication. This is useful to be able to expose API endpoints that are already protected by some kind of token to applications that can't deal with the auth redirect the proxy will impose.
The rules included in the list will get a "domain" key added to them to match the subdomain for the service, and will otherwise be added unmodified to Authelia's configuration.
The options under http.ingress.oidc are used to configure OpenID Connect for services that use the ingress auth mode "oidc". These options affect two things:
http.ingress.oidc.id = lib.mkOption {
type = lib.types.str;
default = name;
};An identifier used for this client when creating outputs and secrets in Pulumi. Most of the time, this is inferred from the service name and should be left alone. It only exists because I used to set this stuff up by hand, and I wasn't consistently making sure the service name matched the identifiers I was using in Pulumi.
http.ingress.oidc.name = lib.mkOption { type = lib.types.str; };The display name of this client. It is shown on login/consent prompts.
http.ingress.oidc.clientId = lib.mkOption { type = lib.types.str; };The automatically generated ID for the client.
When adding a new OIDC client, this should initially be set to the empty string. Then infra changes can be applied, which will use Pulumi to generate a new client ID which will be displayed in the output of the command. That ID should be copied here so that it can be included in Authelia's configuration.
The client ID will also need to be configured in the service itself. If that is happening declaratively in Nix, it can be referenced from this option via the config argument to avoid duplication.
http.ingress.oidc.clientSecret = lib.mkOption { type = lib.types.str; };The argon2id hash of the automatically generated secret for the client.
When adding a new OIDC client, this should initially be set to the empty string. Then infra changes can be applied, which will use Pulumi to generate a new client secret and add it to Vault. The secret won't be displayed, but it can be looked up from Vault. The command "authelia crypto hash generate argon2" can be used to hash the text, and the result should be copied to this option so it can be included in Authelia's configuration.
While Authelia needs only the hashed version of the secret, the service itself will need the actual contents. This should be loaded from Vault and passed along to the service in whatever way makes sense.
For clients that need to use the public authorization flow, this should be set to the empty string.
http.ingress.oidc.redirectUris = lib.mkOption { type = with lib.types; listOf str; };The allowed redirect URIs Authelia can use to send information back to the service. The service's documentation should indicate the correct values for this.
http.ingress.oidc.requirePkce = lib.mkOption {
default = true;
type = lib.types.bool;
};This option determines whether proof key for code exchange is required for the authorization flow. This defaults to true because it should generally be enabled as long as the service supports it. It is especially important for the security of public clients.
http.ingress.oidc.scopes = lib.mkOption {
type = with lib.types; listOf str;
default = [
"openid"
"profile"
"groups"
"email"
];
};The list of scopes that the service is allowed to request from Authelia. "groups" is included here by default because it's generally useful for apps to be able to tell who is an administrator.
http.ingress.oidc.tokenEndpointAuthMethod = lib.mkOption {
default = if config.http.ingress.oidc.clientSecret == "" then "none" else "client_secret_basic";
type = lib.types.enum [
"none"
"client_secret_basic"
"client_secret_post"
"client_secret_jwt"
"private_key_jwt"
];
};The mechanism that the service will use to authenticate with Authelia's token endpoint. The default here mirrors Authelia's defaults which mirror the spec.
http.ingress.oidc.clientConfig = lib.mkOption {
internal = true;
type = lib.types.submodule { freeformType = yamlFormat.type; };
};This is used to produce the actual YAML value that will be included in Authelia's configuration. That value is generated as follows:
http.ingress.oidc.clientConfig =
let
inherit (config.http.ingress) oidc;
in
lib.mkMerge [
{
client_id = oidc.clientId;
client_name = oidc.name;
client_secret = oidc.clientSecret;
redirect_uris = oidc.redirectUris;
scopes = oidc.scopes;
token_endpoint_auth_method = oidc.tokenEndpointAuthMethod;
}
(lib.mkIf oidc.requirePkce {
require_pkce = true;
pkce_challenge_method = "S256";
})
(lib.mkIf (oidc.clientSecret == "") {
public = true;
})
];httpPorts = assignPorts 22100 httpServices; metricsPorts = assignPorts 22200 metricsTunnelServices;
I generally avoid colocating different services: individual services should run in their own microVMs. But it is still possible for me to run more than one service on a machine if they are closely related. Because of this, I need to assign different ports to the tunnels for each service. Rather than force services to hardcode distinct values for these, I do it automatically based on which services are running on the machine. There are two ranges here: one for the normal HTTP tunnels and one for the tunnels serving metrics for services where that happens on a separate port.
httpServices = lib.attrValues (lib.filterAttrs (_: s: s.http.enable) cfg); metricsTunnelServices = lib.filter ( s: s.http.metrics.enable && s.http.metrics.port != null ) httpServices;
Every service with http.enable set to true gets a server tunnel with port numbers starting at 22100.
The subset of those services that have enabled metrics on a separate port get a second tunnel with port numbers starting at 22200.
assignPorts =
start: svcs:
let
names = map (s: s.name) svcs;
ports = lib.range start (start - 1 + lib.length names);
in
lib.listToAttrs (lib.zipListsWith lib.nameValuePair names ports);Ports are assigned by zipping together the names of the services with a range of numbers starting at the given start value. The zipped pairs are made into an attribute set, where the service names are the keys and the ports are the values.
mjm.spire.tunnels =
let
<<httpTunnels>>
<<metricsTunnels>>
in
httpTunnels // metricsTunnels;Each service with HTTP enabled will get an HTTP tunnel and an optional metrics tunnel.
httpTunnels = lib.pipe httpServices [
(map (
{ name, http, ... }:
lib.nameValuePair name {
id = name;
mode = "server";
listen.port = httpPorts.${name};
target.socket = lib.mkIf (http.socket != null) http.socket;
target.port = lib.mkIf (http.port != null) http.port;
allowIngress = http.ingress.subdomain != null;
allowMetrics = http.metrics.enable && http.metrics.port == null;
allowedServices = http.clients;
}
))
lib.listToAttrs
];The HTTP tunnel exposes the service's normal traffic to other machines via mutual TLS. The tunnel listens on the port that was randomly assigned above, and targets the configured Unix socket or TCP port. If an ingress subdomain was configured, then the tunnel will accept connections from Caddy. If metrics are enabled for the service and they aren't going to be served from a separate tunnel on a different port, then connections from Prometheus are allowed as well. And finally, the configured list of clients (and the ones automatically determined from upstreams) are allowed to connect.
metricsTunnels = lib.pipe metricsTunnelServices [
(map (
{ name, http, ... }:
lib.nameValuePair "${name}-metrics" {
id = name;
mode = "server";
listen.port = metricsPorts.${name};
target.port = http.metrics.port;
allowMetrics = true;
}
))
lib.listToAttrs
];The metrics tunnels are a bit simpler. They only allow Prometheus to connect: anyone else will be rejected. They listen on the assigned port and target the configured metrics port.
mjm.consul.services = lib.pipe httpServices [
(map (
{ name, http, ... }:
lib.nameValuePair name {
port = httpPorts.${name};
metrics =
http.metrics
// {
tls = true;
}
// lib.optionalAttrs (http.metrics.port != null) { port = metricsPorts.${name}; };
blockDeploy = http.health.blockDeploy;
checks.up = lib.mkIf (http.health.path != null) {
http.path = http.health.path;
http.port = lib.mkIf (http.health.port != null) http.health.port;
http.socket = lib.mkIf (http.health.port == null && http.socket != null) http.socket;
checkConfig = {
failures_before_warning = 2;
failures_before_critical = 6;
};
};
}
))
lib.listToAttrs
];With the tunnels in place, they need to be advertised to the lab via Consul. A single Consul service is generated for each service with HTTP enabled. It advertises an instance on the assigned port for the HTTP tunnel.
The metrics configuration is largely passed along as-is to the Consul module, with two exceptions. TLS is always enabled for metrics: the service is not listening externally in plaintext, so metrics will always be scraped from one of the two tunnels. And if there is a separate tunnel for metrics, then the metrics port set on the service needs to be the port that was assigned to the tunnel, not the local one that the service configured.
If a health check path was set, then a Consul check is configured based on that. It uses the same local port or Unix socket that the tunnel targets, which is fine since Consul checks happen locally on the machine. This module also sets adds extra tolerance for failures in the checks by default.
mjm.ingress.vhosts = lib.pipe cfg [
lib.attrValues
(lib.filter (s: s.http.ingress.subdomain != null))
(map (
{ name, http, ... }:
lib.nameValuePair http.ingress.subdomain {
upstream.service.name = name;
enableAuthProxy = http.ingress.authMode == "proxy";
useIPv4Proxy = http.ingress.ipv4;
}
))
lib.listToAttrs
];Finally, any services that set a subdomain for ingress get a virtual host configured. The authMode and ipv4 options as well as the service name are used to configure the virtual host.
Note that a virtual host can be set up even if a Consul service wasn't registered in the config here. This is to support at least one service (Vault) that registers itself in Consul automatically.
text/gemini;lang=en-USThis content has been proxied by September (UNKNO).