Erase-your-darlings with preservation

{
  config,
  lib,
  inputs,
  ...
}:
let
  cfg = config.mjm.state;
  fileSystemsCfg = config.fileSystems;
in
{
  imports = [ "${inputs.preservation}/module.nix" ];

  options.mjm.state = {
    <<options>>
  };

  config = lib.mkIf (cfg.persistDir != null) {
    <<config>>
  };

  _class = "nixos";
}

This module is used to configure a machine to use an ephemeral root filesystem, with content that needs to be persistent being bind-mounted into place using the preservation module for NixOS.

In a previous version, service modules would use the options in this module to define the directories where their persistent state was kept. Now I just expect the /var directory to hold most persistent data, so /var is persisted in its entirety. This coarser-grained approach really simplifies things, and I adopted it after looking and finding there wasn't much in my /var directories that wasn't supposed to be kept in this way.

So these days, this module is not used by service modules. The service modules will usually put their data under /var without any special care, or will be modified to put it there if needed. Anything in my config can assume that /var will be preserved between boots, and if something is important, that's where it should live.

Options

persistDir = lib.mkOption {
  type = with lib.types; nullOr path;
  default = null;
};

Setting the persistDir option to a non-null value enables using preservation. This option defines the root where persistent data will be mounted from. Usually it is "/persist", meaning that e.g. /var will be bind-mounted from /persist/var.

directories = lib.mkOption {
  type = lib.types.attrsOf (
    lib.types.submodule (
      { name, config, ... }: {
        options = {
          enable = lib.mkOption {
            type = lib.types.bool;
            default = !(lib.hasAttr config.directory fileSystemsCfg);
          };
          directory = lib.mkOption {
            type = lib.types.path;
            default = name;
          };
          inInitrd = lib.mkOption {
            type = lib.types.bool;
            default = false;
          };
        };
      }
    )
  );
  default = { };
};

The directories option is an attribute set defining directories that will be preserved. Using this structure will make it possible to pre-define some common directories but leave them disabled by default.

Preservation supports some more options than this on the directories, particularly for setting permissions, but I'm not relying on those so they are not replicated here.

files = lib.mkOption {
  type = lib.types.attrsOf (
    lib.types.submodule (
      { name, ... }: {
        options = {
          enable = lib.mkOption {
            type = lib.types.bool;
            default = true;
          };
          file = lib.mkOption {
            type = lib.types.path;
            default = name;
          };
          inInitrd = lib.mkOption {
            type = lib.types.bool;
            default = false;
          };
        };
      }
    )
  );
  default = { };
};

The files option is structured identically to directories, but is meant for bind-mounting individual files rather than whole directories.

tmpfsRoot.size = lib.mkOption {
  type = with lib.types; nullOr str;
  default = null;
};

If preservation is enabled, then a default root filesystem will be set up using tmpfs. For some systems, the default size of half the total memory capacity may not be enough. This option lets a machine override the size.

Config

mjm.state.directories.var = {
  directory = "/var";
  inInitrd = true;
};

Persisting the /var directory is exposed under the "var" key. /var is needed early in boot, so inInitrd is enabled. This directory is enabled by default.

mjm.state.directories.nix = {
  directory = "/nix";
  inInitrd = true;
};

Persisting the /nix directory is exposed under the "nix" key. Like /var, it is also needed early in boot, to access store contents. Because all of my machines that use this setup need this mount, it is enabled by default. It could be disabled for a machine with a different structure for the persisted directory (e.g. I've occasionally used a setup where persistDir was /nix/persist).

mjm.state.directories.home = {
  enable = lib.mkDefault false;
  directory = "/home";
};

Persisting the /home directory is exposed under the "home" key. /home is not needed early in boot, so it can be mounted in stage 2. I only find preserving /home useful for workstations, not servers, so I do not enable this one by default.

mjm.state.files.machine-id = {
  file = "/etc/machine-id";
  inInitrd = true;
};

/etc/machine-id is usually the only individual file that needs to be preserved, since it is not practical to move it somewhere else. It is needed early in boot. It is always necessary, so it is enabled by default.

preservation.enable = true;
preservation.preserveAt.${cfg.persistDir} =
  let
    toPreservationList = lib.flip lib.pipe [
      lib.attrValues
      (lib.filter (v: v.enable))
      (lib.map (lib.flip lib.removeAttrs [ "enable" ]))
    ];
  in
  {
    directories = toPreservationList cfg.directories;
    files = toPreservationList cfg.files;
  };

If a persistDir is set, then the preservation module is enabled and that directory is configured as a preservation root. The directories and files options from this module need a little transformation into what preservation expects: the enable option needs to be respected and then dropped.

services.openssh.hostKeys = [
  {
    path = "${cfg.persistDir}/etc/ssh/ssh_host_ed25519_key";
    type = "ed25519";
  }
];

Rather than bind-mounting the SSH host keys, I choose to just reconfigure sshd to generate and expect to find them in the persisted location. I also take this opportunity to drop the RSA host key that is there by default, since I generally don't really need it and would prefer to use the Ed25519 one.

boot.initrd.systemd.suppressedUnits = [ "systemd-machine-id-commit.service" ];
systemd.suppressedSystemUnits = [ "systemd-machine-id-commit.service" ];

The default systemd unit for committing the machine-id to disk gets confused by this setup. It's not really necessary if we already have one on disk, so I just suppress it. I'm not generally setting up a machine fresh with this module enabled. Instead, I'm converting an existing machine, which already has a machine-id on disk.

fileSystems."/" = lib.mkDefault {
  device = "none";
  fsType = "tmpfs";
  options = [
    "defaults"
    "mode=755"
  ]
  ++ lib.optional (cfg.tmpfsRoot.size != null) "size=${cfg.tmpfsRoot.size}";
};

The most no-hassle way to get an ephemeral root filesystem is to use a tmpfs, which will be backed by RAM and/or swap. Some machines may wish to do a more complex setup where the root filesystem is still backed by disk, but is erased on every boot. They can do that by setting their own value for fileSystems."/", but if a machine does nothing, it will get a tmpfs root for free.