/
multi.nix
233 lines (216 loc) · 9.59 KB
/
multi.nix
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
# Module similar to (flake).agent-service or the NixOS-bundled module, but
# supports running multiple instance of the agent, each with their own files,
# user, etc.
systemArgs@{ pkgs, config, lib, ... }:
let
inherit (lib) mkIf mkDefault types mkOption;
inherit (lib.strings) match;
literalDocBook = lib.literalDocBook or lib.literalExample;
literalExpression = lib.literalExpression or lib.literalExample;
submodule = { config, options, name, ... }:
let
inherit (import ../settings.nix { inherit pkgs lib; }) format makeSettingsOptions;
configFile = format.generate "hercules-ci-agent${suffix}.json" config.settings;
command = "${config.package}/bin/hercules-ci-agent --config ${configFile}";
testCommand = "${command} --test-configuration";
suffix = if name == "" then "" else "-${name}";
user = if name == "" then "hercules-ci-agent" else "hci-${name}";
uniqueStrings = strs: lib.attrValues (lib.genAttrs strs (name: name));
wholeStringMatches = regex: str: match regex str != null;
needsDevices = devices != [ ];
devices = uniqueStrings (
lib.lists.concatLists
(lib.mapAttrsToList
(name: mountable@{ source, ... }:
let
sourcePath = "${source}";
isDev =
lib.strings.hasPrefix "/dev/" sourcePath
# Systemd provides these regardless. If they're used, it's
# probably as a placeholder, and that should just work without
# explicitly allowing it.
&& ! wholeStringMatches "/dev/null" sourcePath
&& ! wholeStringMatches "/dev/zero" sourcePath
&& ! wholeStringMatches "/dev/random" sourcePath
&& ! wholeStringMatches "/dev/urandom" sourcePath
# Let systemd deal with these directories. Maybe they should just
# be forwarded, but without knowing the use case, let's not allow
# anything extra.
&& ! wholeStringMatches "/dev/shm(/.*|$)" sourcePath
&& ! wholeStringMatches "/dev/pts(/.*|$)" sourcePath
# No shenanigans with .. or .
# (this may look like input validation, but don't put untrusted data in your config!)
&& ! wholeStringMatches ".*/\\.\\.(/.*|$)" sourcePath
&& ! wholeStringMatches ".*/\\.(/.*|$)" sourcePath
# Not all of /dev
&& ! wholeStringMatches "/dev/+" sourcePath;
in
lib.optional isDev sourcePath
)
(config.settings.effectMountables or { })
)
);
in
{
options = {
systemConfig = lib.mkOption {
internal = true;
type = types.unspecified; # A function from module arguments to config.
};
package = mkOption {
description = ''
Package containing the bin/hercules-ci-agent executable.
'';
type = types.package;
default = pkgs.hercules-ci-agent;
defaultText = literalExpression "pkgs.hercules-ci-agent";
};
} // makeSettingsOptions { cfg = config; opt = options; };
config = let cfg = config; in
{
settings = {
_module.args = {
packageOption = options.package;
inherit pkgs;
};
baseDirectory = "/var/lib/hercules-ci-agent${suffix}";
nixUserIsTrusted = true;
labels =
let
mkIfNotNull = x: mkIf (x != null) x;
in
{
nixos.configurationRevision = mkIfNotNull systemArgs.config.system.configurationRevision;
nixos.release = systemArgs.config.system.nixos.release;
nixos.label = mkIfNotNull systemArgs.config.system.nixos.label;
nixos.codeName = systemArgs.config.system.nixos.codeName;
nixos.tags = systemArgs.config.system.nixos.tags;
nixos.systemName = mkIfNotNull systemArgs.config.system.name;
};
};
systemConfig = { config, ... }: {
systemd.services."hercules-ci-agent${suffix}" = {
wantedBy = [ "multi-user.target" ];
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
path = [ config.nix.package ];
startLimitBurst = 30 * 1000000; # practically infinite
serviceConfig = {
User = user;
ExecStart = command;
ExecStartPre = testCommand;
Restart = "on-failure";
RestartSec = 120;
# If a worker goes OOM, don't kill the main process. It needs to
# report the failure and it's unlikely to be part of the problem.
OOMPolicy = "continue";
# Work around excessive stack use by libstdc++ regex
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86164
# A 256 MiB stack allows between 400 KiB and 1.5 MiB file to be matched by ".*".
LimitSTACK = 256 * 1024 * 1024;
# Hardening
CapabilityBoundingSet = "";
AmbientCapabilities = "";
ProtectSystem = "full";
LockPersonality = true;
PrivateMounts = true;
PrivateTmp = true;
PrivateUsers = true;
ProtectClock = true;
ProtectHome = true;
ProtectKernelModules = true;
RemoveIPC = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
SystemCallArchitectures = "native";
UMask = "077";
# Hardening / devices
PrivateDevices = ! needsDevices;
DevicePolicy = "closed";
DeviceAllow = if needsDevices then devices else "";
# Hardening / failed
# # Blocks /proc remount in effect container
# ProtectKernelLogs = true;
# ProtectKernelTunables = true;
# # workDirectory may not exist yet.
# WorkingDirectory = cfg.settings.workDirectory;
};
};
# Changes in the secrets do not affect the unit in any way that would cause
# a restart, which is currently necessary to reload the secrets.
systemd.paths."hercules-ci-agent${suffix}-restart-files" = {
wantedBy = [ "hercules-ci-agent${suffix}.service" ];
pathConfig = {
Unit = "hercules-ci-agent${suffix}-restarter.service";
PathChanged = [ cfg.settings.clusterJoinTokenPath cfg.settings.binaryCachesPath ];
};
};
systemd.services."hercules-ci-agent-restarter${suffix}" = {
serviceConfig.Type = "oneshot";
script = ''
# Wait a bit, with the effect of bundling up file changes into a single
# run of this script and hopefully a single restart.
sleep 10
if systemctl is-active --quiet hercules-ci-agent${suffix}.service; then
if ${testCommand}; then
systemctl restart hercules-ci-agent${suffix}.service
else
echo 1>&2 "WARNING: Not restarting agent because config is not valid at this time."
fi
else
echo 1>&2 "Not restarting hercules-ci-agent despite config file update, because it is not already active."
fi
'';
};
# Trusted user allows simplified configuration and better performance
# when operating in a cluster.
nix.settings.trusted-users = [ config.systemd.services."hercules-ci-agent${suffix}".serviceConfig.User ];
users.users.${user} = {
home = cfg.settings.baseDirectory;
createHome = true;
group = user;
description = "Hercules CI Agent system user${lib.optionalString (name != "") " for ${name}"}";
isSystemUser = true;
};
users.groups.${user} = { };
};
};
};
mergeSub =
f: lib.mkMerge (map (sub: f (sub.systemConfig systemArgs)) (lib.attrValues config.services.hercules-ci-agents));
in
{
options = {
services.hercules-ci-agents = lib.mkOption {
type = types.attrsOf (types.submoduleWith {
modules = [ submodule ];
});
default = { };
description = ''
Multiple instances of hercules-ci-agent can be specified.
If you specify an instance named `""`, it will behave just as the `services.hercules-ci-agent` options did.
- User: `hercules-ci-agent`
- Default base directory: `/var/lib/hercules-ci-agent`
Otherwise:
- User: `hci-''${name}`
- Default base directory: `/var/lib/hercules-ci-agent-''${name}`
'';
};
};
config = lib.mkMerge [
{
nix = mergeSub (c: c.nix);
systemd = mergeSub (c: c.systemd);
users = mergeSub (c: c.users);
}
{
nix.extraOptions = lib.mkIf (config.services.hercules-ci-agents != { }) ''
# A store path that was missing at first may well have finished building,
# even shortly after the previous lookup. This *also* applies to the daemon.
narinfo-cache-negative-ttl = 0
'';
}
];
meta.maintainers = [ lib.maintainers.roberth ];
}