Skip to content

Commit

Permalink
Merge pull request #210 from MartinNowak/fixup_agent_scaling
Browse files Browse the repository at this point in the history 
fixup agent scaling
merged-on-behalf-of: Martin Nowak <code@dawg.eu>
  • Loading branch information
@dlang-bot
dlang-bot committed Nov 4, 2018
2 parents ca1cea4 + 11201cc commit 23678ad
Show file tree
Hide file tree
Showing 5 changed files with 204 additions and 15 deletions.
168 changes: 168 additions & 0 deletions data/payloads/hcloud_servers_post
View file
@@ -0,0 +1,168 @@
{
"action": {
"id": 7520113,
"command": "create_server",
"finished": null,
"progress": 0,
"started": "2018-11-04T14:27:32+00:00",
"resources": [
{
"type": "server",
"id": 1327609
}
],
"error": null,
"status": "running"
},
"root_password": "GtcME1gnfIyRot7PPNTw",
"server": {
"outgoing_traffic": 0,
"id": 1327609,
"rescue_enabled": false,
"image": {
"protection": {
"delete": false
},
"image_size": 1.13463800878906,
"id": 1461991,
"created_from": {
"id": 1323136,
"name": "gen-ci-agent-image"
},
"disk_size": 20,
"deprecated": null,
"bound_to": null,
"name": null,
"labels": {},
"os_version": null,
"type": "snapshot",
"description": "ci-agent",
"created": "2018-11-03T01:28:33+00:00",
"os_flavor": "ubuntu",
"rapid_deploy": false,
"status": "available"
},
"locked": false,
"datacenter": {
"description": "Helsinki 1 DC 2",
"location": {
"country": "FI",
"id": 3,
"name": "hel1",
"city": "Helsinki",
"longitude": 24.938379,
"description": "Helsinki DC Park 1",
"latitude": 60.169855
},
"server_types": {
"supported": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15
],
"available": [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15
]
},
"id": 3,
"name": "hel1-dc2"
},
"iso": null,
"included_traffic": 21990232555520,
"name": "ci-agent-d9f239eb-1434-4a5c-9568-4897b36ff061",
"server_type": {
"prices": [
{
"location": "fsn1",
"price_hourly": {
"gross": "0.0047600000000000",
"net": "0.0040000000"
},
"price_monthly": {
"gross": "2.9631000000000000",
"net": "2.4900000000"
}
},
{
"location": "nbg1",
"price_hourly": {
"gross": "0.0047600000000000",
"net": "0.0040000000"
},
"price_monthly": {
"gross": "2.9631000000000000",
"net": "2.4900000000"
}
},
{
"location": "hel1",
"price_hourly": {
"gross": "0.0047600000000000",
"net": "0.0040000000"
},
"price_monthly": {
"gross": "2.9631000000000000",
"net": "2.4900000000"
}
}
],
"id": 1,
"memory": 2,
"name": "cx51",
"cpu_type": "shared",
"description": "CX51",
"storage_type": "local",
"disk": 20,
"cores": 1
},
"labels": {},
"volumes": [],
"ingoing_traffic": 0,
"public_net": {
"ipv6": {
"blocked": false,
"ip": "2a01:4f9:c010:292b::/64",
"dns_ptr": []
},
"ipv4": {
"blocked": false,
"ip": "95.216.192.196",
"dns_ptr": "static.196.192.216.95.clients.your-server.de"
},
"floating_ips": []
},
"backup_window": null,
"created": "2018-11-04T14:27:32+00:00",
"protection": {
"rebuild": false,
"delete": false
},
"status": "initializing"
}
}
10 changes: 6 additions & 4 deletions source/dlangbot/app.d
View file
Expand Up @@ -47,7 +47,7 @@ void startServer(HTTPServerSettings settings)
.get("/", (req, res) => res.render!"index.dt")
.get("*", serveStaticFiles("public"))
.post("/github_hook", &githubHook)
.match(HTTPMethod.HEAD, "/trello_hook", (HTTPServerRequest req, HTTPServerResponse res) => res.writeVoidBody)
.match(HTTPMethod.HEAD, "/trello_hook", (HTTPServerRequest req, HTTPServerResponse res) => res.writeBody(""))
.post("/trello_hook", &trelloHook)
.post("/codecov_hook", &codecovHook)
.post("/buildkite_hook", &buildkiteHook)
Expand Down Expand Up @@ -144,7 +144,7 @@ void githubHook(HTTPServerRequest req, HTTPServerResponse res)
return res.writeBody("handled");

default:
return res.writeVoidBody();
return res.writeBody("");
}
}

Expand Down Expand Up @@ -309,7 +309,7 @@ void buildkiteHook(HTTPServerRequest req, HTTPServerResponse res)
break;

default:
return res.writeVoidBody();
return res.writeBody("");
}
return res.writeBody("handled");
}
Expand All @@ -322,7 +322,9 @@ void agentShutdownCheck(HTTPServerRequest req, HTTPServerResponse res)
import std.algorithm.searching : startsWith;

verifyAgentRequest(req.headers.get("Authentication"));
agentShutdownCheck(req.form.get("hostname"));
auto hostname = req.form.get("hostname");
logInfo("agentShutdownCheck hostname:%s", hostname);
agentShutdownCheck(hostname);
res.writeBody("");
}

Expand Down
5 changes: 4 additions & 1 deletion source/dlangbot/buildkite.d
View file
Expand Up @@ -5,7 +5,7 @@ import std.datetime.systime;
import std.range : empty, front, walkLength;
import std.uuid : randomUUID;

import vibe.core.log : logWarn;
import vibe.core.log : logInfo, logWarn;
import vibe.data.json : Name = name;
import vibe.data.json;
import vibe.http.client : HTTPClientRequest;
Expand Down Expand Up @@ -62,6 +62,7 @@ private void provisionReleaseBuilder(uint nbuilds)
auto servers = scw.servers;
immutable nservers = servers.filter!(s => s.name.startsWith("release-builder-")).walkLength;

logInfo("check provision release-builder nservers: %s, nbuilds: %s", nservers, nbuilds);
if (nservers >= nbuilds)
return;

Expand Down Expand Up @@ -92,6 +93,7 @@ private void provisionCIAgent(uint nbuilds)
auto servers = hc.servers;
immutable nservers = servers.filter!(s => s.name.startsWith("ci-agent-")).walkLength;

logInfo("check provision ci-agent nservers: %s, nbuilds: %s", nservers, nbuilds);
if (nservers >= nbuilds)
return;

Expand Down Expand Up @@ -126,6 +128,7 @@ private uint numReleaseBuilds()
private uint numCIBuilds()
{
import std.algorithm : filter, fold;
import std.conv : to;

return pipelines.filter!(p => p.defaultQueue)
.fold!((sum, p) => sum + p.scheduledBuildsCount + 0.99f * p.runningBuildsCount)(0.0f)
Expand Down
2 changes: 1 addition & 1 deletion source/dlangbot/hcloud_api.d
View file
Expand Up @@ -60,7 +60,7 @@ Server createServer(string name, string serverType, Image image)
"server_type": serverType,
"image": image.id.to!string,
])
.readJson
.readJson["server"]
.deserializeJson!(typeof(return));
}

Expand Down
34 changes: 25 additions & 9 deletions test/buildkite.d
View file
Expand Up @@ -93,6 +93,28 @@ unittest
// CI-Agents on Hetzner Cloud
//==============================================================================

Json hcloudCreateServerResp(ulong id, string name)
{
import core.time : Duration;
import std.datetime.systime : Clock;
import std.datetime.timezone : SimpleTimeZone;
import std.file : readText;
import vibe.data.json : parseJsonString;

// use zulu to get +00:00 instead of Z suffix
static zulu = new immutable SimpleTimeZone(Duration.zero, "Etc/Zulu");
auto now = Clock.currTime(zulu);
now.fracSecs = Duration.zero;
auto time = now.toISOExtString;

auto json = "data/payloads/hcloud_servers_post".readText.parseJsonString;
json["server"]["id"] = id;
json["server"]["name"] = name;
json["server"]["created"] = time;
json["action"]["started"] = time;
return json;
}

@("spawns-ci-agent")
unittest
{
Expand All @@ -110,9 +132,7 @@ unittest
auto name = req.json["name"].get!string;
assert(name.startsWith("ci-agent-"));
assert(req.json["image"] == "1456126");
auto resp = serializeToJson(["name": name, "status": "initializing"]);
resp["id"] = 1321993;
res.writeJsonBody(resp);
res.writeJsonBody(hcloudCreateServerResp(1321993, name));
},
);

Expand Down Expand Up @@ -147,9 +167,7 @@ unittest
auto name = req.json["name"].get!string;
assert(name.startsWith("ci-agent-"));
assert(req.json["image"] == "1456126");
auto resp = serializeToJson(["name": name, "status": "initializing"]);
resp["id"] = 1321994;
res.writeJsonBody(resp);
res.writeJsonBody(hcloudCreateServerResp(1321994, name));
},
);

Expand All @@ -172,9 +190,7 @@ unittest
auto name = req.json["name"].get!string;
assert(name.startsWith("ci-agent-"));
assert(req.json["image"] == "1456126");
auto resp = serializeToJson(["name": name, "status": "initializing"]);
resp["id"] = 1321994;
res.writeJsonBody(resp);
res.writeJsonBody(hcloudCreateServerResp(1321994, name));
},
);

Expand Down

0 comments on commit 23678ad

Please sign in to comment.