Skip to content

Commit

Permalink
Merge branch 'develop' into workorch-zos
Browse files Browse the repository at this point in the history
  • Loading branch information
steveej committed Apr 25, 2024
2 parents e787f57 + 246083c commit 07c49a6
Show file tree
Hide file tree
Showing 18 changed files with 677 additions and 69 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/turn-readiness-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ on:
workflow_dispatch:
inputs: {}

env:
env: {}

jobs:
lints:
Expand Down
12 changes: 11 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,4 +134,14 @@ nix run .#deploy-linux-builder-01 test
If the runners are back online then update the default profile
```shell
nix run .#deploy-linux-builder-01 switch
```
```

## TURN stack

* [Adding a new machine]

* Double-check that all TURN/Signal servers are ready:

```
nix run .#turn-readiness-check
```
114 changes: 114 additions & 0 deletions docs/adding-a-turn-signal-bootstrap-combo-server.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
This example adds the machine configuration `turn-2`.

1. order a VPS on hetzner

2. add 2 additional floating IPv4 addresses to it

3. copy the flake-parts config and adjust the names
```
cp -r modules/flake-parts/nixosConfigurations.turn-{1,2}
git add modules/flake-parts/nixosConfigurations.turn-2
sed -i 's/-1/-2/g' modules/flake-parts/nixosConfigurations.turn-2/*.nix
```

4. change the IP addresses in *modules/flake-parts/nixosConfigurations.turn-2/configuration.nix*.
these are the main and floating IPs of the VPS:

```nix
(...)
}: let
hostName = "turn-2";
turnIpv4 = "65.109.140.0";
turnFqdn = "${hostName}.infra.holochain.org";
signalIpv4 = "95.217.25.40";
signalFqdn = "signal-2.infra.holochain.org";
bootstrapIpv4 = "95.216.176.124";
bootstrapFqdn = "bootstrap-2.infra.holochain.org";
in {
(...)
5. add the DNS and reverse-proxy entries to *modules/flake-parts/nixosConfigurations.dweb-reverse-tls-proxy/configuration.nix*
* add entries for `{turn,signal,bootstrap}-2` to the bind config:
```nix
(...)
environment.etc."bind/zones/${fqdn2domain}.zone" = {
enable = true;
user = "named";
group = "named";
mode = "0644";
text = ''
(...)
turn-2.${fqdn2domain}. A ${self.nixosConfigurations.turn-2.config.services.holochain-turn-server.address}
signal-2.${fqdn2domain}. A ${self.nixosConfigurations.turn-2.config.services.tx5-signal-server.address}
bootstrap-2.${fqdn2domain}. A ${self.nixosConfigurations.turn-2.config.services.kitsune-bootstrap.address}
'';
};
(...)
```

* add a reverse proxy entry to the caddy config:
```nix
(...)
services.caddy.virtualHosts = {
(...)
"acme-turn-2.${fqdn2domain}:80" = {
extraConfig = ''
reverse_proxy http://turn-2.${fqdn2domain}:${builtins.toString self.nixosConfigurations.turn-2.config.services.holochain-turn-server.nginx-http-port}
'';
};
};
(...)
```

6. rebuild the DNS server and restart the bind service
```
nix run .\#deploy-dweb-reverse-tls-proxy switch
nix run .\#ssh-dweb-reverse-tls-proxy "systemctl restart bind"
```

7. verify the records
```
nix run nixpkgs#dig +short {turn,signal,bootstrap}-2.infra.holochain.org @infra.holochain.org
65.109.140.0
95.217.25.40
95.216.176.124
```
8. deploy nixos on the new VPS. replace $IP with the primary IP of the VPS:

```
nix run .#nixos-anywhere -- --flake .\#turn-2 root@$IP
(...)
### Waiting for the maching to become reachable again ###
Warning: Permanently added '65.109.140.0' (ED25519) to the list of known hosts.
### Done! ###
```

this should take about 2-3 minutes.

9. verify the functionality of the signal and turn stack.

```
nix shell .#tx5 --command bash -c '
set -e;
turn-stress turn-2.infra.holochain.org 443 test test;
turn_doctor wss://signal-2.infra.holochain.org;
echo success
'
```

10. commit the changes to git and get them to develop
27 changes: 22 additions & 5 deletions flake.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,11 @@
sops-nix.url = "github:Mic92/sops-nix";
sops-nix.inputs.nixpkgs.follows = "nixpkgs";

# have the latest rust version available
craneNixpkgs = {url = "github:nixos/nixpkgs/nixos-unstable";};
crane = {
url = "github:ipetkov/crane";
inputs.nixpkgs.follows = "nixpkgs-23-11";
inputs.nixpkgs.follows = "craneNixpkgs";
};

keys_steveej = {
Expand Down Expand Up @@ -81,7 +83,7 @@

cachix_for_watch_store.url = "github:cachix/cachix/v1.5";

tx5.url = "github:holochain/tx5/tx5-signal-srv-v0.0.7-alpha";
tx5.url = "github:holochain/tx5/tx5-signal-srv-v0.0.8-alpha";
tx5.flake = false;

holochain-versions.url = "github:holochain/holochain?dir=versions/weekly";
Expand Down
46 changes: 46 additions & 0 deletions modules/flake-parts/hardware.hetzner-cloud-ccx.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
{
# System independent arguments.
...
}: {
flake.nixosModules.hardware-hetzner-cloud-ccx = {...}: {
boot.loader.grub.enable = false;
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;

disko.devices.disk.sda = {
device = "/dev/sda";
type = "disk";
content = {
type = "gpt";
partitions = {
ESP = {
type = "EF00";
size = "1G";
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
};
};
root = {
size = "100%";
content = {
type = "btrfs";
extraArgs = ["-f"]; # Override existing partition
subvolumes = {
# Subvolume name is different from mountpoint
"/rootfs" = {
mountpoint = "/";
};
"/nix" = {
mountOptions = ["noatime"];
mountpoint = "/nix";
};
};
};
};
};
};
};
};
}
46 changes: 46 additions & 0 deletions modules/flake-parts/hardware.hetzner-cloud-cpx.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
{
# System independent arguments.
...
}: {
flake.nixosModules.hardware-hetzner-cloud-cpx = {lib, ...}: {
boot.loader.systemd-boot.enable = false;
boot.loader.grub.efiSupport = false;

# forcing seems required or else there's an error about duplicated devices
boot.loader.grub.devices = lib.mkForce ["/dev/sda"];

disko.devices.disk.sda = {
device = "/dev/sda";
type = "disk";
content = {
type = "gpt";
partitions = {
boot = {
size = "1M";
type = "EF02"; # for grub MBR
};
root = {
size = "100%";
content = {
type = "btrfs";
extraArgs = ["-f"]; # Override existing partition
subvolumes = {
# Subvolume name is different from mountpoint
"/rootfs" = {
mountpoint = "/";
};
"/nix" = {
mountOptions = ["noatime"];
mountpoint = "/nix";
};
"/boot" = {
mountpoint = "/boot";
};
};
};
};
};
};
};
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,9 @@ in {
services.zerotierone = {
enable = lib.mkDefault true;
};

nixpkgs.config.allowUnfreePredicate = pkg:
builtins.elem (lib.getName pkg) [
builtins.elem (builtins.trace (lib.getName pkg) (lib.getName pkg)) [
"zerotierone"
"nomad"
];
Expand Down Expand Up @@ -184,9 +185,21 @@ in {
sj-bm-hostkey0.dev.${fqdn2domain}. A 185.130.224.33
turn.${fqdn2domain}. A ${self.nixosConfigurations.turn-infra-holochain-org.config.services.holochain-turn-server.address}
signal.${fqdn2domain}. A ${self.nixosConfigurations.turn-infra-holochain-org.config.services.tx5-signal-server.address}
bootstrap.${fqdn2domain}. A ${self.nixosConfigurations.turn-infra-holochain-org.config.services.kitsune-bootstrap.address}
turn-0.${fqdn2domain}. A ${self.nixosConfigurations.turn-0.config.services.holochain-turn-server.address}
signal-0.${fqdn2domain}. A ${self.nixosConfigurations.turn-0.config.services.tx5-signal-server.address}
bootstrap-0.${fqdn2domain}. A ${self.nixosConfigurations.turn-0.config.services.kitsune-bootstrap.address}
turn-1.${fqdn2domain}. A ${self.nixosConfigurations.turn-1.config.services.holochain-turn-server.address}
signal-1.${fqdn2domain}. A ${self.nixosConfigurations.turn-1.config.services.tx5-signal-server.address}
bootstrap-1.${fqdn2domain}. A ${self.nixosConfigurations.turn-1.config.services.kitsune-bootstrap.address}
turn-2.${fqdn2domain}. A ${self.nixosConfigurations.turn-2.config.services.holochain-turn-server.address}
signal-2.${fqdn2domain}. A ${self.nixosConfigurations.turn-2.config.services.tx5-signal-server.address}
bootstrap-2.${fqdn2domain}. A ${self.nixosConfigurations.turn-2.config.services.kitsune-bootstrap.address}
turn-3.${fqdn2domain}. A ${self.nixosConfigurations.turn-3.config.services.holochain-turn-server.address}
signal-3.${fqdn2domain}. A ${self.nixosConfigurations.turn-3.config.services.tx5-signal-server.address}
bootstrap-3.${fqdn2domain}. A ${self.nixosConfigurations.turn-3.config.services.kitsune-bootstrap.address}
'';
};

Expand Down Expand Up @@ -308,9 +321,27 @@ in {
'';
};

"acme-turn.${fqdn2domain}:80" = {
"acme-turn-0.${fqdn2domain}:80" = {
extraConfig = ''
reverse_proxy http://turn-0.${fqdn2domain}:${builtins.toString self.nixosConfigurations.turn-0.config.services.holochain-turn-server.nginx-http-port}
'';
};

"acme-turn-1.${fqdn2domain}:80" = {
extraConfig = ''
reverse_proxy http://turn-1.${fqdn2domain}:${builtins.toString self.nixosConfigurations.turn-1.config.services.holochain-turn-server.nginx-http-port}
'';
};

"acme-turn-2.${fqdn2domain}:80" = {
extraConfig = ''
reverse_proxy http://turn-2.${fqdn2domain}:${builtins.toString self.nixosConfigurations.turn-2.config.services.holochain-turn-server.nginx-http-port}
'';
};

"acme-turn-3.${fqdn2domain}:80" = {
extraConfig = ''
reverse_proxy http://turn.${fqdn2domain}:${builtins.toString self.nixosConfigurations.turn-infra-holochain-org.config.services.holochain-turn-server.nginx-http-port}
reverse_proxy http://turn-3.${fqdn2domain}:${builtins.toString self.nixosConfigurations.turn-3.config.services.holochain-turn-server.nginx-http-port}
'';
};
};
Expand Down

0 comments on commit 07c49a6

Please sign in to comment.