Skip to content

Commit

Permalink
Merge branch 'master' into anemish/coinbase
Browse files Browse the repository at this point in the history
  • Loading branch information
nau committed Oct 28, 2020
2 parents 2e73516 + b0295aa commit e2fbca6
Show file tree
Hide file tree
Showing 74 changed files with 8,376 additions and 8,158 deletions.
2 changes: 2 additions & 0 deletions .ignore
@@ -0,0 +1,2 @@
nix/stack.materialized
nix/sources.nix
19 changes: 11 additions & 8 deletions ci.nix
@@ -1,8 +1,8 @@
# 'supportedSystems' restricts the set of systems that we will evaluate for. Useful when you're evaluting
# on a machine with e.g. no way to build the Darwin IFDs you need!
{ supportedSystems ? [ "x86_64-linux" "x86_64-darwin" ]
# This will be used by the packages that get the git revision in lieu of actually trying to find it,
# which doesn't work in all situations. Set to null to get it from git.
# This will be used by the packages that get the git revision in lieu of actually trying to find it,
# which doesn't work in all situations. Set to null to get it from git.
, rev ? "fake"
}:
let
Expand All @@ -14,14 +14,16 @@ let
};
sources = import ./nix/sources.nix;
# Useful for generic library functions: do not use for anything platform dependent
genericPkgs = import sources.nixpkgs {};
in dimension "System" (systems genericPkgs) (systemName: system:
genericPkgs = import sources.nixpkgs { };
in
dimension "System" (systems genericPkgs) (systemName: system:
let
packageSet = import ./default.nix { inherit system rev; checkMaterialization = true; };
pkgs = packageSet.pkgs;
lib = pkgs.lib;
platformFilter = platformFilterGeneric pkgs system;
in filterAttrsOnlyRecursive (_: v: platformFilter v) {
in
filterAttrsOnlyRecursive (_: v: platformFilter v) {
inherit (packageSet) docs papers dev tests plutus-playground marlowe-playground plutus-scb marlowe-symbolic-lambda;
inherit (packageSet.haskell.project) roots;
# build the shell expression to be sure it works on all platforms
Expand All @@ -32,10 +34,11 @@ in dimension "System" (systems genericPkgs) (systemName: system:
# all the checks.
collectChecks = _: ps: pkgs.recurseIntoAttrs (builtins.mapAttrs (_: p: p.checks) ps);
collectComponents = type: ps: packageSet.pkgs.haskell-nix.haskellLib.collectComponents' type ps;
# This computes the Haskell package set sliced by component type
in pkgs.recurseIntoAttrs (dimension
# This computes the Haskell package set sliced by component type
in
pkgs.recurseIntoAttrs (dimension
"Haskell component"
{"library" = collectComponents; "tests" = collectComponents; "benchmarks" = collectComponents; "exes" = collectComponents; "checks" = collectChecks;}
{ "library" = collectComponents; "tests" = collectComponents; "benchmarks" = collectComponents; "exes" = collectComponents; "checks" = collectChecks; }
# Apply the selector to the Haskell package set
(type: selector: (selector type) packageSet.haskell.projectPackages));
})
212 changes: 112 additions & 100 deletions default.nix

Large diffs are not rendered by default.

11 changes: 6 additions & 5 deletions deployment/default.nix
Expand Up @@ -18,7 +18,7 @@ let
${awscli}/bin/aws sts get-session-token --serial-number "arn:aws:iam::454236594309:mfa/$1" --output text --duration-seconds 86400 --token-code "$2" \
| awk '{printf("export AWS_ACCESS_KEY_ID=%s\nexport AWS_SECRET_ACCESS_KEY=\"%s\"\nexport AWS_SESSION_TOKEN=\"%s\"\n",$2,$4,$5)}'
'';
'';

terraform-locals = env:
writeTextFile {
Expand Down Expand Up @@ -49,7 +49,7 @@ let
${awscli}/bin/aws s3 cp --recursive ${plutus-playground.client} s3://plutus-playground-website-${env}/
${awscli}/bin/aws s3 cp --recursive ${marlowe-playground.client} s3://marlowe-playground-website-${env}/
${awscli}/bin/aws s3 cp --recursive ${marlowe-playground.tutorial} s3://marlowe-playground-website-${env}/tutorial
'';
'';

syncPlutusTutorial = env:
writeShellScript "syncPlutusTutorial" ''
Expand All @@ -58,7 +58,7 @@ let
echo "sync plutus tutorial with S3"
${awscli}/bin/aws s3 sync --delete ${plutus-playground.tutorial} s3://plutus-playground-website-${env}/tutorial
${awscli}/bin/aws s3 cp --recursive ${plutus-playground.tutorial} s3://plutus-playground-website-${env}/tutorial
'';
'';

applyTerraform = env: region:
writeShellScript "deploy" ''
Expand Down Expand Up @@ -113,7 +113,7 @@ let
${syncS3 env}
${syncPlutusTutorial env}
echo "done"
'';
'';

destroy = env: region:
writeShellScript "destroy" ''
Expand Down Expand Up @@ -159,4 +159,5 @@ let
pablo = mkEnv "pablo" "eu-west-3";
wyohack = mkEnv "wyohack" "us-west-2";
};
in envs // { inherit getCreds; }
in
envs // { inherit getCreds; }
86 changes: 44 additions & 42 deletions deployment/nixops/default-machine.nix
@@ -1,46 +1,48 @@
{ machines, stdOverlays, nixpkgsLocation, ... }: node: pkgs:
{
nixpkgs.overlays = stdOverlays;
nix = {
nixPath = [ "nixpkgs=${nixpkgsLocation}"
];
binaryCaches = [ https://hydra.iohk.io https://cache.nixos.org ];
requireSignedBinaryCaches = false;
extraOptions = ''
build-cores = 8
auto-optimise-store = true
'';
trustedBinaryCaches = [ https://hydra.iohk.io ];
binaryCachePublicKeys = [
"hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
];
gc.automatic = true;
gc.options = "--delete-older-than 7d";
};

## This was temporarily required when upgrading to 20.03 (weirdly from 19.09)
## In future upgrades you may need to set this to 20.03 (or 29.03, not sure) and deploy then remove and deploy again
# system.stateVersion = "19.03";

imports = [ <nixpkgs/nixos/modules/virtualisation/amazon-image.nix> ];

systemd.services.amazon-init.wantedBy = pkgs.lib.mkForce [ ];

ec2.hvm = true;

networking.timeServers = [ "1.amazon.pool.ntp.org" "2.amazon.pool.ntp.org" "3.amazon.pool.ntp.org" ];

## Disable journald ratelimiting.
services.journald.rateLimitBurst = 0;

## This makes our networking stack ignore the AWS MTU advertisement of 9001,
## that silently breaks intra-VPC, for some reason.
## The intent of this is to reduce the MTU to 1500.
networking.dhcpcd.extraConfig = ''
nooption interface_mtu
'';

users.extraUsers.root.openssh.authorizedKeys.keys = machines.rootSshKeys;
services.fail2ban.enable = true;
nixpkgs.overlays = stdOverlays;
nix = {
nixPath = [
"nixpkgs=${nixpkgsLocation}"
];
binaryCaches = [ https://hydra.iohk.io https://cache.nixos.org ];
requireSignedBinaryCaches = false;
extraOptions = ''
build-cores = 8
auto-optimise-store = true
'';
trustedBinaryCaches = [ https://hydra.iohk.io ];
binaryCachePublicKeys = [
"hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ="
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
];
gc.automatic = true;
gc.options = "--delete-older-than 7d";
};

## This was temporarily required when upgrading to 20.03 (weirdly from 19.09)
## In future upgrades you may need to set this to 20.03 (or 29.03, not sure) and deploy then remove and deploy again
# system.stateVersion = "19.03";

imports = [ <nixpkgs/nixos/modules/virtualisation/amazon-image.nix> ];

systemd.services.amazon-init.wantedBy = pkgs.lib.mkForce [ ];

ec2.hvm = true;

networking.timeServers = [ "1.amazon.pool.ntp.org" "2.amazon.pool.ntp.org" "3.amazon.pool.ntp.org" ];

## Disable journald ratelimiting.
services.journald.rateLimitBurst = 0;

## This makes our networking stack ignore the AWS MTU advertisement of 9001,
## that silently breaks intra-VPC, for some reason.
## The intent of this is to reduce the MTU to 1500.
networking.dhcpcd.extraConfig = ''
nooption interface_mtu
'';

users.extraUsers.root.openssh.authorizedKeys.keys = machines.rootSshKeys;
services.fail2ban.enable = true;

}
63 changes: 33 additions & 30 deletions deployment/nixops/default.nix
@@ -1,33 +1,33 @@
let
plutus = import ../../. {};
plutus = import ../../. { };
serverTemplate = import ./server.nix;
prometheusTemplate = import ./prometheus.nix;
webghc = import ./webghc.nix;
machines = (plutus.pkgs.lib.importJSON ./machines.json);
overlays = import ./overlays.nix;
secrets = (plutus.pkgs.lib.importJSON ./secrets.json);
enableGithubHooks = plutus.pkgs.lib.hasAttr "githubWebhookKey" secrets;
deploymentConfigDir = plutus.pkgs.copyPathToStore ../nixops ;
deploymentConfigDir = plutus.pkgs.copyPathToStore ../nixops;
deploymentServer = plutus.haskell.packages.deployment-server.components.exes.deployment-server-exe;
plutusUrl = "https://${machines.environment}.${machines.plutusTld}";
mkConfig = ghsecrets: redirectUrl: callbackUrl: name: plutus.pkgs.writeTextFile {
name = name;
text = ''
auth:
# Maintainers' notes:
# 1) Github keys and URL in here *must* match the ones set up for this app on
# github.
# 2) If you change the JWT signature, it will break all existing logins.
# Don't change it unless that's something you specifically want!
github-client-id: ${ghsecrets.githubClientId}
github-client-secret: ${ghsecrets.githubClientSecret}
jwt-signature: ${ghsecrets.jwtSignature}
redirect-url: ${redirectUrl}
marlowe:
# The API Gateway url to trigger the marlowe symbolic lambda
symbolic-url: "${machines.marloweSymbolicUrl}"
api-key: "${secrets.apiGatewayKey}"
callback-url: "${callbackUrl}"
auth:
# Maintainers' notes:
# 1) Github keys and URL in here *must* match the ones set up for this app on
# github.
# 2) If you change the JWT signature, it will break all existing logins.
# Don't change it unless that's something you specifically want!
github-client-id: ${ghsecrets.githubClientId}
github-client-secret: ${ghsecrets.githubClientSecret}
jwt-signature: ${ghsecrets.jwtSignature}
redirect-url: ${redirectUrl}
marlowe:
# The API Gateway url to trigger the marlowe symbolic lambda
symbolic-url: "${machines.marloweSymbolicUrl}"
api-key: "${secrets.apiGatewayKey}"
callback-url: "${callbackUrl}"
'';
};
playgroundConfig = mkConfig secrets.plutus plutusUrl "" "playground.yaml";
Expand All @@ -39,19 +39,22 @@ let
deploymentName = "playgrounds";
options = { inherit stdOverlays machines defaultMachine plutus secrets nixpkgsLocation nixosLocation slackChannel nixopsStateFile deploymentName; };
defaultMachine = (import ./default-machine.nix) options;
playgroundOptions = options // { serviceConfig = playgroundConfig;
serviceName = "plutus-playground";
server-invoker = plutus.plutus-playground.server-invoker;
client = plutus.plutus-playground.client;
};
playgroundOptions = options // {
serviceConfig = playgroundConfig;
serviceName = "plutus-playground";
server-invoker = plutus.plutus-playground.server-invoker;
client = plutus.plutus-playground.client;
};
playgroundA = serverTemplate.mkInstance playgroundOptions machines.playgroundA;
playgroundB = serverTemplate.mkInstance playgroundOptions machines.playgroundB;
webGhcA = webghc.mkInstance (options // {web-ghc = plutus.web-ghc; }) machines.webGhcA;
webGhcB = webghc.mkInstance (options // {web-ghc = plutus.web-ghc; }) machines.webGhcA;
nixops = prometheusTemplate.mkInstance
(options // {configDir = deploymentConfigDir; inherit deploymentServer enableGithubHooks;})
{dns = "nixops.internal.${machines.environment}.${machines.plutusTld}";
ip = "127.0.0.1";
name = "nixops"; };
webGhcA = webghc.mkInstance (options // { web-ghc = plutus.web-ghc; }) machines.webGhcA;
webGhcB = webghc.mkInstance (options // { web-ghc = plutus.web-ghc; }) machines.webGhcA;
nixops = prometheusTemplate.mkInstance
(options // { configDir = deploymentConfigDir; inherit deploymentServer enableGithubHooks; })
{
dns = "nixops.internal.${machines.environment}.${machines.plutusTld}";
ip = "127.0.0.1";
name = "nixops";
};
in
{ inherit playgroundA playgroundB nixops webGhcA webGhcB; }
{ inherit playgroundA playgroundB nixops webGhcA webGhcB; }
15 changes: 8 additions & 7 deletions deployment/nixops/network.nix
@@ -1,18 +1,19 @@
let
playground = import ../../. {};
playground = import ../../. { };
machines = (playground.pkgs.lib.importJSON ./machines.json);
mkInstance = node:
{
{
deployment.targetHost = node.ip;
deployment.hasFastConnection = true;
};
};
playgroundA = mkInstance machines.playgroundA;
playgroundB = mkInstance machines.playgroundB;
webGhcA = mkInstance machines.webghcA;
webGhcB = mkInstance machines.webghcB;
nixops = { deployment.targetHost = "localhost"; };
in
{ inherit playgroundA playgroundB webGhcA webGhcB nixops;
network.description = "Plutus Playground";
network.enableRollback = true;
}
{
inherit playgroundA playgroundB webGhcA webGhcB nixops;
network.description = "Plutus Playground";
network.enableRollback = true;
}
2 changes: 1 addition & 1 deletion deployment/nixops/overlays.nix
Expand Up @@ -10,7 +10,7 @@
rev = "v${version}";
sha256 = "1vgpwnwqjc93nvdpcd52748bwl3r371jb55l17bsgdzrmlcyfm8a";
};
GOCACHE="off";
GOCACHE = "off";
});
};
}

0 comments on commit e2fbca6

Please sign in to comment.