Deployment

build/default.nix

@@ -14,6 +14,8 @@ let
     platform-process
     platform-packaging
     platform-visual
+    platform-terraform
+    platform-deployment
   ]);
 
 in

build/haskell.nix

@@ -16,24 +16,34 @@ rec {
     rev = "64e7bfb3abcad278e6160cd411abdd21a485a671";
   };
 
+  terraformHsSrc = fetchGit {
+    url = https://github.com/atidot/terraform-hs;
+    rev = "4815301f4d5cf8343907ea55e41f4f491930dc69";
+  };
+
   platformTypesSrc     = ../platform-types;
   platformDSLSrc       = ../platform-dsl;
   platformAWSSrc       = ../platform-aws;
   platformKubeSrc      = ../platform-kube;
   platformProcessSrc   = ../platform-process;
   platformPackagingSrc = ../platform-packaging;
   platformVisualSrc    = ../platform-visual;
+  platformTerraformSrc = ../platform-terraform;
+  platformDeploymentSrc = ../platform-deployment;
 
   projectPackages = hspkgs: {
-    executor           = ease hspkgs.executor;
-    stratosphere       = hspkgs.callCabal2nix "stratosphere"       "${stratosphereSrc}" {};
-    platform-types     = hspkgs.callCabal2nix "platform-types"     "${platformTypesSrc}" {};
-    platform-dsl       = hspkgs.callCabal2nix "platform-dsl"       "${platformDSLSrc}" {};
-    platform-aws       = hspkgs.callCabal2nix "platform-aws"       "${platformAWSSrc}" {};
-    platform-kube      = hspkgs.callCabal2nix "platform-kube"      "${platformKubeSrc}" {};
-    platform-packaging = hspkgs.callCabal2nix "platform-packaging" "${platformPackagingSrc}" {};
-    platform-process   = hspkgs.callCabal2nix "platform-process"   "${platformProcessSrc}" {};
-    platform-visual    = hspkgs.callCabal2nix "platform-visual"    "${platformVisualSrc}" {};
+    executor            = ease hspkgs.executor;
+    stratosphere        = hspkgs.callCabal2nix "stratosphere"       "${stratosphereSrc}" {};
+    terraform-hs        = hspkgs.callCabal2nix "terraform-hs"       "${terraformHsSrc}"  {};
+    platform-types      = hspkgs.callCabal2nix "platform-types"     "${platformTypesSrc}" {};
+    platform-dsl        = hspkgs.callCabal2nix "platform-dsl"       "${platformDSLSrc}" {};
+    platform-aws        = hspkgs.callCabal2nix "platform-aws"       "${platformAWSSrc}" {};
+    platform-kube       = hspkgs.callCabal2nix "platform-kube"      "${platformKubeSrc}" {};
+    platform-packaging  = hspkgs.callCabal2nix "platform-packaging" "${platformPackagingSrc}" {};
+    platform-process    = hspkgs.callCabal2nix "platform-process"   "${platformProcessSrc}" {};
+    platform-visual     = hspkgs.callCabal2nix "platform-visual"    "${platformVisualSrc}" {};
+    platform-terraform  = hspkgs.callCabal2nix "platform-terraform" "${platformTerraformSrc}" {};
+    platform-deployment = hspkgs.callCabal2nix "platform-deployment" "${platformDeploymentSrc}" {};
   };
 
   packages = haskellPackages.override (old: {

platform-deployment/.gitignore

@@ -0,0 +1,3 @@
+.stack-work/
+platform-deployment.cabal
+*~

platform-deployment/LICENSE

@@ -0,0 +1,30 @@
+Copyright Author name here (c) 2019
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+    * Neither the name of Author name here nor the names of other
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

platform-deployment/README.md

@@ -0,0 +1,77 @@
+# platform-deployment
+
+## Intro
+
+Allows us to declare a deployment inside an aws instance, with dockers, disks and secrets declarative management
+
+Platform deployment package holds two parts:
+
+1. Deployment-dsl - High language implemented with free monad, that describes the things we need in order to deploy software configuration (containers, secrets and data). [Here](src/Atidot/Platform/Deployment.hs)
+1. Interpreters for deployment dsl - there are two interpreters
+1. 1. AMI - starts a machine in aws with terraform configuration, then apply changes, and eventually, saves an AMI containing all the changes supplied. This is dynamic, which means that the machine state is changed after applying each line. [Here](src/Atidot/Platform/Deployment/Interpreter/AMI.hs)
+1. 1. Terraform - Generates a terraform deployment file, that once applied, will start and then configure a full machine, on `terraform init`. This is static, which means that the interpreter is run, and then a file is generated. There is no AMI ready at the end of the interpreter run. It is more mature of the two. [here](https://github.com/Atidot/platform/blob/deployment/platform-deployment/src/Atidot/Platform/Deployment/Interpreter/Terraform.hs)
+
+## Terraform requirements and notes
+
+* [Terraform](https://www.terraform.io/) needs to be installed.
+* In the terraform configuration, there is an sh key [path](src/Atidot/Platform/Deployment/Interpreter/AMI/Types/Default.hs#L23). It is needed to be supplied for the remote provisioners and for ssh connection into the deployed machine
+* AWS credentials for configuring the aws cli are needed but are prompted during `terraform init`
+* AWS EBS volumes are needed in order for mounting to work properly
+* Volume is needed to be created before running the interpreter.It’s (Volume) name/id should be be added to the configuration [Here](src/Atidot/Platform/Deployment/Interpreter/Terraform/Template.hs#L26)
+* Terraform deploys a single virtual machine with network components required for ssh connection into that machine. Then it uses a series of remote provisioners to change the state of the machine:
+* 1. Installation of software and secrets initialization [Here](src/Atidot/Platform/Deployment/Interpreter/Terraform/Template.hs#L85)
+* 1.  secrets mounting code [Here](src/Atidot/Platform/Deployment/Interpreter/Terraform/Template.hs#L110)
+* 1. Mounting and pulling dockers [Here](src/Atidot/Platform/Deployment/Interpreter/Terraform/Template.hs#L156)
+* 1. Running dockers (Run command in the deployment DSL) [Here](src/Atidot/Platform/Deployment/Interpreter/Terraform/Template.hs#L189)
+* In order to run new scripts, they should be added in [Main](app/Main.hs#L27)
+Nes,nesa,std,sat,cdne - are error checking scripts, they should fail.
+* All secrets used must be properly defined in advance in the AWS secrets manager
+
+## Next Steps
+
+* In order to avoid credentials for aws-cli completely, we can add an IAM role for the AMI [Here](https://dzone.com/articles/aws-secret-manager-protect-your-secrets-in-applica)
+* CLI שrgument for terraform configurations (using json, from file?)
+* Separate failed scripts (Nes,nesa,std,sat,cdne) to tests, to ensure that behavior is stable
+* Instead of provisioner for running dockers, a provisioner for preparing a script that run dockers on computer start
+* Better name giving to the ebs volumes (will be relevant if we want more than one volume) [Here](src/Atidot/Platform/Deployment/Interpreter/Terraform/Template.hs#L44)
+* Improving the description of secrets in the dsl (they come in tuples at least, and are in a json format),
+
+## example script
+
+``` haskell
+nsss :: DeploymentM Bool
+nsss = do
+    secret <- secret "tutorials/MyFirstTutorialSecret"
+    dir <- mount "data"
+    c <- container "hello-world"
+    attachSecret secret c
+    attachVolume dir c
+    execute [] c []
+    return b
+```
+
+This script Declares:
+
+1. A secret
+2. An external disk "data"
+3. a container named hello world
+4. attaches the secret to the container
+5. attaches the mount to the container
+6. executes the container
+
+## How to Run
+
+``` bash
+$ cd ../build
+..
+$ make shell
+... # in the newly opened nix shell
+$ platform-deployment-exe --script nsss
+{initializes terraform and generates files from example}
+$ cd terraform_dep
+...
+$ ls
+aws_cli_config.tf  example.tf
+$ terraform apply # applies genetated configuration
+...
+```

platform-deployment/app/Main.hs

@@ -0,0 +1,35 @@
+{-# LANGUAGE DeriveGeneric     #-}
+module Main where
+
+import "optparse-generic" Options.Generic
+import "data-default"     Data.Default
+import                    Atidot.Platform.Deployment
+import                    Atidot.Platform.Deployment.Interpreter.Terraform
+import                    Atidot.Platform.Deployment.Interpreter.AMI.Types
+
+data CLI = CLI
+    {script :: String
+    } deriving (Generic, Show)
+
+instance ParseRecord CLI
+
+
+main :: IO ()
+main = do
+    x <- script <$> getRecord "Platform Deployment"
+    case lookup x scripts of
+        Just s -> runTerraform def s
+        Nothing -> error $ unlines
+            [ "script '" ++ x ++ "' not found"
+            , "available scripts are: " ++ show (map fst scripts)
+            ]
+
+scripts =
+    [ ("nsss",nsss)
+    , ("kiss",kiss)
+    , ("nes",noneExistentSecret)
+    , ("nesa",noneExistentSecretAttached)
+    , ("sdt", secretDeclaredTwice)
+    , ("sat",secretAttachedTwice)
+    , ("cdne",containerDoesNotExists)
+    ]

platform-deployment/platform-deployment.cabal

@@ -0,0 +1,114 @@
+cabal-version: 1.12
+
+-- This file has been generated from package.yaml by hpack version 0.31.0.
+--
+-- see: https://github.com/sol/hpack
+--
+-- hash: 61e356af67999cee1223314d87fdfdd592f960368b933db1b8f345df60667e82
+
+name:           platform-deployment
+version:        0.1.0.0
+description:    Please see the README on GitHub at <https://github.com/githubuser/platform-deployment#readme>
+homepage:       https://github.com/githubuser/platform-deployment#readme
+bug-reports:    https://github.com/githubuser/platform-deployment/issues
+author:         Author name here
+maintainer:     example@example.com
+copyright:      2019 Author name here
+license:        BSD3
+license-file:   LICENSE
+build-type:     Simple
+extra-source-files:
+    README.md
+    ChangeLog.md
+
+source-repository head
+  type: git
+  location: https://github.com/githubuser/platform-deployment
+
+library
+  exposed-modules:
+      Atidot.Platform.Deployment
+      Atidot.Platform.Deployment.Interpreter.AMI
+      Atidot.Platform.Deployment.Interpreter.AMI.Template
+      Atidot.Platform.Deployment.Interpreter.AMI.Types
+      Atidot.Platform.Deployment.Interpreter.AMI.Types.Default
+      Atidot.Platform.Deployment.Interpreter.AMI.Types.Types
+      Atidot.Platform.Deployment.Interpreter.Terraform
+      Atidot.Platform.Deployment.Interpreter.Terraform.Template
+      Atidot.Platform.Deployment.Interpreter.Test
+      Atidot.Platform.Deployment.Interpreter.Utils
+  other-modules:
+      Paths_platform_deployment
+  hs-source-dirs:
+      src
+  default-extensions: PackageImports OverloadedStrings ScopedTypeVariables
+  ghc-options: -Wall -Werror
+  build-depends:
+      aeson
+    , base >=4.7 && <5
+    , containers
+    , data-default
+    , directory
+    , exceptions
+    , free
+    , ginger
+    , mtl
+    , optparse-generic
+    , raw-strings-qq
+    , text
+    , turtle
+    , uuid
+  default-language: Haskell2010
+
+executable platform-deployment-exe
+  main-is: Main.hs
+  other-modules:
+      Paths_platform_deployment
+  hs-source-dirs:
+      app
+  default-extensions: PackageImports OverloadedStrings ScopedTypeVariables
+  ghc-options: -threaded -rtsopts -with-rtsopts=-N
+  build-depends:
+      aeson
+    , base >=4.7 && <5
+    , containers
+    , data-default
+    , directory
+    , exceptions
+    , free
+    , ginger
+    , mtl
+    , optparse-generic
+    , platform-deployment
+    , raw-strings-qq
+    , text
+    , turtle
+    , uuid
+  default-language: Haskell2010
+
+test-suite platform-deployment-test
+  type: exitcode-stdio-1.0
+  main-is: Spec.hs
+  other-modules:
+      Paths_platform_deployment
+  hs-source-dirs:
+      test
+  default-extensions: PackageImports OverloadedStrings ScopedTypeVariables
+  ghc-options: -threaded -rtsopts -with-rtsopts=-N
+  build-depends:
+      aeson
+    , base >=4.7 && <5
+    , containers
+    , data-default
+    , directory
+    , exceptions
+    , free
+    , ginger
+    , mtl
+    , optparse-generic
+    , platform-deployment
+    , raw-strings-qq
+    , text
+    , turtle
+    , uuid
+  default-language: Haskell2010