Merge pull request #30 from bravetools/release-1.55

Release 1.55

.gitignore

@@ -17,4 +17,8 @@ install/darwin/brave
 install/ubuntu/brave
 
 db/bravetoolsdb/*.db
-db/bravetoolsdb/bravetoolsdb
+db/bravetoolsdb/bravetoolsdb*
+db/bravetoolsdb/bravetoolsdb
+
+brave*
+bravetools*

VERSION

@@ -1 +1 @@
-release-1.54
+release-1.55

commands/base.go

@@ -13,10 +13,10 @@ import (
 )
 
 var baseBuild = &cobra.Command{
-	Use:   "base NAME",
-	Short: "Build a base unit",
-	Long: `Build a base unit from images available at https://images.linuxcontainers.org.
-Command accepts image names in the format Distribution/Release/Architecture`,
+	Use:   "base DISTRIBUTION/RELEASE/ARCH",
+	Short: "Pull a base image from LXD Image Server or public Github Bravefile",
+	Long: `Import images available at https://images.linuxcontainers.org or
+from Bravefiles stored in public GitHub repositories`,
 	Run: buildBase,
 }
 

commands/configure.go

@@ -9,8 +9,8 @@ import (
 
 var configureHost = &cobra.Command{
 	Use:   "configure",
-	Short: "Configure local host parameters such as storage",
-	Long:  `Bravetools reads configuration specifications from ~/.bravetools/config.yaml and configures host accordingly`,
+	Short: "Configure local host parameters",
+	Long:  `Update host configuration using settings in $HOME/.bravetools/config.yaml`,
 	Run:   configure,
 }
 

db/db_operations.go

@@ -4,25 +4,26 @@ import (
 	"database/sql"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"log"
 	"os"
 
+	"github.com/bravetools/bravetools/shared"
+
 	// import sqlite driver
 	_ "github.com/mattn/go-sqlite3"
 )
 
 // OpenDB opens database
-func OpenDB(filepath string) *sql.DB {
-	//log.Println("Initialising SQlite database " + filepath)
-	db, err := sql.Open("sqlite3", filepath)
-	if err != nil {
-		log.Fatal(err)
-	}
-	if db == nil {
-		log.Fatalln("db nil")
+func OpenDB(filepath string) (db *sql.DB, err error) {
+	//log.Println("Connecting to SQlite database " + filepath)
+
+	if !shared.FileExists(filepath) {
+		return nil, fmt.Errorf("Database file %s not present", filepath)
 	}
 
-	return db
+	db, err = sql.Open("sqlite3", filepath)
+	return db, err
 }
 
 // InitDB creates an empty database
@@ -37,7 +38,10 @@ func InitDB(filepath string) error {
 	file.Close()
 	log.Println("Database file created")
 
-	db := OpenDB(filepath)
+	db, err := OpenDB(filepath)
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	defer db.Close()
 	log.Println("Creating units table ..")
@@ -68,10 +72,7 @@ func InitDB(filepath string) error {
 func InsertUnitDB(db *sql.DB, unit BraveUnit) (int64, error) {
 	defer db.Close()
 
-	u, _ := unitByName(db, unit.Name)
-	if u.Name == unit.Name {
-		return 0, errors.New("Unit already exists")
-	}
+	// TODO: duplicate unit names could exist in DB. If unit name required to be unique it should be checked earlier.
 
 	//log.Println("Inserting unit ..")
 	insertUnit := `INSERT INTO units(uid, 

db/db_operations_test.go

@@ -10,7 +10,10 @@ import (
 
 func Test_GetAllUnits(t *testing.T) {
 
-	db := OpenDB("brave_test.db")
+	db, err := OpenDB("brave_test.db")
+	if err != nil {
+		t.Log("Failed to open db")
+	}
 
 	units, err := GetAllUnitsDB(db)
 	if err != nil {
@@ -23,7 +26,10 @@ func Test_GetAllUnits(t *testing.T) {
 
 func Test_GetUnit(t *testing.T) {
 
-	db := OpenDB("brave_test.db")
+	db, err := OpenDB("brave_test.db")
+	if err != nil {
+		t.Log("Failed to open db")
+	}
 
 	unit, err := GetUnitDB(db, "test")
 	if err != nil {
@@ -43,9 +49,12 @@ func Test_GetUnit(t *testing.T) {
 
 func Test_DeleteUnit(t *testing.T) {
 
-	db := OpenDB("brave_test.db")
+	db, err := OpenDB("brave_test.db")
+	if err != nil {
+		t.Log("Failed to open db")
+	}
 
-	err := DeleteUnitDB(db, "test")
+	err = DeleteUnitDB(db, "test")
 	if err != nil {
 		t.Log("Error deleting unit")
 		t.Log("Error: ", err)
@@ -75,7 +84,10 @@ func Test_InsertUnit(t *testing.T) {
 		Data: data,
 	}
 
-	db := OpenDB("brave_test.db")
+	db, err := OpenDB("brave_test.db")
+	if err != nil {
+		t.Log("Failed to open db")
+	}
 
 	id, err := InsertUnitDB(db, unit)
 	if err != nil {

docs/docs/cli/brave_base.md

@@ -16,13 +16,40 @@ brave base NAME
 
 ## Description
 
-Build a base unit from images available at https://images.linuxcontainers.org.
-Command accepts image names in the format Distribution/Release/Architecture
+Bravetools can import base images either from LXD Image Server (https://images.linuxcontainers.org) or from Bravefiles stored in public GitHub repositories. This creates an efficient way of levereging multiple pre-built images to speed up creation of a bespoke system container.
+
+### Pulling from LXD Image server
+
+Clean OS images can be imported into Bravetools directly from LXD Image server:
+
+```bash
+brave base alpine/edge/amd64
+```
+
+### Pulling from a GiHub repository
+
+Container images can be quite large and downloading their distributions is not efficient. Bravetools has a mechanism to access a remotely-stored Bravefile and build an image directly from it:
+
+```bash
+brave base github.com/beringresearch/bravefiles/ubuntu/ubuntu-bionic-py3
+```
+
+This will create an Ubuntu 18.04 image with Python3 installation, using a Bravefile located on a [GitHub repository](https://github.com/beringresearch/bravefiles/tree/master/ubuntu/ubuntu-bionic-py3)
+
+### Usage inside a Bravefile
+All local images can be utilised inside a Bravefile using the `local` location option:
+
+
+```yaml
+base:
+  image: brave-alpine-edge-base-1.0
+  location: local
+```
 
 ## Options
 
 ```
   -h, --help   help for base
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_build.md

@@ -25,4 +25,4 @@ Build an image from a Bravefile
   -p, --path string   Absolute path to Bravefile [OPTIONAL]
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_configure.md

@@ -24,4 +24,4 @@ Bravetools reads configuration specifications from ~/.bravetools/config.yaml and
   -h, --help   help for configure
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_deploy.md

@@ -30,4 +30,4 @@ deployment options e.g. CPU and RAM should be configured through [Bravefile](../
   -p, --port string     Publish Unit port to host [OPTIONAL]
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_images.md

@@ -24,4 +24,4 @@ List images
   -h, --help   help for images
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_import.md

@@ -24,4 +24,4 @@ Import a tarball into local Bravetools image repository
   -h, --help   help for import
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_info.md

@@ -25,4 +25,4 @@ Display workspace information
       --short   Returns host IP address
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_init.md

@@ -47,4 +47,4 @@ These steps ensure that Bravetools establishes a connection with LXD server and
   -s, --storage string   Host storage size [OPTIONAL]
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_mount.md

@@ -36,4 +36,4 @@ printf "uid $(id -u) 1000\ngid $(id -g) 1000" | lxc config set test raw.idmap -
   -h, --help   help for mount
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_publish.md

@@ -24,4 +24,4 @@ Published Unit will be saved in the current working directory as *.tar.gz file
   -h, --help   help for publish
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_remove.md

@@ -25,4 +25,4 @@ Remove a Unit or an Image
   -i, --image   Toggle to delete a local image
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_start.md

@@ -24,4 +24,4 @@ Start Unit
   -h, --help   help for start
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_stop.md

@@ -24,4 +24,4 @@ Stop Unit
   -h, --help   help for stop
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_umount.md

@@ -24,4 +24,4 @@ Unmount <disk> from UNIT
   -h, --help   help for umount
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_units.md

@@ -24,4 +24,4 @@ This function returns a list of all Units deployed on a remote Bravetools host
   -h, --help   help for units
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/docs/cli/brave_version.md

@@ -24,4 +24,4 @@ Show current bravetools version
   -h, --help   help for version
 ```
 
-###### Auto generated by spf13/cobra on 4-Oct-2020
+###### Auto generated by spf13/cobra on 19-Oct-2020

docs/examples/reprodibility-bioinformatics.md

@@ -139,7 +139,7 @@ To restore the original environment, simply import the image archive into Bravet
 
 ```bash
 > brave import ubuntu-bionic-rstudio-server-TIMESTAMP.tar.gz
-> brave deploy ubuntu-bionic-rstudio-server-TIMESTAMP --ip 10.0.0.23 --ports 8787:8787 --name ubuntu-bionic-rstudio-server
+> brave deploy ubuntu-bionic-rstudio-server-TIMESTAMP --ip 10.0.0.23 --port 8787:8787 --name ubuntu-bionic-rstudio-server
 ```
 
 ## Conclusions

docs/intro/why_bravetools.md

@@ -19,4 +19,15 @@ Bravetools addresses these limitations. It has a simple and lightweight command
 
 **Focus on code not infrastructure**. Maintaining and configuring infrastructure is difficult! With any application built and deployed using Bravetools infrastructure and environment have to be configured just once. Developers can spend more time on creating and improving software and less time on managing production environments.
 
-Let's [get started](../../installation)!
+## Why not Docker?
+
+Docker is the industry-standard way for building **Application Containers**. Surprising as it may seem, Bravetools was not designed to compete with Docker! In fact, you can [run Docker containers inside Bravetools Units](../docs/../docker). The main reasons we decided to use System Containers for our infrastructure and core are:
+
+* **Docker containers were being treated like VMs**. Our team was pushing Vim, python, RStudio, R, libraries, and Shiny webservers into the same Docker container and treating it like a lightweight VM. Docker containers are simply not designed to run multiple processes and our approach was an anti-pattern.
+
+* **Valuable time was being spent on devops and not research**. As our container practices improved, the number of Docker containers required to run our research pipelines increased drastically. We found that we were spending more time on managing containers instead of doing iterative research.
+
+* **Persistent data storage and data sharing is complicated**. By design, all files created inside a Docker container are stored on a writable container layer. This means that the data doesn’t persist when that container no longer exists, and it can be difficult to get the data out of the container if another process needs it. Furthermore, a container’s writable layer is tightly coupled to the host machine where the container is running - you can’t easily move the data somewhere else. Docker volumes are a good workaround, but are difficult to share and reproduce across collaborating teams.
+
+## Let's ...
+[get started](../../installation)!

examples/go-service/Bravefile

@@ -38,7 +38,7 @@ service:
   version: 1.0
   ip: 10.0.0.100
   ports:
-    - 3000:300
+    - 3000:3000
   resources:
-    ram: 2GB
+    ram: 1GB
     cpu: 1

platform/helpers.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"os"
 	"os/signal"
+	"path"
 	"path/filepath"
 	"strings"
 	"syscall"
@@ -340,30 +341,32 @@ func bravefileCopy(copy []shared.CopyCommand, service string, remote Remote) err
 	dir, _ := os.Getwd()
 	for _, c := range copy {
 		source := c.Source
-		source = dir + "/" + source
+		source = path.Join(dir, source)
+		sourcePath := filepath.FromSlash(source)
+
 		target := c.Target
 		_, err := Exec(service, []string{"mkdir", "-p", target}, remote)
 		if err != nil {
 			return errors.New("Failed to create target directory: " + err.Error())
 		}
 
-		fi, err := os.Lstat(source)
+		fi, err := os.Lstat(sourcePath)
 		if err != nil {
-			return errors.New("Failed to read file " + source + ": " + err.Error())
+			return errors.New("Failed to read file " + sourcePath + ": " + err.Error())
 		}
 
 		if fi.IsDir() {
-			err = Push(service, source, target, remote)
+			err = Push(service, sourcePath, target, remote)
 			if err != nil {
 				return errors.New("Failed to push symlink: " + err.Error())
 			}
 		} else if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
-			err = SymlinkPush(service, source, target, remote)
+			err = SymlinkPush(service, sourcePath, target, remote)
 			if err != nil {
 				return errors.New("Failed to push directory: " + err.Error())
 			}
 		} else {
-			err = FilePush(service, source, target, remote)
+			err = FilePush(service, sourcePath, target, remote)
 			if err != nil {
 				return errors.New("Failed to push file: " + err.Error())
 			}
@@ -414,14 +417,6 @@ func cleanUnusedStoragePool(name string, remote Remote) {
 	}
 }
 
-func fileExists(filename string) bool {
-	info, err := os.Stat(filename)
-	if os.IsNotExist(err) {
-		return false
-	}
-	return !info.IsDir()
-}
-
 // addIPRules adds firewall rule to the host iptable
 func addIPRules(ct string, hostPort string, ctPort string, bh *BraveHost) error {