Improvements for lib packages (#16)

* fixed bat I/O redirects, added copyrights, adjustments to library

* parse /etc/hosts for host resolution

updated bat to make use of new shttp client

* updated examples/documentation

* fix for host resolution

* removed comment

* updated regexp

* updated host parsing

* Manually fixed some build issues.

After rebasing on top of the new master with the next version changes, there were
several build issues. Those are fixed here.

* Fix build

* Changes from review

README.md

@@ -11,9 +11,14 @@ To build:
 2. run `make` to build all projects
 
 
+## bat
+
+bat is a CLI cURL-like tool for testing, debugging, and generally interacting with HTTP servers over SCION/QUIC. Documentation of the code is available in the [README.md](https://github.com/netsec-ethz/scion-apps/blob/master/bat/README.md)
+
+
 ## camerapp
 
-Camerapp contains image fetcher and server applications, using the SCION network. Documentation on the code is available in the [README.md](https://github.com/netsec-ethz/scion-apps/blob/master/camerapp/README.md)
+Camerapp contains image fetcher and server applications, using the SCION network. Documentation of the code is available in the [README.md](https://github.com/netsec-ethz/scion-apps/blob/master/camerapp/README.md)
 
 Installation and usage information is available on the [SCION Tutorials web page for camerapp](https://netsec-ethz.github.io/scion-tutorials/sample_projects/access_camera/).
 

bat/README.md

@@ -1,5 +1,7 @@
 # bat
 
+![](images/bat_output.png "sample output of bat application")
+
 Go implemented CLI cURL-like tool for humans. Bat can be used for testing, debugging, and generally interacting with HTTP servers.
 
 This repository is a fork of [astaxie/bat](https://github.com/astaxie/bat) making it available for SCION/QUIC.
@@ -22,29 +24,35 @@ If you experience problems with the `govendor` commands above:
 
 ### Usage
 
-In contrast to the original tool, we require a remote SCION address and a URL path instead of a full URL.
-For example:
+In contrast to the original tool, we require a local SCION address.
+If you're running bat in a VM, your local address can be inferred:
 
 ```
-bat -r ISD-AS,[IP]:port GET /api/download
+bat GET https://server/api/download
 ```
 
-If you're running bat in a VM, your local address can be inferred. In case this fails or you are running a different setup, provide the local address using the ```-l``` flag:
+In case this fails or you are running a different setup, provide the local address using the ```-l``` flag:
 
 ```
-bat -l ISD-AS,[IP]:port -r ISD-AS,[IP]:port GET /api/download
+bat -l ISD-AS,[IP]:port GET https://server/api/download
 ```
 
-The HTTP method defaults to GET in case there is no data to be sent and to POST otherwise.
+The scheme defaults to HTTPS. The method defaults to GET in case there is no data to be sent and to POST otherwise.
 
-### Examples
+Hostnames are resolved by parsing the `/etc/hosts` file. Known hosts can be added by adding lines like this:
 
 ```
-bat -r ISD-AS,[IP]:port /api/download
-
-bat -r ISD-AS,[IP]:port /api/upload foo=bar
+# The following lines are SCION hosts
+17-ffaa:1:10,[10.0.8.100]	server1
+18-ffaa:0:11,[10.0.8.120]	server2
+```
 
-bat -r ISD-AS,[IP]:port -f /api/upload foo=bar
+### Examples
 
-bat -r ISD-AS,[IP]:port -body "Hello World" POST /api/upload
-```
+| Request                                             | Explanation                                                        |
+| --------------------------------------------------- | ------------------------------------------------------------------ |
+| bat server:8080/api/download                        | HTTPS GET request to server:8080/download                          |
+| bat -b server:8080/api/download                     | Run a benchmark against server:8080/download                       |
+| bat server:8080/api/upload foo=bar                  | HTTPS POST request with JSON encoded data<br>to server:8080/upload |
+| bat -f server:8080/api/upload foo=bar               | HTTPS POST request with URL encoded data<br>to server:8080/upload  |
+| bat -body "Hello World" POST server:8080/api/upload | HTTPS POST request with raw data<br>to server:8080/upload          |

bat/bat.go

@@ -1,4 +1,5 @@
 // Copyright 2015 bat authors
+// Modifications copyright 2018 ETH Zurich
 //
 // Licensed under the Apache License, Version 2.0 (the "License"): you may
 // not use this file except in compliance with the License. You may obtain
@@ -36,6 +37,7 @@ import (
 	"strings"
 
 	"github.com/netsec-ethz/scion-apps/lib/shttp"
+	slog "github.com/scionproto/scion/go/lib/log"
 	"github.com/scionproto/scion/go/lib/snet"
 )
 
@@ -66,8 +68,7 @@ var (
 	benchC           int
 	isjson           = flag.Bool("json", true, "Send the data as a JSON object")
 	method           = flag.String("method", "GET", "HTTP method")
-	route            = flag.String("route", "", "HTTP request route")
-	URL              *string
+	URL              = flag.String("url", "", "HTTP request URL")
 	jsonmap          map[string]interface{}
 	contentJsonRegex = `application/(.*)json`
 )
@@ -104,9 +105,6 @@ func init() {
 	flag.Parse()
 
 	// SCION: add shttp Transport to defaultSetting
-	// use a dummy.com for pointing to remote
-	// TODO: get rid of this as soon as RAINS is deployed
-
 	if local == "" {
 		var err error
 		local, err = readIsdAS()
@@ -115,21 +113,17 @@ func init() {
 		}
 	}
 
-	raddr, err := snet.AddrFromString(remote)
-	laddr, err2 := snet.AddrFromString(local)
-	if err != nil || err2 != nil {
-		if err != nil {
-			usage()
-		}
+	laddr, err := snet.AddrFromString(local)
+	if err != nil {
 		usage()
 	}
 
-	dns := map[string]*snet.Addr{"dummy.com": raddr}
-
 	defaultSetting.Transport = &shttp.Transport{
-		DNS:   dns,
 		LAddr: laddr,
 	}
+
+	// redirect SCION log to a log file
+	slog.SetupLogFile("scion", "log", "info", 10, 10, 0)
 }
 
 func parsePrintOption(s string) {
@@ -180,30 +174,23 @@ func main() {
 			}
 		}
 	}
-
 	// TODO: uncomment when RAINS is deployed and remove belows URL creation
-	/* if *URL == "" {
-		usage()
-	} */
-
 	/* start of URL creation: Remove when RAINS is deployed*/
-	if *route == "" {
+	if *URL == "" {
 		usage()
 	}
-	temp := strings.Join([]string{"https://dummy.com", *route}, "")
-	URL = &temp
-	/* end of URL creation*/
 
 	if strings.HasPrefix(*URL, ":") {
 		urlb := []byte(*URL)
 		if *URL == ":" {
-			*URL = "http://localhost/"
+			*URL = "https://localhost/"
 		} else if len(*URL) > 1 && urlb[1] != '/' {
-			*URL = "http://localhost" + *URL
+			*URL = "https://localhost" + *URL
 		} else {
-			*URL = "http://localhost" + string(urlb[1:])
+			*URL = "https://localhost" + string(urlb[1:])
 		}
 	}
+
 	if !strings.HasPrefix(*URL, "http://") && !strings.HasPrefix(*URL, "https://") {
 		*URL = "https://" + *URL
 	}
@@ -266,7 +253,7 @@ func main() {
 	}
 	res, err := httpreq.Response()
 	if err != nil {
-		log.Fatalln("can't get the url", err)
+		log.Fatalln("Error", err)
 	}
 
 	// download file
@@ -329,24 +316,20 @@ func main() {
 		if err != nil {
 			panic(err)
 		}
-		if fi.Mode()&os.ModeDevice == os.ModeDevice {
-			var dumpHeader, dumpBody []byte
+		if fi.Mode()&os.ModeDevice == os.ModeDevice { // console output
+			var dumpHeader, dumpBody string
 			dump := httpreq.DumpRequest()
-			dps := strings.Split(string(dump), "\n")
-			for i, line := range dps {
-				if len(strings.Trim(line, "\r\n ")) == 0 {
-					dumpHeader = []byte(strings.Join(dps[:i], "\n"))
-					dumpBody = []byte(strings.Join(dps[i:], "\n"))
-					break
-				}
-			}
+			dps := strings.Split(string(dump), "\r\n\r\n")
+			dumpHeader = dps[0]
+			dumpBody = dps[1]
+
 			if printOption&printReqHeader == printReqHeader {
-				fmt.Println(ColorfulRequest(string(dumpHeader)))
+				fmt.Println(ColorfulRequest(dumpHeader))
 				fmt.Println("")
 			}
 			if printOption&printReqBody == printReqBody {
-				if string(dumpBody) != "\r\n" {
-					fmt.Println(string(dumpBody))
+				if dumpBody != "" {
+					fmt.Println(dumpBody)
 					fmt.Println("")
 				}
 			}
@@ -361,30 +344,26 @@ func main() {
 				body := formatResponseBody(res, httpreq, pretty)
 				fmt.Println(ColorfulResponse(body, res.Header.Get("Content-Type")))
 			}
-		} else {
+		} else { // IO file redirect
 			body := formatResponseBody(res, httpreq, pretty)
 			_, err = os.Stdout.WriteString(body)
 			if err != nil {
 				log.Fatal(err)
 			}
 		}
 	} else {
-		var dumpHeader, dumpBody []byte
+		var dumpHeader, dumpBody string
 		dump := httpreq.DumpRequest()
-		dps := strings.Split(string(dump), "\n")
-		for i, line := range dps {
-			if len(strings.Trim(line, "\r\n ")) == 0 {
-				dumpHeader = []byte(strings.Join(dps[:i], "\n"))
-				dumpBody = []byte(strings.Join(dps[i:], "\n"))
-				break
-			}
-		}
+		dps := strings.Split(string(dump), "\r\n\r\n")
+		dumpHeader = dps[0]
+		dumpBody = dps[1]
+
 		if printOption&printReqHeader == printReqHeader {
-			fmt.Println(string(dumpHeader))
+			fmt.Println(dumpHeader)
 			fmt.Println("")
 		}
 		if printOption&printReqBody == printReqBody {
-			fmt.Println(string(dumpBody))
+			fmt.Println(dumpBody)
 			fmt.Println("")
 		}
 		if printOption&printRespHeader == printRespHeader {
@@ -408,13 +387,13 @@ Usage:
 	bat [flags] [METHOD] URL [ITEM [ITEM]]
 
 flags:
-  -r                          Remote SCION address of web server
   -l                          Local SCION address, for VMs this can be omitted
   -a, -auth=USER[:PASS]       Pass a username:password pair as the argument
   -b, -bench=false            Sends bench requests to URL
   -b.N=1000                   Number of requests to run
   -b.C=100                    Number of requests to run concurrently
   -body=""                    Send RAW data as body
+  -d	                      Fetch a large file in download mode, provides a progress bar
   -f, -form=false             Submitting the data as a form
   -j, -json=true              Send the data in a JSON object
   -p, -pretty=true            Print Json Pretty Format
@@ -430,10 +409,6 @@ flags:
 METHOD:
   bat defaults to either GET (if there is no request data) or POST (with request data).
 
-Route:
-  The route on the server to access (e.g. /api/upload).
-  The route flag can be omitted.
-
 ITEM:
   Can be any of:
     Query string   key=value
@@ -443,7 +418,8 @@ ITEM:
 
 Example:
 
-	bat -r 17-ffaa:1:1 /download
+	bat -l 17-ffaa:1:1[IP]:0 https://server:8080/download
+	The protocol can be omitted, bat defaults to HTTPS
 
 For more help information please refer to https://github.com/netsec-ethz/scion-apps/bat
 `

bat/bench.go

@@ -1,3 +1,4 @@
+// Modifications copyright 2018 ETH Zurich
 // This file has been modified to make it compatible with SCION
 
 package main

bat/example_server/server.go

@@ -1,3 +1,17 @@
+// Copyright 2018 ETH Zurich
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.package main
+
 package main
 
 import (

bat/filter.go

@@ -1,3 +1,4 @@
+// Modifications copyright 2018 ETH Zurich
 // This file has been modified to make it compatible with SCION
 
 package main
@@ -36,10 +37,7 @@ func filter(args []string) []string {
 	if len(args) <= i {
 		log.Fatal("Miss the URL")
 	}
-
-	// TODO: revert when RAINS is deployed
-	//*URL = args[i]
-	*route = args[i]
+	*URL = args[i]
 	i++
 
 	return args[i:]

bat/http.go

@@ -1,11 +1,11 @@
+// Modifications copyright 2018 ETH Zurich
 // This file has been modified to make it compatible with SCION
 
 package main
 
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -112,7 +112,6 @@ func formatResponseBody(res *http.Response, httpreq *httplib.BeegoHttpRequest, p
 	if err != nil {
 		log.Fatalln("can't get the url", err)
 	}
-	fmt.Println("")
 	match, err := regexp.MatchString(contentJsonRegex, res.Header.Get("Content-Type"))
 	if err != nil {
 		log.Fatalln("failed to compile regex", err)

bat/httplib/httplib.go

@@ -1,5 +1,6 @@
 // Copyright 2014 beego Author. All Rights Reserved.
 // Copyright 2015 bat authors
+// Modifications copyright 2018 ETH Zurich
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

bat/utils.go

@@ -1,3 +1,4 @@
+// Modifications copyright 2018 ETH Zurich
 // This file has been modified to make it compatible with SCION
 
 package main

bwtester/bwtestclient/bwtestclient.go

@@ -17,6 +17,7 @@ import (
 	"unicode"
 
 	. "github.com/netsec-ethz/scion-apps/bwtester/bwtestlib"
+	"github.com/netsec-ethz/scion-apps/lib/scionutil"
 	"github.com/scionproto/scion/go/lib/sciond"
 	"github.com/scionproto/scion/go/lib/snet"
 	"github.com/scionproto/scion/go/lib/spath"
@@ -330,12 +331,20 @@ func main() {
 	} else if sciondPath == "" {
 		sciondPath = sciond.GetDefaultSCIONDPath(nil)
 	}
-	err = snet.Init(clientCCAddr.IA, sciondPath, dispatcherPath)
-	Check(err)
 
 	var pathEntry *sciond.PathReplyEntry
 	if !serverCCAddr.IA.Eq(clientCCAddr.IA) {
-		pathEntry = ChoosePath(interactive, pathAlgo, *clientCCAddr, *serverCCAddr)
+		if interactive {
+			pathEntry = scionutil.ChoosePathInteractive(clientCCAddr, serverCCAddr)
+		} else {
+			var metric int
+			if pathAlgo == "mtu" {
+				metric = scionutil.MTU
+			} else if pathAlgo == "shortest" {
+				metric = scionutil.Shortest
+			}
+			pathEntry = scionutil.ChoosePathByMetric(metric, clientCCAddr, serverCCAddr)
+		}
 		if pathEntry == nil {
 			LogFatal("No paths available to remote destination")
 		}