Merge branch 'master' into feature/linkedql

.gitignore

@@ -8,9 +8,9 @@ cayley.json
 .cayley_history
 .DS_Store
 
-# The built binary
-cayley/
 dist/
 vendor/
+ui/
 packrd/packed-*
-cayley
+# The build binary
+/cayley

.travis.yml

@@ -16,6 +16,7 @@ matrix:
 install:
   - sudo snap install snapcraft --classic
   - go mod download
+  - go run cmd/download_ui/download_ui.go
 
 cache:
   directories:

Dockerfile

@@ -19,6 +19,10 @@ WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download
 
+# Copy UI download script and execute
+COPY cmd/download_ui/ ./cmd/download_ui/
+RUN go run cmd/download_ui/download_ui.go
+
 # Add all the other files
 ADD . .
 
@@ -54,6 +58,8 @@ VOLUME [ "/data" ]
 
 EXPOSE 64210
 
+HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD [ "cayley", "health" ]
+
 # Adding everything to entrypoint allows us to init, load and serve only with
 # arguments passed to docker run. For example:
 # `docker run cayleygraph/cayley --init -i /data/my_data.nq`

cmd/cayley/cayley.go

@@ -143,6 +143,7 @@ func init() {
 		command.NewConvertCmd(),
 		command.NewDedupCommand(),
 		command.NewLinkedQLSchemaCommand(),
+		command.NewHealthCmd(),
 	)
 	rootCmd.PersistentFlags().StringP("config", "c", "", "path to an explicit configuration file")
 

cmd/cayley/command/health.go

@@ -0,0 +1,39 @@
+package command
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+
+	"github.com/spf13/cobra"
+)
+
+const defaultAddress = "http://localhost:64210/"
+
+func NewHealthCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:     "health",
+		Aliases: []string{},
+		Short:   "Health check HTTP server",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) > 1 {
+				return fmt.Errorf("Too many arguments provided, expected 0 or 1")
+			}
+			address := defaultAddress
+			if len(args) == 1 {
+				address = args[0]
+			}
+			healthAddress := address + "health"
+			resp, err := http.Get(healthAddress)
+			if err != nil {
+				return err
+			}
+			defer resp.Body.Close()
+			if resp.StatusCode != 204 {
+				return fmt.Errorf("/health responded with status code %d, expected 204", resp.StatusCode)
+			}
+			log.Printf("%s ok", healthAddress)
+			return nil
+		},
+	}
+}

cmd/download_ui/download_ui.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	version       = "v0.8.0"
+	fileURL       = "https://github.com/cayleygraph/web/releases/download/" + version + "/web.zip"
+	fileName      = "web.zip"
+	directoryName = "ui"
+)
+
+func main() {
+	log.Printf("Downloading %s to %s...", fileURL, fileName)
+	if err := DownloadFile(fileName, fileURL); err != nil {
+		panic(err)
+	}
+	log.Printf("Downloaded %s to %s", fileURL, fileName)
+
+	log.Printf("Extracting %s to %s...", fileName, directoryName)
+	err := Unzip(fileName, directoryName)
+	if err != nil {
+		panic(err)
+	}
+	log.Printf("Extracted %s to %s/", fileName, directoryName)
+	err = os.Remove(fileName)
+	if err != nil {
+		panic(err)
+	}
+	log.Printf("Removed %s", fileName)
+}
+
+// DownloadFile will download a url to a local file. It's efficient because it will
+// write as it downloads and not load the whole file into memory.
+func DownloadFile(filepath string, url string) error {
+
+	// Get the data
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("Received %v status code instead of 200 for %v", resp.Status, url)
+	}
+
+	// Create the file
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	// Write the body to file
+	_, err = io.Copy(out, resp.Body)
+	return err
+}
+
+// Unzip will decompress a zip archive, moving all files and folders
+// within the zip file (parameter 1) to an output directory (parameter 2).
+func Unzip(src string, dest string) error {
+	r, err := zip.OpenReader(src)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	for _, f := range r.File {
+
+		// Store filename/path for returning and using later on
+		fpath := filepath.Join(dest, f.Name)
+
+		// Check for ZipSlip. More Info: http://bit.ly/2MsjAWE
+		if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) {
+			return fmt.Errorf("%s: illegal file path", fpath)
+		}
+
+		if f.FileInfo().IsDir() {
+			// Make Folder
+			os.MkdirAll(fpath, 0755)
+			continue
+		}
+
+		// Make File
+		if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil {
+			return err
+		}
+
+		outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+		if err != nil {
+			return err
+		}
+
+		rc, err := f.Open()
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(outFile, rc)
+
+		if err != nil {
+			return err
+		}
+
+		// Close the file without defer to close before next iteration of loop
+		err = outFile.Close()
+
+		if err != nil {
+			return err
+		}
+
+		err = rc.Close()
+
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

docs/contributing.md

@@ -19,9 +19,13 @@ Follow the instructions for running Cayley locally:
 git clone https://github.com/cayleygraph/cayley
 cd cayley
 
-# download dependencies
+# Download dependencies
 go mod download
 
+# Download web files (optional)
+
+go run cmd/download_ui/download_ui.go
+
 # Install packr 2
 
 go get -u github.com/gobuffalo/packr/v2/packr2

graph/gaedatastore/config.go

@@ -120,7 +120,7 @@ func (d *duration) MarshalJSON() ([]byte, error) {
 	return []byte(fmt.Sprintf("%q", *d)), nil
 }
 
-// Load reads a JSON-encoded config contained in the given file. A zero value
+// LoadConf reads a JSON-encoded config contained in the given file. A zero value
 // config is returned if the filename is empty.
 func LoadConf(file string) (*Config, error) {
 	config := &Config{}

graph/graphtest/graphtest.go

@@ -98,7 +98,7 @@ func BenchmarkAll(b *testing.B, gen testutil.DatabaseFunc, conf *Config) {
 	})
 }
 
-// This is a simple test graph.
+// MakeQuadSet makes a simple test graph.
 //
 //    +---+                        +---+
 //    | A |-------               ->| F |<--
@@ -842,6 +842,7 @@ func TestLoadTypedQuads(t testing.TB, gen testutil.DatabaseFunc, conf *Config) {
 	require.Equal(t, exp, st, "Unexpected quadstore size")
 }
 
+// TestAddRemove tests add and remove
 // TODO(dennwc): add tests to verify that QS behaves in a right way with IgnoreOptions,
 // returns ErrQuadExists, ErrQuadNotExists is doing rollback.
 func TestAddRemove(t testing.TB, gen testutil.DatabaseFunc, conf *Config) {

graph/hasa.go

@@ -52,7 +52,7 @@ type HasA struct {
 	dir     quad.Direction
 }
 
-// Construct a new HasA iterator, given the quad subiterator, and the quad
+// NewHasA construct a new HasA iterator, given the quad subiterator, and the quad
 // direction for which it stands.
 func NewHasA(qs QuadIndexer, subIt iterator.Shape, d quad.Direction) *HasA {
 	return &HasA{
@@ -70,15 +70,15 @@ func (it *HasA) Lookup() iterator.Index {
 	return newHasAContains(it.qs, it.primary.Lookup(), it.dir)
 }
 
-// Return our sole subiterator.
+// SubIterators returns our sole subiterator.
 func (it *HasA) SubIterators() []iterator.Shape {
 	return []iterator.Shape{it.primary}
 }
 
 // Direction accessor.
 func (it *HasA) Direction() quad.Direction { return it.dir }
 
-// Pass the Optimize() call along to the subiterator. If it becomes Null,
+// Optimize pass the Optimize() call along to the subiterator. If it becomes Null,
 // then the HasA becomes Null (there are no quads that have any directions).
 func (it *HasA) Optimize(ctx context.Context) (iterator.Shape, bool) {
 	newPrimary, changed := it.primary.Optimize(ctx)
@@ -95,7 +95,7 @@ func (it *HasA) String() string {
 	return fmt.Sprintf("HasA(%v)", it.dir)
 }
 
-// GetStats() returns the statistics on the HasA iterator. This is curious. Next
+// Stats returns the statistics on the HasA iterator. This is curious. Next
 // cost is easy, it's an extra call or so on top of the subiterator Next cost.
 // ContainsCost involves going to the graph.QuadStore, iterating out values, and hoping
 // one sticks -- potentially expensive, depending on fanout. Size, however, is

graph/iterator/and_optimize.go

@@ -297,7 +297,7 @@ func getStatsForSlice(ctx context.Context, its, opt []Shape) (Costs, []Costs, er
 	}, arr, last
 }
 
-// and.Stats() lives here in and-iterator-optimize.go because it may
+// Stats lives here in and-iterator-optimize.go because it may
 // in the future return different statistics based on how it is optimized.
 // For now, however, it's pretty static.
 //

graph/iterator/fixed.go

@@ -35,7 +35,7 @@ type Fixed struct {
 	values []refs.Ref
 }
 
-// Creates a new Fixed iterator with a custom comparator.
+// NewFixed creates a new Fixed iterator with a custom comparator.
 func NewFixed(vals ...refs.Ref) *Fixed {
 	return &Fixed{
 		values: append([]refs.Ref{}, vals...),

graph/iterator/iterator.go

@@ -97,7 +97,7 @@ type Index interface {
 	Contains(ctx context.Context, v refs.Ref) bool
 }
 
-// Tagger is an interface for iterators that can tag values. Tags are returned as a part of TagResults call.
+// TaggerShape is an interface for iterators that can tag values. Tags are returned as a part of TagResults call.
 type TaggerShape interface {
 	Shape
 	TaggerBase
@@ -172,20 +172,23 @@ func Height(it Shape, filter func(Shape) bool) int {
 	return maxDepth + 1
 }
 
-// Here we define the simplest iterator -- the Null iterator. It contains nothing.
+// Null is the simplest iterator -- the Null iterator. It contains nothing.
 // It is the empty set. Often times, queries that contain one of these match nothing,
 // so it's important to give it a special iterator.
 type Null struct{}
 
+// NewNull creates a new Null iterator
 // Fairly useless New function.
 func NewNull() *Null {
 	return &Null{}
 }
 
+// Iterate implements Iterator
 func (it *Null) Iterate() Scanner {
 	return it
 }
 
+// Lookup implements Iterator
 func (it *Null) Lookup() Index {
 	return it
 }

graph/iterator/resolver.go

@@ -29,7 +29,7 @@ type Resolver struct {
 	order []quad.Value
 }
 
-// Creates a new Resolver iterator.
+// NewResolver creates a new Resolver iterator.
 func NewResolver(qs refs.Namer, nodes ...quad.Value) *Resolver {
 	it := &Resolver{
 		qs:    qs,

graph/linksto.go

@@ -48,7 +48,7 @@ type LinksTo struct {
 	size    refs.Size
 }
 
-// Construct a new LinksTo iterator around a direction and a subiterator of
+// NewLinksTo construct a new LinksTo iterator around a direction and a subiterator of
 // nodes.
 func NewLinksTo(qs QuadIndexer, it iterator.Shape, d quad.Direction) *LinksTo {
 	return &LinksTo{
@@ -58,7 +58,7 @@ func NewLinksTo(qs QuadIndexer, it iterator.Shape, d quad.Direction) *LinksTo {
 	}
 }
 
-// Return the direction under consideration.
+// Direction returns the direction under consideration.
 func (it *LinksTo) Direction() quad.Direction { return it.dir }
 
 func (it *LinksTo) Iterate() iterator.Scanner {
@@ -73,7 +73,7 @@ func (it *LinksTo) String() string {
 	return fmt.Sprintf("LinksTo(%v)", it.dir)
 }
 
-// Return a list containing only our subiterator.
+// SubIterators returns a list containing only our subiterator.
 func (it *LinksTo) SubIterators() []iterator.Shape {
 	return []iterator.Shape{it.primary}
 }
@@ -90,7 +90,7 @@ func (it *LinksTo) Optimize(ctx context.Context) (iterator.Shape, bool) {
 	return it, false
 }
 
-// Return a guess as to how big or costly it is to next the iterator.
+// Stats returns a guess as to how big or costly it is to next the iterator.
 func (it *LinksTo) Stats(ctx context.Context) (iterator.Costs, error) {
 	subitStats, err := it.primary.Stats(ctx)
 	// TODO(barakmich): These should really come from the quadstore itself