sql: implement memory management system for caches (#802)

sql: implement memory management system for caches

.travis.yml

@@ -10,21 +10,11 @@ env:
 addons:
   apt:
     packages:
-      - libonig-dev
       - libmysqlclient-dev
 
 matrix:
   fast_finish: true
 
-sudo: required
-
-services:
-  - docker
-
-install:
-  - go get ./...
-  - make dependencies
-
 before_script:
   - sudo service mysql stop
 
@@ -45,44 +35,34 @@ jobs:
       python: '3.6'
       before_install:
         - eval "$(gimme 1.12.4)"
-      install:
-        - go get ./...
       script:
         - make TEST=python-pymysql integration
 
     - language: php
       php: '7.1'
       before_install:
         - eval "$(gimme 1.12.4)"
-      install:
-        - go get ./...
       script:
         - make TEST=php integration
 
     - language: ruby
       ruby: '2.3'
       before_install:
         - eval "$(gimme 1.12.4)"
-      install:
-        - go get ./...
       script:
         - make TEST=ruby integration
 
     - language: java
       jdk: openjdk8
       before_install:
         - eval "$(gimme 1.12.4)"
-      install:
-        - go get ./...
       script:
         - make TEST=jdbc-mariadb integration
 
     - language: node_js
       node_js: '7'
       before_install:
         - eval "$(gimme 1.12.4)"
-      install:
-        - go get ./...
       script:
         - make TEST=javascript integration
 
@@ -91,16 +71,12 @@ jobs:
       dotnet: '2.1'
       before_install:
         - eval "$(gimme 1.12.4)"
-      install:
-        - go get ./...
       script:
         - make TEST=dotnet integration
 
     - language: c
       compiler: clang
       before_install:
         - eval "$(gimme 1.12.4)"
-      install:
-        - go get ./...
       script:
         - make TEST=c integration

README.md

@@ -141,8 +141,7 @@ SET <variable name> = <value>
 |:-----|:-----|:------------|
 |`INMEMORY_JOINS`|environment|If set it will perform all joins in memory. Default is off.|
 |`inmemory_joins`|session|If set it will perform all joins in memory. Default is off. This has precedence over `INMEMORY_JOINS`.|
-|`MAX_MEMORY_JOIN`|environment|The maximum number of memory, in megabytes, that can be consumed by go-mysql-server before switching to multipass mode in joins. Default is the 20% of all available physical memory.|
-|`max_memory_joins`|session|The maximum number of memory, in megabytes, that can be consumed by go-mysql-server before switching to multipass mode in joins. Default is the 20% of all available physical memory. This has precedence over `MAX_MEMORY_JOIN`.|
+|`MAX_MEMORY`|environment|The maximum number of memory, in megabytes, that can be consumed by go-mysql-server. Any in-memory caches or computations will no longer try to use memory when the limit is reached. Note that this may cause certain queries to fail if there is not enough memory available, such as queries using DISTINCT, ORDER BY or GROUP BY with groupings.|
 |`DEBUG_ANALYZER`|environment|If set, the analyzer will print debug messages. Default is off.|
 |`PILOSA_INDEX_THREADS`|environment|Number of threads used in index creation. Default is the number of cores available in the machine.|
 |`pilosa_index_threads`|environment|Number of threads used in index creation. Default is the number of cores available in the machine. This has precedence over `PILOSA_INDEX_THREADS`.|
@@ -176,14 +175,14 @@ func main() {
     s.Start()
 }
 
-func createTestDatabase() *mem.Database {
+func createTestDatabase() *memory.Database {
     const (
         dbName    = "test"
         tableName = "mytable"
     )
 
-    db := mem.NewDatabase(dbName)
-    table := mem.NewTable(tableName, sql.Schema{
+    db := memory.NewDatabase(dbName)
+    table := memory.NewTable(tableName, sql.Schema{
         {Name: "name", Type: sql.Text, Nullable: false, Source: tableName},
         {Name: "email", Type: sql.Text, Nullable: false, Source: tableName},
         {Name: "phone_numbers", Type: sql.JSON, Nullable: false, Source: tableName},

_example/main.go

@@ -5,7 +5,7 @@ import (
 
 	sqle "github.com/src-d/go-mysql-server"
 	"github.com/src-d/go-mysql-server/auth"
-	"github.com/src-d/go-mysql-server/mem"
+	"github.com/src-d/go-mysql-server/memory"
 	"github.com/src-d/go-mysql-server/server"
 	"github.com/src-d/go-mysql-server/sql"
 )
@@ -42,14 +42,14 @@ func main() {
 	s.Start()
 }
 
-func createTestDatabase() *mem.Database {
+func createTestDatabase() *memory.Database {
 	const (
 		dbName    = "mydb"
 		tableName = "mytable"
 	)
 
-	db := mem.NewDatabase(dbName)
-	table := mem.NewTable(tableName, sql.Schema{
+	db := memory.NewDatabase(dbName)
+	table := memory.NewTable(tableName, sql.Schema{
 		{Name: "name", Type: sql.Text, Nullable: false, Source: tableName},
 		{Name: "email", Type: sql.Text, Nullable: false, Source: tableName},
 		{Name: "phone_numbers", Type: sql.JSON, Nullable: false, Source: tableName},

_integration/ruby/Makefile

@@ -1,9 +1,10 @@
 vendor/bundle:
+	gem install bundler --version=1.16.5
 	bundler install --path vendor/bundle
 
 dependencies: vendor/bundle
 
 test: dependencies
 	bundler exec ruby mysql_test.rb
 
-.PHONY: test
+.PHONY: test

auth/common_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/stretchr/testify/require"
 	sqle "github.com/src-d/go-mysql-server"
 	"github.com/src-d/go-mysql-server/auth"
-	"github.com/src-d/go-mysql-server/mem"
+	"github.com/src-d/go-mysql-server/memory"
 	"github.com/src-d/go-mysql-server/server"
 	"github.com/src-d/go-mysql-server/sql"
 	"github.com/src-d/go-mysql-server/sql/analyzer"
@@ -21,13 +21,13 @@ import (
 const port = 3336
 
 func authEngine(au auth.Auth) (string, *sqle.Engine, error) {
-	db := mem.NewDatabase("test")
+	db := memory.NewDatabase("test")
 	catalog := sql.NewCatalog()
 	catalog.AddDatabase(db)
 
 	tblName := "test"
 
-	table := mem.NewTable(tblName, sql.Schema{
+	table := memory.NewTable(tblName, sql.Schema{
 		{Name: "id", Type: sql.Text, Nullable: false, Source: tblName},
 		{Name: "name", Type: sql.Text, Nullable: false, Source: tblName},
 	})

benchmark/tpc_h_test.go

@@ -10,9 +10,9 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/src-d/go-mysql-server"
+	sqle "github.com/src-d/go-mysql-server"
 
-	"github.com/src-d/go-mysql-server/mem"
+	"github.com/src-d/go-mysql-server/memory"
 	"github.com/src-d/go-mysql-server/sql"
 )
 
@@ -83,11 +83,11 @@ func executeQueries(b *testing.B, e *sqle.Engine) error {
 }
 
 func genDB(b *testing.B) (sql.Database, error) {
-	db := mem.NewDatabase("tpch")
+	db := memory.NewDatabase("tpch")
 
 	for _, m := range tpchTableMetadata {
 		b.Log("generating table", m.name)
-		t := mem.NewTable(m.name, m.schema)
+		t := memory.NewTable(m.name, m.schema)
 		if err := insertDataToTable(m.name, t, len(m.schema)); err != nil {
 			return nil, err
 		}
@@ -98,7 +98,7 @@ func genDB(b *testing.B) (sql.Database, error) {
 	return db, nil
 }
 
-func insertDataToTable(name string, t *mem.Table, columnCount int) error {
+func insertDataToTable(name string, t *memory.Table, columnCount int) error {
 	f, err := os.Open(name + ".tbl")
 	if err != nil {
 		return err