sql is out

Makefile

@@ -166,15 +166,12 @@ fetch-depends:
 	$(GO) get github.com/asdine/storm/codec/protobuf
 	$(GO) get github.com/ryanuber/go-glob
 	$(GO) get github.com/gorilla/sessions
-	$(GO) get github.com/mattn/go-sqlite3
-	$(GO) get github.com/jinzhu/gorm
 	$(GO) get github.com/stretchr/testify/assert
 	$(GO) get github.com/spf13/viper
 	$(GO) get -t github.com/RoaringBitmap/roaring
 	$(GO) get github.com/elastic/go-ucfg
 	$(GO) get github.com/jasonlvhit/gocron
 	$(GO) get github.com/quipo/statsd
-	$(GO) get github.com/go-sql-driver/mysql
 	$(GO) get github.com/jbowles/cld2_nlpt
 	$(GO) get github.com/mafredri/cdp
 	$(GO) get github.com/ararog/timeago

gopa.yml

@@ -268,12 +268,6 @@ modules:
     index_prefix: gopa-
     username: elastic
     password: changeme
-#  // before use MySQL, you should create database first:
-#  // DROP DATABASE gopa;
-#  // CREATE DATABASE IF NOT EXISTS gopa DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
-#  driver: mysql
-#  mysql:
-#    connection: root:password@tcp(localhost:3306)/gopa?charset=utf8&parseTime=true&loc=Local
 
 - name: stats
   enabled: true

model/host.go

@@ -8,7 +8,7 @@ import (
 
 // Host is host struct
 type Host struct {
-	Host        string        `json:"host,omitempty" gorm:"not null;unique;primary_key" index:"id"`
+	Host        string        `json:"host,omitempty" index:"id"`
 	Favicon     string        `json:"favicon,omitempty"`
 	Enabled     bool          `json:"enabled"`
 	HostConfigs *[]HostConfig `json:"host_configs,omitempty"`

model/host_config.go

@@ -25,17 +25,17 @@ import (
 )
 
 type HostConfig struct {
-	ID         string `json:"id,omitempty" gorm:"not null;unique;primary_key" index:"id"`
-	Host       string `gorm:"index" json:"host"`
-	UrlPattern string `gorm:"index" json:"url_pattern"`
-	Runner     string `gorm:"index" json:"runner"`
-	SortOrder  int    `gorm:"index" json:"sort_order"`
+	ID         string `json:"id,omitempty" index:"id"`
+	Host       string `json:"host"`
+	UrlPattern string `json:"url_pattern"`
+	Runner     string `json:"runner"`
+	SortOrder  int    `json:"sort_order"`
 
-	PipelineID string `gorm:"index" json:"pipeline_id"`
+	PipelineID string `json:"pipeline_id"`
 	Cookies    string `json:"cookies,omitempty"`
 
-	Created time.Time `gorm:"index" json:"created,omitempty"`
-	Updated time.Time `gorm:"index" json:"updated,omitempty"`
+	Created time.Time `json:"created,omitempty"`
+	Updated time.Time `json:"updated,omitempty"`
 }
 
 func CreateHostConfig(config *HostConfig) error {

model/project.go

@@ -25,7 +25,7 @@ import (
 
 // Project is a definition, include a collection of Host
 type Project struct {
-	ID          string    `storm:"id,unique" json:"id,omitempty" gorm:"not null;unique;primary_key" index:"id"`
+	ID          string    `storm:"id,unique" json:"id,omitempty" index:"id"`
 	Name        string    `json:"name,omitempty"`
 	Description string    `json:"description,omitempty"`
 	Enabled     bool      `json:"enabled"`

model/snapshot.go

@@ -35,50 +35,50 @@ type LinkGroup struct {
 }
 
 type Snapshot struct {
-	ID      string `json:"id,omitempty" gorm:"not null;unique;primary_key" index:"id"`
+	ID      string `json:"id,omitempty" index:"id"`
 	Version int    `json:"version,omitempty"`
-	Url     string `json:"url,omitempty" gorm:"type:varchar(500)"`
+	Url     string `json:"url,omitempty"`
 	TaskID  string `json:"task_id,omitempty"`
-	Path    string `json:"path,omitempty"  gorm:"-"` //path of this file
-	File    string `json:"file,omitempty"  gorm:"-"` //filename of this page
-	Ext     string `json:"ext,omitempty"  gorm:"-"`  //extension of filename
+	Path    string `json:"path,omitempty"` //path of this file
+	File    string `json:"file,omitempty"` //filename of this page
+	Ext     string `json:"ext,omitempty"`  //extension of filename
 
-	StatusCode int    `json:"-" gorm:"-"`
-	Payload    []byte `json:"-" gorm:"-"`
+	StatusCode int    `json:"-"`
+	Payload    []byte `json:"-"`
 	Size       uint64 `json:"size,omitempty"`
 
 	ScreenshotID string `json:"screenshot_id,omitempty"`
 
-	Headers    map[string][]string     `json:"-" gorm:"-"`
-	Metadata   *map[string]interface{} `json:"-" gorm:"-"`
-	Parameters []KV                    `json:"-" gorm:"-"`
+	Headers    map[string][]string     `json:"-"`
+	Metadata   *map[string]interface{} `json:"-"`
+	Parameters []KV                    `json:"-"`
 
-	Language string `json:"lang,omitempty" gorm:"-"`
+	Language string `json:"lang,omitempty"`
 
-	Title       string `json:"title,omitempty" gorm:"type:varchar(500)"`
-	Summary     string `json:"summary,omitempty" gorm:"-"`
-	Text        string `json:"text,omitempty" gorm:"-"`
+	Title       string `json:"title,omitempty"`
+	Summary     string `json:"summary,omitempty"`
+	Text        string `json:"text,omitempty"`
 	ContentType string `json:"content_type,omitempty"`
 
-	Tags []string `json:"tags,omitempty" gorm:"-"`
+	Tags []string `json:"tags,omitempty"`
 
-	Links LinkGroup `json:"links,omitempty" gorm:"-"`
+	Links LinkGroup `json:"links,omitempty"`
 
 	Images struct {
 		Internal []PageLink `json:"internal,omitempty"`
 		External []PageLink `json:"external,omitempty"`
-	} `json:"images,omitempty" gorm:"-"`
-
-	H1     []string `json:"h1,omitempty" gorm:"-"`
-	H2     []string `json:"h2,omitempty" gorm:"-"`
-	H3     []string `json:"h3,omitempty" gorm:"-"`
-	H4     []string `json:"h4,omitempty" gorm:"-"`
-	H5     []string `json:"h5,omitempty" gorm:"-"`
-	Bold   []string `json:"bold,omitempty" gorm:"-"`
-	Italic []string `json:"italic,omitempty" gorm:"-"`
-
-	Classifications  []string                `json:"classifications,omitempty" gorm:"-"`
-	EnrichedFeatures *map[string]interface{} `json:"enriched_features,omitempty" gorm:"-"`
+	} `json:"images,omitempty"`
+
+	H1     []string `json:"h1,omitempty"`
+	H2     []string `json:"h2,omitempty"`
+	H3     []string `json:"h3,omitempty"`
+	H4     []string `json:"h4,omitempty"`
+	H5     []string `json:"h5,omitempty"`
+	Bold   []string `json:"bold,omitempty"`
+	Italic []string `json:"italic,omitempty"`
+
+	Classifications  []string                `json:"classifications,omitempty"`
+	EnrichedFeatures *map[string]interface{} `json:"enriched_features,omitempty"`
 
 	Hash    string `json:"hash,omitempty"`
 	SimHash string `json:"sim_hash,omitempty"`

model/task.go

@@ -59,22 +59,22 @@ func NewTask(url, ref string, depth int, breadth int) *Task {
 }
 
 type Task struct {
-	ID string `gorm:"not null;unique;primary_key" json:"id" index:"id"`
+	ID string `json:"id" index:"id"`
 	// the url may not cleaned, may miss the host part, need reference to provide the complete url information
-	Url         string    `storm:"index" json:"url,omitempty" gorm:"type:varchar(500)"`
+	Url         string    `storm:"index" json:"url,omitempty"`
 	Reference   string    `json:"reference_url,omitempty"`
 	Depth       int       `storm:"index" json:"depth"`
 	Breadth     int       `storm:"index" json:"breadth"`
-	Host        string    `gorm:"index" json:"host"`
+	Host        string    `json:"host"`
 	Schema      string    `json:"schema,omitempty"`
-	OriginalUrl string    `json:"original_url,omitempty" gorm:"type:varchar(500)"`
-	Status      int       `gorm:"index" json:"status"`
-	Message     string    `json:"message,omitempty" gorm:"type:varchar(500)"`
-	Created     time.Time `gorm:"index" json:"created,omitempty"`
-	Updated     time.Time `gorm:"index" json:"updated,omitempty"`
-	LastFetch   time.Time `gorm:"index" json:"last_fetch,omitempty"`
-	LastCheck   time.Time `gorm:"index" json:"last_check,omitempty"`
-	NextCheck   time.Time `gorm:"index" json:"next_check,omitempty"`
+	OriginalUrl string    `json:"original_url,omitempty"`
+	Status      int       `json:"status"`
+	Message     string    `json:"message,omitempty"`
+	Created     time.Time `json:"created,omitempty"`
+	Updated     time.Time `json:"updated,omitempty"`
+	LastFetch   time.Time `json:"last_fetch,omitempty"`
+	LastCheck   time.Time `json:"last_check,omitempty"`
+	NextCheck   time.Time `json:"next_check,omitempty"`
 
 	SnapshotVersion  int       `json:"snapshot_version,omitempty"`
 	SnapshotID       string    `json:"snapshot_id,omitempty"`