Merge pull request #54 from GrappigPanda/devel

[WIP] Milestone 0.2.0 + bug fixes.

Dockerfile

@@ -1,15 +1,35 @@
 FROM golang
 
+# Install redis server
 RUN apt-get update && apt-get install -y redis-server
+
+# Install and configure mysql
+RUN echo 'mysql-server mysql-server/root_password password secret_password' | debconf-set-selections
+RUN echo 'mysql-server mysql-server/root_password_again password secret_password' | debconf-set-selections
+RUN apt-get install -y mysql-server
+ADD build/my.cnf /etc/mysql/my.cnf
+RUN mkdir -p /var/lib/mysql
+RUN chmod -R 755 /var/lib/mysql
+
+# Install and get supervisord so that we can run multiple processes.
 RUN apt-get install -y supervisor
 RUN mkdir -p /var/log/supervisor
 
+# Move local files to the docker image
 ADD . /go/src/github.com/GrappigPanda/notorious
-COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+COPY build/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 
+# Install dependencies
 RUN go get gopkg.in/redis.v3
+RUN go get github.com/jinzhu/gorm
+RUN go get github.com/go-sql-driver/mysql
+RUN go get github.com/NotoriousTracker/viper
+
+# Build notorious
 RUN go install github.com/GrappigPanda/notorious
 
+# Set the entry command
 CMD ["/usr/bin/supervisord"]
 
+# Allow remote connections into notorious
 EXPOSE 3000

build/my.cnf

@@ -0,0 +1,127 @@
+#
+# The MySQL database server configuration file.
+#
+# You can copy this to one of:
+# - "/etc/mysql/my.cnf" to set global options,
+# - "~/.my.cnf" to set user-specific options.
+# 
+# One can use all long options that the program supports.
+# Run program with --help to get a list of available options and with
+# --print-defaults to see which it would actually understand and use.
+#
+# For explanations see
+# http://dev.mysql.com/doc/mysql/en/server-system-variables.html
+
+# This will be passed to all mysql clients
+# It has been reported that passwords should be enclosed with ticks/quotes
+# escpecially if they contain "#" chars...
+# Remember to edit /etc/mysql/debian.cnf when changing the socket location.
+[client]
+port        = 3306
+socket      = /var/run/mysqld/mysqld.sock
+
+# Here is entries for some specific programs
+# The following values assume you have at least 32M ram
+
+# This was formally known as [safe_mysqld]. Both versions are currently parsed.
+[mysqld_safe]
+socket      = /var/run/mysqld/mysqld.sock
+nice        = 0
+
+[mysqld]
+#
+# * Basic Settings
+#
+user        = mysql
+pid-file    = /var/run/mysqld/mysqld.pid
+socket      = /var/run/mysqld/mysqld.sock
+port        = 3306
+basedir     = /usr
+datadir     = /var/lib/mysql
+tmpdir      = /tmp
+lc-messages-dir = /usr/share/mysql
+skip-external-locking
+#
+# Instead of skip-networking the default is now to listen only on
+# localhost which is more compatible and is not less secure.
+# bind-address       = 0.0.0.0
+#
+# * Fine Tuning
+#
+key_buffer      = 16M
+max_allowed_packet  = 16M
+thread_stack        = 192K
+thread_cache_size       = 8
+# This replaces the startup script and checks MyISAM tables if needed
+# the first time they are touched
+myisam-recover         = BACKUP
+#max_connections        = 100
+#table_cache            = 64
+#thread_concurrency     = 10
+#
+# * Query Cache Configuration
+#
+query_cache_limit   = 1M
+query_cache_size        = 16M
+#
+# * Logging and Replication
+#
+# Both location gets rotated by the cronjob.
+# Be aware that this log type is a performance killer.
+# As of 5.1 you can enable the log at runtime!
+# general_log_file        = /var/log/mysql/mysql.log
+# general_log             = 1
+#
+# Error log - should be very few entries.
+#
+log_error = /var/log/mysql/error.log
+#
+# Here you can see queries with especially long duration
+#log_slow_queries   = /var/log/mysql/mysql-slow.log
+#long_query_time = 2
+#log-queries-not-using-indexes
+#
+# The following can be used as easy to replay backup logs or for replication.
+# note: if you are setting up a replication slave, see README.Debian about
+#       other settings you may need to change.
+#server-id      = 1
+#log_bin            = /var/log/mysql/mysql-bin.log
+expire_logs_days    = 10
+max_binlog_size         = 100M
+#binlog_do_db       = include_database_name
+#binlog_ignore_db   = include_database_name
+#
+# * InnoDB
+#
+# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/.
+# Read the manual for more InnoDB related options. There are many!
+#
+# * Security Features
+#
+# Read the manual, too, if you want chroot!
+# chroot = /var/lib/mysql/
+#
+# For generating SSL certificates I recommend the OpenSSL GUI "tinyca".
+#
+# ssl-ca=/etc/mysql/cacert.pem
+# ssl-cert=/etc/mysql/server-cert.pem
+# ssl-key=/etc/mysql/server-key.pem
+
+
+
+[mysqldump]
+quick
+quote-names
+max_allowed_packet  = 16M
+
+[mysql]
+no-auto-rehash # faster start of mysql but no tab completition
+
+[isamchk]
+key_buffer      = 16M
+
+#
+# * IMPORTANT: Additional settings that can override those from this file!
+#   The files must end with '.cnf', otherwise they'll be ignored.
+#
+!includedir /etc/mysql/conf.d/

supervisord.conf → build/supervisord.conf

@@ -8,3 +8,6 @@ command=/usr/bin/redis-server
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 command=/go/bin/notorious
+
+[program:mysql]
+command=/usr/bin/mysqld_safe

config/config.go

@@ -0,0 +1,32 @@
+package config
+
+import (
+	"github.com/NotoriousTracker/viper"
+)
+
+type ConfigStruct struct {
+	MySQLHost string
+	MySQLPort string
+	MySQLUser string
+	MySQLPass string
+	MySQLDB   string
+}
+
+func LoadConfig() ConfigStruct {
+	viper.SetConfigName("config")
+	viper.SetConfigType("yaml")
+	viper.AddConfigPath(".")
+
+	err := viper.ReadInConfig()
+	if err != nil {
+		panic("Failed to open config file")
+	}
+
+	return ConfigStruct{
+		viper.Get("MySQLHost").(string),
+		viper.Get("MySQLPort").(string),
+		viper.Get("MySQLUser").(string),
+		viper.Get("MySQLPass").(string),
+		viper.Get("MySQLDB").(string),
+	}
+}

database/database.go

@@ -0,0 +1,33 @@
+package db
+
+import (
+	"fmt"
+	"github.com/GrappigPanda/notorious/config"
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/mysql"
+)
+
+func formatConnectString(c config.ConfigStruct) string {
+	return fmt.Sprintf("%s:%s@%s/%s",
+		c.MySQLUser,
+		c.MySQLPass,
+		c.MySQLHost,
+		c.MySQLDB,
+	)
+}
+
+func OpenConnection() *gorm.DB {
+	c := config.LoadConfig()
+
+	db, err := gorm.Open("mysql", formatConnectString(c))
+	if err != nil {
+		panic(err)
+	}
+
+	return db
+}
+
+func ScrapeTorrent(db *gorm.DB, infoHash string) interface{} {
+	var torrent Torrent
+	return db.Where("infoHash = ?", infoHash).Find(&torrent).Value
+}

database/schemas.go

@@ -0,0 +1,16 @@
+package db
+
+import (
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/mysql"
+)
+
+type Torrent struct {
+	gorm.Model
+	id         int    `gorm:"AUTO_INCREMENT, unique"`
+	infoHash   string `gorm:"varchar(32), not null"`
+	name       string `gorm:"not null"`
+	Downloaded int    `gorm:"not null"`
+	Seeders    int    `gorm:"not null"`
+	Leechers   int    `gorm:"not null"`
+}

main.go

@@ -1,9 +1,18 @@
 package main
 
 import (
-    "github.com/GrappigPanda/notorious/server"
+	"github.com/GrappigPanda/notorious/reaper"
+	"github.com/GrappigPanda/notorious/server"
+	"time"
 )
 
 func main() {
-    server.RunServer()
+	c := server.OpenClient()
+	_, err := c.Ping().Result()
+	if err != nil {
+		panic("No Redis instance detected. If deploying without Docker, install redis-server")
+	}
+
+	go reaper.StartReapingScheduler(1 * time.Minute)
+	server.RunServer()
 }

reaper/reaper.go

@@ -0,0 +1,76 @@
+package reaper
+
+import (
+	"fmt"
+	"gopkg.in/redis.v3"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func reapInfoHash(c *redis.Client, infoHash string, out chan int) {
+	// Fan-out method to reap peers who have outlived their TTL.
+	keys, err := c.SMembers(infoHash).Result()
+	if err != nil {
+		panic(err)
+	}
+
+	count := 0
+	currTime := int64(time.Now().UTC().Unix())
+
+	for i := range keys {
+		if x := strings.Split(keys[i], ":"); len(x) != 3 {
+			c.SRem(infoHash, keys[i])
+
+		} else {
+			endTime := convertTimeToUnixTimeStamp(x[2])
+			if currTime >= endTime {
+				c.SRem(infoHash, keys[i])
+				count += 1
+			}
+		}
+	}
+
+	out <- count
+}
+
+func convertTimeToUnixTimeStamp(time string) (endTime int64) {
+	endTime, err := strconv.ParseInt(time, 10, 64)
+	if err != nil {
+		panic(err)
+	}
+
+	return
+}
+
+func reapPeers() (peersReaped int) {
+	// Fans out each info in `keys *` from the Redis DB to the `reapInfoHash`
+	// function.
+	client := OpenClient()
+
+	keys, err := getAllKeys(client, "*")
+	if err != nil {
+		panic(err)
+	}
+
+	out := make(chan int)
+	for i := range keys {
+		go reapInfoHash(client, keys[i], out)
+		peersReaped += <-out
+	}
+
+	return
+}
+
+func StartReapingScheduler(waitTime time.Duration) {
+	// The timer which sets off the peer reaping every `waitTime` seconds.
+	reapedPeers := 0
+	go func() {
+		for {
+			time.Sleep(waitTime)
+			fmt.Println("Starting peer reaper")
+			reapedPeers += reapPeers()
+			fmt.Printf("%v peers reaped total\n", reapedPeers)
+		}
+	}()
+}

reaper/reaperRedis.go

@@ -0,0 +1,31 @@
+package reaper
+
+import (
+	"fmt"
+	"gopkg.in/redis.v3"
+)
+
+func OpenClient() (client *redis.Client) {
+	// Opens a connection to the redis connection.
+	// TODO(ian): Add a config option for redis host:port
+	// TODO(ian): Add error checking here.
+	client = redis.NewClient(&redis.Options{
+		Addr:     "localhost:6379",
+		Password: "",
+		DB:       0,
+	})
+
+	return
+}
+
+func getAllKeys(c *redis.Client, keymember string) (keys []string, err error) {
+	// getAllKeys gets all the keys for a specified `keymember`
+
+	allKeys := c.Keys(keymember)
+	keys, err = allKeys.Result()
+	if err != nil {
+		fmt.Errorf("Failed to reap peers")
+	}
+
+	return
+}