Disqus migration added (#64)

* stuff * added disqus migration * update documentation * updated config readme with bindaddress
vkuznecovas · Apr 27, 2018 · 0fd22c1 · 0fd22c1
1 parent 60045b2
commit 0fd22c1
Show file tree

Hide file tree

Showing 6 changed files with 344 additions and 3 deletions.
diff --git a/.gitignore b/.gitignore
@@ -10,5 +10,6 @@ goreleaser-build/
 .DS_Store
 main
 coverage.txt
+disqus.xml
 _mouthful_db
-data/
+data/
diff --git a/README.md b/README.md
@@ -23,6 +23,7 @@ There's a demo hosted at [mouthful.dizzy.zone](https://mouthful.dizzy.zone). Che
     * [Data sources](#data-source)
     * [Config file from Docker image](#config-file-from-docker)
     * [Nginx configuration](#nginx-config)
+* [Migrations](#migrations)
 * [Contributing](#contributing)
 * [Wish list](#wish-list)
 * [Get in touch](#get-in-touch)
@@ -35,7 +36,7 @@ There's a demo hosted at [mouthful.dizzy.zone](https://mouthful.dizzy.zone). Che
 * Server side caching to prevent excessive database calls
 * Rate limiting
 * Honeypot feature, to prevent bots from posting comments
-* Migrations from existing commenting engines(isso)
+* Migrations from existing commenting engines(isso, disqus)
 * Configuration - most of the features can be turned on or off, as well as customized to your preferences.
 
 # Installation
@@ -203,14 +204,22 @@ location /mouthful-demo/ {
 }
 ```
 
+# Migrations
+
+Mouthful supports migrating existing data from the following commenting engines:
+* [Isso](./cmd/migration/isso/README.md)
+* [Disqus](./cmd/migration/disqus/README.md)
+
+Click on them to read the documentation on how that is done.
+
 
 # Contributing
 
 Contributions are more than welcome. If you've found a bug, raise an issue. If you've got a feature request, open up an issue as well. I'll try and keep the api stable, as well as tag each release with a semantic version.
 
 # Wish list
 
-I'm a keen backender and not too sharp on the frontend part. If you're willing to contribute, front end(both client and admin) are not in the best of shapes, especially the admin panel. Frontend might require a refactor. Any addition of tests would be great as well. Migrations from other commenting engines would be encouraged as well. If someone could send me a disqus dump, I'd make a migration for that.
+I'm a keen backender and not too sharp on the frontend part. If you're willing to contribute, front end(both client and admin) are not in the best of shapes, especially the admin panel. Frontend might require a refactor. Any addition of tests would be great as well. Migrations from other commenting engines would be encouraged as well. 
 
 # Get in touch
 

diff --git a/cmd/migration/disqus/README.md b/cmd/migration/disqus/README.md
@@ -0,0 +1,13 @@
+# Migration tool from disqus xml to mouthful sqlite
+
+This tool will migrate all the threads and comments from disqus xml dump to mouthful sqlite.
+
+## Usage
+
+Simply run the main.go, providing 1 argument: path to disqus xml dump file
+
+`go run main.go ./disqus.xml`
+
+This will create a mouthful.db file in the current directory as output
+
+> special thanks to Reddit user [doenietzomoeilijk](https://www.reddit.com/user/doenietzomoeilijk) for providing the dump to make this possible
diff --git a/cmd/migration/disqus/main.go b/cmd/migration/disqus/main.go
@@ -0,0 +1,177 @@
+package main
+
+import (
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"time"
+
+	"github.com/vkuznecovas/mouthful/global"
+
+	uuid "github.com/satori/go.uuid"
+	"github.com/vkuznecovas/mouthful/api"
+	configModel "github.com/vkuznecovas/mouthful/config/model"
+	"github.com/vkuznecovas/mouthful/db"
+	dbModel "github.com/vkuznecovas/mouthful/db/model"
+
+	"github.com/vkuznecovas/mouthful/db/sqlxDriver"
+
+	"github.com/vkuznecovas/mouthful/cmd/migration/disqus/model"
+)
+
+type cpm struct {
+	Uid    uuid.UUID
+	Parent *string
+}
+
+var commentParentMap map[string]cpm
+var toDelete []uuid.UUID
+
+func getThread(threads *[]*model.Cthread, id string) *model.Cthread {
+	for _, v := range *threads {
+		if (v.AttrDsqSpaceid) == id {
+			return v
+		}
+	}
+	return nil
+}
+
+func insertComment(comment *model.Cpost, comments *[]*model.Cpost, threads *[]*model.Cthread, database sqlxDriver.Database) error {
+	// Insert parent if parent exists, and all its parents if needed
+	if _, ok := commentParentMap[comment.AttrDsqSpaceid]; ok {
+		if commentParentMap[comment.AttrDsqSpaceid].Parent != nil {
+			var c *model.Cpost
+			for _, v := range *comments {
+				if v.AttrDsqSpaceid == *commentParentMap[comment.AttrDsqSpaceid].Parent {
+					c = v
+					break
+				}
+			}
+			err := insertComment(c, comments, threads, database)
+			if err != nil {
+				panic(err)
+			}
+
+		}
+		// insert the comment itself
+		t := comment.Cthread
+		if len(t) > 0 {
+			thread := getThread(threads, t[0].AttrDsqSpaceid)
+			u, err := url.Parse(thread.Clink.SValue)
+			if err != nil {
+				panic(err)
+			}
+			path := api.NormalizePath(u.Path)
+			author := ""
+			if comment.Cauthor.Cname.SValue != "" {
+				author = comment.Cauthor.Cname.SValue
+			} else if comment.Cauthor.Cusername.SValue != "" {
+				author = comment.Cauthor.Cusername.SValue
+			} else if comment.Cauthor.Cemail.SValue != "" {
+				author = comment.Cauthor.Cemail.SValue
+			}
+			var replyTo *uuid.UUID
+			replyTo = nil
+			if commentParentMap[comment.AttrDsqSpaceid].Parent != nil {
+				cp := commentParentMap[*commentParentMap[comment.AttrDsqSpaceid].Parent]
+				replyTo = &cp.Uid
+			}
+			t, err := database.GetThread(path)
+			if err != nil {
+				if err == global.ErrThreadNotFound {
+					tuid, err := database.CreateThread(path)
+					if err != nil {
+						panic(err)
+					}
+					t = dbModel.Thread{
+						Id: *tuid,
+					}
+				}
+			}
+			uid := uuid.NewV4()
+			createdAt, err := time.Parse(time.RFC3339, comment.CcreatedAt.SValue)
+			if err != nil {
+				panic(err)
+			}
+			_, err = database.DB.Exec(database.DB.Rebind("INSERT INTO comment(id, threadId, body, author, confirmed, createdAt, replyTo, deletedAt) VALUES(?,?,?,?,?,?,?,?)"), uid, t.Id, comment.Cmessage.SValue, author, true, createdAt, replyTo, nil)
+			if err != nil {
+				panic(err)
+			}
+			if comment.CisSpam.SValue == "true" || comment.CisDeleted.SValue == "true" {
+				toDelete = append(toDelete, uid)
+			}
+			editedMapValue := commentParentMap[comment.AttrDsqSpaceid]
+			editedMapValue.Uid = uid
+			commentParentMap[comment.AttrDsqSpaceid] = editedMapValue
+		} else {
+			err := fmt.Errorf("Comment %v has no threads", comment.AttrDsqSpaceid)
+			return err
+		}
+	} else {
+		err := fmt.Errorf("Comment %v not found", comment.AttrDsqSpaceid)
+		return err
+	}
+
+	return nil
+}
+
+func main() {
+	argsWithoutProg := os.Args[1:]
+	if len(argsWithoutProg) == 0 {
+		panic(errors.New("Please provide a source database filename"))
+	}
+	// read disqus.xml
+	contents, err := ioutil.ReadFile(argsWithoutProg[0])
+	if err != nil {
+		panic(err)
+	}
+
+	commentParentMap = make(map[string]cpm, 0)
+	dbFile := "./mouthful.db"
+	mouthfulDB, err := db.GetDBInstance(configModel.Database{
+		Database: &dbFile,
+		Dialect:  "sqlite3",
+	})
+	st := mouthfulDB.GetUnderlyingStruct()
+	driverCasted := st.(*sqlxDriver.Database)
+
+	if err != nil {
+		panic(err)
+	}
+
+	var dis model.Cdisqus
+	err = xml.Unmarshal([]byte(contents), &dis)
+	if err != nil {
+		panic(err)
+	}
+
+	// first we form a map for comments, we'll need this to get their parent
+	for _, v := range dis.Cpost {
+		m := cpm{
+			Uid: uuid.NewV4(),
+		}
+		if v.Cparent != nil {
+			m.Parent = &v.Cparent.AttrDsqSpaceid
+		} else {
+			m.Parent = nil
+		}
+		commentParentMap[v.AttrDsqSpaceid] = m
+	}
+
+	for _, v := range dis.Cpost {
+		err = insertComment(v, &dis.Cpost, &dis.Cthread, *driverCasted)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	for _, v := range toDelete {
+		err = driverCasted.DeleteComment(v)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
diff --git a/cmd/migration/disqus/model/disqus.go b/cmd/migration/disqus/model/disqus.go
@@ -0,0 +1,140 @@
+// package model contains all the required
+package model
+
+import "encoding/xml"
+
+type Cauthor struct {
+	XMLName      xml.Name      `xml:"author,omitempty" json:"author,omitempty"`
+	Cemail       *Cemail       `xml:"http://disqus.com email,omitempty" json:"email,omitempty"`
+	CisAnonymous *CisAnonymous `xml:"http://disqus.com isAnonymous,omitempty" json:"isAnonymous,omitempty"`
+	Cname        *Cname        `xml:"http://disqus.com name,omitempty" json:"name,omitempty"`
+	Cusername    *Cusername    `xml:"http://disqus.com username,omitempty" json:"username,omitempty"`
+}
+
+type Ccategory struct {
+	XMLName        xml.Name    `xml:"category,omitempty" json:"category,omitempty"`
+	AttrDsqSpaceid string      `xml:"http://disqus.com/disqus-internals id,attr"  json:",omitempty"`
+	Cforum         *Cforum     `xml:"http://disqus.com forum,omitempty" json:"forum,omitempty"`
+	CisDefault     *CisDefault `xml:"http://disqus.com isDefault,omitempty" json:"isDefault,omitempty"`
+	Ctitle         *Ctitle     `xml:"http://disqus.com title,omitempty" json:"title,omitempty"`
+}
+
+type CcreatedAt struct {
+	XMLName xml.Name `xml:"createdAt,omitempty" json:"createdAt,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type Cdisqus struct {
+	XMLName                    xml.Name   `xml:"disqus,omitempty" json:"disqus,omitempty"`
+	AttrXmlnsdsq               string     `xml:"xmlns dsq,attr"  json:",omitempty"`
+	AttrXsiSpaceschemaLocation string     `xml:"http://www.w3.org/2001/XMLSchema-instance schemaLocation,attr"  json:",omitempty"`
+	Attrxmlns                  string     `xml:"xmlns,attr"  json:",omitempty"`
+	AttrXmlnsxsi               string     `xml:"xmlns xsi,attr"  json:",omitempty"`
+	Ccategory                  *Ccategory `xml:"http://disqus.com category,omitempty" json:"category,omitempty"`
+	Cpost                      []*Cpost   `xml:"http://disqus.com post,omitempty" json:"post,omitempty"`
+	Cthread                    []*Cthread `xml:"http://disqus.com thread,omitempty" json:"thread,omitempty"`
+}
+
+type Cemail struct {
+	XMLName xml.Name `xml:"email,omitempty" json:"email,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type Cforum struct {
+	XMLName xml.Name `xml:"forum,omitempty" json:"forum,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type Cid struct {
+	XMLName xml.Name `xml:"id,omitempty" json:"id,omitempty"`
+}
+
+type CipAddress struct {
+	XMLName xml.Name `xml:"ipAddress,omitempty" json:"ipAddress,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type CisAnonymous struct {
+	XMLName xml.Name `xml:"isAnonymous,omitempty" json:"isAnonymous,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type CisClosed struct {
+	XMLName xml.Name `xml:"isClosed,omitempty" json:"isClosed,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type CisDefault struct {
+	XMLName xml.Name `xml:"isDefault,omitempty" json:"isDefault,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type CisDeleted struct {
+	XMLName xml.Name `xml:"isDeleted,omitempty" json:"isDeleted,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type CisSpam struct {
+	XMLName xml.Name `xml:"isSpam,omitempty" json:"isSpam,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type Clink struct {
+	XMLName xml.Name `xml:"link,omitempty" json:"link,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type Cmessage struct {
+	XMLName xml.Name `xml:"message,omitempty" json:"message,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type Cname struct {
+	XMLName xml.Name `xml:"name,omitempty" json:"name,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type Cparent struct {
+	XMLName        xml.Name `xml:"parent,omitempty" json:"parent,omitempty"`
+	AttrDsqSpaceid string   `xml:"http://disqus.com/disqus-internals id,attr"  json:",omitempty"`
+}
+
+type Cpost struct {
+	XMLName        xml.Name    `xml:"post,omitempty" json:"post,omitempty"`
+	AttrDsqSpaceid string      `xml:"http://disqus.com/disqus-internals id,attr"  json:",omitempty"`
+	Cauthor        *Cauthor    `xml:"http://disqus.com author,omitempty" json:"author,omitempty"`
+	CcreatedAt     *CcreatedAt `xml:"http://disqus.com createdAt,omitempty" json:"createdAt,omitempty"`
+	Cid            *Cid        `xml:"http://disqus.com id,omitempty" json:"id,omitempty"`
+	CipAddress     *CipAddress `xml:"http://disqus.com ipAddress,omitempty" json:"ipAddress,omitempty"`
+	CisDeleted     *CisDeleted `xml:"http://disqus.com isDeleted,omitempty" json:"isDeleted,omitempty"`
+	CisSpam        *CisSpam    `xml:"http://disqus.com isSpam,omitempty" json:"isSpam,omitempty"`
+	Cmessage       *Cmessage   `xml:"http://disqus.com message,omitempty" json:"message,omitempty"`
+	Cparent        *Cparent    `xml:"http://disqus.com parent,omitempty" json:"parent,omitempty"`
+	Cthread        []*Cthread  `xml:"http://disqus.com thread,omitempty" json:"thread,omitempty"`
+}
+
+type Cthread struct {
+	XMLName        xml.Name    `xml:"thread,omitempty" json:"thread,omitempty"`
+	AttrDsqSpaceid string      `xml:"http://disqus.com/disqus-internals id,attr"  json:",omitempty"`
+	Cauthor        *Cauthor    `xml:"http://disqus.com author,omitempty" json:"author,omitempty"`
+	Ccategory      *Ccategory  `xml:"http://disqus.com category,omitempty" json:"category,omitempty"`
+	CcreatedAt     *CcreatedAt `xml:"http://disqus.com createdAt,omitempty" json:"createdAt,omitempty"`
+	Cforum         *Cforum     `xml:"http://disqus.com forum,omitempty" json:"forum,omitempty"`
+	Cid            *Cid        `xml:"http://disqus.com id,omitempty" json:"id,omitempty"`
+	CipAddress     *CipAddress `xml:"http://disqus.com ipAddress,omitempty" json:"ipAddress,omitempty"`
+	CisClosed      *CisClosed  `xml:"http://disqus.com isClosed,omitempty" json:"isClosed,omitempty"`
+	CisDeleted     *CisDeleted `xml:"http://disqus.com isDeleted,omitempty" json:"isDeleted,omitempty"`
+	Clink          *Clink      `xml:"http://disqus.com link,omitempty" json:"link,omitempty"`
+	Cmessage       *Cmessage   `xml:"http://disqus.com message,omitempty" json:"message,omitempty"`
+	Ctitle         *Ctitle     `xml:"http://disqus.com title,omitempty" json:"title,omitempty"`
+}
+
+type Ctitle struct {
+	XMLName xml.Name `xml:"title,omitempty" json:"title,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
+
+type Cusername struct {
+	XMLName xml.Name `xml:"username,omitempty" json:"username,omitempty"`
+	SValue  string   `xml:",chardata" json:",omitempty"`
+}
diff --git a/examples/configs/README.md b/examples/configs/README.md
@@ -45,6 +45,7 @@ Api section changes the behaviour of the back-end API.
 | Variables     | Use           | Type | Required  | Default value | Recommended setting |
 | ------------- |:-------------:| :---:| :-------: |  :----------: |  :----------------: |
 | port     | sets the port for API to run on | bool | false | 8080 | up to you |
+| bindAddress | sets the address that the api will listen on | string | 0.0.0.0 | up to you |
 | logging     | determines if gin logging will be enabled for the api | bool | false | true | true
 | debug     | enables or disables the gin debug mode with more verbal logging. | bool | false | false | false |
 | cache     | cache settings for the api | object | true |  | [see below](#api.cache) |