attachments on s3 and numerous other fixes.

including
  - fixes on deleteCard and moveCardTo/FromBoard
  - fixes on checklists and checkItems
  - renaming and other quibbles

missing
  - comments
  - custom fields
  - testing? I guess I'll just test manually.

allowed.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"strings"
 	"time"
 
 	"github.com/jmoiron/sqlx/types"
@@ -10,19 +9,32 @@ import (
 	"github.com/rs/zerolog"
 )
 
-func onAllowed(logger zerolog.Logger, trello trelloClient, wh Webhook) {
+func onAllowed(logger zerolog.Logger, token string, wh Webhook) {
 	b := wh.Action.Data.Board.Id
 
 	switch wh.Action.Type {
 	case "createCard", "copyCard", "convertToCardFromCheckItem", "moveCardToBoard":
-		// if a card is moved from another tracked board to this board
-		// this will give time to the other webhook to delete everything from the backups
-		// table so we can recreate everything here.
-		time.Sleep(time.Second * 2)
+		if wh.Action.Type == "moveCardToBoard" {
+			// if a card is moved from another tracked board to this board
+			// this will give time to the other webhook to delete everything from the backups
+			// table so we can recreate everything here.
+			time.Sleep(time.Second * 2)
+		} else if wh.Action.Type == "convertToCardFromCheckItem" {
+			// we must proceed as if deleting the checkItem here
+			checkItemId, err := itemJustConvertedIntoCard(
+				wh.Action.Data.Card.Name,
+				wh.Action.Data.Checklist.Id,
+			)
+			if err == nil {
+				wh.Action.Type = "deleteCheckItem"
+				wh.Action.Data.CheckItem.Id = checkItemId
+				onAllowed(logger, token, wh)
+			}
+		}
 
 		saveBackupData(b, wh.Action.Data.Card.Id, wh.Action.Data.Card)
 	case "deleteCard", "moveCardFromBoard":
-		// delete card, checklists and checkitems
+		// delete card, checklists and checkItems
 		var card Card
 		fetchBackupData(wh.Action.Data.Card.Id, &card)
 		for _, idChecklist := range card.IdChecklists {
@@ -86,7 +98,7 @@ func onAllowed(logger zerolog.Logger, trello trelloClient, wh Webhook) {
 	case "updateChecklist":
 		err = saveBackupData(b, wh.Action.Data.Checklist.Id, wh.Action.Data.Checklist)
 	case "removeChecklistFromCard":
-		// delete checkitems and checklist
+		// delete checkItems and checklist
 		var checklist Checklist
 		fetchBackupData(wh.Action.Data.Checklist.Id, &checklist)
 		for _, idCheckItem := range checklist.IdCheckItems {
@@ -95,7 +107,7 @@ func onAllowed(logger zerolog.Logger, trello trelloClient, wh Webhook) {
 				pretty.Log(err)
 			}
 		}
-		deleteBackupData(b, wh.Action.Data.Checklist.Id)
+		go deleteBackupData(b, wh.Action.Data.Checklist.Id)
 
 		// update card
 		err = updateBackupData(b, wh.Action.Data.Card.Id, wh.Action.Data.Card,
@@ -104,7 +116,7 @@ func onAllowed(logger zerolog.Logger, trello trelloClient, wh Webhook) {
 			wh.Action.Data.Checklist.Id,
 		)
 	case "createCheckItem":
-		// create checkitem on database
+		// create checkItem on database
 		go saveBackupData(b, wh.Action.Data.CheckItem.Id, wh.Action.Data.CheckItem)
 
 		// update checklist
@@ -116,7 +128,7 @@ func onAllowed(logger zerolog.Logger, trello trelloClient, wh Webhook) {
 	case "updateCheckItem", "updateCheckItemStateOnCard":
 		err = saveBackupData(b, wh.Action.Data.CheckItem.Id, wh.Action.Data.CheckItem)
 	case "deleteCheckItem":
-		// delete checkitem
+		// delete checkItem
 		deleteBackupData(b, wh.Action.Data.CheckItem.Id)
 
 		// update checklist
@@ -126,43 +138,29 @@ func onAllowed(logger zerolog.Logger, trello trelloClient, wh Webhook) {
 			wh.Action.Data.CheckItem.Id,
 		)
 	case "addAttachmentToCard":
-		primary := Attachment{
-			Name: wh.Action.Data.Attachment.Name,
-			Url:  wh.Action.Data.Attachment.Url,
-		}
-
-		if strings.Split(primary.Url, "/")[2] == "trello-attachments.s3.amazonaws.com" {
+		if attachmentIsUploaded(wh.Action.Data.Attachment) {
 			// this file was uploaded on Trello, we must save a
-			// secondary copy (on the same card)
-			var secondary Attachment
-
-			// ensure we don't enter an infinite loop of backup saving
-			key := "replicate-attachment:" + wh.Action.Data.Card.Id +
-				"/" + wh.Action.Data.Attachment.Name
-			if stored := rds.Get(key).Val(); stored != "t" {
-				err = trello("post", "/1/cards/"+wh.Action.Data.Card.Id+"/attachments",
-					primary, &secondary)
-
-				if err != nil {
-					break
-				}
-
-				err = rds.Set(key, "t", time.Minute).Err()
-
-				primary.Id = wh.Action.Data.Attachment.Id
-
-				// save the primary as a backup to the secondary
-				saveBackupData(b, secondary.Id, primary)
-
-				// and vice-versa
-				saveBackupData(b, primary.Id, secondary)
+			// secondary copy (on s3, same path)
+			err = saveToS3(wh.Action.Data.Attachment.Id, wh.Action.Data.Attachment.Url)
+			if err != nil {
+				break
 			}
-		} else {
-			// just save the primary data as a backup to itself
-			saveBackupData(b, wh.Action.Data.Attachment.Id, primary)
 		}
+
+		go saveBackupData(b, wh.Action.Data.Attachment.Id, wh.Action.Data.Attachment)
+		err = updateBackupData(b, wh.Action.Data.Card.Id, wh.Action.Data.Card,
+			`'{"idAttachments": []}'::jsonb || $init || data`,
+			`jsonb_set(data, '{idAttachments}', (data->'idAttachments') || $arg)`,
+			wh.Action.Data.Attachment.Id)
 	case "deleteAttachmentFromCard":
-		err = deleteBackupData(b, wh.Action.Data.Attachment.Id)
+		go deleteBackupData(b, wh.Action.Data.Attachment.Id)
+		go deleteFromS3(wh.Action.Data.Attachment.Id)
+
+		err = updateBackupData(b, wh.Action.Data.Card.Id, wh.Action.Data.Card,
+			`'{"idAttachments": []}'::jsonb || $init || data`,
+			`jsonb_set(data, '{idAttachments}', (data->'idAttachments') - ($arg::jsonb#>>'{}'))`,
+			wh.Action.Data.Attachment.Id,
+		)
 	}
 
 	if err != nil {

attachments.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"os"
+
+	"github.com/minio/minio-go"
+)
+
+func saveToS3(id, trelloURL string) (err error) {
+	// download file from trello
+	file, err := ioutil.TempFile("", "trello-permissions-")
+	if err != nil {
+		file.Close()
+		return
+	}
+
+	resp, err := http.Get(trelloURL)
+	if err != nil {
+		file.Close()
+		return
+	}
+
+	_, err = io.Copy(file, resp.Body)
+	if err != nil {
+		file.Close()
+		return
+	}
+	file.Close()
+
+	// upload to s3
+	_, err = ms3.FPutObject(s.S3BucketName, id, file.Name(),
+		minio.PutObjectOptions{})
+
+	return
+}
+
+func restoreFromS3(attId, attName, cardId, token string) (err error) {
+	file, err := ioutil.TempFile("", "trello-permissions-")
+	if err != nil {
+		return
+	}
+
+	path := file.Name()
+	file.Close()
+
+	// download file from s3
+	err = ms3.FGetObject(s.S3BucketName,
+		attId,
+		path,
+		minio.GetObjectOptions{})
+	if err != nil {
+		return
+	}
+
+	// upload files to trello
+	file, err = os.Open(path)
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	post := &bytes.Buffer{}
+	writer := multipart.NewWriter(post)
+	part, err := writer.CreateFormFile("file", path)
+	if err != nil {
+		return
+	}
+
+	_, err = io.Copy(part, file)
+	if err != nil {
+		return
+	}
+
+	writer.WriteField("name", attName)
+	writer.WriteField("key", s.TrelloApiKey)
+	writer.WriteField("token", token)
+
+	err = writer.Close()
+	if err != nil {
+		return
+	}
+
+	req, err := http.NewRequest("POST",
+		"https://api.trello.com/1/cards/"+cardId+"/attachments", post)
+	if err != nil {
+		return
+	}
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+	_, err = http.DefaultClient.Do(req)
+	return
+}
+
+func deleteFromS3(id string) (err error) {
+	return ms3.RemoveObject(s.S3BucketName, id)
+}

helpers.go

@@ -186,3 +186,27 @@ func deleteBackupData(boardId, id string) (err error) {
 	_, err = pg.Exec(`DELETE FROM backups WHERE id = $1 AND board = $2`, id, boardId)
 	return
 }
+
+func itemJustConvertedIntoCard(cardName, parentChecklistId string) (id string, err error) {
+	err = pg.Get(&id, `
+WITH
+potential_checkitems AS (
+  SELECT id, data->'id' AS json_id FROM backups
+  WHERE data->>'name' = $1 AND NOT data ? 'shortLink'
+),
+parent_checklist AS (
+  SELECT id, data->'idCheckItems' AS idCheckItems FROM backups
+  WHERE id = $2
+)
+SELECT potential_checkitems.id
+FROM potential_checkitems
+INNER JOIN parent_checklist ON potential_checkitems.json_id <@ idCheckItems
+LIMIT 1
+    `, cardName, parentChecklistId)
+	return
+}
+
+func attachmentIsUploaded(attachment Attachment) bool {
+	attHost := strings.Split(attachment.Url, "/")[2]
+	return attHost == "trello-attachments.s3.amazonaws.com"
+}

main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/jmoiron/sqlx"
 	"github.com/kelseyhightower/envconfig"
 	_ "github.com/lib/pq"
+	"github.com/minio/minio-go"
 	"github.com/rs/zerolog"
 	"gopkg.in/redis.v5"
 	"gopkg.in/tylerb/graceful.v1"
@@ -29,12 +30,16 @@ type Settings struct {
 	TrelloApiKey    string `envconfig:"TRELLO_API_KEY" required:"true"`
 	TrelloApiSecret string `envconfig:"TRELLO_API_SECRET"`
 	RedisURL        string `envconfig:"REDIS_URL"`
+	AWSKeyId        string `envconfig:"AWS_KEY_ID" required:"true"`
+	AWSSecretKey    string `envconfig:"AWS_SECRET_KEY" required:"true"`
+	S3BucketName    string `envconfig:"S3_BUCKET_NAME" required:"true"`
 }
 
 var err error
 var s Settings
 var pg *sqlx.DB
 var rds *redis.Client
+var ms3 *minio.Client
 var router *mux.Router
 var schema graphql.Schema
 var log = zerolog.New(os.Stderr).Output(zerolog.ConsoleWriter{Out: os.Stderr})
@@ -47,6 +52,14 @@ func main() {
 
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 
+	// minio s3 client
+	ms3, _ = minio.New(
+		"s3.amazonaws.com",
+		s.AWSKeyId,
+		s.AWSSecretKey,
+		true,
+	)
+
 	// graphql schema
 	schema, err = graphql.NewSchema(schemaConfig)
 	if err != nil {

postgres.sql

@@ -17,25 +17,3 @@ CREATE TABLE backups (
 table boards;
 table backups;
 delete from backups;
-
-WITH
-init AS (
-  SELECT ('{"id":"5b4a910f0aa8cfb949529fb9","name":"Checklist"}' || '{"idCheckItems": []}'::jsonb || data) AS data
-  FROM (
-    SELECT 0 AS idx, data FROM backups WHERE id = '5b4a910f0aa8cfb949529fb9'
-    UNION ALL
-    SELECT 1 AS idx, '{}'::jsonb
-  ) AS whatever
-  ORDER BY idx LIMIT 1
-),
-new AS (
-  SELECT jsonb_set(
-    data,
-    '{idCheckItems}',
-    data->'idCheckItems' || '"5b4a9232a495da35ccca80ea"'
-  ) AS data
-    FROM init
-)
-INSERT INTO backups (id, board, data) VALUES ('5b4a910f0aa8cfb949529fb9', '5b1980e92c9e71f5e06ad718', (SELECT data FROM new))
-  ON CONFLICT (id) DO UPDATE
-    SET data = (SELECT data FROM new);

types.go

@@ -39,11 +39,13 @@ type Card struct {
 	Desc              string            `json:"desc,omitempty"`
 	Due               string            `json:"due,omitempty"`
 	DueComplete       bool              `json:"dueComplete,omitempty"`
+	Closed            bool              `json:"closed,omitempty"`
 	Pos               float64           `json:"pos,omitempty"`
 	IdAttachmentCover string            `json:"idAttachmentCover,omitempty"`
 	IdMembers         []string          `json:"idMembers,omitempty"`
 	IdLabels          []string          `json:"idLabels,omitempty"`
 	IdChecklists      []string          `json:"idChecklists,omitempty"`
+	IdAttachments     []string          `json:"idAttachments,omitempty"`
 	Checklists        []Checklist       `json:"checklists,omitempty"`
 	Attachments       []Attachment      `json:"attachments,omitempty"`
 	CustomFieldItems  []CustomFieldItem `json:"customFieldItems,omitempty"`
@@ -57,10 +59,11 @@ type Checklist struct {
 }
 
 type CheckItem struct {
-	Id    string  `json:"id,omitempty"`
-	Name  string  `json:"name,omitempty"`
-	State string  `json:"state,omitempty"`
-	Pos   float64 `json:"pos,omitempty"`
+	Id      string  `json:"id,omitempty"`
+	Name    string  `json:"name,omitempty"`
+	State   string  `json:"state,omitempty"`
+	Checked bool    `json:"checked,omitempty"`
+	Pos     float64 `json:"pos,omitempty"`
 }
 
 type Attachment struct {