Skip to content
Permalink
Browse files

attachments on s3 and numerous other fixes.

including
  - fixes on deleteCard and moveCardTo/FromBoard
  - fixes on checklists and checkItems
  - renaming and other quibbles

missing
  - comments
  - custom fields
  - testing? I guess I'll just test manually.
  • Loading branch information...
fiatjaf committed Jul 18, 2018
1 parent aba74f6 commit 7f77de94e4935e9d98520d98ec3bcdc4e09e2f12
Showing with 289 additions and 120 deletions.
  1. +42 −44 allowed.go
  2. +100 −0 attachments.go
  3. +24 −0 helpers.go
  4. +13 −0 main.go
  5. +0 −22 postgres.sql
  6. +7 −4 types.go
  7. +101 −48 unallowed.go
  8. +2 −2 webhooks.go
@@ -1,7 +1,6 @@
package main

import (
"strings"
"time"

"github.com/jmoiron/sqlx/types"
@@ -10,19 +9,32 @@ import (
"github.com/rs/zerolog"
)

func onAllowed(logger zerolog.Logger, trello trelloClient, wh Webhook) {
func onAllowed(logger zerolog.Logger, token string, wh Webhook) {
b := wh.Action.Data.Board.Id

switch wh.Action.Type {
case "createCard", "copyCard", "convertToCardFromCheckItem", "moveCardToBoard":
// if a card is moved from another tracked board to this board
// this will give time to the other webhook to delete everything from the backups
// table so we can recreate everything here.
time.Sleep(time.Second * 2)
if wh.Action.Type == "moveCardToBoard" {
// if a card is moved from another tracked board to this board
// this will give time to the other webhook to delete everything from the backups
// table so we can recreate everything here.
time.Sleep(time.Second * 2)
} else if wh.Action.Type == "convertToCardFromCheckItem" {
// we must proceed as if deleting the checkItem here
checkItemId, err := itemJustConvertedIntoCard(
wh.Action.Data.Card.Name,
wh.Action.Data.Checklist.Id,
)
if err == nil {
wh.Action.Type = "deleteCheckItem"
wh.Action.Data.CheckItem.Id = checkItemId
onAllowed(logger, token, wh)
}
}

saveBackupData(b, wh.Action.Data.Card.Id, wh.Action.Data.Card)
case "deleteCard", "moveCardFromBoard":
// delete card, checklists and checkitems
// delete card, checklists and checkItems
var card Card
fetchBackupData(wh.Action.Data.Card.Id, &card)
for _, idChecklist := range card.IdChecklists {
@@ -86,7 +98,7 @@ func onAllowed(logger zerolog.Logger, trello trelloClient, wh Webhook) {
case "updateChecklist":
err = saveBackupData(b, wh.Action.Data.Checklist.Id, wh.Action.Data.Checklist)
case "removeChecklistFromCard":
// delete checkitems and checklist
// delete checkItems and checklist
var checklist Checklist
fetchBackupData(wh.Action.Data.Checklist.Id, &checklist)
for _, idCheckItem := range checklist.IdCheckItems {
@@ -95,7 +107,7 @@ func onAllowed(logger zerolog.Logger, trello trelloClient, wh Webhook) {
pretty.Log(err)
}
}
deleteBackupData(b, wh.Action.Data.Checklist.Id)
go deleteBackupData(b, wh.Action.Data.Checklist.Id)

// update card
err = updateBackupData(b, wh.Action.Data.Card.Id, wh.Action.Data.Card,
@@ -104,7 +116,7 @@ func onAllowed(logger zerolog.Logger, trello trelloClient, wh Webhook) {
wh.Action.Data.Checklist.Id,
)
case "createCheckItem":
// create checkitem on database
// create checkItem on database
go saveBackupData(b, wh.Action.Data.CheckItem.Id, wh.Action.Data.CheckItem)

// update checklist
@@ -116,7 +128,7 @@ func onAllowed(logger zerolog.Logger, trello trelloClient, wh Webhook) {
case "updateCheckItem", "updateCheckItemStateOnCard":
err = saveBackupData(b, wh.Action.Data.CheckItem.Id, wh.Action.Data.CheckItem)
case "deleteCheckItem":
// delete checkitem
// delete checkItem
deleteBackupData(b, wh.Action.Data.CheckItem.Id)

// update checklist
@@ -126,43 +138,29 @@ func onAllowed(logger zerolog.Logger, trello trelloClient, wh Webhook) {
wh.Action.Data.CheckItem.Id,
)
case "addAttachmentToCard":
primary := Attachment{
Name: wh.Action.Data.Attachment.Name,
Url: wh.Action.Data.Attachment.Url,
}

if strings.Split(primary.Url, "/")[2] == "trello-attachments.s3.amazonaws.com" {
if attachmentIsUploaded(wh.Action.Data.Attachment) {
// this file was uploaded on Trello, we must save a
// secondary copy (on the same card)
var secondary Attachment

// ensure we don't enter an infinite loop of backup saving
key := "replicate-attachment:" + wh.Action.Data.Card.Id +
"/" + wh.Action.Data.Attachment.Name
if stored := rds.Get(key).Val(); stored != "t" {
err = trello("post", "/1/cards/"+wh.Action.Data.Card.Id+"/attachments",
primary, &secondary)

if err != nil {
break
}

err = rds.Set(key, "t", time.Minute).Err()

primary.Id = wh.Action.Data.Attachment.Id

// save the primary as a backup to the secondary
saveBackupData(b, secondary.Id, primary)

// and vice-versa
saveBackupData(b, primary.Id, secondary)
// secondary copy (on s3, same path)
err = saveToS3(wh.Action.Data.Attachment.Id, wh.Action.Data.Attachment.Url)
if err != nil {
break
}
} else {
// just save the primary data as a backup to itself
saveBackupData(b, wh.Action.Data.Attachment.Id, primary)
}

go saveBackupData(b, wh.Action.Data.Attachment.Id, wh.Action.Data.Attachment)
err = updateBackupData(b, wh.Action.Data.Card.Id, wh.Action.Data.Card,
`'{"idAttachments": []}'::jsonb || $init || data`,
`jsonb_set(data, '{idAttachments}', (data->'idAttachments') || $arg)`,
wh.Action.Data.Attachment.Id)
case "deleteAttachmentFromCard":
err = deleteBackupData(b, wh.Action.Data.Attachment.Id)
go deleteBackupData(b, wh.Action.Data.Attachment.Id)
go deleteFromS3(wh.Action.Data.Attachment.Id)

err = updateBackupData(b, wh.Action.Data.Card.Id, wh.Action.Data.Card,
`'{"idAttachments": []}'::jsonb || $init || data`,
`jsonb_set(data, '{idAttachments}', (data->'idAttachments') - ($arg::jsonb#>>'{}'))`,
wh.Action.Data.Attachment.Id,
)
}

if err != nil {
@@ -0,0 +1,100 @@
package main

import (
"bytes"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"os"

"github.com/minio/minio-go"
)

func saveToS3(id, trelloURL string) (err error) {
// download file from trello
file, err := ioutil.TempFile("", "trello-permissions-")
if err != nil {
file.Close()
return
}

resp, err := http.Get(trelloURL)
if err != nil {
file.Close()
return
}

_, err = io.Copy(file, resp.Body)
if err != nil {
file.Close()
return
}
file.Close()

// upload to s3
_, err = ms3.FPutObject(s.S3BucketName, id, file.Name(),
minio.PutObjectOptions{})

return
}

func restoreFromS3(attId, attName, cardId, token string) (err error) {
file, err := ioutil.TempFile("", "trello-permissions-")
if err != nil {
return
}

path := file.Name()
file.Close()

// download file from s3
err = ms3.FGetObject(s.S3BucketName,
attId,
path,
minio.GetObjectOptions{})
if err != nil {
return
}

// upload files to trello
file, err = os.Open(path)
if err != nil {
return
}
defer file.Close()

post := &bytes.Buffer{}
writer := multipart.NewWriter(post)
part, err := writer.CreateFormFile("file", path)
if err != nil {
return
}

_, err = io.Copy(part, file)
if err != nil {
return
}

writer.WriteField("name", attName)
writer.WriteField("key", s.TrelloApiKey)
writer.WriteField("token", token)

err = writer.Close()
if err != nil {
return
}

req, err := http.NewRequest("POST",
"https://api.trello.com/1/cards/"+cardId+"/attachments", post)
if err != nil {
return
}
req.Header.Set("Content-Type", writer.FormDataContentType())
_, err = http.DefaultClient.Do(req)
return
}

func deleteFromS3(id string) (err error) {
return ms3.RemoveObject(s.S3BucketName, id)
}
@@ -186,3 +186,27 @@ func deleteBackupData(boardId, id string) (err error) {
_, err = pg.Exec(`DELETE FROM backups WHERE id = $1 AND board = $2`, id, boardId)
return
}

func itemJustConvertedIntoCard(cardName, parentChecklistId string) (id string, err error) {
err = pg.Get(&id, `
WITH
potential_checkitems AS (
SELECT id, data->'id' AS json_id FROM backups
WHERE data->>'name' = $1 AND NOT data ? 'shortLink'
),
parent_checklist AS (
SELECT id, data->'idCheckItems' AS idCheckItems FROM backups
WHERE id = $2
)
SELECT potential_checkitems.id
FROM potential_checkitems
INNER JOIN parent_checklist ON potential_checkitems.json_id <@ idCheckItems
LIMIT 1
`, cardName, parentChecklistId)
return
}

func attachmentIsUploaded(attachment Attachment) bool {
attHost := strings.Split(attachment.Url, "/")[2]
return attHost == "trello-attachments.s3.amazonaws.com"
}
13 main.go
@@ -16,6 +16,7 @@ import (
"github.com/jmoiron/sqlx"
"github.com/kelseyhightower/envconfig"
_ "github.com/lib/pq"
"github.com/minio/minio-go"
"github.com/rs/zerolog"
"gopkg.in/redis.v5"
"gopkg.in/tylerb/graceful.v1"
@@ -29,12 +30,16 @@ type Settings struct {
TrelloApiKey string `envconfig:"TRELLO_API_KEY" required:"true"`
TrelloApiSecret string `envconfig:"TRELLO_API_SECRET"`
RedisURL string `envconfig:"REDIS_URL"`
AWSKeyId string `envconfig:"AWS_KEY_ID" required:"true"`
AWSSecretKey string `envconfig:"AWS_SECRET_KEY" required:"true"`
S3BucketName string `envconfig:"S3_BUCKET_NAME" required:"true"`
}

var err error
var s Settings
var pg *sqlx.DB
var rds *redis.Client
var ms3 *minio.Client
var router *mux.Router
var schema graphql.Schema
var log = zerolog.New(os.Stderr).Output(zerolog.ConsoleWriter{Out: os.Stderr})
@@ -47,6 +52,14 @@ func main() {

zerolog.SetGlobalLevel(zerolog.DebugLevel)

// minio s3 client
ms3, _ = minio.New(
"s3.amazonaws.com",
s.AWSKeyId,
s.AWSSecretKey,
true,
)

// graphql schema
schema, err = graphql.NewSchema(schemaConfig)
if err != nil {
@@ -17,25 +17,3 @@ CREATE TABLE backups (
table boards;
table backups;
delete from backups;

WITH
init AS (
SELECT ('{"id":"5b4a910f0aa8cfb949529fb9","name":"Checklist"}' || '{"idCheckItems": []}'::jsonb || data) AS data
FROM (
SELECT 0 AS idx, data FROM backups WHERE id = '5b4a910f0aa8cfb949529fb9'
UNION ALL
SELECT 1 AS idx, '{}'::jsonb
) AS whatever
ORDER BY idx LIMIT 1
),
new AS (
SELECT jsonb_set(
data,
'{idCheckItems}',
data->'idCheckItems' || '"5b4a9232a495da35ccca80ea"'
) AS data
FROM init
)
INSERT INTO backups (id, board, data) VALUES ('5b4a910f0aa8cfb949529fb9', '5b1980e92c9e71f5e06ad718', (SELECT data FROM new))
ON CONFLICT (id) DO UPDATE
SET data = (SELECT data FROM new);
@@ -39,11 +39,13 @@ type Card struct {
Desc string `json:"desc,omitempty"`
Due string `json:"due,omitempty"`
DueComplete bool `json:"dueComplete,omitempty"`
Closed bool `json:"closed,omitempty"`
Pos float64 `json:"pos,omitempty"`
IdAttachmentCover string `json:"idAttachmentCover,omitempty"`
IdMembers []string `json:"idMembers,omitempty"`
IdLabels []string `json:"idLabels,omitempty"`
IdChecklists []string `json:"idChecklists,omitempty"`
IdAttachments []string `json:"idAttachments,omitempty"`
Checklists []Checklist `json:"checklists,omitempty"`
Attachments []Attachment `json:"attachments,omitempty"`
CustomFieldItems []CustomFieldItem `json:"customFieldItems,omitempty"`
@@ -57,10 +59,11 @@ type Checklist struct {
}

type CheckItem struct {
Id string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
State string `json:"state,omitempty"`
Pos float64 `json:"pos,omitempty"`
Id string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
State string `json:"state,omitempty"`
Checked bool `json:"checked,omitempty"`
Pos float64 `json:"pos,omitempty"`
}

type Attachment struct {
Oops, something went wrong.

0 comments on commit 7f77de9

Please sign in to comment.
You can’t perform that action at this time.