From a53ff76d0fcdbeba5fcf1b7914c42ed85972f377 Mon Sep 17 00:00:00 2001 From: Utku Ufuk Date: Sun, 31 May 2020 02:23:53 +0300 Subject: [PATCH] delete stale cards if in strict mode & handle sources concurrently --- cmd/entrello/main.go | 50 ++++++++++++++++++++++- cmd/entrello/source.go | 26 +++++++++--- internal/trello/trello.go | 86 +++++++++++++++++++++------------------ 3 files changed, 115 insertions(+), 47 deletions(-) diff --git a/cmd/entrello/main.go b/cmd/entrello/main.go index 9c882e3..f8d5dfc 100644 --- a/cmd/entrello/main.go +++ b/cmd/entrello/main.go @@ -2,35 +2,81 @@ package main import ( "log" + "sync" "github.com/utkuufuk/entrello/internal/config" "github.com/utkuufuk/entrello/internal/trello" ) +type CardQueue struct { + add chan trello.Card + del chan trello.Card + err chan error +} + func main() { + // read config params cfg, err := config.ReadConfig("config.yml") if err != nil { // @todo: send telegram notification instead if enabled log.Fatalf("[-] could not read config variables: %v", err) } + // get a list of enabled sources and the corresponding labels for each source sources, labels := getEnabledSourcesAndLabels(cfg.Sources) if len(sources) == 0 { return } + // initialize the Trello client client, err := trello.NewClient(cfg) if err != nil { // @todo: send telegram notification instead if enabled log.Fatalf("[-] could not create trello client: %v", err) } - if err := client.LoadExistingCards(labels); err != nil { + // within the Trello client, load the existing cards (only with relevant labels) + if err := client.LoadCards(labels); err != nil { // @todo: send telegram notification instead if enabled log.Fatalf("[-] could not load existing cards from the board: %v", err) } + // initialize channels, then start listening each source for cards to create/delete and errors + q := CardQueue{ + add: make(chan trello.Card), + del: make(chan trello.Card), + err: make(chan error), + } + + var wg sync.WaitGroup + wg.Add(len(sources)) for _, src := range sources { - process(client, src) + go queueActionables(src, client, q, &wg) } + + go func() { + for { + select { + case c := <-q.add: + // @todo: send telegram notification instead if enabled + if err := client.CreateCard(c); err != nil { + log.Printf("[-] error occurred while creating card: %v", err) + break + } + log.Printf("[+] created new card: %s", c.Name) + case c := <-q.del: + // @todo: send telegram notification instead if enabled + if err := client.ArchiveCard(c); err != nil { + log.Printf("[-] error occurred while archiving card: %v", err) + break + } + log.Printf("[+] archived stale card: %s", c.Name) + case err := <-q.err: + // @todo: send telegram notification instead if enabled + log.Printf("[-] %v", err) + } + } + }() + + wg.Wait() } diff --git a/cmd/entrello/source.go b/cmd/entrello/source.go index e48af98..e62df13 100644 --- a/cmd/entrello/source.go +++ b/cmd/entrello/source.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "sync" "time" "github.com/utkuufuk/entrello/internal/config" @@ -89,16 +90,29 @@ func shouldQuery(src source, now time.Time) (bool, error) { return false, fmt.Errorf("unrecognized source period type: '%s'", src.GetPeriod().Type) } -func process(client trello.Client, src source) { +// queueActionables fetches new cards from the source, then pushes those to be created and +// to be deleted into the corresponding channels, as well as any errors encountered. +func queueActionables(src source, client trello.Client, q CardQueue, wg *sync.WaitGroup) { + defer wg.Done() + cards, err := src.FetchNewCards() if err != nil { - // @todo: send telegram notification instead if enabled - log.Printf("[-] could not get cards for source '%s': %v", src.GetName(), err) + q.err <- fmt.Errorf("could not fetch cards for source '%s': %v", src.GetName(), err) + return + } + + new, stale := client.CompareWithExisting(cards, src.GetLabel()) + fmt.Printf("%s\nnew: %v\nstale:%v\n", src.GetName(), new, stale) + + for _, c := range new { + q.add <- c + } + + if !src.IsStrict() { return } - if err := client.UpdateCards(cards, src.IsStrict()); err != nil { - // @todo: send telegram notification instead if enabled - log.Printf("[-] error occurred while processing source '%s': %v", src.GetName(), err) + for _, c := range stale { + q.del <- c } } diff --git a/internal/trello/trello.go b/internal/trello/trello.go index 1c9aa99..c9b656b 100644 --- a/internal/trello/trello.go +++ b/internal/trello/trello.go @@ -8,10 +8,12 @@ import ( "github.com/utkuufuk/entrello/internal/config" ) +type Card *trello.Card + // Client represents a Trello client model type Client struct { // client is the Trello API client - client *trello.Client + api *trello.Client // boardId is the ID of the board to read & write cards boardId string @@ -20,26 +22,19 @@ type Client struct { listId string // a map of existing cards in the board, where the key is the label ID and value is the card name - existingCards map[string][]string -} - -// Card represents a Trello card -type Card struct { - name string - label string - description string - dueDate *time.Time + existingCards map[string][]Card } -func NewClient(c config.Config) (Client, error) { +func NewClient(c config.Config) (client Client, err error) { if c.BoardId == "" || c.ListId == "" || c.TrelloApiKey == "" || c.TrelloApiToken == "" { - return Client{}, fmt.Errorf("could not create trello client, missing configuration parameter(s)") + return client, fmt.Errorf("could not create trello client, missing configuration parameter(s)") } + return Client{ - client: trello.NewClient(c.TrelloApiKey, c.TrelloApiToken), + api: trello.NewClient(c.TrelloApiKey, c.TrelloApiToken), boardId: c.BoardId, listId: c.ListId, - existingCards: make(map[string][]string), + existingCards: make(map[string][]Card), }, nil } @@ -57,13 +52,17 @@ func NewCard(name, label, description string, dueDate *time.Time) (card Card, er if description == "" { return card, fmt.Errorf("description cannot be blank") } - return Card{name, label, description, dueDate}, nil + return &trello.Card{ + Name: name, + Desc: description, + Due: dueDate, + IDLabels: []string{label}, + }, nil } -// LoadExistingCards retrieves and saves all existing cards from the board that has at least one -// of the given label IDs -func (c Client) LoadExistingCards(labels []string) error { - board, err := c.client.GetBoard(c.boardId, trello.Defaults()) +// LoadCards retrieves existing cards from the board that have at least one of the given label IDs +func (c Client) LoadCards(labels []string) error { + board, err := c.api.GetBoard(c.boardId, trello.Defaults()) if err != nil { return fmt.Errorf("could not get board data: %w", err) } @@ -74,43 +73,52 @@ func (c Client) LoadExistingCards(labels []string) error { } for _, label := range labels { - c.existingCards[label] = make([]string, 0, len(cards)) + c.existingCards[label] = make([]Card, 0, len(cards)) } for _, card := range cards { for _, label := range card.IDLabels { - c.existingCards[label] = append(c.existingCards[label], card.Name) + c.existingCards[label] = append(c.existingCards[label], card) } } return nil } -// UpdateCards creates the given cards except the ones that already exist. -// Also deletes the stale cards if strict mode is enabled. -func (c Client) UpdateCards(cards []Card, strict bool) error { +// CompareWithExisting +func (c Client) CompareWithExisting(cards []Card, label string) (new, stale []Card) { + m := make(map[string]*trello.Card) + for _, card := range c.existingCards[label] { + m[card.Name] = card + } + for _, card := range cards { - if contains(c.existingCards[card.label], card.name) { + _, ok := m[card.Name] + m[card.Name] = nil + if ok { continue } + new = append(new, card) + } - if err := c.createCard(card); err != nil { - return fmt.Errorf("[-] could not create card '%s': %v", card.name, err) + for _, card := range m { + if card == nil { + continue } - - // @todo: send telegram notification if enabled + stale = append(stale, card) } - return nil + + return new, stale +} + +// CreateCard creates a Trello card using the the Trello API +func (c Client) CreateCard(card Card) error { + card.IDList = c.listId + return c.api.CreateCard(card, trello.Defaults()) } -// createCard creates a Trello card using the the API client -func (c Client) createCard(card Card) error { - return c.client.CreateCard(&trello.Card{ - Name: card.name, - Desc: card.description, - Due: card.dueDate, - IDList: c.listId, - IDLabels: []string{card.label}, - }, trello.Defaults()) +// ArchiveCard archives a Trello card using the the Trello API +func (c Client) ArchiveCard(card Card) error { + return (*trello.Card)(card).Update(trello.Arguments{"closed": "true"}) } // contains returns true if the list of strings contain the given string