Options data format & Deribit options scraper (#194)

* Coinflex scraper + refactors for Deribit, Huobi, FTX (#190) (#1)

* [+] added huobi futures scraper

* [*] 1. refactored futures huobi scraper && 2. ftx futures scraper

* [+] futures ftx: not writing the pongs & subscription & unsubscription messages

* [+] futures scraper: added deribit & refactors to all other futures scrapers

* [*] refactors: ftx, deribit, huobi

* [+] futures scarper: coinflex

* [+] deribit scraper: 1. unique refresh token for each scraper; 2. better handling of failure to refresh the token; 3. better handling of errors; 4. better saving of trades; 5. only futures markets allowed in the futures scraper (excluded options);

* [+] futures-scrapers: 1. better retry after error; 2. better file writing

* [+] deribit: generalized code to accommodate for options

* [+] deribit: options scraper introduced & options data

internal/pkg/exchange-scrapers/APIDerivativesScraper.go

@@ -0,0 +1,35 @@
+package scrapers
+
+import (
+	"log"
+	"sync"
+
+	writers "github.com/diadata-org/diadata/internal/pkg/scraper-writers"
+)
+
+// DeribitScraperKind - used to distinguish between the futures and options scrapers
+type DeribitScraperKind int
+
+const (
+	// DeribitFuture - constant to signal the futures scraper
+	DeribitFuture DeribitScraperKind = iota + 1
+	// DeribitOption - constant to signal the options scraper
+	DeribitOption
+)
+
+// DeribitScraper - used in conjunction with the DeribitScraperKind in a new struct to define futures and options scrapers
+type DeribitScraper struct {
+	Markets   []string
+	WaitGroup *sync.WaitGroup
+	Writer    writers.Writer
+	Logger    *log.Logger
+
+	// required for deribit to:
+	// 1. authenticate (trades is a private channel)
+	// 2. referesh the token from step 1., so that the channel isn't closed
+	AccessKey    string
+	AccessSecret string
+
+	RefreshTokenEvery int16 // how often we refresh the token (in seconds)
+	MarketKind        DeribitScraperKind
+}

internal/pkg/exchange-scrapers/APIOptionsScraper.go

@@ -0,0 +1,21 @@
+package scrapers
+
+import "time"
+
+// OptionsScraper is an interface for all of the Options Contracts scrapers
+type OptionsScraper interface {
+	Scrape(market string) // a self-sustained goroutine that scrapes a single market
+	ScrapeMarkets()       // will scrape the options markets defined during instantiation of the scraper
+	ScraperClose(market string, websocketConnection interface{}) error
+	Authenticate(market string, websocketConnection interface{}) error
+}
+
+// OptionOrderbookDatum is a unit of option order book data. Other meta thing like expiration time and strike price can be queried from the meta files / db.
+type OptionOrderbookDatum struct {
+	InstrumentName  string
+	ObservationTime time.Time
+	AskPrice        float64
+	BidPrice        float64
+	AskSize         float64
+	BidSize         float64
+}

internal/pkg/exchange-scrapers/FuturesCoinflexScraper.go

@@ -139,6 +139,7 @@ func (s *CoinflexFuturesScraper) Scrape(market string) {
 			ws, _, err := websocket.DefaultDialer.Dial(u.String(), nil)
 			if err != nil {
 				s.Logger.Printf("[ERROR] dial: %s", err)
+				time.Sleep(time.Duration(retryIn) * time.Second)
 				return
 			}
 			defer s.ScraperClose(market, ws)
@@ -190,7 +191,7 @@ func (s *CoinflexFuturesScraper) Scrape(market string) {
 				s.Logger.Printf("[DEBUG] received a message: %s", message)
 				if msg.Notice == "OrdersMatched" {
 					s.Logger.Printf("[DEBUG] received new match message on [%s]: %s", market, message)
-					_, err = s.Writer.Write(string(message)+"|", scrapeDataSaveLocationCoinflex+s.Writer.GetWriteFileName("coinflex", market))
+					_, err = s.Writer.Write(string(message)+"\n", scrapeDataSaveLocationCoinflex+s.Writer.GetWriteFileName("coinflex", market))
 					if err != nil {
 						s.Logger.Printf("[ERROR] could not save to file: %s, on market: [%s], err: %s", scrapeDataSaveLocationCoinflex+s.Writer.GetWriteFileName("coinflex", market), market, err)
 						return