Expand env variables in the api url

.gitignore

@@ -1,11 +1,9 @@
 */saved.json
-.env
-*/.env
 coverage.out
 
 pkg/pow/kernelSource.go
 pkg/contracts
 
-/tellor
+configs/.env
 
-/configs/myconfig*
+/tellor

CHANGELOG.md

@@ -9,6 +9,11 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 
 We use *breaking :warning:* to mark changes that are not backward compatible (relates only to v0.y.z releases.)
 
+## Unreleased
+- [#254](https://github.com/tellor-io/TellorMiner/pull/254)
+    - Added support for expanding variables in the indexer api url.
+    - Added config to specify the `.env` file location. The default is now `configs/.env` so users should either specify a custom location in the `config.json` or move it inside the default config folder.
+
 ## [v5.0.0](https://github.com/tellor-io/TellorMiner/releases) - 2020.11.02
 
 ### Added

configs/.env.example

@@ -0,0 +1,2 @@
+PRIVATE_KEY=00000000000000000000000000000000000000000000000000
+VIXEOD_KEY=xxxx

configs/config.json

@@ -8,5 +8,6 @@
     "serverPort": 5000,
     "gasMax": 10,
     "dbFile": "/tellorDB",
-    "configFolder": "/configs"
+    "configFolder": "/configs",
+    "envFile": "/configs/.env.example"
 }

configs/indexes.json

@@ -189,5 +189,8 @@
 	],
 	"ZRX/USD": [
 		"json(https://api.binance.com/api/v1/klines?symbol=ZRXUSDT&interval=1d&limit=1).0.4"
+	],
+	"VIXEOD": [
+		"json(https://www.quandl.com/api/v3/datasets/CHRIS/CBOE_VX1.json?api_key=${VIXEOD_KEY}).dataset.data.0.4"
 	]
 }

pkg/config/config.go

@@ -9,10 +9,12 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"strings"
 	"time"
 
 	"github.com/joho/godotenv"
+	"github.com/pkg/errors"
 )
 
 // Unfortunate hack to enable json parsing of human readable time strings
@@ -93,8 +95,12 @@ type Config struct {
 	ProfitThreshold uint64 `json:"profitThreshold"`
 	// Config parameters excluded from the json config file.
 	PrivateKey string `json:"privateKey"`
+	// EnvFile location that include all private details like private key etc.
+	EnvFile string `json:"envFile"`
 }
 
+const ConfigFolder = "configs"
+
 var config = Config{
 	GasMax:                       10,
 	GasMultiplier:                1,
@@ -118,6 +124,8 @@ var config = Config{
 		"indexers":         true,
 		"disputeChecker":   false,
 	},
+	ConfigFolder: ConfigFolder,
+	EnvFile:      path.Join(ConfigFolder, ".env"),
 }
 
 const PrivateKeyEnvName = "ETH_PRIVATE_KEY"
@@ -139,10 +147,9 @@ func ParseConfigBytes(data []byte) error {
 	}
 	// Check if the env is already set, only try loading .env if its not there.
 	if config.PrivateKey == "" {
-		// Load the env
-		err = godotenv.Load()
+		err = godotenv.Load(config.EnvFile)
 		if err != nil {
-			return fmt.Errorf("error reading .env file: %v", err.Error())
+			return errors.Wrap(err, "loading .env file")
 		}
 
 		config.PrivateKey = os.Getenv(PrivateKeyEnvName)

pkg/tracker/index.go

@@ -9,11 +9,14 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/url"
+	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 
 	"github.com/benbjohnson/clock"
+	"github.com/joho/godotenv"
+	"github.com/pkg/errors"
 	"github.com/tellor-io/TellorMiner/pkg/apiOracle"
 	"github.com/tellor-io/TellorMiner/pkg/config"
 	"github.com/tellor-io/TellorMiner/pkg/util"
@@ -56,10 +59,21 @@ func BuildIndexTrackers() ([]Tracker, error) {
 
 	for symbol, apis := range baseIndexes {
 		for _, api := range apis {
-			//did we already have a tracker for this API string?
+			// Tracker for this API already added?
 			_, ok := indexers[api]
 			if !ok {
 				pathStr, args := util.ParseQueryString(api)
+
+				// Expand any env variables with their values from the .env file.
+				vars, err := godotenv.Read(cfg.EnvFile)
+				// Ignore file doesn't exist errors.
+				if _, ok := err.(*os.PathError); err != nil && !ok {
+					return nil, errors.Wrap(err, "reading .env file")
+				}
+				pathStr = os.Expand(pathStr, func(key string) string {
+					return vars[key]
+				})
+
 				var name string
 				var source DataSource
 				if strings.HasPrefix(pathStr, "http") {