skip downloading when file protocol is file:// (#201)

* skip downloading local files

* updated csv and json clients to separate reading remote and local files

Co-authored-by: Maziar <maziar@appbase.io>

importer/adaptor/csv/client.go

@@ -114,14 +114,17 @@ func (c *Client) initConnection() error {
 	var name string
 	name = strings.Replace(c.uri, "file://", "", 1)
 
-	// read file from remote url
-	if url, err := url.ParseRequestURI(c.uri); err == nil {
-		name = fmt.Sprintf("%s/%s.csv", common.DefaultDownloadDirectory, fmt.Sprintf("%d", time.Now().Unix()))
-		if err := common.DownloadFile(name, url.String()); err != nil {
-			return err
+	// read file from remote url if protocol contains http
+	proto := strings.Split(c.uri, "//")
+	if strings.Contains(proto[0], `http`) {
+		if url, err := url.ParseRequestURI(c.uri); err == nil {
+			name = fmt.Sprintf("%s/%s.csv", common.DefaultDownloadDirectory, fmt.Sprintf("%d", time.Now().Unix()))
+			if err := common.DownloadFile(name, url.String()); err != nil {
+				return err
+			}
+			log.Infoln("Download complete for:", name)
+			c.deleteFileAfterUsage = true
 		}
-		log.Infoln("Download complete for:", name)
-		c.deleteFileAfterUsage = true
 	}
 	f, err := os.OpenFile(name, os.O_RDWR, 0666)
 	if err != nil {

importer/adaptor/json/client.go

@@ -99,15 +99,18 @@ func (c *Client) initFile() error {
 	var name string
 	name = strings.Replace(c.uri, "file://", "", 1)
 
-	// read file from remote url
-	if url, err := url.ParseRequestURI(c.uri); err == nil {
-		name = fmt.Sprintf("%s/%s.csv", common.DefaultDownloadDirectory, fmt.Sprintf("%d", time.Now().Unix()))
-		if err := common.DownloadFile(name, url.String()); err != nil {
-			return err
+	// read file from remote url if protocol contains http
+	proto := strings.Split(c.uri, "//")
+	if strings.Contains(proto[0], `http`) {
+		if url, err := url.ParseRequestURI(c.uri); err == nil {
+			name = fmt.Sprintf("%s/%s.csv", common.DefaultDownloadDirectory, fmt.Sprintf("%d", time.Now().Unix()))
+			if err := common.DownloadFile(name, url.String()); err != nil {
+				return err
+			}
+			log.Infoln("Download complete for:", name)
+			c.uri = name
+			c.deleteFileAfterUsage = true
 		}
-		log.Infoln("Download complete for:", name)
-		c.uri = name
-		c.deleteFileAfterUsage = true
 	}
 
 	f, err := os.OpenFile(name, os.O_RDWR, 0666)

importer/adaptor/jsonl/client.go

@@ -95,13 +95,16 @@ func (c *Client) initFile() error {
 		return nil
 	}
 	name := strings.Replace(c.uri, "file://", "", 1)
-	if url, err := url.ParseRequestURI(c.uri); err == nil {
-		name = fmt.Sprintf("%s/%s.csv", common.DefaultDownloadDirectory, fmt.Sprintf("%d", time.Now().Unix()))
-		if err := common.DownloadFile(name, url.String()); err != nil {
-			return err
+	proto := strings.Split(c.uri, "//")
+	if strings.Contains(proto[0], `http`) {
+		if url, err := url.ParseRequestURI(c.uri); err == nil {
+			name = fmt.Sprintf("%s/%s.csv", common.DefaultDownloadDirectory, fmt.Sprintf("%d", time.Now().Unix()))
+			if err := common.DownloadFile(name, url.String()); err != nil {
+				return err
+			}
+			log.Infoln("Download complete for:", name)
+			c.deleteFileAfterUsage = true
 		}
-		log.Infoln("Download complete for:", name)
-		c.deleteFileAfterUsage = true
 	}
 
 	f, err := os.OpenFile(name, os.O_RDWR, 0666)