自动拉取数据源

.gitignore

@@ -1,3 +1,4 @@
 gonews
 nohup.out
-*.log
+*.log
+.idea/

README.md

@@ -12,7 +12,7 @@ gonews是基于`go+vue`实现的golang每日新闻浏览与检索平台
 ## 项目截图
 
 ![gonews](gonews.jpg)
-  
+
 ## 部署
 
 
@@ -28,12 +28,14 @@ git clone https://github.com/gocn/news /data/news
 go get -u github.com/mikemintang/gonews
 ```
 
-- 解析数据
+- 启动后台服务，拉取、解析数据
 
 ```
 nohup gonews -d /data/news > /data/log/gonews.log 2>&1 
 ```
 
+可选参数：-pull <minutes> ，拉取数据源的时间间隔，单位分钟，默认值60
+
 - 启动Api
 
 ```
@@ -69,23 +71,6 @@ server {
 }
 ```
 
-- Shell脚本
-
-```
-#!/bin/sh
-
-cd /data/news
-git pull origin master
-```
-
-- 定时任务
-
-```
-crontab -e
-*/1 * * * * /bin/sh /data/shell/cache_news.sh
-*/2 * * * * nohup gonews -d /data/news/ > /data/log/gonews.log 2>&1
-```
-
 
 ## 用到的技术
 

main.go

@@ -19,13 +19,15 @@ const (
 var dir string
 var act string
 var port int
+var pullInterval int64 // 自动拉取间隔
 
 var wg sync.WaitGroup
 
 func init() {
 	flag.StringVar(&dir, "d", "", "the path of news to parse")
 	flag.StringVar(&act, "a", "cache", "the action to run service, values 'api' or 'cache'")
 	flag.IntVar(&port, "p", 8017, "the port to listen for api service")
+	flag.Int64Var(&pullInterval, "pull", 60, "the time interval (minutes) to pull data source, ")
 }
 
 func main() {
@@ -40,13 +42,8 @@ func main() {
 		}
 		fmt.Fprintf(os.Stdout, "%s", "Success to run api service")
 	} else { // 缓存数据操作
-		files := getFileList(dir)
-		for _, file := range files {
-			wg.Add(1)
-			go cacheNews(file)
-		}
-		wg.Wait()
-		fmt.Fprintf(os.Stdout, "%s", "Success to cache news")
+		initDataPuller(dir, pullInterval)
+		select {}
 	}
 }
 

pull.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"time"
+	"os/exec"
+	"fmt"
+	"os"
+	"strings"
+)
+
+// 自动获取数据, interval单位为分钟
+func initDataPuller(dir string, interval int64) {
+	ticker := time.NewTicker(time.Duration(interval) * time.Minute)
+	go func() {
+		for {
+			// 拉取数据
+			cmd := exec.Command("git", "pull", "origin", "master")
+			cmd.Dir = dir
+			out, err := cmd.Output()
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "%s\n", "Pull failed")
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				fmt.Fprintf(os.Stderr, "%s\n", out)
+			} else {
+				if strings.Contains(string(out),"Already up to date") {
+					continue
+				}
+				fmt.Fprintf(os.Stdout, "%s\n", "Pull success")
+
+				// 缓存数据操作
+				files := getFileList(dir)
+				for _, file := range files {
+					wg.Add(1)
+					go cacheNews(file)
+				}
+				wg.Wait()
+				fmt.Fprintf(os.Stdout, "%s\n", "Success to cache news")
+			}
+			// 定时器
+			<-ticker.C
+		}
+	}()
+}