log error url, not panic

bold881 · Dec 21, 2017 · 0c9196f · 0c9196f
1 parent 18cf014
commit 0c9196f
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 4 deletions.
diff --git a/crawler.go b/crawler.go
@@ -132,7 +132,10 @@ func CrawlGoQuery(szurl string, chPI chan PageItem, ch2Crawl chan string) {
 
 	doc, err := goquery.NewDocument(szurl)
 	if err != nil {
-		log.Fatal(err)
+		//log.Fatal(err)
+		log.Println(err)
+		crwedUrls.Del(szurl)
+		return
 	}
 
 	// news article

diff --git a/news.go b/news.go
@@ -78,12 +78,10 @@ func main() {
 		if len(ch2Crawl) > 0 {
 			url := <-ch2Crawl
 			go CrawlGoQuery(url, chPageItem, ch2Crawl)
-			fmt.Println("sleep 10 milliseconds...")
 			time.Sleep(10 * time.Millisecond)
 			continue
 		} else if counter < 100 {
 			counter++
-			fmt.Println("sleep 10 seconds...")
 			time.Sleep(10 * time.Second)
 			continue
 		}

diff --git a/pageitem.go b/pageitem.go
@@ -65,3 +65,12 @@ func (cURLs *CrawledURLs) Add(url string) {
 	cURLs.crawled[url] = true
 	cURLs.Unlock()
 }
+
+func (cURLs *CrawledURLs) Del(url string) {
+	cURLs.Lock()
+	if cURLs.crawled == nil {
+		cURLs.crawled = make(map[string]bool)
+	}
+	delete(cURLs.crawled, url)
+	cURLs.Unlock()
+}
diff --git a/saver.go b/saver.go
@@ -52,7 +52,7 @@ func string2Time(rawstr string) time.Time {
 	} else if strings.Contains(rawstr, "來源") {
 		strarr = strings.Split(rawstr, "來源")
 	} else {
-		return
+		return time.Now()
 	}
 	rawstr = strarr[0]
 	rawstr = strings.TrimSpace(rawstr)