-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.go
71 lines (56 loc) · 1.7 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
package main
import (
"encoding/csv"
"flag"
"fmt"
"log"
"os"
"strings"
"github.com/gocolly/colly"
)
func main() {
var pokemonType1 string
var pokemonType2 string
flag.StringVar(&pokemonType1, "type1", "", "Type of Pokemon to scrape")
flag.StringVar(&pokemonType2, "type2", "", "Type of Pokemon to scrape")
flag.Parse()
pageToScrape := "https://pokemondb.net/pokedex/all/"
pokemonData := [][]string{{"id", "Name", "Type", "Total", "HP", "Attack", "Defense", "Sp.Atk", "Sp.Def", "Speed"}}
c := colly.NewCollector()
c.OnError(func(_ *colly.Response, err error) {
log.Println("Something went wrong: ", err)
})
c.OnHTML("a", func(e *colly.HTMLElement) {
fmt.Printf("%v\n", e.Attr("href"))
})
c.OnHTML(".data-table", func(e *colly.HTMLElement) {
// For each row in the table, extract Pokemon data
e.ForEach("tbody tr", func(_ int, el *colly.HTMLElement) {
var pokemonInfo []string
el.ForEach("td", func(index int, elem *colly.HTMLElement) {
pokemonInfo = append(pokemonInfo, elem.Text)
})
if strings.Contains(strings.ToLower(pokemonInfo[2]), strings.ToLower(pokemonType1)) && strings.Contains(strings.ToLower(pokemonInfo[2]), pokemonType2) {
pokemonData = append(pokemonData, pokemonInfo)
}
})
})
c.OnScraped(func(r *colly.Response) {
fmt.Println(r.Request.URL, " scraped!")
})
c.Visit(pageToScrape)
file, err := os.Create("Pokemon.csv")
if err != nil {
log.Fatalf("Failed creating file: %s", err)
}
defer file.Close()
writer := csv.NewWriter(file)
defer writer.Flush()
for _, value := range pokemonData {
err := writer.Write(value)
if err != nil {
log.Fatalf("Error writing record to file: %s", err)
}
}
fmt.Println("CSV file saved successfully.")
}