-
Notifications
You must be signed in to change notification settings - Fork 0
/
recursivedownload.go
67 lines (65 loc) · 1.5 KB
/
recursivedownload.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
package workers
import (
"fmt"
"github.com/Vexvain/Git-Dump/internal/utils"
"github.com/valyala/fasthttp"
"io/ioutil"
"os"
"strings"
"sync"
"time"
)
func RecursiveDownloadWorker(c *fasthttp.Client, queue chan string, baseUrl, baseDir string, wg *sync.WaitGroup) {
wg.Add(1)
defer wg.Done()
var ctr int
for {
select {
case f := <-queue:
if f == "" {
continue
}
ctr = 0
uri := utils.Url(baseUrl, f)
code, body, err := c.Get(nil, uri)
fmt.Printf("[-] Fetching %s [%d]\n", uri, code)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
continue
}
if strings.HasSuffix(f, "/") {
if !utils.IsHtml(body) {
fmt.Printf("warning: %s doesn't appear to be an index", uri)
continue
}
indexedFiles, err := utils.GetIndexedFiles(body)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
continue
}
for _, idxf := range indexedFiles {
queue <- utils.Url(f, idxf)
}
} else {
if utils.IsHtml(body) {
fmt.Printf("warning: %s doesn't appear to be a git file", uri)
continue
}
if err := utils.CreateParentFolders(utils.Url(baseDir, f)); err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
continue
}
if err := ioutil.WriteFile(utils.Url(baseDir, f), body, os.ModePerm); err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
}
}
default:
// TODO: get rid of dirty hack somehow
if ctr >= graceTimes {
return
}
ctr++
time.Sleep(gracePeriod)
}
}
}