-
Notifications
You must be signed in to change notification settings - Fork 0
/
files.go
96 lines (84 loc) · 2.52 KB
/
files.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
package util
import (
"flag"
"fmt"
"github.com/its-my-data/doubak/proto"
"github.com/mengzhuo/cookiestxt"
"html"
"io/fs"
"log"
"net/http"
"os"
"path/filepath"
"strings"
)
const CollectorPathPrefix = "collector/"
// GetPathWithCreation returns the concatenated path with output path and have them created in advance.
func GetPathWithCreation(subdirs string) (string, error) {
baseDir := flag.Lookup(proto.Flag_output_dir.String()).Value.String()
return GetPathWithCreationWithBase(baseDir, subdirs)
}
// GetPathWithCreationWithBase returns the concatenated path with base path and have them created in advance.
func GetPathWithCreationWithBase(base, subdirs string) (string, error) {
// Try to create the base directory first if not exists.
if _, err := os.Stat(base); os.IsNotExist(err) {
// Only create the top-most base directory.
err := os.Mkdir(base, os.ModePerm)
if err != nil {
return "", err
}
}
// Recursively create sub-directories.
newPath := filepath.Join(base, subdirs)
return newPath, os.MkdirAll(newPath, os.ModePerm)
}
// ReadEntireFile reads the entire file to a string.
func ReadEntireFile(fullPath string) string {
b, err := os.ReadFile(fullPath)
if err != nil {
log.Fatal(err)
}
return string(b)
}
// GetFilePathListWithPattern returns the full paths for files matching the pattern in the base path.
func GetFilePathListWithPattern(basePath, fileNamePattern string) []string {
var files []string
filepath.WalkDir(basePath, func(s string, d fs.DirEntry, e error) error {
if e != nil {
return e
}
if matched, _ := filepath.Match(fileNamePattern, d.Name()); matched {
files = append(files, s)
}
return nil
})
log.Println("Found", len(files), "matched files with pattern:", fileNamePattern)
return files
}
// LoadCookiesFile loads the external cookies file.
func LoadCookiesFile(filePath string) ([]*http.Cookie, error) {
f, err := os.Open(filePath)
if err != nil {
log.Fatal(err)
}
defer f.Close()
return cookiestxt.Parse(f)
}
// LoadCookiesFileToString loads the external cookies file.
func LoadCookiesFileToString(filePath string) (string, error) {
cookies, err := LoadCookiesFile(filePath)
if err != nil {
return "", nil
}
log.Println("Loaded", len(cookies), "cookies")
c := make([]string, 0)
for _, cookie := range cookies {
if len(cookie.Name) == 0 || len(cookie.Value) == 0 {
continue
}
c = append(c, fmt.Sprintf("%s=%s", cookie.Name, html.UnescapeString(cookie.Value)))
}
result := strings.Join(c, "; ")
log.Println("Cookies are", result)
return result, nil
}