-
Notifications
You must be signed in to change notification settings - Fork 18
/
batchdl.go
466 lines (455 loc) · 17.1 KB
/
batchdl.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
package batchdl
// 批量下载站点的种子
import (
"encoding/csv"
"fmt"
"os"
"os/signal"
"syscall"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/sagan/ptool/client"
"github.com/sagan/ptool/cmd"
"github.com/sagan/ptool/cmd/common"
"github.com/sagan/ptool/config"
"github.com/sagan/ptool/site"
"github.com/sagan/ptool/util"
"github.com/sagan/ptool/util/torrentutil"
)
var command = &cobra.Command{
Use: "batchdl {site} [--action add|download|...] [--base-url torrents_page_url]",
Annotations: map[string]string{"cobra-prompt-dynamic-suggestions": "batchdl"},
Aliases: []string{"ebookgod"},
Short: "Batch download the smallest (or by any other order) torrents from a site.",
Long: `Batch download the smallest (or by any other order) torrents from a site.
--rename <name> flag supports the following variable placeholders:
* [size] : Torrent size
* [id] : Torrent id in site
* [site] : Torrent site
* [filename] : Original torrent filename without ".torrent" extension
* [filename128] : The prefix of [filename] which is at max 128 bytes
* [name] : Torrent name
* [name128] : The prefix of torrent name which is at max 128 bytes`,
Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
RunE: batchdl,
}
var (
downloadAll = false
onePage = false
addPaused = false
dense = false
addRespectNoadd = false
includeDownloaded = false
freeOnly = false
noPaid = false
noNeutral = false
nohr = false
allowBreak = false
addCategoryAuto = false
largestFlag = false
newestFlag = false
maxTorrents = int64(0)
minSeeders = int64(0)
maxSeeders = int64(0)
addCategory = ""
addClient = ""
addTags = ""
filter = ""
excludes = ""
savePath = ""
minTorrentSizeStr = ""
maxTorrentSizeStr = ""
maxTotalSizeStr = ""
freeTimeAtLeastStr = ""
startPage = ""
downloadDir = ""
exportFile = ""
baseUrl = ""
rename = ""
action = ""
sortFlag = ""
orderFlag = ""
includes = []string{}
)
func init() {
command.Flags().BoolVarP(&downloadAll, "all", "a", false,
`Download all torrents. Equivalent to "--include-downloaded --min-seeders -1"`)
command.Flags().BoolVarP(&onePage, "one-page", "", false, "Only fetch one page torrents")
command.Flags().BoolVarP(&addPaused, "add-paused", "", false, "Add torrents to client in paused state")
command.Flags().BoolVarP(&dense, "dense", "d", false, "Dense mode: show full torrent title & subtitle")
command.Flags().BoolVarP(&freeOnly, "free", "", false, "Skip none-free torrent")
command.Flags().BoolVarP(&noPaid, "no-paid", "", false, "Skip paid (use bonus points) torrent")
command.Flags().BoolVarP(&noNeutral, "no-neutral", "", false,
"Skip neutral (do not count uploading & downloading & seeding bonus) torrent")
command.Flags().BoolVarP(&largestFlag, "largest", "l", false,
`Sort site torrents by size in desc order. Equivalent to "--sort size --order desc"`)
command.Flags().BoolVarP(&newestFlag, "newest", "n", false,
`Download newest torrents of site. Equivalent to "--sort time --order desc --one-page"`)
command.Flags().BoolVarP(&addRespectNoadd, "add-respect-noadd", "", false,
`Used with "--action add". Check and respect _noadd flag in client`)
command.Flags().BoolVarP(&nohr, "no-hr", "", false,
"Skip torrent that has any type of HnR (Hit and Run) restriction")
command.Flags().BoolVarP(&allowBreak, "break", "", false,
"Break (stop finding more torrents) if all torrents of current page do not meet criterion")
command.Flags().BoolVarP(&includeDownloaded, "include-downloaded", "", false,
"Do NOT skip torrent that has been downloaded before")
command.Flags().BoolVarP(&addCategoryAuto, "add-category-auto", "", false,
"Automatically set category of added torrent to corresponding sitename")
command.Flags().Int64VarP(&maxTorrents, "max-torrents", "", -1,
"Number limit of torrents handled. -1 == no limit (Press Ctrl+C to stop)")
command.Flags().StringVarP(&minTorrentSizeStr, "min-torrent-size", "", "-1",
"Skip torrent with size smaller than (<) this value. -1 == no limit")
command.Flags().StringVarP(&maxTorrentSizeStr, "max-torrent-size", "", "-1",
"Skip torrent with size larger than (>) this value. -1 == no limit")
command.Flags().StringVarP(&maxTotalSizeStr, "max-total-size", "", "-1",
"Will at most download torrents with total contents size of this value. -1 == no limit")
command.Flags().Int64VarP(&minSeeders, "min-seeders", "", 1,
"Skip torrent with seeders less than (<) this value. -1 == no limit")
command.Flags().Int64VarP(&maxSeeders, "max-seeders", "", -1,
"Skip torrent with seeders more than (>) this value. -1 == no limit")
command.Flags().StringVarP(&freeTimeAtLeastStr, "free-time", "", "",
"Used with --free. Set the allowed minimal remaining torrent free time. e.g.: 12h, 1d")
command.Flags().StringVarP(&filter, "filter", "", "",
"If set, skip torrent which title or subtitle does NOT contains this string")
command.Flags().StringArrayVarP(&includes, "includes", "", nil,
"Comma-separated list that ONLY torrent which title or subtitle contains any one in the list will be downloaded. "+
"Can be provided multiple times, in which case every list MUST be matched")
command.Flags().StringVarP(&excludes, "excludes", "", "",
"Comma-separated list that torrent which title of subtitle contains any one in the list will be skipped")
command.Flags().StringVarP(&startPage, "start-page", "", "",
"Start fetching torrents from here (should be the returned LastPage value last time you run this command)")
command.Flags().StringVarP(&downloadDir, "download-dir", "", ".",
`Used with "--action download". Set the local dir of downloaded torrents. Default == current dir`)
command.Flags().StringVarP(&addClient, "add-client", "", "",
`Used with "--action add". Set the client. Required in this action`)
command.Flags().StringVarP(&addCategory, "add-category", "", "",
`Used with "--action add". Set the category when adding torrent to client`)
command.Flags().StringVarP(&addTags, "add-tags", "", "",
`Used with "--action add". Set the tags when adding torrent to client (comma-separated)`)
command.Flags().StringVarP(&savePath, "add-save-path", "", "",
"Set contents save path of added torrents")
command.Flags().StringVarP(&exportFile, "export-file", "", "",
`Used with "--action export|printid". Set the output file. (If not set, will use stdout)`)
command.Flags().StringVarP(&baseUrl, "base-url", "", "",
`Manually set the base url of torrents list page. e.g.: "special.php", "torrents.php?cat=100"`)
command.Flags().StringVarP(&rename, "rename", "", "", "Rename downloaded or added torrents (supports variables)")
cmd.AddEnumFlagP(command, &action, "action", "", ActionEnumFlag)
cmd.AddEnumFlagP(command, &sortFlag, "sort", "", common.SiteTorrentSortFlag)
cmd.AddEnumFlagP(command, &orderFlag, "order", "", common.OrderFlag)
cmd.RootCmd.AddCommand(command)
}
func batchdl(command *cobra.Command, args []string) error {
sitename := args[0]
siteInstance, err := site.CreateSite(sitename)
if err != nil {
return err
}
if downloadAll {
includeDownloaded = true
minSeeders = -1
}
if largestFlag && newestFlag {
return fmt.Errorf("--largest and --newest flags are NOT compatible")
}
if largestFlag {
sortFlag = "size"
orderFlag = "desc"
} else if newestFlag {
sortFlag = "time"
orderFlag = "desc"
onePage = true
}
var includesList [][]string
var excludesList []string
for _, include := range includes {
includesList = append(includesList, util.SplitCsv(include))
}
if excludes != "" {
excludesList = util.SplitCsv(excludes)
}
minTorrentSize, _ := util.RAMInBytes(minTorrentSizeStr)
maxTorrentSize, _ := util.RAMInBytes(maxTorrentSizeStr)
maxTotalSize, _ := util.RAMInBytes(maxTotalSizeStr)
desc := false
if orderFlag == "desc" {
desc = true
}
freeTimeAtLeast := int64(0)
if freeTimeAtLeastStr != "" {
t, err := util.ParseTimeDuration(freeTimeAtLeastStr)
if err != nil {
return fmt.Errorf("invalid --free-time value %s: %v", freeTimeAtLeastStr, err)
}
freeTimeAtLeast = t
}
if nohr && siteInstance.GetSiteConfig().GlobalHnR {
log.Errorf("No torrents will be downloaded: site %s enforces global HnR restrictions",
siteInstance.GetName(),
)
return nil
}
var clientInstance client.Client
var clientAddTorrentOption *client.TorrentOption
var clientAddFixedTags []string
var outputFileFd *os.File = os.Stdout
var csvWriter *csv.Writer
if action == "add" {
if addClient == "" {
return fmt.Errorf("you much specify the client used to add torrents to via --add-client flag")
}
clientInstance, err = client.CreateClient(addClient)
if err != nil {
return fmt.Errorf("failed to create client %s: %v", addClient, err)
}
status, err := clientInstance.GetStatus()
if err != nil {
return fmt.Errorf("failed to get client %s status: %v", clientInstance.GetName(), err)
}
if addRespectNoadd && status.NoAdd {
log.Warnf("Client has _noadd flag and --add-respect-noadd flag is set. Abort task")
return nil
}
clientAddTorrentOption = &client.TorrentOption{
Pause: addPaused,
SavePath: savePath,
}
clientAddFixedTags = []string{client.GenerateTorrentTagFromSite(siteInstance.GetName())}
if addTags != "" {
clientAddFixedTags = append(clientAddFixedTags, util.SplitCsv(addTags)...)
}
} else if action == "export" || action == "printid" {
if exportFile != "" {
outputFileFd, err = os.OpenFile(exportFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
return fmt.Errorf("failed to create output file %s: %v", exportFile, err)
}
}
if action == "export" {
csvWriter = csv.NewWriter(outputFileFd)
csvWriter.Write([]string{"name", "size", "time", "id"})
}
}
flowControlInterval := config.DEFAULT_SITE_FLOW_CONTROL_INTERVAL
if siteInstance.GetSiteConfig().FlowControlInterval > 0 {
flowControlInterval = siteInstance.GetSiteConfig().FlowControlInterval
}
cntTorrents := int64(0)
cntAllTorrents := int64(0)
totalSize := int64(0)
totalAllSize := int64(0)
errorCnt := int64(0)
var torrents []site.Torrent
var marker = startPage
var lastMarker = ""
doneHandle := func() {
fmt.Printf("\nDone. Torrents(Size/Cnt) | AllTorrents(Size/Cnt) | LastPage: %s/%d | %s/%d | \"%s\"; ErrorCnt: %d\n",
util.BytesSize(float64(totalSize)),
cntTorrents,
util.BytesSize(float64(totalAllSize)),
cntAllTorrents,
lastMarker,
errorCnt,
)
if csvWriter != nil {
csvWriter.Flush()
}
}
sigs := make(chan os.Signal, 1)
go func() {
sig := <-sigs
log.Debugf("Received signal %v", sig)
doneHandle()
if errorCnt > 0 {
cmd.Exit(1)
} else {
cmd.Exit(0)
}
}()
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
mainloop:
for {
now := util.Now()
lastMarker = marker
log.Printf("Get torrents with page parker '%s'", marker)
torrents, marker, err = siteInstance.GetAllTorrents(sortFlag, desc, marker, baseUrl)
cntTorrentsThisPage := 0
if err != nil {
log.Errorf("Failed to fetch page %s torrents: %v", lastMarker, err)
break
}
if len(torrents) == 0 {
log.Warnf("No torrents found in page %s (may be an error). Abort", lastMarker)
break
}
cntAllTorrents += int64(len(torrents))
for _, torrent := range torrents {
totalAllSize += torrent.Size
if minTorrentSize >= 0 && torrent.Size < minTorrentSize {
log.Tracef("Skip torrent %s due to size %d < minTorrentSize", torrent.Name, torrent.Size)
if sortFlag == "size" && desc {
break mainloop
} else {
continue
}
}
if maxTorrentSize >= 0 && torrent.Size > maxTorrentSize {
log.Tracef("Skip torrent %s due to size %d > maxTorrentSize", torrent.Name, torrent.Size)
if sortFlag == "size" && !desc {
break mainloop
} else {
continue
}
}
if !includeDownloaded && torrent.IsActive {
log.Tracef("Skip active torrent %s", torrent.Name)
continue
}
if minSeeders >= 0 && torrent.Seeders < minSeeders {
log.Tracef("Skip torrent %s due to too few seeders", torrent.Name)
if sortFlag == "seeders" && desc {
break mainloop
} else {
continue
}
}
if maxSeeders >= 0 && torrent.Seeders > maxSeeders {
log.Tracef("Skip torrent %s due to too more seeders", torrent.Name)
if sortFlag == "seeders" && !desc {
break mainloop
} else {
continue
}
}
if filter != "" && !torrent.MatchFilter(filter) {
log.Tracef("Skip torrent %s due to filter %s does NOT match", torrent.Name, filter)
continue
}
if torrent.MatchFiltersOr(excludesList) {
log.Tracef("Skip torrent %s due to excludes matches", torrent.Name)
continue
}
if !torrent.MatchFiltersAndOr(includesList) {
log.Tracef("Skip torrent %s due to includes does NOT match", torrent.Name)
continue
}
if freeOnly {
if torrent.DownloadMultiplier != 0 {
log.Tracef("Skip none-free torrent %s", torrent.Name)
continue
}
if freeTimeAtLeast > 0 && torrent.DiscountEndTime > 0 && torrent.DiscountEndTime < now+freeTimeAtLeast {
log.Tracef("Skip torrent %s which remaining free time is too short", torrent.Name)
continue
}
}
if nohr && torrent.HasHnR {
log.Tracef("Skip HR torrent %s", torrent.Name)
continue
}
if noPaid && torrent.Paid && !torrent.Bought {
log.Tracef("Skip paid torrent %s", torrent.Name)
continue
}
if noNeutral && torrent.Neutral {
log.Tracef("Skip neutral torrent %s", torrent.Name)
continue
}
if maxTotalSize >= 0 && totalSize+torrent.Size > maxTotalSize {
log.Tracef("Skip torrent %s which would break max total size limit", torrent.Name)
if sortFlag == "size" && !desc {
break mainloop
} else {
continue
}
}
cntTorrents++
cntTorrentsThisPage++
totalSize += torrent.Size
var err error
if action == "show" {
site.PrintTorrents([]site.Torrent{torrent}, "", now, cntTorrents != 1, dense, nil)
} else if action == "export" {
csvWriter.Write([]string{torrent.Name, fmt.Sprint(torrent.Size), fmt.Sprint(torrent.Time), torrent.Id})
} else if action == "printid" {
fmt.Fprintf(outputFileFd, "%s\n", torrent.Id)
} else {
var torrentContent []byte
var filename string
if torrent.DownloadUrl != "" {
torrentContent, filename, _, err = siteInstance.DownloadTorrent(torrent.DownloadUrl)
} else {
torrentContent, filename, _, err = siteInstance.DownloadTorrent(torrent.Id)
}
if err != nil {
fmt.Printf("torrent %s (%s): failed to download: %v\n", torrent.Id, torrent.Name, err)
} else if tinfo, err := torrentutil.ParseTorrent(torrentContent, 99); err != nil {
fmt.Printf("torrent %s (%s): failed to parse: %v\n", torrent.Id, torrent.Name, err)
} else {
if action == "download" {
fileName := ""
if rename == "" {
fileName = filename
} else {
fileName = torrentutil.RenameTorrent(rename, sitename, torrent.Id, filename, tinfo)
}
err = os.WriteFile(downloadDir+"/"+fileName, torrentContent, 0666)
if err != nil {
fmt.Printf("torrent %s: failed to write to %s/file %s: %v\n", torrent.Id, downloadDir, filename, err)
} else {
fmt.Printf("torrent %s - %s (%s): downloaded to %s/%s\n", torrent.Id, torrent.Name,
util.BytesSize(float64(torrent.Size)), downloadDir, fileName)
}
} else if action == "add" {
tags := []string{}
tags = append(tags, clientAddFixedTags...)
if torrent.HasHnR || siteInstance.GetSiteConfig().GlobalHnR {
tags = append(tags, "_hr")
}
clientAddTorrentOption.Tags = tags
if addCategoryAuto {
clientAddTorrentOption.Category = sitename
} else {
clientAddTorrentOption.Category = addCategory
}
if rename != "" {
clientAddTorrentOption.Name = torrentutil.RenameTorrent(rename, sitename, torrent.Id, filename, tinfo)
}
err = clientInstance.AddTorrent(torrentContent, clientAddTorrentOption, nil)
if err != nil {
fmt.Printf("torrent %s (%s): failed to add to client: %v\n", torrent.Id, torrent.Name, err)
} else {
fmt.Printf("torrent %s - %s (%s) (seeders=%d, time=%s): added to client\n", torrent.Id, torrent.Name,
util.BytesSize(float64(torrent.Size)), torrent.Seeders, util.FormatDuration(now-torrent.Time))
}
}
}
}
if err != nil {
errorCnt++
}
if maxTorrents >= 0 && cntTorrents >= maxTorrents {
break mainloop
}
if maxTotalSize >= 0 && maxTotalSize-totalSize <= maxTotalSize/100 {
break mainloop
}
}
if onePage || marker == "" {
break
}
if cntTorrentsThisPage == 0 {
if allowBreak {
break
} else {
log.Warnf("Warning, current page %s has no required torrents.", lastMarker)
}
}
log.Warnf("Finish handling page %s. Torrents(Size/Cnt) | AllTorrents(Size/Cnt) till now: %s/%d | %s/%d. "+
"Will process next page %s in %d seconds. Press Ctrl + C to stop",
lastMarker, util.BytesSize(float64(totalSize)), cntTorrents,
util.BytesSize(float64(totalAllSize)), cntAllTorrents, marker, flowControlInterval)
util.Sleep(flowControlInterval)
}
doneHandle()
return nil
}