Skip to content

Commit

Permalink
add support for ttg site
Browse files Browse the repository at this point in the history
  • Loading branch information
@sagan
sagan committed Nov 30, 2023
1 parent 2d09bc0 commit b7b9134
Show file tree
Hide file tree
Showing 8 changed files with 161 additions and 35 deletions.
2 changes: 1 addition & 1 deletion cmd/status/status.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func status(cmd *cobra.Command, args []string) error {
}
if showAll || showAllSites {
for _, site := range config.Get().Sites {
if site.Disabled {
if site.Disabled || site.Hidden {
continue
}
names = append(names, site.Name)
Expand Down
11 changes: 9 additions & 2 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ type SiteConfigStruct struct {
Aliases []string // for internal use only
Comment string `yaml:"comment"`
Disabled bool `yaml:"disabled"`
Hidden bool `yaml:"hidden"` // exclude from default groups (like "_all")
Url string `yaml:"url"`
Domains []string `yaml:"domains"` // other site domains (do not include subdomain part)
TorrentsUrl string `yaml:"torrentsUrl"`
Expand Down Expand Up @@ -99,15 +100,21 @@ type SiteConfigStruct struct {
SelectorTorrentFree string `yaml:"SelectorTorrentFree"`
SelectorTorrentNoTraffic string `yaml:"selectorTorrentNoTraffic"`
SelectorTorrentNeutral string `yaml:"selectorTorrentNeutral"`
SelectorTorrentHnR string `yaml:"selectorTorrentHnR"`
SelectorTorrentPaid string `yaml:"selectorTorrentPaid"`
SelectorTorrentDiscountEndTime string `yaml:"selectorTorrentDiscountEndTime"`
SelectorUserInfo string `yaml:"selectorUserInfo"`
SelectorUserInfoUserName string `yaml:"selectorUserInfoUserName"`
SelectorUserInfoUploaded string `yaml:"selectorUserInfoUploaded"`
SelectorUserInfoDownloaded string `yaml:"selectorUserInfoDownloaded"`
UseCuhash bool `yaml:"useCuhash"` // hdcity 使用机制。种子下载地址里必须有cuhash参数。
TorrentDownloadUrl string `yaml:"torrentDownloadUrl"` // use {id} placeholders in url
TorrentDownloadUrlPrefix string `yaml:"torrentDownloadUrlPrefix"`
Passkey string `yaml:"passkey"`
UseCuhash bool `yaml:"useCuhash"` // hdcity 使用机制。种子下载地址里必须有cuhash参数。
UseDigitHash bool `yaml:"useDigitHash"` // ttg 使用机制。种子下载地址末段必须有4位数字校验码或Passkey参数(即使有 Cookie)。
TorrentUrlIdRegexp string `yaml:"torrentUrlIdRegexp"`
FlowControlInterval int64 `yaml:"flowControlInterval"` // 暂定名。两次请求种子列表页间隔时间(秒)
NexusphpNoLetDown bool `yaml:"nexusphpNoLetDown"`
TorrentUploadSpeedLimitValue int64
BrushTorrentMinSizeLimitValue int64
BrushTorrentMaxSizeLimitValue int64
Expand Down Expand Up @@ -302,7 +309,7 @@ func GetGroupSites(name string) []string {
if name == "_all" { // special group of all sites
sitenames := []string{}
for _, siteConfig := range Get().Sites {
if siteConfig.Disabled {
if siteConfig.Disabled || siteConfig.Hidden {
continue
}
sitenames = append(sitenames, siteConfig.GetName())
Expand Down
101 changes: 89 additions & 12 deletions site/nexusphp/nexusphp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,16 @@ type Site struct {
datatime int64
datetimeExtra int64
cuhash string
digitHashPasskey string
digitHashErr error
idRegexp *regexp.Regexp
torrentsParserOption *TorrentsParserOption
}

const (
DEFAULT_TORRENTS_URL = "torrents.php"
)

var sortFields = map[string]string{
"name": "1",
"time": "4",
Expand Down Expand Up @@ -62,7 +68,7 @@ func (npclient *Site) SearchTorrents(keyword string, baseUrl string) ([]site.Tor
if npclient.SiteConfig.SearchUrl != "" {
baseUrl = npclient.SiteConfig.SearchUrl
} else {
baseUrl = "torrents.php"
baseUrl = DEFAULT_TORRENTS_URL
}
}
searchUrl := npclient.SiteConfig.ParseSiteUrl(baseUrl, true)
Expand All @@ -87,24 +93,42 @@ func (npclient *Site) DownloadTorrent(torrentUrl string) ([]byte, string, error)
id := strings.TrimPrefix(torrentUrl, npclient.GetName()+".")
return npclient.DownloadTorrentById(id)
}
if !strings.Contains(torrentUrl, "/download") {
urlObj, err := url.Parse(torrentUrl)
if err != nil {
return nil, "", fmt.Errorf("invalid torrent url: %v", err)
}
id := ""
if npclient.idRegexp != nil {
m := npclient.idRegexp.FindStringSubmatch(torrentUrl)
if m != nil {
return npclient.DownloadTorrentById(m[npclient.idRegexp.SubexpIndex("id")])
id = m[npclient.idRegexp.SubexpIndex("id")]
}
}
urlObj, err := url.Parse(torrentUrl)
id := ""
if err == nil {
if id == "" {
id = urlObj.Query().Get("id")
}
downloadUrlPrefix := strings.TrimPrefix(npclient.SiteConfig.TorrentDownloadUrlPrefix, "/")
if downloadUrlPrefix == "" {
downloadUrlPrefix = "download"
}
if !strings.HasPrefix(urlObj.Path, "/"+downloadUrlPrefix) && id != "" {
return npclient.DownloadTorrentById(id)
}
// skip NP download notice. see https://github.com/xiaomlove/nexusphp/blob/php8/public/download.php
torrentUrl = util.AppendUrlQueryString(torrentUrl, "letdown=1")
if !npclient.SiteConfig.NexusphpNoLetDown {
torrentUrl = util.AppendUrlQueryString(torrentUrl, "letdown=1")
}
return site.DownloadTorrentByUrl(npclient, npclient.HttpClient, torrentUrl, id)
}

func (npclient *Site) DownloadTorrentById(id string) ([]byte, string, error) {
torrentUrl := npclient.SiteConfig.Url + "download.php?https=1&letdown=1&id=" + id
torrentUrl := ""
if npclient.SiteConfig.TorrentDownloadUrl != "" {
torrentUrl = strings.ReplaceAll(npclient.SiteConfig.TorrentDownloadUrl, "{id}", id)
} else {
torrentUrl = "download.php?https=1&letdown=1&id=" + id
}
torrentUrl = npclient.SiteConfig.ParseSiteUrl(torrentUrl, false)
if npclient.SiteConfig.UseCuhash {
if npclient.cuhash == "" {
// update cuhash by side effect of sync (fetching latest torrents)
Expand All @@ -115,10 +139,59 @@ func (npclient *Site) DownloadTorrentById(id string) ([]byte, string, error) {
} else {
log.Warnf("Failed to get site cuhash. torrent download may fail")
}
} else if npclient.SiteConfig.UseDigitHash {
passkey := ""
if npclient.SiteConfig.Passkey != "" {
passkey = npclient.SiteConfig.Passkey
} else if npclient.digitHashPasskey != "" {
passkey = npclient.digitHashPasskey
} else if npclient.digitHashErr == nil { // only try to fetch passkey once
npclient.digitHashPasskey, npclient.digitHashErr = npclient.getDigithash(id)
if npclient.digitHashErr != nil {
log.Warnf("Failed to get site passkey. torrent download may fail")
} else {
passkey = npclient.digitHashPasskey
log.Infof(`Found site passkey. Add the passkey = "%s" line to site config block of ptool.toml to speed up the next visit`, passkey)
}
}
if passkey != "" {
torrentUrl = strings.TrimSuffix(torrentUrl, "/")
torrentUrl += "/" + passkey
}
}
return site.DownloadTorrentByUrl(npclient, npclient.HttpClient, torrentUrl, id)
}

func (npclient *Site) getDigithash(id string) (string, error) {
detailsUrl := npclient.SiteConfig.ParseSiteUrl(fmt.Sprintf("t/%s/", id), false)
doc, _, err := util.GetUrlDoc(detailsUrl, npclient.HttpClient, npclient.SiteConfig.Cookie, npclient.SiteConfig.UserAgent, nil)
if err != nil {
return "", fmt.Errorf("failed to get torrent detail page: %v", err)
}
downloadUrlPrefix := strings.TrimPrefix(npclient.SiteConfig.TorrentDownloadUrlPrefix, "/")
if downloadUrlPrefix == "" {
downloadUrlPrefix = "download"
}
torrentDownloadLinks := doc.Find(fmt.Sprintf(`a[href^="/%s"],a[href^="%s"],a[href^="%s%s"]`,
downloadUrlPrefix, downloadUrlPrefix, npclient.SiteConfig.Url, downloadUrlPrefix))
passkey := ""
torrentDownloadLinks.EachWithBreak(func(i int, el *goquery.Selection) bool {
urlPathes := strings.Split(el.AttrOr("href", ""), "/")
if len(urlPathes) > 2 {
key := urlPathes[len(urlPathes)-1]
if util.IsHexString(key, 32) {
passkey = key
return false
}
}
return true
})
if passkey == "" {
return "", fmt.Errorf("no passkey found in torrent detail page")
}
return passkey, nil
}

func (npclient *Site) GetStatus() (*site.Status, error) {
err := npclient.sync()
if err != nil {
Expand Down Expand Up @@ -160,7 +233,11 @@ func (npclient *Site) GetAllTorrents(sort string, desc bool, pageMarker string,
page = util.ParseInt(pageMarker)
}
if baseUrl == "" {
baseUrl = "torrents.php"
if npclient.SiteConfig.TorrentsUrl != "" {
baseUrl = npclient.SiteConfig.TorrentsUrl
} else {
baseUrl = DEFAULT_TORRENTS_URL
}
}
pageUrl := npclient.SiteConfig.ParseSiteUrl(baseUrl, true)
queryString := ""
Expand Down Expand Up @@ -251,8 +328,9 @@ func (npclient *Site) sync() error {
}
url := npclient.SiteConfig.TorrentsUrl
if url == "" {
url = npclient.SiteConfig.Url + "torrents.php"
url = DEFAULT_TORRENTS_URL
}
url = npclient.SiteConfig.ParseSiteUrl(url, false)
doc, res, err := util.GetUrlDoc(url, npclient.HttpClient,
npclient.SiteConfig.Cookie, npclient.SiteConfig.UserAgent, nil)
if err != nil {
Expand Down Expand Up @@ -394,6 +472,7 @@ func NewSite(name string, siteConfig *config.SiteConfigStruct, config *config.Co
selectorTorrentSize: siteConfig.SelectorTorrentSize,
selectorTorrentProcessBar: siteConfig.SelectorTorrentProcessBar,
selectorTorrentFree: siteConfig.SelectorTorrentFree,
selectorTorrentHnR: siteConfig.SelectorTorrentHnR,
selectorTorrentNeutral: siteConfig.SelectorTorrentNeutral,
selectorTorrentNoTraffic: siteConfig.SelectorTorrentNoTraffic,
selectorTorrentPaid: siteConfig.SelectorTorrentPaid,
Expand All @@ -402,8 +481,6 @@ func NewSite(name string, siteConfig *config.SiteConfigStruct, config *config.Co
}
if siteConfig.TorrentUrlIdRegexp != "" {
site.idRegexp = regexp.MustCompile(siteConfig.TorrentUrlIdRegexp)
} else {
site.idRegexp = regexp.MustCompile(`\bid=(?P<id>\d+)\b`)
}
return site, nil
}
Expand Down
40 changes: 23 additions & 17 deletions site/nexusphp/parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type TorrentsParserOption struct {
selectorTorrentSize string
selectorTorrentProcessBar string
selectorTorrentFree string
selectorTorrentHnR string
selectorTorrentNoTraffic string
selectorTorrentNeutral string
selectorTorrentPaid string
Expand Down Expand Up @@ -285,7 +286,7 @@ func parseTorrents(doc *goquery.Document, option *TorrentsParserOption,
case "leechers":
leechers = util.ParseInt(text)
case "snatched":
snatched = util.ParseInt(text)
snatched = parseCountString(text)
case "time":
time = util.DomTime(s, option.location)
case "name":
Expand Down Expand Up @@ -351,7 +352,11 @@ func parseTorrents(doc *goquery.Document, option *TorrentsParserOption,
}
downloadEl := s.Find(option.selectorTorrentDownloadLink)
if downloadEl.Length() > 0 {
downloadUrl = option.siteurl + downloadEl.AttrOr("href", "")
downloadUrl = downloadEl.AttrOr("href", "")
// @todo : add support for relative URL
if !util.IsUrl(downloadUrl) {
downloadUrl = option.siteurl + strings.TrimPrefix(downloadUrl, "/")
}
m := idRegexp.FindStringSubmatch(downloadUrl)
if m != nil {
id = m[idRegexp.SubexpIndex("id")]
Expand All @@ -366,33 +371,30 @@ func parseTorrents(doc *goquery.Document, option *TorrentsParserOption,
time, _ = util.ExtractTime(text, option.location)
}
}
if fieldColumIndex["seeders"] == -1 {
if option.selectorTorrentSeeders != "" {
seeders = util.ParseInt(util.DomSelectorText(s, option.selectorTorrentSeeders))
}
zeroSeederLeechers := seeders == 0 && leechers == 0
if (fieldColumIndex["seeders"] == -1 || zeroSeederLeechers) && option.selectorTorrentSeeders != "" {
seeders = util.ParseInt(util.DomSelectorText(s, option.selectorTorrentSeeders))
}
if fieldColumIndex["leechers"] == -1 {
if option.selectorTorrentLeechers != "" {
leechers = util.ParseInt(util.DomSelectorText(s, option.selectorTorrentLeechers))
}
if (fieldColumIndex["leechers"] == -1 || zeroSeederLeechers) && option.selectorTorrentLeechers != "" {
leechers = util.ParseInt(util.DomSelectorText(s, option.selectorTorrentLeechers))
}
if fieldColumIndex["snatched"] == -1 {
if option.selectorTorrentSnatched != "" {
snatched = util.ParseInt(util.DomSelectorText(s, option.selectorTorrentSnatched))
}
if fieldColumIndex["snatched"] == -1 && option.selectorTorrentSnatched != "" {
snatched = util.ParseInt(util.DomSelectorText(s, option.selectorTorrentSnatched))
}
if fieldColumIndex["size"] == -1 {
if option.selectorTorrentSize != "" {
size, _ = util.RAMInBytes(util.DomSelectorText(s, option.selectorTorrentSize))
}
}
if s.Find(`*[title="H&R"],*[alt="H&R"]`).Length() > 0 {
if s.Find(`*[title="H&R"],*[alt="H&R"],*[title="Hit and Run"]`).Length() > 0 {
hnr = true
} else if option.selectorTorrentHnR != "" && s.Find(option.selectorTorrentHnR).Length() > 0 {
hnr = true
}
if s.Find(`*[alt="2X Free"]`).Length() > 0 {
downloadMultiplier = 0
uploadMultiplier = 2
} else if s.Find(`*[title="免费"],*[title="免費"],*[alt="Free"],*[alt="FREE"]`).Length() > 0 ||
} else if s.Find(`*[title="免费"],*[title="免費"],*[alt="Free"],*[alt="FREE"],*[alt="free"]`).Length() > 0 ||
domCheckTextTagExisting(s, "free") {
downloadMultiplier = 0
} else if domCheckTextTagExisting(s, "2xfree") {
Expand Down Expand Up @@ -420,7 +422,7 @@ func parseTorrents(doc *goquery.Document, option *TorrentsParserOption,
if option.selectorTorrentDiscountEndTime != "" {
discountEndTime, _ = util.ParseFutureTime(util.DomRemovedSpecialCharsText(s.Find(option.selectorTorrentDiscountEndTime)))
} else {
re := regexp.MustCompile(`(?i)(?P<free>(^|\s)(免费|免費|FREE)\s*)?(剩余|剩餘|限时|限時)(时间|時間)?\s*(?P<time>[YMDHMSymdhms年月周天小时時分种鐘秒\d]+)`)
re := regexp.MustCompile(`(?i)(?P<free>(^|\s)(免费|免費|FREE)\s*)?(剩余|剩餘|限时|限時)(时间|時間)?\s*(?P<time>\d[\sYMDHMSymdhms年月周天小时時分种鐘秒\d]+[YMDHMSymdhms年月周天小时時分种鐘秒])`)
m := re.FindStringSubmatch(util.DomRemovedSpecialCharsText(s))
if m != nil {
if m[re.SubexpIndex("free")] != "" {
Expand Down Expand Up @@ -485,3 +487,7 @@ func domCheckTextTagExisting(node *goquery.Selection, str string) (existing bool
})
return
}

func parseCountString(str string) int64 {
return util.ParseInt(strings.TrimSuffix(str, "次"))
}
2 changes: 1 addition & 1 deletion site/site.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ type Status struct {
type Site interface {
GetName() string
GetSiteConfig() *config.SiteConfigStruct
// download torrent by id (eg. 12345), sitename.id (eg. mteam.12345), or torrent download url (eg. https://kp.m-team.cc/download.php?id=12345)
// download torrent by id (eg. 12345), sitename.id (eg. mteam.12345), or absolute download url (eg. https://kp.m-team.cc/download.php?id=12345)
DownloadTorrent(url string) (content []byte, filename string, err error)
// download torrent by torrent id (eg. 12345)
DownloadTorrentById(id string) (content []byte, filename string, err error)
Expand Down
24 changes: 23 additions & 1 deletion site/tpl/tpl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package tpl

// 站点模板。
// CSS选择器使用 goquery 解析，支持 jQuery 的扩展语法(例如 :contains("txt") )。
// 除 Url 以外的所有 ***Url (例如 TorrentsUrl) 均应当使用相对路径

import (
"sort"
Expand Down Expand Up @@ -212,6 +213,7 @@ var (
SelectorUserInfoUploaded: `#bottomnav a[href="userdetails"] i[title="上传量："]@after`,
SelectorUserInfoDownloaded: `#bottomnav a[href="userdetails"] i[title="下载量："]@after`,
UseCuhash: true,
TorrentDownloadUrl: `download?id={id}`,
TorrentUrlIdRegexp: `\bt-(?P<id>\d+)\b`,
Comment: "城市",
},
Expand Down Expand Up @@ -340,7 +342,7 @@ var (
Aliases: []string{"leaguehd", "lemon"},
Url: "https://lemonhd.org/",
Domains: []string{"leaguehd.com"},
TorrentsUrl: "https://lemonhd.org/torrents_new.php",
TorrentsUrl: "torrents_new.php",
SelectorTorrentFree: `div:contains("免費")`,
Comment: "柠檬",
},
Expand Down Expand Up @@ -459,6 +461,26 @@ var (
Url: "https://pt.soulvoice.club/",
Comment: "聆音",
},
"totheglory": {
Type: "nexusphp",
Aliases: []string{"ttg"},
Url: "https://totheglory.im/",
TorrentsUrl: "browse.php?c=M",
TorrentsExtraUrls: []string{"browse.php?c=G"},
SelectorTorrent: `a.dl_a[href^="/dl/"]`,
SelectorTorrentDownloadLink: `a.dl_a[href^="/dl/"]`,
SelectorTorrentDetailsLink: `.name_left a[href^="/t/"]`,
SelectorTorrentSeeders: `a[href$="&toseeders=1"]`,
SelectorTorrentLeechers: `a[href$="&todlers=1"]`,
SelectorTorrentSnatched: `td:nth-child(8)@text`, // it's ugly
SelectorTorrentProcessBar: `.process`,
TorrentDownloadUrl: `dl/{id}`,
TorrentDownloadUrlPrefix: `dl/`,
TorrentUrlIdRegexp: `\b(t|dl)/(?P<id>\d+)\b`,
UseDigitHash: true,
NexusphpNoLetDown: true,
Comment: "听听歌、套",
},
"tu88": {
Type: "nexusphp",
Url: "http://pt.tu88.men/",
Expand Down
9 changes: 9 additions & 0 deletions util/datetime.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,15 @@ func ParseTime(str string, location *time.Location) (int64, error) {
}

func ParseTimeDuration(str string) (int64, error) {
// remove inside spaces like the one in "9 小时"
var re = regexp.MustCompile(`^(.*?)\s*(\D+)\s*(.*?)$`)
for {
str1 := re.ReplaceAllString(str, `$1$2$3`)
if str1 == str {
break
}
str = str1
}
str = strings.ReplaceAll(str, "年", "y")
str = strings.ReplaceAll(str, "月", "M")
str = strings.ReplaceAll(str, "周", "w")
Expand Down

0 comments on commit b7b9134

Please sign in to comment.