Skip to content

Commit

Permalink
feat: better device filtering for download beta
Browse files Browse the repository at this point in the history 
fixes #11
  • Loading branch information
@blacktop
blacktop committed May 9, 2020
1 parent 7335244 commit 607853d
Show file tree
Hide file tree
Showing 2 changed files with 98 additions and 56 deletions.
37 changes: 21 additions & 16 deletions cmd/ipsw/cmd/download_beta.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ var betaCmd = &cobra.Command{
Short: "Download beta IPSWs from the iphone wiki",
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
var filteredIPSWs []download.BetaIPSW
var filteredURLS []string
if Verbose {
log.SetLevel(log.DebugLevel)
}
Expand All @@ -58,55 +58,60 @@ var betaCmd = &cobra.Command{

ipsws, err := download.ScrapeURLs(args[0])
if err != nil {
return errors.Wrap(err, "failed to query ipsw.me api")
return errors.Wrap(err, "failed to query www.theiphonewiki.com")
}

if len(ipsws) < 1 {
log.Errorf("no ipsws found for build %s", args[0])
return nil
}

for _, i := range ipsws {
for url, ipsw := range ipsws {
if len(device) > 0 {
if strings.EqualFold(device, i.Device) {
filteredIPSWs = append(filteredIPSWs, i)
if utils.StrSliceContains(ipsw.Devices, device) {
filteredURLS = append(filteredURLS, url)
}
} else {
filteredIPSWs = append(filteredIPSWs, i)
filteredURLS = append(filteredURLS, url)
}
}

if len(filteredURLS) < 1 {
log.Errorf("no ipsws match device %s", device)
return nil
}

log.Debug("URLs to Download:")
for _, i := range ipsws {
utils.Indent(log.Debug, 2)(i.URL)
for _, url := range filteredURLS {
utils.Indent(log.Debug, 2)(url)
}

cont := true
if !skip {
cont = false
prompt := &survey.Confirm{
Message: fmt.Sprintf("You are about to download %d ipsw files. Continue?", len(filteredIPSWs)),
Message: fmt.Sprintf("You are about to download %d ipsw files. Continue?", len(filteredURLS)),
}
survey.AskOne(prompt, &cont)
}

if cont {
downloader := download.NewDownload(proxy, insecure)
for _, i := range filteredIPSWs {
for _, url := range filteredURLS {
var destName string
if removeCommas {
destName = strings.Replace(path.Base(i.URL), ",", "_", -1)
destName = strings.Replace(path.Base(url), ",", "_", -1)
} else {
destName = path.Base(i.URL)
destName = path.Base(url)
}
if _, err := os.Stat(destName); os.IsNotExist(err) {
log.WithFields(log.Fields{
"device": i.Device,
"build": i.BuildID,
"version": i.Version,
"devices": ipsws[url].Devices,
"build": ipsws[url].BuildID,
"version": ipsws[url].Version,
}).Info("Getting IPSW")
// download file
downloader.URL = i.URL
downloader.URL = url
downloader.RemoveCommas = removeCommas
err = downloader.Do()
if err != nil {
Expand Down
117 changes: 77 additions & 40 deletions internal/download/scrape.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package download

import (
"regexp"
"sort"
"strings"

"github.com/gocolly/colly/v2"
Expand All @@ -18,64 +19,100 @@ var devices = []string{"iPad", "iPad_Air", "iPad_Pro", "iPad_mini", "iPhone", "i

// BetaIPSW object
type BetaIPSW struct {
Device string `json:"device,omitempty"`
Version string `json:"version,omitempty"`
BuildID string `json:"buildid,omitempty"`
URL string `json:"url,omitempty"`
Devices []string `json:"devices,omitempty"`
Version string `json:"version,omitempty"`
BuildID string `json:"buildid,omitempty"`
}

func unique(ipsws []BetaIPSW) []BetaIPSW {
unique := make(map[string]bool, len(ipsws))
us := make([]BetaIPSW, len(unique))
for _, elem := range ipsws {
if len(elem.URL) != 0 {
if !unique[elem.URL] {
us = append(us, elem)
unique[elem.URL] = true
}
func trimQuotes(s string) string {
if len(s) > 0 && s[0] == '"' {
s = s[1:]
}
if len(s) > 0 && s[len(s)-1] == '"' {
s = s[:len(s)-1]
}
return s
}

func appendIfMissing(slice []string, i string) []string {
for _, ele := range slice {
if ele == i {
return slice
}
}
return us
return append(slice, i)
}

// ScrapeURLs will scrape the iPhone Wiki for beta firmwares
func ScrapeURLs(build string) ([]BetaIPSW, error) {
ipsws := []BetaIPSW{}
func ScrapeURLs(build string) (map[string]BetaIPSW, error) {
ipsws := map[string]BetaIPSW{}

c := colly.NewCollector(
colly.AllowedDomains("www.theiphonewiki.com"),
colly.URLFilters(
regexp.MustCompile("https://www.theiphonewiki.com/wiki/Beta_Firmware/(.+)$"),
),
colly.Async(true),
colly.MaxDepth(1),
)

// On every a element which has href attribute call callback
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
link := e.Attr("href")

if strings.Contains(link, "apple.com") {
r := regexp.MustCompile(`\/(?P<device>i.+)_(?P<version>.+)_(?P<build>\w+)_Restore.ipsw$`)
names := r.SubexpNames()

result := r.FindAllStringSubmatch(link, -1)
if result != nil {
m := map[string]string{}
for i, n := range result[0] {
m[names[i]] = n
}
if strings.EqualFold(m["build"], build) {
ipsws = append(ipsws, BetaIPSW{
Device: m["device"],
Version: m["version"],
BuildID: m["build"],
URL: link,
})
}
}
}
c.Visit(e.Request.AbsoluteURL(e.Attr("href")))
})

c.OnHTML("body", func(e *colly.HTMLElement) {
e.ForEach("table.wikitable", func(_ int, ta *colly.HTMLElement) {
var cols []string
possibleIPSW := BetaIPSW{Devices: []string{}}

ta.ForEach("tr", func(_ int, row *colly.HTMLElement) {
row.ForEach("th", func(_ int, el *colly.HTMLElement) {
cols = append(cols, trimQuotes(strings.TrimSpace(el.Text)))
})
row.ForEach("td", func(_ int, el *colly.HTMLElement) {
switch cols[el.Index] {
case "Version":
possibleIPSW.Version = strings.TrimSpace(el.Text)
case "Build":
possibleIPSW.BuildID = strings.TrimSpace(el.Text)
case "Keys":
el.ForEach("a", func(_ int, device *colly.HTMLElement) {
possibleIPSW.Devices = appendIfMissing(possibleIPSW.Devices, strings.TrimSpace(device.Text))
})
}

if el.ChildAttr("a", "class") == "external text" {
link := el.ChildAttr("a", "href")

if strings.Contains(link, "apple.com") {
r := regexp.MustCompile(`\/(?P<device>i.+)_(?P<version>.+)_(?P<build>\w+)_Restore.ipsw$`)
names := r.SubexpNames()

c.Visit(e.Request.AbsoluteURL(link))
result := r.FindAllStringSubmatch(link, -1)
if result != nil {
m := map[string]string{}
for i, n := range result[0] {
m[names[i]] = n
}
if strings.EqualFold(m["build"], build) {
if _, ok := ipsws[link]; ok {
oldIPSW := ipsws[link]
for _, dev := range possibleIPSW.Devices {
oldIPSW.Devices = appendIfMissing(oldIPSW.Devices, dev)
}
sort.Strings(oldIPSW.Devices)
ipsws[link] = oldIPSW
} else {
ipsws[link] = possibleIPSW
}
}
}
}
}
})
})
})
})

for _, device := range devices {
Expand All @@ -87,5 +124,5 @@ func ScrapeURLs(build string) ([]BetaIPSW, error) {

c.Wait()

return unique(ipsws), nil
return ipsws, nil
}

0 comments on commit 607853d

Please sign in to comment.