Skip to content

Commit

Permalink
feat: add GITHUB_TOKEN support to download git and update cmd to …
Browse files Browse the repository at this point in the history 
…battle rate-limiting
  • Loading branch information
@blacktop
blacktop committed Feb 23, 2022
1 parent 28cef74 commit a6dec39
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 17 deletions.
46 changes: 37 additions & 9 deletions cmd/ipsw/cmd/download_git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
Expand All @@ -43,7 +44,11 @@ func init() {
downloadCmd.AddCommand(gitCmd)

gitCmd.Flags().StringP("product", "p", "", "macOS product to download (i.e. dyld)")
gitCmd.Flags().StringP("output", "o", "", "Folder to download files to")
gitCmd.Flags().StringP("api", "a", "", "Github API Token")
viper.BindPFlag("download.git.product", gitCmd.Flags().Lookup("product"))
viper.BindPFlag("download.git.output", gitCmd.Flags().Lookup("output"))
viper.BindPFlag("download.git.api", gitCmd.Flags().Lookup("api"))
}

const githubApiURL = "https://api.github.com/orgs/apple-oss-distributions/repos?sort=updated&per_page=100"
Expand Down Expand Up @@ -90,13 +95,16 @@ type githubTag struct {

type GithubTags []githubTag

func queryAppleGithubRepo(prod, proxy string, insecure bool) (*githubRepo, error) {
func queryAppleGithubRepo(prod, proxy string, insecure bool, api string) (*githubRepo, error) {
var repo githubRepo
req, err := http.NewRequest("GET", "https://api.github.com/repos/apple-oss-distributions/"+prod, nil)
if err != nil {
return nil, fmt.Errorf("cannot create http request: %v", err)
}
req.Header.Set("Accept", "application/vnd.github.v3+json")
if len(api) > 0 {
req.Header.Add("Authorization", "token "+api)
}

client := &http.Client{
Transport: &http.Transport{
Expand Down Expand Up @@ -127,7 +135,7 @@ func queryAppleGithubRepo(prod, proxy string, insecure bool) (*githubRepo, error
return &repo, nil
}

func queryAppleGithubRepos(proxy string, insecure bool) (GithubRepos, error) {
func queryAppleGithubRepos(proxy string, insecure bool, api string) (GithubRepos, error) {
var resp *http.Response
var page GithubRepos
var repos GithubRepos
Expand All @@ -137,6 +145,9 @@ func queryAppleGithubRepos(proxy string, insecure bool) (GithubRepos, error) {
return nil, fmt.Errorf("cannot create http request: %v", err)
}
req.Header.Set("Accept", "application/vnd.github.v3+json")
if len(api) > 0 {
req.Header.Add("Authorization", "token "+api)
}

client := &http.Client{
Transport: &http.Transport{
Expand Down Expand Up @@ -273,7 +284,7 @@ func populatePageValues(r *http.Response) pageInfo {
return pinfo
}

func queryAppleGithubTags(url, proxy string, insecure bool) (GithubTags, error) {
func queryAppleGithubTags(url, proxy string, insecure bool, api string) (GithubTags, error) {

var tags GithubTags

Expand All @@ -282,6 +293,9 @@ func queryAppleGithubTags(url, proxy string, insecure bool) (GithubTags, error)
return nil, fmt.Errorf("cannot create http request: %v", err)
}
req.Header.Set("Accept", "application/vnd.github.v3+json")
if len(api) > 0 {
req.Header.Add("Authorization", "token "+api)
}

client := &http.Client{
Transport: &http.Transport{
Expand Down Expand Up @@ -334,10 +348,22 @@ var gitCmd = &cobra.Command{
insecure := viper.GetBool("download.insecure")
// flags
downloadProduct := viper.GetString("download.git.product")
outputFolder := viper.GetString("download.git.output")
apiToken := viper.GetString("download.git.api")

if len(apiToken) == 0 {
if val, ok := os.LookupEnv("GITHUB_TOKEN"); ok {
apiToken = val
} else {
if val, ok := os.LookupEnv("GITHUB_API_TOKEN"); ok {
apiToken = val
}
}
}

if len(downloadProduct) == 0 {
log.Info("Querying github.com/orgs/apple-oss-distributions for repositories...")
repos, err = queryAppleGithubRepos(proxy, insecure)
repos, err = queryAppleGithubRepos(proxy, insecure, apiToken)
if err != nil {
return err
}
Expand All @@ -346,15 +372,15 @@ var gitCmd = &cobra.Command{
return fmt.Errorf("no repos found")
}
} else {
repo, err := queryAppleGithubRepo(downloadProduct, proxy, insecure)
repo, err := queryAppleGithubRepo(downloadProduct, proxy, insecure, apiToken)
if err != nil {
return err
}
repos = append(repos, *repo)
}

for _, repo := range repos {
tags, err := queryAppleGithubTags(strings.TrimSuffix(repo.TagsURL, "{/id}"), proxy, insecure)
tags, err := queryAppleGithubTags(strings.TrimSuffix(repo.TagsURL, "{/id}"), proxy, insecure, apiToken)
if err != nil {
return err
}
Expand All @@ -366,16 +392,18 @@ var gitCmd = &cobra.Command{

latestTag := tags[0]

destName := getDestName(latestTag.TarURL, false)
destName += ".tar.gz"
tarURL := fmt.Sprintf("https://github.com/apple-oss-distributions/%s/archive/refs/tags/%s.tar.gz", repo.Name, latestTag.Name)

destName := getDestName(tarURL, false)
destName = filepath.Join(outputFolder, destName)

if _, err := os.Stat(destName); os.IsNotExist(err) {
log.WithFields(log.Fields{
"file": destName,
}).Info("Downloading")
// download file
downloader := download.NewDownload(proxy, insecure, false, false, false, false)
downloader.URL = latestTag.TarURL
downloader.URL = tarURL
downloader.DestName = destName

err = downloader.Do()
Expand Down
19 changes: 17 additions & 2 deletions cmd/ipsw/cmd/update.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,12 @@ func init() {
updateCmd.Flags().Bool("detect", false, "detect my platform")
updateCmd.Flags().Bool("replace", false, "overwrite current ipsw")
// updateCmd.Flags().BoolP("yes", "y", false, "do not prompt user")
updateCmd.Flags().StringP("api", "a", "", "Github API Token (incase you get rate limited)")

updateCmd.Flags().StringP("platform", "p", "", "ipsw platform binary to update")
}

func queryGithub(proxy string, insecure bool) (GithubReleases, error) {
func queryGithub(proxy string, insecure bool, api string) (GithubReleases, error) {

var releases GithubReleases

Expand All @@ -94,6 +95,9 @@ func queryGithub(proxy string, insecure bool) (GithubReleases, error) {
return nil, fmt.Errorf("cannot create http request: %v", err)
}
req.Header.Set("Accept", "application/vnd.github.v3+json")
if len(api) > 0 {
req.Header.Add("Authorization", "token "+api)
}

client := &http.Client{
Transport: &http.Transport{
Expand Down Expand Up @@ -141,6 +145,7 @@ var updateCmd = &cobra.Command{

platform, _ := cmd.Flags().GetString("platform")
detectPlatform, _ := cmd.Flags().GetBool("detect")
apiToken, _ := cmd.Flags().GetString("api")

if detectPlatform {
os := runtime.GOOS
Expand Down Expand Up @@ -171,7 +176,17 @@ var updateCmd = &cobra.Command{
destPath = filepath.Clean(args[0])
}

releases, err := queryGithub(proxy, insecure)
if len(apiToken) == 0 {
if val, ok := os.LookupEnv("GITHUB_TOKEN"); ok {
apiToken = val
} else {
if val, ok := os.LookupEnv("GITHUB_API_TOKEN"); ok {
apiToken = val
}
}
}

releases, err := queryGithub(proxy, insecure, apiToken)
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@ require (
github.com/spf13/cobra v1.3.0
github.com/spf13/viper v1.10.1
github.com/ulikunitz/xz v0.5.10
github.com/vbauerster/mpb/v7 v7.4.1
github.com/vbauerster/mpb/v7 v7.3.2
golang.org/x/crypto v0.0.0-20220214200702-86341886e292
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
golang.org/x/sys v0.0.0-20220209214540-3681064d5158
golang.org/x/sys v0.0.0-20220222200937-f2425489ef4c
)

// replace github.com/blacktop/go-macho => ../go-macho
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -539,8 +539,8 @@ github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKw
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/vbauerster/mpb/v7 v7.4.1 h1:NhLMWQ3gNg2KJR8oeA9lO8Xvq+eNPmixDmB6JEQOUdA=
github.com/vbauerster/mpb/v7 v7.4.1/go.mod h1:Ygg2mV9Vj9sQBWqsK2m2pidcf9H3s6bNKtqd3/M4gBo=
github.com/vbauerster/mpb/v7 v7.3.2 h1:tCuxMy8G9cLdjb61b6wO7I1vRT/LyMEzRbr3xCC0JPU=
github.com/vbauerster/mpb/v7 v7.3.2/go.mod h1:wfxIZcOJq/bG1/lAtfzMXcOiSvbqVi/5GX5WCSi+IsA=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down Expand Up @@ -771,8 +771,8 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220222200937-f2425489ef4c h1:sSIdNI2Dd6vGv47bKc/xArpfxVmEz2+3j0E6I484xC4=
golang.org/x/sys v0.0.0-20220222200937-f2425489ef4c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
Expand Down

0 comments on commit a6dec39

Please sign in to comment.