|
| 1 | +// Copyright 2020 Google LLC |
| 2 | +// |
| 3 | +// Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +// you may not use this file except in compliance with the License. |
| 5 | +// You may obtain a copy of the License at |
| 6 | +// |
| 7 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +// |
| 9 | +// Unless required by applicable law or agreed to in writing, software |
| 10 | +// distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +// See the License for the specific language governing permissions and |
| 13 | +// limitations under the License. |
| 14 | + |
| 15 | +// +build go1.15 |
| 16 | + |
| 17 | +package main |
| 18 | + |
| 19 | +import ( |
| 20 | + "bufio" |
| 21 | + "context" |
| 22 | + "encoding/json" |
| 23 | + "log" |
| 24 | + "net/http" |
| 25 | + "strings" |
| 26 | + "time" |
| 27 | +) |
| 28 | + |
| 29 | +// indexer gets a limited list of entries from index.golang.org. |
| 30 | +type indexer interface { |
| 31 | + get(prefix string, since time.Time) (entries []indexEntry, last time.Time, err error) |
| 32 | +} |
| 33 | + |
| 34 | +// indexClient is used to access index.golang.org. |
| 35 | +type indexClient struct{} |
| 36 | + |
| 37 | +var _ indexer = indexClient{} |
| 38 | + |
| 39 | +// indexEntry represents a line in the output of index.golang.org/index. |
| 40 | +type indexEntry struct { |
| 41 | + Path string |
| 42 | + Version string |
| 43 | + Timestamp time.Time |
| 44 | +} |
| 45 | + |
| 46 | +// newModules returns the new modules with the given prefix. |
| 47 | +// |
| 48 | +// newModules uses index.golang.org/index?since=timestamp to find new module |
| 49 | +// versions since the given timestamp. |
| 50 | +// |
| 51 | +// newModules stores the timestamp of the last successful run with tSaver. |
| 52 | +func newModules(ctx context.Context, i indexer, tSaver timeSaver, prefix string) ([]indexEntry, error) { |
| 53 | + since, err := tSaver.get(ctx) |
| 54 | + if err != nil { |
| 55 | + return nil, err |
| 56 | + } |
| 57 | + fiveMinAgo := time.Now().Add(-5 * time.Minute).UTC() // When to stop processing. |
| 58 | + entries := []indexEntry{} |
| 59 | + log.Printf("Fetching index.golang.org entries since %s", since.Format(time.RFC3339)) |
| 60 | + count := 0 |
| 61 | + for { |
| 62 | + count++ |
| 63 | + var cur []indexEntry |
| 64 | + cur, since, err = i.get(prefix, since) |
| 65 | + if err != nil { |
| 66 | + return nil, err |
| 67 | + } |
| 68 | + entries = append(entries, cur...) |
| 69 | + if since.After(fiveMinAgo) { |
| 70 | + break |
| 71 | + } |
| 72 | + } |
| 73 | + log.Printf("Parsed %d index.golang.org pages up to %s", count, since.Format(time.RFC3339)) |
| 74 | + if err := tSaver.put(ctx, since); err != nil { |
| 75 | + return nil, err |
| 76 | + } |
| 77 | + |
| 78 | + return entries, nil |
| 79 | +} |
| 80 | + |
| 81 | +// get fetches a single chronological page of modules from |
| 82 | +// index.golang.org/index. |
| 83 | +func (indexClient) get(prefix string, since time.Time) ([]indexEntry, time.Time, error) { |
| 84 | + entries := []indexEntry{} |
| 85 | + sinceString := since.Format(time.RFC3339) |
| 86 | + resp, err := http.Get("https://index.golang.org/index?since=" + sinceString) |
| 87 | + if err != nil { |
| 88 | + return nil, time.Time{}, err |
| 89 | + } |
| 90 | + |
| 91 | + s := bufio.NewScanner(resp.Body) |
| 92 | + last := time.Time{} |
| 93 | + for s.Scan() { |
| 94 | + e := indexEntry{} |
| 95 | + if err := json.Unmarshal(s.Bytes(), &e); err != nil { |
| 96 | + return nil, time.Time{}, err |
| 97 | + } |
| 98 | + last = e.Timestamp // Always update the last timestamp. |
| 99 | + if !strings.HasPrefix(e.Path, prefix) || |
| 100 | + strings.Contains(e.Path, "internal") || |
| 101 | + strings.Contains(e.Path, "third_party") || |
| 102 | + strings.Contains(e.Version, "-") { // Filter out pseudo-versions. |
| 103 | + continue |
| 104 | + } |
| 105 | + entries = append(entries, e) |
| 106 | + } |
| 107 | + return entries, last, nil |
| 108 | +} |
0 commit comments