Browse files

Add found_way of github updates

  • Loading branch information...
1 parent 29e9d91 commit fa828d0ee1d2833b79670a39637ba86ce967a985 @daviddengcn committed Mar 10, 2016
@@ -65,6 +65,20 @@ func (cdb *CrawlerDB) SchedulePackage(pkg string, sTime time.Time, etag string)
return nil
+// SchedulePackage schedules a package to be crawled at a specific time if not specified earlier.
+func (cdb *CrawlerDB) PushToCrawlPackage(pkg string) {
+ now := time.Now()
+ var ent CrawlingEntry
+ if cdb.PackageDB.Get(pkg, &ent) {
+ if ent.ScheduleTime.Before(now) {
+ // The package has been scheduled to an earlier time.
+ return
+ }
+ }
+ ent.ScheduleTime = now
+ cdb.PackageDB.Put(pkg, ent)
func TrimPackageName(pkg string) string {
return strings.TrimFunc(strings.TrimSpace(pkg), func(r rune) bool {
return r > rune(128)
@@ -79,8 +93,7 @@ func (cdb *CrawlerDB) AppendPackage(pkg string, inDocs func(pkg string) bool) {
var ent CrawlingEntry
- exists := cdb.PackageDB.Get(pkg, &ent)
- if exists {
+ if cdb.PackageDB.Get(pkg, &ent) {
if ent.ScheduleTime.Before(time.Now()) || inDocs(pkg) {
@@ -7,8 +7,11 @@ import (
+ ""
+ sppb ""
// touchPackage forces a package to update if it was not crawled before a
@@ -26,7 +29,7 @@ func touchPackage(pkg string, crawledBefore time.Time, pkgUTs map[string]time.Ti
// set Etag to "" to force updating
- cDB.SchedulePackage(pkg, time.Now(), "")
+ cDB.PushToCrawlPackage(pkg)
func touchByGithubUpdates(pkgUTs map[string]time.Time) {
@@ -38,6 +41,7 @@ func touchByGithubUpdates(pkgUTs map[string]time.Time) {
count := 0
+ now := time.Now()
for _, r := range rs {
if r.Owner == nil || r.UpdatedAt == nil {
@@ -48,6 +52,9 @@ func touchByGithubUpdates(pkgUTs map[string]time.Time) {
touchPackage(fmt.Sprintf("", user, path), r.UpdatedAt.Time, pkgUTs)
+ if err := store.AppendPackageEvent("", user+"/"+path, "githubhupdate", now, sppb.HistoryEvent_Action_None); err != nil {
+ log.Printf("UpdatePackageHistory %s %s failed: %v", "", user+"/"+path, err)
+ }
log.Printf("%d updates found!", count)
@@ -19,6 +19,8 @@ import (
+ sppb ""
var (
@@ -102,11 +102,13 @@ func (*GoFileInfo) ProtoMessage() {}
func (*GoFileInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
type RepoInfo struct {
- LastCrawled *google_protobuf.Timestamp `protobuf:"bytes,1,opt,name=last_crawled,json=lastCrawled" json:"last_crawled,omitempty"`
- Stars int32 `protobuf:"varint,2,opt,name=stars" json:"stars,omitempty"`
- Description string `protobuf:"bytes,3,opt,name=description" json:"description,omitempty"`
+ // The timestamp this repo-info is crawled
+ CrawlingTime *google_protobuf.Timestamp `protobuf:"bytes,1,opt,name=crawling_time,json=crawlingTime" json:"crawling_time,omitempty"`
+ Stars int32 `protobuf:"varint,2,opt,name=stars" json:"stars,omitempty"`
+ Description string `protobuf:"bytes,3,opt,name=description" json:"description,omitempty"`
// Where this project was forked from, full path
- Source string `protobuf:"bytes,5,opt,name=source" json:"source,omitempty"`
+ Source string `protobuf:"bytes,5,opt,name=source" json:"source,omitempty"`
+ // As far as we know, when this repo was updated
LastUpdated *google_protobuf.Timestamp `protobuf:"bytes,4,opt,name=last_updated,json=lastUpdated" json:"last_updated,omitempty"`
@@ -115,9 +117,9 @@ func (m *RepoInfo) String() string { return proto.CompactTextString(m
func (*RepoInfo) ProtoMessage() {}
func (*RepoInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
-func (m *RepoInfo) GetLastCrawled() *google_protobuf.Timestamp {
+func (m *RepoInfo) GetCrawlingTime() *google_protobuf.Timestamp {
if m != nil {
- return m.LastCrawled
+ return m.CrawlingTime
return nil
@@ -129,7 +131,7 @@ func (m *RepoInfo) GetLastUpdated() *google_protobuf.Timestamp {
return nil
-// Information for non-repository folder.
+// Information for a non-repository folder.
type FolderInfo struct {
// E.g. "sub"
Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
@@ -259,46 +261,45 @@ func init() {
var fileDescriptor0 = []byte{
- // 645 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x53, 0xc1, 0x6e, 0xd4, 0x3c,
- 0x10, 0xfe, 0xb3, 0xbb, 0x4d, 0xb7, 0x93, 0x6d, 0xff, 0x95, 0x85, 0x60, 0x5b, 0x24, 0x54, 0x45,
- 0x42, 0xe2, 0x94, 0x15, 0x45, 0x54, 0x20, 0x84, 0xa0, 0xaa, 0x5a, 0x28, 0x07, 0x84, 0xd2, 0x56,
- 0x1c, 0x23, 0x37, 0xf1, 0x66, 0x23, 0x12, 0x3b, 0xb2, 0x9d, 0x56, 0xfb, 0x2e, 0x9c, 0x78, 0x1b,
- 0xde, 0x80, 0x57, 0xe1, 0xc6, 0xd8, 0x4e, 0xba, 0x8b, 0x10, 0xb0, 0x9c, 0xec, 0x99, 0xf9, 0xc6,
- 0x33, 0xdf, 0x37, 0x63, 0x78, 0x9a, 0x17, 0x7a, 0xde, 0x5c, 0x45, 0xa9, 0xa8, 0xa6, 0x19, 0xbd,
- 0x2e, 0xb2, 0x8c, 0xf1, 0x3c, 0xe5, 0xd3, 0x3c, 0x55, 0x6c, 0x5a, 0x4b, 0xa1, 0xc5, 0x54, 0xd5,
- 0x45, 0xc6, 0x64, 0x7b, 0x44, 0xd6, 0x47, 0x02, 0x13, 0x8f, 0x9c, 0x6b, 0xef, 0xc5, 0xca, 0x1b,
- 0xb9, 0x28, 0x29, 0xcf, 0x5d, 0xe6, 0x55, 0x33, 0x9b, 0xd6, 0x7a, 0x51, 0x33, 0x35, 0xd5, 0x45,
- 0xc5, 0x94, 0xa6, 0x55, 0xbd, 0xbc, 0xb9, 0x97, 0xc2, 0xef, 0x1e, 0xc0, 0x1b, 0x71, 0x5a, 0x94,
- 0xec, 0x8c, 0xcf, 0x04, 0x39, 0x04, 0x1f, 0xa3, 0xba, 0x51, 0x13, 0x6f, 0xdf, 0x7b, 0xb4, 0x73,
- 0xf0, 0x20, 0x5a, 0xa9, 0x14, 0x2d, 0x81, 0xd1, 0xb9, 0x45, 0xc5, 0x2d, 0x9a, 0x10, 0x18, 0x70,
- 0x5a, 0xb1, 0x49, 0x0f, 0xb3, 0xb6, 0x62, 0x7b, 0x27, 0xfb, 0x10, 0x64, 0x4c, 0xa5, 0xb2, 0xa8,
- 0x75, 0x21, 0xf8, 0xa4, 0x6f, 0x43, 0xab, 0x2e, 0x72, 0x0f, 0x36, 0x0b, 0x95, 0x68, 0x6c, 0x68,
- 0x32, 0xc0, 0xe8, 0x30, 0xf6, 0x0b, 0x75, 0x81, 0x16, 0x99, 0x60, 0xa0, 0xaa, 0x85, 0xd4, 0x6a,
- 0xb2, 0xb1, 0xdf, 0xc7, 0xb4, 0xce, 0x0c, 0xdf, 0x81, 0xef, 0x4a, 0x93, 0x00, 0x36, 0x2f, 0xf9,
- 0x27, 0x2e, 0x6e, 0xf8, 0xf8, 0x3f, 0x32, 0x86, 0xd1, 0x07, 0x2a, 0x15, 0x3b, 0x6f, 0xd2, 0x94,
- 0x29, 0x35, 0xf6, 0xc8, 0xff, 0x10, 0x58, 0xcf, 0x29, 0xc5, 0x96, 0xb3, 0x71, 0xcf, 0x40, 0xce,
- 0xe7, 0xa2, 0x29, 0xb3, 0xb3, 0x9c, 0x0b, 0xc9, 0xc6, 0xfd, 0xf0, 0x9b, 0x07, 0xc3, 0x98, 0xd5,
- 0xc2, 0x32, 0x7f, 0x09, 0xa3, 0x92, 0x2a, 0x9d, 0xa4, 0x92, 0xde, 0x60, 0x82, 0xe5, 0x1f, 0x1c,
- 0xec, 0x45, 0xb9, 0x10, 0x79, 0xc9, 0xa2, 0x4e, 0xd1, 0xe8, 0xa2, 0x13, 0x30, 0x0e, 0x0c, 0xfe,
- 0xd8, 0xc1, 0xc9, 0x1d, 0xd8, 0x40, 0xaf, 0x54, 0x56, 0x81, 0x8d, 0xd8, 0x19, 0x6b, 0x48, 0x70,
- 0x17, 0x05, 0x17, 0x8d, 0x4c, 0x19, 0x12, 0x35, 0xc1, 0xd6, 0xba, 0x6d, 0xa7, 0xa9, 0x33, 0xaa,
- 0xb1, 0x9d, 0xc1, 0x7a, 0xed, 0x5c, 0x3a, 0x78, 0xf8, 0x05, 0xc7, 0x7a, 0x2a, 0x4a, 0x1c, 0x9a,
- 0x25, 0xd7, 0x8d, 0xc7, 0x5b, 0x19, 0x0f, 0xfa, 0x6a, 0xaa, 0xe7, 0xdd, 0xc8, 0xcc, 0x1d, 0x35,
- 0xea, 0xab, 0x39, 0x6d, 0xfb, 0x34, 0x57, 0xb2, 0x0b, 0xc3, 0xb9, 0xae, 0xca, 0xa4, 0x91, 0xa5,
- 0xed, 0x01, 0x47, 0x61, 0xec, 0x4b, 0x59, 0x92, 0x57, 0xb0, 0x6d, 0xc5, 0x2a, 0x78, 0x9e, 0x98,
- 0xb5, 0xb2, 0x0c, 0xfe, 0xdc, 0xe3, 0xa8, 0x4b, 0x30, 0xae, 0x30, 0x85, 0xd1, 0x71, 0x6b, 0xdb,
- 0x2e, 0x7f, 0x79, 0xd0, 0xfb, 0xb7, 0x07, 0x0d, 0x25, 0xa6, 0x69, 0xde, 0x51, 0x32, 0xf7, 0xf0,
- 0xab, 0x07, 0xa3, 0xb7, 0x85, 0xd2, 0x42, 0x2e, 0x4e, 0xae, 0x19, 0xd7, 0xe4, 0x19, 0x6c, 0xdd,
- 0x7e, 0x82, 0x35, 0x2a, 0x2c, 0xc1, 0x38, 0x13, 0x9f, 0xa6, 0x76, 0x90, 0x3d, 0xfb, 0x39, 0x1e,
- 0xfe, 0xf4, 0x39, 0x56, 0x8b, 0x44, 0x47, 0x16, 0x17, 0x9d, 0xf0, 0xa6, 0x8a, 0xdb, 0xa4, 0xbd,
- 0xd7, 0xe0, 0x3b, 0x77, 0x78, 0x08, 0x03, 0x13, 0x21, 0x43, 0x18, 0xbc, 0x17, 0x9c, 0xe1, 0xfe,
- 0xe2, 0x32, 0x2f, 0x57, 0x17, 0xc0, 0xbf, 0xdd, 0x5a, 0x0c, 0x9c, 0xf1, 0x6b, 0x5a, 0x16, 0x19,
- 0x2e, 0xec, 0xe7, 0x1e, 0x04, 0x6d, 0x19, 0x2b, 0xd8, 0x63, 0xf0, 0x99, 0x29, 0x67, 0x7e, 0x6b,
- 0x1f, 0x79, 0xec, 0xfe, 0xb6, 0xa1, 0xb8, 0x05, 0x92, 0xe7, 0x00, 0x33, 0xd1, 0xf0, 0xcc, 0x09,
- 0xdc, 0xfb, 0x3b, 0x7d, 0x8b, 0xb6, 0xea, 0xde, 0x07, 0x67, 0x24, 0x37, 0x74, 0xd1, 0xae, 0xc8,
- 0xd0, 0x3a, 0x3e, 0xd2, 0x05, 0x39, 0x82, 0x9d, 0x92, 0x9a, 0x9f, 0x9c, 0x28, 0xc7, 0x63, 0x8d,
- 0x8d, 0xdd, 0x76, 0x19, 0x2d, 0x71, 0x33, 0xfe, 0xf6, 0x89, 0x99, 0x65, 0xbf, 0xce, 0x3e, 0xb9,
- 0x04, 0xa7, 0xd6, 0x95, 0x6f, 0x11, 0x4f, 0x7e, 0x04, 0x00, 0x00, 0xff, 0xff, 0x81, 0x50, 0x44,
- 0xe1, 0x55, 0x05, 0x00, 0x00,
+ // 636 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x53, 0xdd, 0x6a, 0x14, 0x4d,
+ 0x10, 0xfd, 0x66, 0x7f, 0x26, 0x9b, 0x9a, 0x4d, 0xbe, 0xa5, 0x11, 0xdd, 0x44, 0x90, 0x30, 0x20,
+ 0x78, 0x35, 0x8b, 0x11, 0x83, 0x22, 0xa2, 0x41, 0x12, 0x8d, 0x17, 0x22, 0x93, 0x04, 0x2f, 0x97,
+ 0xce, 0x4c, 0xef, 0x6c, 0x63, 0x4f, 0xf7, 0xd0, 0xdd, 0x93, 0x65, 0xdf, 0xc5, 0x2b, 0xdf, 0xc6,
+ 0x57, 0xf0, 0x4d, 0xbc, 0xb3, 0x7f, 0x66, 0xb2, 0x2b, 0xa2, 0xae, 0xe0, 0xd5, 0x74, 0x55, 0x9d,
+ 0x9a, 0xaa, 0x3a, 0xa7, 0x0a, 0x1e, 0x17, 0x54, 0xcf, 0xeb, 0xab, 0x24, 0x13, 0xe5, 0x24, 0xc7,
+ 0xd7, 0x34, 0xcf, 0x09, 0x2f, 0x32, 0x3e, 0x29, 0x32, 0x45, 0x26, 0x95, 0x14, 0x5a, 0x4c, 0x54,
+ 0x45, 0x73, 0x22, 0x9b, 0x4f, 0xe2, 0x7c, 0x28, 0xb2, 0xf1, 0xc4, 0xbb, 0xf6, 0x9f, 0xad, 0xfd,
+ 0xa3, 0x10, 0x0c, 0xf3, 0xc2, 0x67, 0x5e, 0xd5, 0xb3, 0x49, 0xa5, 0x97, 0x15, 0x51, 0x13, 0x4d,
+ 0x4b, 0xa2, 0x34, 0x2e, 0xab, 0xd5, 0xcb, 0xff, 0x29, 0xfe, 0x16, 0x00, 0xbc, 0x16, 0xa7, 0x94,
+ 0x91, 0x33, 0x3e, 0x13, 0xe8, 0x08, 0x42, 0x13, 0xd5, 0xb5, 0x1a, 0x07, 0x07, 0xc1, 0x83, 0xdd,
+ 0xc3, 0x7b, 0xc9, 0x5a, 0xa5, 0x64, 0x05, 0x4c, 0xce, 0x1d, 0x2a, 0x6d, 0xd0, 0x08, 0x41, 0x8f,
+ 0xe3, 0x92, 0x8c, 0x3b, 0x26, 0x6b, 0x3b, 0x75, 0x6f, 0x74, 0x00, 0x51, 0x4e, 0x54, 0x26, 0x69,
+ 0xa5, 0xa9, 0xe0, 0xe3, 0xae, 0x0b, 0xad, 0xbb, 0xd0, 0x1d, 0xd8, 0xa2, 0x6a, 0xaa, 0x4d, 0x43,
+ 0xe3, 0x9e, 0x89, 0x0e, 0xd2, 0x90, 0xaa, 0x0b, 0x63, 0xa1, 0xb1, 0x09, 0x94, 0x95, 0x90, 0x5a,
+ 0x8d, 0xfb, 0x07, 0x5d, 0x93, 0xd6, 0x9a, 0xf1, 0x5b, 0x08, 0x7d, 0x69, 0x14, 0xc1, 0xd6, 0x25,
+ 0xff, 0xc8, 0xc5, 0x82, 0x8f, 0xfe, 0x43, 0x23, 0x18, 0xbe, 0xc7, 0x52, 0x91, 0xf3, 0x3a, 0xcb,
+ 0x88, 0x52, 0xa3, 0x00, 0xfd, 0x0f, 0x91, 0xf3, 0x9c, 0x62, 0xd3, 0x72, 0x3e, 0xea, 0x58, 0xc8,
+ 0xf9, 0x5c, 0xd4, 0x2c, 0x3f, 0x2b, 0xb8, 0x90, 0x64, 0xd4, 0x8d, 0xbf, 0x06, 0x30, 0x48, 0x49,
+ 0x25, 0xdc, 0xe4, 0x2f, 0x60, 0x27, 0x93, 0x78, 0xc1, 0x28, 0x2f, 0xa6, 0x96, 0x24, 0x47, 0x40,
+ 0x74, 0xb8, 0x9f, 0x14, 0x42, 0x14, 0x8c, 0x24, 0x2d, 0xa5, 0xc9, 0x45, 0xcb, 0x60, 0x3a, 0x6c,
+ 0x13, 0xac, 0x0b, 0xdd, 0x82, 0xbe, 0x71, 0x4b, 0xe5, 0x38, 0xe8, 0xa7, 0xde, 0xd8, 0x80, 0x84,
+ 0xdb, 0x86, 0x72, 0x51, 0xcb, 0x8c, 0x98, 0x51, 0x6d, 0xb0, 0xb1, 0xd0, 0x73, 0x18, 0x32, 0xac,
+ 0xf4, 0xb4, 0xae, 0x72, 0xac, 0x49, 0xee, 0x18, 0xfa, 0x7d, 0x3f, 0x91, 0xc5, 0x5f, 0x7a, 0x78,
+ 0xfc, 0xd9, 0x08, 0x7b, 0x2a, 0x98, 0x91, 0xcd, 0x8d, 0xd7, 0x0a, 0x14, 0xac, 0x09, 0x64, 0x7c,
+ 0x15, 0xd6, 0xf3, 0x56, 0x34, 0xfb, 0x36, 0x2c, 0x75, 0xd5, 0x1c, 0x37, 0x7d, 0xda, 0x27, 0xda,
+ 0x83, 0xc1, 0x5c, 0x97, 0x6c, 0x5a, 0x4b, 0xe6, 0x7a, 0x30, 0x62, 0x58, 0xfb, 0x52, 0xb2, 0x9f,
+ 0x39, 0xeb, 0xff, 0x1d, 0x67, 0x71, 0x06, 0xc3, 0x57, 0x8d, 0xfd, 0x6f, 0x44, 0x30, 0x23, 0x11,
+ 0x8d, 0x8b, 0x76, 0x24, 0xfb, 0x8e, 0xbf, 0x04, 0x30, 0x7c, 0x43, 0x95, 0x16, 0x72, 0x79, 0x72,
+ 0x4d, 0xb8, 0x46, 0x4f, 0x60, 0xfb, 0xe6, 0x0c, 0x36, 0xa8, 0xb0, 0x02, 0x1b, 0x4d, 0x42, 0x9c,
+ 0x39, 0x21, 0x3b, 0xee, 0x3c, 0xee, 0xff, 0x70, 0x1e, 0xeb, 0x45, 0x92, 0x63, 0x87, 0x4b, 0x4e,
+ 0x78, 0x5d, 0xa6, 0x4d, 0xd2, 0xfe, 0x4b, 0x08, 0xbd, 0x3b, 0x3e, 0x82, 0x9e, 0x8d, 0xa0, 0x01,
+ 0xf4, 0xde, 0x09, 0x4e, 0xcc, 0x06, 0x9b, 0x75, 0x5e, 0x2d, 0x2f, 0x40, 0x78, 0xb3, 0xb7, 0x26,
+ 0x70, 0xc6, 0xaf, 0x31, 0xa3, 0xb9, 0x59, 0xd9, 0x4f, 0x1d, 0x88, 0x9a, 0x32, 0x8e, 0xb0, 0x87,
+ 0x10, 0x12, 0x5b, 0xce, 0xde, 0x6b, 0xd7, 0xcc, 0xb1, 0xf7, 0xcb, 0x86, 0xd2, 0x06, 0x88, 0x9e,
+ 0x02, 0xcc, 0x44, 0xcd, 0x73, 0x4f, 0x70, 0xe7, 0xcf, 0xe3, 0x3b, 0xb4, 0x63, 0xf7, 0x2e, 0x78,
+ 0x63, 0xba, 0xc0, 0xcb, 0x66, 0x45, 0x06, 0xce, 0xf1, 0x01, 0x2f, 0xd1, 0x31, 0xec, 0x32, 0x6c,
+ 0x6f, 0x79, 0xaa, 0xfc, 0x1c, 0x1b, 0x6c, 0xec, 0x8e, 0xcf, 0x68, 0x06, 0xb7, 0xf2, 0x37, 0xbf,
+ 0x98, 0xb9, 0xe9, 0x37, 0xd9, 0x27, 0x9f, 0xe0, 0xd9, 0xba, 0x0a, 0x1d, 0xe2, 0xd1, 0xf7, 0x00,
+ 0x00, 0x00, 0xff, 0xff, 0x97, 0x0e, 0x34, 0x31, 0x57, 0x05, 0x00, 0x00,
@@ -20,16 +20,18 @@ message GoFileInfo {
message RepoInfo {
- google.protobuf.Timestamp last_crawled = 1;
+ // The timestamp this repo-info is crawled
+ google.protobuf.Timestamp crawling_time = 1;
int32 stars = 2;
string description = 3;
// Where this project was forked from, full path
string source = 5;
+ // As far as we know, when this repo was updated
google.protobuf.Timestamp last_updated = 4;
-// Information for non-repository folder.
+// Information for a non-repository folder.
message FolderInfo {
// E.g. "sub"
string name = 1;
@@ -70,7 +70,7 @@ func repoInfoFromGithub(repo *github.Repository) *sppb.RepoInfo {
Description: stringsp.Get(repo.Description),
Stars: int32(getInt(repo.StargazersCount)),
- ri.LastCrawled, _ = ptypes.TimestampProto(time.Now())
+ ri.CrawlingTime, _ = ptypes.TimestampProto(time.Now())
ri.LastUpdated, _ = ptypes.TimestampProto(getTimestamp(repo.PushedAt).Time)
if repo.Source != nil {
ri.Source = stringsp.Get(repo.Source.Name)
@@ -41,7 +41,7 @@ var box = &bh.RefCountBox{
func RepoInfoAge(r *sppb.RepoInfo) time.Duration {
- t, _ := ptypes.Timestamp(r.LastCrawled)
+ t, _ := ptypes.Timestamp(r.CrawlingTime)
return time.Now().Sub(t)
@@ -20,7 +20,7 @@ func init() {
func TestRepoInfoAge(t *testing.T) {
ts, _ := ptypes.TimestampProto(time.Now().Add(-time.Hour))
age := RepoInfoAge(&sppb.RepoInfo{
- LastCrawled: ts,
+ CrawlingTime: ts,
assert.ValueShould(t, "age", age, age >= time.Hour && age < time.Hour+time.Minute, "age out of expected range")

0 comments on commit fa828d0

Please sign in to comment.