{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":682283844,"defaultBranch":"main","name":"open_lm","ownerLogin":"mlfoundations","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2023-08-23T20:55:55.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/87461581?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1715886506.0","currentOid":""},"activityList":{"items":[{"before":null,"after":"c165eb0e8cfe7811406df071db2e2e694e3cff98","ref":"refs/heads/gsmynris/dataloader_update","pushedAt":"2024-05-16T19:08:26.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Intermediate progress.","shortMessageHtmlLink":"Intermediate progress."}},{"before":"b864d15957811dae0886f7c6f444e882590f7d39","after":"012bf911b917d0964a818f1116d1a309dd6a776b","ref":"refs/heads/main","pushedAt":"2024-05-16T18:26:32.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"achalddave","name":"Achal Dave","path":"/achalddave","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/1168881?s=80&v=4"},"commit":{"message":"Further improve errors. (#277)\n\n* Update main\r\n\r\n* Update train\r\n\r\n* Adapt to previous tests.","shortMessageHtmlLink":"Further improve errors. (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2296035739\" data-permission-text=\"Title is private\" data-url=\"https://github.com/mlfoundations/open_lm/issues/277\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/mlfoundations/open_lm/pull/277/hovercard\" href=\"https://github.com/mlfoundations/open_lm/pull/277\">#277</a>)"}},{"before":"9c564b28bebafd179bf029866164c7d7df3e8d6b","after":"f32a972f95fd3e04de6837e285b46315af5c09a4","ref":"refs/heads/allow_for_unbalanced_shards","pushedAt":"2024-05-16T05:20:21.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jeffreywpli","name":null,"path":"/jeffreywpli","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/22289197?s=80&v=4"},"commit":{"message":"add check for whether the manifest agrees with the token count","shortMessageHtmlLink":"add check for whether the manifest agrees with the token count"}},{"before":null,"after":"119b4d4b4859bf83841e51b224fa498d9ee810ed","ref":"refs/heads/gsmyrnis/improve_s3_dl","pushedAt":"2024-05-15T19:13:07.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Download files to local storage first.","shortMessageHtmlLink":"Download files to local storage first."}},{"before":"61b7dbe72553d3793b0a9fa94b8dd0b91995c864","after":"9c564b28bebafd179bf029866164c7d7df3e8d6b","ref":"refs/heads/allow_for_unbalanced_shards","pushedAt":"2024-05-15T07:18:50.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jeffreywpli","name":null,"path":"/jeffreywpli","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/22289197?s=80&v=4"},"commit":{"message":"linting","shortMessageHtmlLink":"linting"}},{"before":null,"after":"61b7dbe72553d3793b0a9fa94b8dd0b91995c864","ref":"refs/heads/allow_for_unbalanced_shards","pushedAt":"2024-05-15T07:12:25.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"jeffreywpli","name":null,"path":"/jeffreywpli","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/22289197?s=80&v=4"},"commit":{"message":"allow for unbalanced write","shortMessageHtmlLink":"allow for unbalanced write"}},{"before":"b2f8a9896de675b895cdaec3ab833006221fbf53","after":"f690fbbfe3c7578b75f65225f597acf01d8f670f","ref":"refs/heads/gsmyrnis/improve_errors_further","pushedAt":"2024-05-14T21:01:36.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Adapt to previous tests.","shortMessageHtmlLink":"Adapt to previous tests."}},{"before":"b864d15957811dae0886f7c6f444e882590f7d39","after":"b2f8a9896de675b895cdaec3ab833006221fbf53","ref":"refs/heads/gsmyrnis/improve_errors_further","pushedAt":"2024-05-14T17:45:34.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Update train","shortMessageHtmlLink":"Update train"}},{"before":null,"after":"b864d15957811dae0886f7c6f444e882590f7d39","ref":"refs/heads/gsmyrnis/improve_errors_further","pushedAt":"2024-05-14T17:43:55.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Improve error message. (#275)\n\n* Improve error message.\r\n\r\n* Formatting.","shortMessageHtmlLink":"Improve error message. (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2295760303\" data-permission-text=\"Title is private\" data-url=\"https://github.com/mlfoundations/open_lm/issues/275\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/mlfoundations/open_lm/pull/275/hovercard\" href=\"https://github.com/mlfoundations/open_lm/pull/275\">#275</a>)"}},{"before":"64a8554da511dece32d92652ce00401d7d24102e","after":"39e80efa26cbc77bfd791dd190019cfc4cbcb88b","ref":"refs/heads/gsmyrnis/improve_errors","pushedAt":"2024-05-14T17:42:37.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Formatting.","shortMessageHtmlLink":"Formatting."}},{"before":"5d9d0d05f91be72c4054192453fff8dec0c8e704","after":"64a8554da511dece32d92652ce00401d7d24102e","ref":"refs/heads/gsmyrnis/improve_errors","pushedAt":"2024-05-14T17:39:12.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"More detailed logging, and save failed checkpoints.","shortMessageHtmlLink":"More detailed logging, and save failed checkpoints."}},{"before":"3b4a06385c6b3f892962d7c3fcb1b30a023aaa15","after":"b864d15957811dae0886f7c6f444e882590f7d39","ref":"refs/heads/main","pushedAt":"2024-05-14T15:21:23.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"Vaishaal","name":null,"path":"/Vaishaal","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/1361040?s=80&v=4"},"commit":{"message":"Improve error message. (#275)\n\n* Improve error message.\r\n\r\n* Formatting.","shortMessageHtmlLink":"Improve error message. (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2295760303\" data-permission-text=\"Title is private\" data-url=\"https://github.com/mlfoundations/open_lm/issues/275\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/mlfoundations/open_lm/pull/275/hovercard\" href=\"https://github.com/mlfoundations/open_lm/pull/275\">#275</a>)"}},{"before":"e173dafaa742a18f97e48506ab42d6aba13d22dd","after":"5d9d0d05f91be72c4054192453fff8dec0c8e704","ref":"refs/heads/gsmyrnis/improve_errors","pushedAt":"2024-05-14T15:20:00.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Formatting.","shortMessageHtmlLink":"Formatting."}},{"before":null,"after":"e173dafaa742a18f97e48506ab42d6aba13d22dd","ref":"refs/heads/gsmyrnis/improve_errors","pushedAt":"2024-05-14T15:14:37.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Improve error message.","shortMessageHtmlLink":"Improve error message."}},{"before":"b47fd0512e49af5dbdb5ae9f30fbaae9e3162451","after":"3b4a06385c6b3f892962d7c3fcb1b30a023aaa15","ref":"refs/heads/main","pushedAt":"2024-05-14T03:07:05.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"achalddave","name":"Achal Dave","path":"/achalddave","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/1168881?s=80&v=4"},"commit":{"message":"Improve error handling for s3 read errors. (#273)\n\n* Improve error handling.\r\n\r\n* Renamed args.\r\n\r\n* Bugfix.\r\n\r\n* Another small bugfix.\r\n\r\n* Small relaxation in tests.\r\n\r\n* Another fix in tests.\r\n\r\n* Revert tests and update expected steps count.\r\n\r\n* Fix error in final ckpt\r\n\r\n* Fix name\r\n\r\n* Expand on elements being saved in stats file.\r\n\r\n* Formatting\r\n\r\n* Add more detailed logging on next shard per source.\r\n\r\n* Even more detailed logging.\r\n\r\n* Add percent of data seen.\r\n\r\n---------\r\n\r\nCo-authored-by: George Smyrnis <gsmyrnis@utexas.edu>","shortMessageHtmlLink":"Improve error handling for s3 read errors. (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2291749166\" data-permission-text=\"Title is private\" data-url=\"https://github.com/mlfoundations/open_lm/issues/273\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/mlfoundations/open_lm/pull/273/hovercard\" href=\"https://github.com/mlfoundations/open_lm/pull/273\">#273</a>)"}},{"before":"e10d61da112cb0e63884479a3e526975a914b600","after":"06cb29ed2b854496fa7b7745a56647ac14c3f9ec","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-14T02:19:13.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Add percent of data seen.","shortMessageHtmlLink":"Add percent of data seen."}},{"before":"236817a3ab5429036a90f1be690f75324f03e557","after":"e10d61da112cb0e63884479a3e526975a914b600","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-14T01:42:25.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Even more detailed logging.","shortMessageHtmlLink":"Even more detailed logging."}},{"before":"ae3b481d14fd9dc2aebb6a1fc32bf8ee49dad31b","after":"236817a3ab5429036a90f1be690f75324f03e557","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-14T01:35:50.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Add more detailed logging on next shard per source.","shortMessageHtmlLink":"Add more detailed logging on next shard per source."}},{"before":"d17c147d35a11fc74baafcd0461b8504e514c2f6","after":"ae3b481d14fd9dc2aebb6a1fc32bf8ee49dad31b","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-13T22:47:46.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Formatting","shortMessageHtmlLink":"Formatting"}},{"before":"91b467ee18977af137079dc09b6435c0537f4cef","after":"d17c147d35a11fc74baafcd0461b8504e514c2f6","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-13T22:36:51.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Expand on elements being saved in stats file.","shortMessageHtmlLink":"Expand on elements being saved in stats file."}},{"before":"afee0c7aae34e6836f6446a5b93017d2ac745c74","after":"91b467ee18977af137079dc09b6435c0537f4cef","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-13T22:11:18.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Fix name","shortMessageHtmlLink":"Fix name"}},{"before":"543ae63d2468b61565c20cd76d95ee2cfb053635","after":"afee0c7aae34e6836f6446a5b93017d2ac745c74","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-13T21:52:20.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Fix error in final ckpt","shortMessageHtmlLink":"Fix error in final ckpt"}},{"before":"d510f9c4da2dd01f3e7a138f3259316918b5a174","after":"543ae63d2468b61565c20cd76d95ee2cfb053635","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-13T21:39:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Revert tests and update expected steps count.","shortMessageHtmlLink":"Revert tests and update expected steps count."}},{"before":"2b0355e896eaa3facfe91e7030d3aa9e086f42d2","after":"2ad4845d0b4f9a55c1b5fd7f57f446551ce6fcf5","ref":"refs/heads/faster_tokenization_and_retry","pushedAt":"2024-05-13T20:45:01.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jeffreywpli","name":null,"path":"/jeffreywpli","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/22289197?s=80&v=4"},"commit":{"message":"ensure duplicate sequences don't get same hash","shortMessageHtmlLink":"ensure duplicate sequences don't get same hash"}},{"before":"75d984d51eb50c8a6e4200e693b3fbf45609e151","after":"d510f9c4da2dd01f3e7a138f3259316918b5a174","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-13T08:02:09.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Another fix in tests.","shortMessageHtmlLink":"Another fix in tests."}},{"before":"9d9708cfce4df7f44d7ed7c5d78c13ca4a53e01a","after":"75d984d51eb50c8a6e4200e693b3fbf45609e151","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-13T07:43:41.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Small relaxation in tests.","shortMessageHtmlLink":"Small relaxation in tests."}},{"before":"feee5b6ac24cf7d2f1ce1c86b3bc0ade3dd5b915","after":"9d9708cfce4df7f44d7ed7c5d78c13ca4a53e01a","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-13T04:45:23.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Another small bugfix.","shortMessageHtmlLink":"Another small bugfix."}},{"before":"f01a34eeae489780ef8ffbca5345f557e39b0a5b","after":"feee5b6ac24cf7d2f1ce1c86b3bc0ade3dd5b915","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-13T04:34:00.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Bugfix.","shortMessageHtmlLink":"Bugfix."}},{"before":"1ef62e230e4d456d23d34720878b8bc83715ab2f","after":"f01a34eeae489780ef8ffbca5345f557e39b0a5b","ref":"refs/heads/gsmyrnis/fix_s3_error_handling","pushedAt":"2024-05-13T04:19:34.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"GeorgiosSmyrnis","name":null,"path":"/GeorgiosSmyrnis","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26143743?s=80&v=4"},"commit":{"message":"Renamed args.","shortMessageHtmlLink":"Renamed args."}},{"before":"f67de76099706642f57a32ca99699f00b1d8a3a0","after":"23bf4d464e32176d8c1e6cee85de8d06be796c69","ref":"refs/heads/allow-duplicate-input-paths","pushedAt":"2024-05-13T04:06:38.000Z","pushType":"push","commitsCount":16,"pusher":{"login":"achalddave","name":"Achal Dave","path":"/achalddave","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/1168881?s=80&v=4"},"commit":{"message":"Merge remote-tracking branch 'origin/controlling_buffer_for_larger_files' into allow-duplicate-input-paths","shortMessageHtmlLink":"Merge remote-tracking branch 'origin/controlling_buffer_for_larger_fi…"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAES_xvWAA","startCursor":null,"endCursor":null}},"title":"Activity · mlfoundations/open_lm"}