{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":731759196,"defaultBranch":"main","name":"lm-evaluation-harness","ownerLogin":"mozilla-ai","currentUserCanPush":false,"isFork":true,"isEmpty":false,"createdAt":"2023-12-14T20:12:56.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/129804596?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1710264669.0","currentOid":""},"activityList":{"items":[{"before":"fcf69884cbc4954f1675b90396ed316b364aeb4a","after":"9fe457037e8c51114d4e918cb054c6e5507b2f53","ref":"refs/heads/parse_cli_args","pushedAt":"2024-03-14T21:00:58.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"remove defaults for store_true","shortMessageHtmlLink":"remove defaults for store_true"}},{"before":"5e96a8b431a6f4a617fa9bc7c7e375c315a5fccb","after":"fcf69884cbc4954f1675b90396ed316b364aeb4a","ref":"refs/heads/parse_cli_args","pushedAt":"2024-03-14T15:54:59.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"add tests, fix parser","shortMessageHtmlLink":"add tests, fix parser"}},{"before":"4251e24d48957babd79d19b68c3aabc0e5d3204d","after":"5e96a8b431a6f4a617fa9bc7c7e375c315a5fccb","ref":"refs/heads/parse_cli_args","pushedAt":"2024-03-14T14:27:00.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"change tests for parsing","shortMessageHtmlLink":"change tests for parsing"}},{"before":null,"after":"4251e24d48957babd79d19b68c3aabc0e5d3204d","ref":"refs/heads/parse_cli_args","pushedAt":"2024-03-12T17:31:09.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"fix spacing","shortMessageHtmlLink":"fix spacing"}},{"before":"4eba9cf390e2bcee120c3612119fd9b13407f290","after":"49695e8d94c3ab011b7ae8814d809de30b1b1182","ref":"refs/heads/main","pushedAt":"2024-03-12T17:20:42.000Z","pushType":"push","commitsCount":20,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"cli_evaluate calls simple_evaluate with the same verbosity. (#1563)","shortMessageHtmlLink":"cli_evaluate calls simple_evaluate with the same verbosity. (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2181211251\" data-permission-text=\"Title is private\" data-url=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/1563\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/EleutherAI/lm-evaluation-harness/pull/1563/hovercard\" href=\"https://github.com/EleutherAI/lm-evaluation-harness/pull/1563\">Eleuther…</a>"}},{"before":"95167926c5d2a751dea5cd9d497da3114ab8a7b5","after":"4eba9cf390e2bcee120c3612119fd9b13407f290","ref":"refs/heads/main","pushedAt":"2024-03-04T01:50:53.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"Cleaning up unused unit tests (#1516)","shortMessageHtmlLink":"Cleaning up unused unit tests (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2165691426\" data-permission-text=\"Title is private\" data-url=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/1516\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/EleutherAI/lm-evaluation-harness/pull/1516/hovercard\" href=\"https://github.com/EleutherAI/lm-evaluation-harness/pull/1516\">EleutherAI#1516</a>)"}},{"before":null,"after":"a938eb4d85de808032d816823dbcb03157915486","ref":"refs/heads/remove_tests","pushedAt":"2024-03-04T01:05:54.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"clean up tests","shortMessageHtmlLink":"clean up tests"}},{"before":"eacb74e387545f0eaccf1862355e7b7a50cf4add","after":"95167926c5d2a751dea5cd9d497da3114ab8a7b5","ref":"refs/heads/main","pushedAt":"2024-03-04T01:05:39.000Z","pushType":"push","commitsCount":19,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"Setting trust_remote_code to True for HuggingFace datasets compatibility (#1487)\n\n* setting trust_remote_code\r\n\r\n* dataset list no notebooks\r\n\r\n* respect trust remote code\r\n\r\n* Address changes, move cli options and change datasets\r\n\r\n* fix task for tests\r\n\r\n* headqa\r\n\r\n* remove kobest\r\n\r\n* pin datasets and address comments\r\n\r\n* clean up space","shortMessageHtmlLink":"Setting trust_remote_code to True for HuggingFace datasets compatibil…"}},{"before":"5d5efe12db2b0b6e675de98d255023f0f5bedd5a","after":"73d908cb7b1156e6c9b6c8df5817939b285609e8","ref":"refs/heads/trust_remote_code","pushedAt":"2024-03-03T20:18:52.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"clean up space","shortMessageHtmlLink":"clean up space"}},{"before":"2a1063e0ba9abee1d0def3bfc8159fbb5c8bdb71","after":"5d5efe12db2b0b6e675de98d255023f0f5bedd5a","ref":"refs/heads/trust_remote_code","pushedAt":"2024-03-03T20:17:33.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"pin datasets and address comments","shortMessageHtmlLink":"pin datasets and address comments"}},{"before":"343c3f0eb24c90d2368c21d89ab7206e2568b2dd","after":"2a1063e0ba9abee1d0def3bfc8159fbb5c8bdb71","ref":"refs/heads/trust_remote_code","pushedAt":"2024-02-28T20:06:00.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"remove kobest","shortMessageHtmlLink":"remove kobest"}},{"before":"0ce51218f86fcdec1e496a84a595f098ffafc014","after":"343c3f0eb24c90d2368c21d89ab7206e2568b2dd","ref":"refs/heads/trust_remote_code","pushedAt":"2024-02-28T19:51:34.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"headqa","shortMessageHtmlLink":"headqa"}},{"before":"87b49011a91c36c90cd0b66c1205b19cd2d311eb","after":"0ce51218f86fcdec1e496a84a595f098ffafc014","ref":"refs/heads/trust_remote_code","pushedAt":"2024-02-28T16:04:07.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"fix task for tests","shortMessageHtmlLink":"fix task for tests"}},{"before":"544731cbb048e15e97e26b2317cfef84489dcc17","after":"87b49011a91c36c90cd0b66c1205b19cd2d311eb","ref":"refs/heads/trust_remote_code","pushedAt":"2024-02-27T21:19:45.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"Address changes, move cli options and change datasets","shortMessageHtmlLink":"Address changes, move cli options and change datasets"}},{"before":"c5b2d5ad6afaea87ceaccf3d0ed774ca87c548fe","after":"544731cbb048e15e97e26b2317cfef84489dcc17","ref":"refs/heads/trust_remote_code","pushedAt":"2024-02-27T16:57:13.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"respect trust remote code","shortMessageHtmlLink":"respect trust remote code"}},{"before":"c7c8b32961532ce6971dc4148e9734bee1612d05","after":"c5b2d5ad6afaea87ceaccf3d0ed774ca87c548fe","ref":"refs/heads/trust_remote_code","pushedAt":"2024-02-27T16:47:14.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"dataset list no notebooks","shortMessageHtmlLink":"dataset list no notebooks"}},{"before":"c7d323cb3d80125c524afe17a001c854357c4e8b","after":"c7c8b32961532ce6971dc4148e9734bee1612d05","ref":"refs/heads/trust_remote_code","pushedAt":"2024-02-27T16:46:06.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"dataset list","shortMessageHtmlLink":"dataset list"}},{"before":"41384cef6542de2f4c6a1e6f7449e5ad4547579f","after":"c7d323cb3d80125c524afe17a001c854357c4e8b","ref":"refs/heads/trust_remote_code","pushedAt":"2024-02-27T16:43:38.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"dataset list","shortMessageHtmlLink":"dataset list"}},{"before":null,"after":"41384cef6542de2f4c6a1e6f7449e5ad4547579f","ref":"refs/heads/trust_remote_code","pushedAt":"2024-02-24T22:30:31.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"setting trust_remote_code","shortMessageHtmlLink":"setting trust_remote_code"}},{"before":"a72babbfbddd9195748351892dced4f82fccbc0d","after":"eacb74e387545f0eaccf1862355e7b7a50cf4add","ref":"refs/heads/main","pushedAt":"2024-02-23T20:47:13.000Z","pushType":"push","commitsCount":4,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"Adding documentation for Weights and Biases CLI interface (#1466)\n\n* interface docs\r\n\r\n* fix link","shortMessageHtmlLink":"Adding documentation for Weights and Biases CLI interface (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2151641306\" data-permission-text=\"Title is private\" data-url=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/1466\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/EleutherAI/lm-evaluation-harness/pull/1466/hovercard\" href=\"https://github.com/EleutherAI/lm-evaluation-harness/pull/1466\">EleutherAI…</a>"}},{"before":"9e0ca2e12102a4ee53d594bf50d6eef6c159030b","after":"51ab2fed072e71aa4474ca24a4b86283c9a1f719","ref":"refs/heads/wandb_interface","pushedAt":"2024-02-23T19:02:02.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"fix link","shortMessageHtmlLink":"fix link"}},{"before":null,"after":"9e0ca2e12102a4ee53d594bf50d6eef6c159030b","ref":"refs/heads/wandb_interface","pushedAt":"2024-02-23T18:58:39.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"interface docs","shortMessageHtmlLink":"interface docs"}},{"before":"af74a93d1abd9a35fee9d628c2637ae922a82541","after":"a72babbfbddd9195748351892dced4f82fccbc0d","ref":"refs/heads/main","pushedAt":"2024-02-22T18:23:15.000Z","pushType":"push","commitsCount":96,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"PR fixing the issue #1391 (wrong contexts in the mgsm task) (#1440)\n\n* fix the issue #1391, wrong contexts in mgsm tasks\r\n\r\n* fix yaml issue for having two target_delimiter lines. For COT tasks, keep the one with a space (default)\r\n\r\n* regenerate all task yaml files\r\n- change naming so that file name will match with task name\r\n- task|file follows a consistent naming way, mgsm_(mode)_(lang) for three modes, i.e., direct, en_cot, and native_cot\r\n\r\n* English CoTs should have a space as target_delimiter\r\n\r\n* Update utils.py\r\n\r\n* Apply suggestions from code review\r\n\r\n---------\r\n\r\nCo-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com>","shortMessageHtmlLink":"PR fixing the issue <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2115652492\" data-permission-text=\"Title is private\" data-url=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/1391\" data-hovercard-type=\"issue\" data-hovercard-url=\"/EleutherAI/lm-evaluation-harness/issues/1391/hovercard\" href=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/1391\">EleutherAI#1391</a> (wrong contexts in the mgsm task) (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2140994636\" data-permission-text=\"Title is private\" data-url=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/1440\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/EleutherAI/lm-evaluation-harness/pull/1440/hovercard\" href=\"https://github.com/EleutherAI/lm-evaluation-harness/pull/1440\">…</a>"}},{"before":"78545d42f2ca95c6fe0ed220d456eeb94f4485e9","after":"af74a93d1abd9a35fee9d628c2637ae922a82541","ref":"refs/heads/main","pushedAt":"2023-12-27T01:52:14.000Z","pushType":"push","commitsCount":23,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"pin vllm at < 0.2.6 (#1212)","shortMessageHtmlLink":"pin vllm at &lt; 0.2.6 (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2055664240\" data-permission-text=\"Title is private\" data-url=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/1212\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/EleutherAI/lm-evaluation-harness/pull/1212/hovercard\" href=\"https://github.com/EleutherAI/lm-evaluation-harness/pull/1212\">EleutherAI#1212</a>)"}},{"before":"b48ea1392714f18de224bbba389140c5608e4c0c","after":"4ad3dc7b0fa40206be5e2e877c7c0e35a7b47fa5","ref":"refs/heads/local-chat-completions","pushedAt":"2023-12-20T20:33:39.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"README","shortMessageHtmlLink":"README"}},{"before":"bd30c9c75febbb90428ec3b5dee19466d1139084","after":"b48ea1392714f18de224bbba389140c5608e4c0c","ref":"refs/heads/local-chat-completions","pushedAt":"2023-12-20T19:50:37.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"README and remove kwargs","shortMessageHtmlLink":"README and remove kwargs"}},{"before":"7dae1523e1d64362127e4ca92c82c0addbe44216","after":"bd30c9c75febbb90428ec3b5dee19466d1139084","ref":"refs/heads/local-chat-completions","pushedAt":"2023-12-20T16:14:37.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"remove unnecessary kwargs","shortMessageHtmlLink":"remove unnecessary kwargs"}},{"before":"abafff5921007525946d7445966cce2e7636db81","after":"7dae1523e1d64362127e4ca92c82c0addbe44216","ref":"refs/heads/local-chat-completions","pushedAt":"2023-12-20T16:12:48.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"generalize to **kwargs","shortMessageHtmlLink":"generalize to **kwargs"}},{"before":"db85da53f03e7de5ab3dfb7ac640fffc7814e48e","after":"abafff5921007525946d7445966cce2e7636db81","ref":"refs/heads/local-chat-completions","pushedAt":"2023-12-20T15:39:49.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"Ruff and README fix","shortMessageHtmlLink":"Ruff and README fix"}},{"before":"51a88417d8be985db8dad2c48c201bef2aac6a0d","after":"db85da53f03e7de5ab3dfb7ac640fffc7814e48e","ref":"refs/heads/local-chat-completions","pushedAt":"2023-12-20T15:32:53.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"veekaybee","name":"Vicki Boykis","path":"/veekaybee","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3837836?s=80&v=4"},"commit":{"message":"add todo on chat completion kwarg validation","shortMessageHtmlLink":"add todo on chat completion kwarg validation"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAEFhdu_wA","startCursor":null,"endCursor":null}},"title":"Activity · mozilla-ai/lm-evaluation-harness"}