{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":764100614,"defaultBranch":"main","name":"Evaluator","ownerLogin":"LlamaTouch","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2024-02-27T13:35:33.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/166697140?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1709705982.0","currentOid":""},"activityList":{"items":[{"before":"e3e986a2cc68e46a2f3a1b1f211f3f8a9a512d1b","after":"4335369c7e271b784718d46b0d04659ed881aca0","ref":"refs/heads/main","pushedAt":"2024-06-24T14:50:56.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"fix: click_match: get xpath from vh_simp_ui_json_path","shortMessageHtmlLink":"fix: click_match: get xpath from vh_simp_ui_json_path"}},{"before":"7dc08b20a6b417a6516529759016acc8bb7cad6a","after":"e3e986a2cc68e46a2f3a1b1f211f3f8a9a512d1b","ref":"refs/heads/main","pushedAt":"2024-06-18T13:44:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"doc: add doc for evaluating collected agent exec traces","shortMessageHtmlLink":"doc: add doc for evaluating collected agent exec traces"}},{"before":"18468232c227fbb572d922986b728044f6ce01c8","after":"7dc08b20a6b417a6516529759016acc8bb7cad6a","ref":"refs/heads/main","pushedAt":"2024-06-18T12:41:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"doc: add how-to-use guide for llamatouch evaluator","shortMessageHtmlLink":"doc: add how-to-use guide for llamatouch evaluator"}},{"before":"600cd215d2ddbd493d5a0443b2fedca4e4c1f9ce","after":"18468232c227fbb572d922986b728044f6ce01c8","ref":"refs/heads/main","pushedAt":"2024-06-17T11:36:47.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"github-actions[bot]","name":null,"path":"/apps/github-actions","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/15368?s=80&v=4"},"commit":{"message":"Apply Python Linter","shortMessageHtmlLink":"Apply Python Linter"}},{"before":"75d69d721840af18f40536ba45dc6ab63fa8f700","after":"600cd215d2ddbd493d5a0443b2fedca4e4c1f9ce","ref":"refs/heads/main","pushedAt":"2024-06-17T11:36:18.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"chore: rm isort workflow","shortMessageHtmlLink":"chore: rm isort workflow"}},{"before":"032f9250dfca122ceef88e0d29f6ffe4104b625a","after":"75d69d721840af18f40536ba45dc6ab63fa8f700","ref":"refs/heads/main","pushedAt":"2024-06-17T11:33:33.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"fix: load agent exec trace/human eval result by paths","shortMessageHtmlLink":"fix: load agent exec trace/human eval result by paths"}},{"before":"aa09d21e45f8d599199a27e189859519f93c3fc5","after":"032f9250dfca122ceef88e0d29f6ffe4104b625a","ref":"refs/heads/main","pushedAt":"2024-06-13T15:31:11.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"refactor: explicitly pass ground-truth dataset path to DatasetHelper class","shortMessageHtmlLink":"refactor: explicitly pass ground-truth dataset path to DatasetHelper …"}},{"before":"9b553fdfafe4c1046d58dde977c1dfbccb1b270c","after":"aa09d21e45f8d599199a27e189859519f93c3fc5","ref":"refs/heads/main","pushedAt":"2024-05-21T09:02:04.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"feat: add image exact match; fix click exact match (#3)\n\n* add image_exact_match and modify click_exact_match logit\r\n\r\n* add click_match and img_match test module\r\n\r\n* add imagehash dependency\r\n\r\n* update test module\r\n\r\n* fix img_exact_match and remove IMAGE\r\n\r\n* modify img_exact_match\r\n\r\n* remove commented-out code\r\n\r\n* modify img_match_test\r\n\r\n* fix null_state","shortMessageHtmlLink":"feat: add image exact match; fix click exact match (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2296990786\" data-permission-text=\"Title is private\" data-url=\"https://github.com/LlamaTouch/Evaluator/issues/3\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/LlamaTouch/Evaluator/pull/3/hovercard\" href=\"https://github.com/LlamaTouch/Evaluator/pull/3\">#3</a>)"}},{"before":"98498746bb1207b0f4ad6fd6a3e0f672cdbacb5c","after":"9b553fdfafe4c1046d58dde977c1dfbccb1b270c","ref":"refs/heads/main","pushedAt":"2024-05-17T02:15:45.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"chore: add isort github action","shortMessageHtmlLink":"chore: add isort github action"}},{"before":"b98b7ac18a8e7a063c799d269baa239fe694a5c8","after":"98498746bb1207b0f4ad6fd6a3e0f672cdbacb5c","ref":"refs/heads/main","pushedAt":"2024-05-17T01:41:28.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"Update python-linter.yml","shortMessageHtmlLink":"Update python-linter.yml"}},{"before":"7fd8b77e9b07c36acfe97980da5983080ea13e0d","after":"b98b7ac18a8e7a063c799d269baa239fe694a5c8","ref":"refs/heads/main","pushedAt":"2024-04-29T00:23:14.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"doc: update README.md","shortMessageHtmlLink":"doc: update README.md"}},{"before":"bba0f9ab936e84d7adf0ca651d9c12ae12a7ccae","after":"7fd8b77e9b07c36acfe97980da5983080ea13e0d","ref":"refs/heads/main","pushedAt":"2024-04-17T01:38:51.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"doc: toggle to show example code in README","shortMessageHtmlLink":"doc: toggle to show example code in README"}},{"before":"39d68e7bf8a54d4bcbad7cbde76db7ddfa14ccd0","after":"bba0f9ab936e84d7adf0ca651d9c12ae12a7ccae","ref":"refs/heads/main","pushedAt":"2024-04-16T14:33:38.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"doc: LlamaTouch Evaluator how-to guide","shortMessageHtmlLink":"doc: LlamaTouch Evaluator how-to guide"}},{"before":"2776da27f498fed7572422a801ab1175ebb5d999","after":"39d68e7bf8a54d4bcbad7cbde76db7ddfa14ccd0","ref":"refs/heads/main","pushedAt":"2024-04-16T13:54:20.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"refactor: remove unused abstract method from Agent base class","shortMessageHtmlLink":"refactor: remove unused abstract method from Agent base class"}},{"before":"4654f4625cd2051c1a586fd1d7fbac802fd23126","after":"2776da27f498fed7572422a801ab1175ebb5d999","ref":"refs/heads/main","pushedAt":"2024-04-16T09:56:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"refactor: a bunch of code for refactoring","shortMessageHtmlLink":"refactor: a bunch of code for refactoring"}},{"before":"88bbbabf83fb8d46b89c4f4eaaa2b7c9553ccb6a","after":"4654f4625cd2051c1a586fd1d7fbac802fd23126","ref":"refs/heads/main","pushedAt":"2024-04-16T08:51:57.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"github-actions[bot]","name":null,"path":"/apps/github-actions","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/15368?s=80&v=4"},"commit":{"message":"Apply black formatter","shortMessageHtmlLink":"Apply black formatter"}},{"before":"eb49c537ec573053d08be558496ee4d585d804cd","after":"88bbbabf83fb8d46b89c4f4eaaa2b7c9553ccb6a","ref":"refs/heads/main","pushedAt":"2024-04-16T08:51:24.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"fix: add missing prefix 'r' in regex's pattern string","shortMessageHtmlLink":"fix: add missing prefix 'r' in regex's pattern string"}},{"before":"86870f6b9d4b854fc26fa84e2a09cfd4064d5c8b","after":"eb49c537ec573053d08be558496ee4d585d804cd","ref":"refs/heads/main","pushedAt":"2024-04-16T08:33:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"feat: allow epi_metadata_path as arg for the DatasetHelper class","shortMessageHtmlLink":"feat: allow epi_metadata_path as arg for the DatasetHelper class"}},{"before":"99c83d733dba102da30f5349f49de9334c921ccf","after":"86870f6b9d4b854fc26fa84e2a09cfd4064d5c8b","ref":"refs/heads/main","pushedAt":"2024-04-13T00:48:10.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"fix: fix essential state key in vis script","shortMessageHtmlLink":"fix: fix essential state key in vis script"}},{"before":"c0b00f5138e677193a800ad4fcd728445e177a10","after":"99c83d733dba102da30f5349f49de9334c921ccf","ref":"refs/heads/main","pushedAt":"2024-04-12T11:41:26.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"github-actions[bot]","name":null,"path":"/apps/github-actions","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/15368?s=80&v=4"},"commit":{"message":"Apply black formatter","shortMessageHtmlLink":"Apply black formatter"}},{"before":"4e673f3a765514ca3930f6c85906074d08068900","after":"c0b00f5138e677193a800ad4fcd728445e177a10","ref":"refs/heads/main","pushedAt":"2024-04-12T11:40:53.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"feat: simplify ess keyword","shortMessageHtmlLink":"feat: simplify ess keyword"}},{"before":"68572b1305ea0506d355a947cc4615e45409ab54","after":"4e673f3a765514ca3930f6c85906074d08068900","ref":"refs/heads/main","pushedAt":"2024-04-03T15:14:32.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"wshprimy","name":"Shihe Wang","path":"/wshprimy","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/44517722?s=80&v=4"},"commit":{"message":"fix: exact match of annotated UI component","shortMessageHtmlLink":"fix: exact match of annotated UI component"}},{"before":"fd1f67af7a848df45e335cbab22114926dc7d656","after":"68572b1305ea0506d355a947cc4615e45409ab54","ref":"refs/heads/main","pushedAt":"2024-04-03T08:44:35.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"wshprimy","name":"Shihe Wang","path":"/wshprimy","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/44517722?s=80&v=4"},"commit":{"message":"feat!: ablation study","shortMessageHtmlLink":"feat!: ablation study"}},{"before":"573d5799e02a9255d8370e8ee095871e77c44fcf","after":"fd1f67af7a848df45e335cbab22114926dc7d656","ref":"refs/heads/main","pushedAt":"2024-04-03T04:02:28.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"github-actions[bot]","name":null,"path":"/apps/github-actions","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/15368?s=80&v=4"},"commit":{"message":"Apply black formatter","shortMessageHtmlLink":"Apply black formatter"}},{"before":"2a5c95c7e3f0fe5e3da45ddae6327892a425631f","after":"573d5799e02a9255d8370e8ee095871e77c44fcf","ref":"refs/heads/main","pushedAt":"2024-04-03T04:01:59.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"wshprimy","name":"Shihe Wang","path":"/wshprimy","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/44517722?s=80&v=4"},"commit":{"message":"fix: metric","shortMessageHtmlLink":"fix: metric"}},{"before":"085b0e1c01047848d1f86bebbab0098ed3ecded9","after":"2a5c95c7e3f0fe5e3da45ddae6327892a425631f","ref":"refs/heads/main","pushedAt":"2024-04-03T03:31:57.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"wshprimy","name":"Shihe Wang","path":"/wshprimy","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/44517722?s=80&v=4"},"commit":{"message":"fix: appagent VH","shortMessageHtmlLink":"fix: appagent VH"}},{"before":"44e9b4a2e5cd55ed0cbe1d1538e3e8807ffe0564","after":"085b0e1c01047848d1f86bebbab0098ed3ecded9","ref":"refs/heads/main","pushedAt":"2024-04-03T03:11:37.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"lizhang20","name":"Li Zhang","path":"/lizhang20","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/28941750?s=80&v=4"},"commit":{"message":"fix: error variale name","shortMessageHtmlLink":"fix: error variale name"}},{"before":"d1e1a20643ea3e0e1f4344cedbfb02989f9539c6","after":"44e9b4a2e5cd55ed0cbe1d1538e3e8807ffe0564","ref":"refs/heads/main","pushedAt":"2024-04-03T02:38:55.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"wshprimy","name":"Shihe Wang","path":"/wshprimy","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/44517722?s=80&v=4"},"commit":{"message":"refactor: function and path","shortMessageHtmlLink":"refactor: function and path"}},{"before":"a24263a34a751d758eaaf4c09770fdaca8bdd99a","after":"d1e1a20643ea3e0e1f4344cedbfb02989f9539c6","ref":"refs/heads/main","pushedAt":"2024-04-03T01:41:27.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"wshprimy","name":"Shihe Wang","path":"/wshprimy","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/44517722?s=80&v=4"},"commit":{"message":"feat: FLAG only_human_eval_positive","shortMessageHtmlLink":"feat: FLAG only_human_eval_positive"}},{"before":"1ff5967eae9ced53859a75d47fa304db075b2848","after":"a24263a34a751d758eaaf4c09770fdaca8bdd99a","ref":"refs/heads/main","pushedAt":"2024-04-02T16:05:20.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"github-actions[bot]","name":null,"path":"/apps/github-actions","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/15368?s=80&v=4"},"commit":{"message":"Apply black formatter","shortMessageHtmlLink":"Apply black formatter"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAEbaR_BAA","startCursor":null,"endCursor":null}},"title":"Activity · LlamaTouch/Evaluator"}