{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":716344776,"defaultBranch":"main","name":"document_vector_search_benchmark","ownerLogin":"nouu-me","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2023-11-09T00:04:51.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/148523619?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1710204975.0","currentOid":""},"activityList":{"items":[{"before":"f56f0bb051642b3aac4f348e05b22310e699a039","after":"c078e9a42deb077dc663f15f9c6cbe6718de4591","ref":"refs/heads/main","pushedAt":"2024-03-12T14:39:48.000Z","pushType":"pr_merge","commitsCount":4,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"Merge pull request #5 from nouu-me/feature/fix-errors\n\nfix lint/run_benchmark errors","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2180517633\" data-permission-text=\"Title is private\" data-url=\"https://github.com/nouu-me/document_vector_search_benchmark/issues/5\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/nouu-me/document_vector_search_benchmark/pull/5/hovercard\" href=\"https://github.com/nouu-me/document_vector_search_benchmark/pull/5\">#5</a> from nouu-me/feature/fix-errors"}},{"before":"7e55a8c7dcb64f98830f02219134f08a6ec9ad7c","after":"1356ae15de39139e064869ea56a189f7ee7bfd46","ref":"refs/heads/feature/fix-errors","pushedAt":"2024-03-12T14:37:40.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"colbert-ir -> colbert-ai","shortMessageHtmlLink":"colbert-ir -&gt; colbert-ai"}},{"before":"addb765790137a4df811be72365603bfaa3b0d1b","after":"7e55a8c7dcb64f98830f02219134f08a6ec9ad7c","ref":"refs/heads/feature/fix-errors","pushedAt":"2024-03-12T01:01:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"fix CI errors","shortMessageHtmlLink":"fix CI errors"}},{"before":null,"after":"addb765790137a4df811be72365603bfaa3b0d1b","ref":"refs/heads/feature/fix-errors","pushedAt":"2024-03-12T00:56:15.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"fix lint/run_benchmark errors","shortMessageHtmlLink":"fix lint/run_benchmark errors"}},{"before":"9e7c65078605e7dde125405362f2111c5e4cdf99","after":"f56f0bb051642b3aac4f348e05b22310e699a039","ref":"refs/heads/main","pushedAt":"2024-03-11T23:01:32.000Z","pushType":"pr_merge","commitsCount":16,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"Merge pull request #3 from bclavie/chore/add_colbert_and_ir_benchmarks\n\nMain changes:\r\n\r\n- Includes JaColBERTv1, JaColBERTv2, and BGE-M3 (dense vector only)\r\n- Evals on JSQuAd (R@{1,3,5,10}) and MIRACL (R@{3,5,10, 100}), using a dataset composed of the top 1000 BM25 results for each query.\r\n- Did not run full evals on TyDi as it's pretty similar to MIRACL and was making the table nearly unreadable, but it's implemented and the top 1000 BM25 candidates are available.\r\n- Updated the table with the results from the full run\r\n- Changed batch sizes so they're adapted to each model, including the remote ones, and added a dynamic sequence length detector so they all get set to 512 unless manually specified in the config.\r\n- Set device to cuda if cuda is detected, otherwise defaults to CPU.\r\n- Modified the CSV to Markdown table to group by Dataset, and drop the columns that are less informative (R@1 is useless for MIRACL and so is R@100 for JSQuAd)","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2055147324\" data-permission-text=\"Title is private\" data-url=\"https://github.com/nouu-me/document_vector_search_benchmark/issues/3\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/nouu-me/document_vector_search_benchmark/pull/3/hovercard\" href=\"https://github.com/nouu-me/document_vector_search_benchmark/pull/3\">#3</a> from bclavie/chore/add_colbert_and_ir_benchmarks"}},{"before":"4cce137ef648a7ec806b549f7479b06205127c08","after":"9e7c65078605e7dde125405362f2111c5e4cdf99","ref":"refs/heads/main","pushedAt":"2023-11-12T10:45:09.000Z","pushType":"pr_merge","commitsCount":3,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"Merge pull request #2 from nouu-me/feature/add-models-20231110\n\nFeature/add models 20231110","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1989346685\" data-permission-text=\"Title is private\" data-url=\"https://github.com/nouu-me/document_vector_search_benchmark/issues/2\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/nouu-me/document_vector_search_benchmark/pull/2/hovercard\" href=\"https://github.com/nouu-me/document_vector_search_benchmark/pull/2\">#2</a> from nouu-me/feature/add-models-20231110"}},{"before":null,"after":"89f1b5084bdcf52d35b83a5a087ff53e1c422da6","ref":"refs/heads/feature/add-models-20231110","pushedAt":"2023-11-12T10:26:00.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"added more models","shortMessageHtmlLink":"added more models"}},{"before":"d94080a7f0e57b916487d632689863602a5749f9","after":null,"ref":"refs/heads/feature/benchmark-for-JSQuAD","pushedAt":"2023-11-09T08:26:07.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"}},{"before":"66022aa37f5d4d32f30087ba9b0e789b3fe0ca38","after":"4cce137ef648a7ec806b549f7479b06205127c08","ref":"refs/heads/main","pushedAt":"2023-11-09T08:24:20.000Z","pushType":"pr_merge","commitsCount":8,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"Merge pull request #1 from nouu-me/feature/benchmark-for-JSQuAD\n\nFeature/benchmark-for-JSQuAD","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1984932171\" data-permission-text=\"Title is private\" data-url=\"https://github.com/nouu-me/document_vector_search_benchmark/issues/1\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/nouu-me/document_vector_search_benchmark/pull/1/hovercard\" href=\"https://github.com/nouu-me/document_vector_search_benchmark/pull/1\">#1</a> from nouu-me/feature/benchmark-for-JSQuAD"}},{"before":"3c7bd3d4486a9558d5aa5ebb1d41192c971422c9","after":"d94080a7f0e57b916487d632689863602a5749f9","ref":"refs/heads/feature/benchmark-for-JSQuAD","pushedAt":"2023-11-09T08:21:19.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"fix","shortMessageHtmlLink":"fix"}},{"before":"289c3839602878bdb3ece94707e894f1df285d2d","after":"3c7bd3d4486a9558d5aa5ebb1d41192c971422c9","ref":"refs/heads/feature/benchmark-for-JSQuAD","pushedAt":"2023-11-09T08:17:29.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"fix ci","shortMessageHtmlLink":"fix ci"}},{"before":"ad8d75158b13674530e98fe95142db1b4484b86b","after":"289c3839602878bdb3ece94707e894f1df285d2d","ref":"refs/heads/feature/benchmark-for-JSQuAD","pushedAt":"2023-11-09T06:36:33.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"fix ci","shortMessageHtmlLink":"fix ci"}},{"before":"f8168dcc01b4ade108d9a9bae63e51ef0480b7e1","after":"ad8d75158b13674530e98fe95142db1b4484b86b","ref":"refs/heads/feature/benchmark-for-JSQuAD","pushedAt":"2023-11-09T06:34:05.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"added ci.yaml","shortMessageHtmlLink":"added ci.yaml"}},{"before":"9d1614eb048b4895edf1b671eee489b8c378e614","after":"f8168dcc01b4ade108d9a9bae63e51ef0480b7e1","ref":"refs/heads/feature/benchmark-for-JSQuAD","pushedAt":"2023-11-09T06:31:14.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"fix lint errors","shortMessageHtmlLink":"fix lint errors"}},{"before":null,"after":"9d1614eb048b4895edf1b671eee489b8c378e614","ref":"refs/heads/feature/benchmark-for-JSQuAD","pushedAt":"2023-11-09T06:30:22.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"added embedding, metric, relevance","shortMessageHtmlLink":"added embedding, metric, relevance"}},{"before":null,"after":"66022aa37f5d4d32f30087ba9b0e789b3fe0ca38","ref":"refs/heads/main","pushedAt":"2023-11-09T00:04:51.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"TatsuyaShirakawa","name":"Tatsuya Shirakawa","path":"/TatsuyaShirakawa","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11359334?s=80&v=4"},"commit":{"message":"Initial commit","shortMessageHtmlLink":"Initial commit"}}],"hasNextPage":false,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAEE37h9gA","startCursor":null,"endCursor":null}},"title":"Activity · nouu-me/document_vector_search_benchmark"}