{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":641670914,"defaultBranch":"main","name":"tokenizers-cpp","ownerLogin":"mlc-ai","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2023-05-16T23:40:16.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/106173866?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1719349122.0","currentOid":""},"activityList":{"items":[{"before":"eec9d6831daa4a64410abf94e8be796be6be6d5d","after":"c0fab1e14a9421c1501acee5b7703e5dafa60479","ref":"refs/heads/main","pushedAt":"2024-06-27T17:33:50.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"[CMake] Support disable SentencePiece tokenizer (#38)\n\nThis PR supports turning off SentencePiece tokenizer to reduce binary\r\nsize.","shortMessageHtmlLink":"[CMake] Support disable SentencePiece tokenizer (#38)"}},{"before":"fc9dde0b2145eb534c600880179217889eb6f5a4","after":"125d072f52290fa6d2944b3d72ccc937786ec631","ref":"refs/heads/disable-sentencepiece","pushedAt":"2024-06-25T20:59:40.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"MasterJH5574","name":"Ruihang Lai","path":"/MasterJH5574","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/45167100?s=80&v=4"},"commit":{"message":"[CMake] Support disable SentencePiece tokenizer\n\nThis PR supports turning off SentencePiece tokenizer to reduce binary\nsize.","shortMessageHtmlLink":"[CMake] Support disable SentencePiece tokenizer"}},{"before":null,"after":"fc9dde0b2145eb534c600880179217889eb6f5a4","ref":"refs/heads/disable-sentencepiece","pushedAt":"2024-06-25T20:58:42.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"MasterJH5574","name":"Ruihang Lai","path":"/MasterJH5574","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/45167100?s=80&v=4"},"commit":{"message":"[CMake] Support disable SentencePiece tokenizer\n\nThis PR supports turning off SentencePiece tokenizer to reduce binary\nsize.","shortMessageHtmlLink":"[CMake] Support disable SentencePiece tokenizer"}},{"before":"a52e33e41c7a033eff83d8636a76969cb9de104d","after":"eec9d6831daa4a64410abf94e8be796be6be6d5d","ref":"refs/heads/main","pushedAt":"2024-06-03T04:35:32.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"CharlieFRuan","name":"Charlie Ruan","path":"/CharlieFRuan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/53290280?s=80&v=4"},"commit":{"message":"[Web] Bump version to 0.1.4","shortMessageHtmlLink":"[Web] Bump version to 0.1.4"}},{"before":"2db668a6ff618636a18143eb5c06e17673b3279b","after":"a52e33e41c7a033eff83d8636a76969cb9de104d","ref":"refs/heads/main","pushedAt":"2024-06-02T19:07:19.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"MasterJH5574","name":"Ruihang Lai","path":"/MasterJH5574","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/45167100?s=80&v=4"},"commit":{"message":"[Rust] Bump HuggingFace tokenizer version (#36)\n\nThis PR bumps the HuggingFace tokenizers package version to work with\r\nsome of the latest models (e.g., Mistral v0.3).","shortMessageHtmlLink":"[Rust] Bump HuggingFace tokenizer version (#36)"}},{"before":null,"after":"e65efa962f6a26e46a71e2930f4fa8ad97af4813","ref":"refs/heads/bump-hf-tokenizers","pushedAt":"2024-06-02T19:07:00.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"MasterJH5574","name":"Ruihang Lai","path":"/MasterJH5574","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/45167100?s=80&v=4"},"commit":{"message":"[Rust] Bump HuggingFace tokenizer version\n\nThis PR bumps the HuggingFace tokenizers package version to work with\nsome of the latest models (e.g., Mistral v0.3).","shortMessageHtmlLink":"[Rust] Bump HuggingFace tokenizer version"}},{"before":"55170e8f9546891bd44ed94338ec0c1400471c0b","after":"2db668a6ff618636a18143eb5c06e17673b3279b","ref":"refs/heads/main","pushedAt":"2024-05-29T20:27:10.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"vinx13","name":"Wuwei Lin","path":"/vinx13","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/7195739?s=80&v=4"},"commit":{"message":"Remove incorrect final for non-virtual func (#34)","shortMessageHtmlLink":"Remove incorrect final for non-virtual func (#34)"}},{"before":"07376b9d8b8d37c6de0e30c5fd2851fdfee57d4d","after":"55170e8f9546891bd44ed94338ec0c1400471c0b","ref":"refs/heads/main","pushedAt":"2024-05-29T18:50:08.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Added EncodeBatch interface (#33)","shortMessageHtmlLink":"Added EncodeBatch interface (#33)"}},{"before":"35bad0e0d08ba2309d45ad2ef479b43b2455c70d","after":"07376b9d8b8d37c6de0e30c5fd2851fdfee57d4d","ref":"refs/heads/main","pushedAt":"2024-04-05T12:33:42.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Set value of TOKENIZERS_CPP_CARGO_TARGET based on ANDROID_ABI (#28)","shortMessageHtmlLink":"Set value of TOKENIZERS_CPP_CARGO_TARGET based on ANDROID_ABI (#28)"}},{"before":"7466de507d3b34ab283ccdd34c6d5fb37237a744","after":"35bad0e0d08ba2309d45ad2ef479b43b2455c70d","ref":"refs/heads/main","pushedAt":"2024-04-04T11:39:44.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"[HFTokenizer] Allow use of \"add_special_tokens\" for HFTokenizer.Encode and \"skip_special_tokens\" for HFTokenizer.Decode methods. (#27)","shortMessageHtmlLink":"[HFTokenizer] Allow use of \"add_special_tokens\" for HFTokenizer.Encod…"}},{"before":"27dbe17d7268801ec720569167af905c88d3db50","after":"7466de507d3b34ab283ccdd34c6d5fb37237a744","ref":"refs/heads/main","pushedAt":"2024-03-30T21:25:49.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"CharlieFRuan","name":"Charlie Ruan","path":"/CharlieFRuan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/53290280?s=80&v=4"},"commit":{"message":"[Web] Expose getVocabSize and idToToken to web, bump version to 0.1.3 (#25)","shortMessageHtmlLink":"[Web] Expose getVocabSize and idToToken to web, bump version to 0.1.3 ("}},{"before":"467d5cd6999fce2fb325d36a18d8e1cfa5e5bc4a","after":"27dbe17d7268801ec720569167af905c88d3db50","ref":"refs/heads/main","pushedAt":"2023-12-19T20:07:07.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Add support for querying vocabulary from tokenizer (#22)","shortMessageHtmlLink":"Add support for querying vocabulary from tokenizer (#22)"}},{"before":"08316ea4d98ca8f5a551ddb1cb5c6f4c5518ac52","after":"467d5cd6999fce2fb325d36a18d8e1cfa5e5bc4a","ref":"refs/heads/main","pushedAt":"2023-11-18T19:56:39.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"CharlieFRuan","name":"Charlie Ruan","path":"/CharlieFRuan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/53290280?s=80&v=4"},"commit":{"message":"Bump version to 0.1.2","shortMessageHtmlLink":"Bump version to 0.1.2"}},{"before":"4456fa8c8a9747074403f1f0804d99bc3061a8fa","after":"08316ea4d98ca8f5a551ddb1cb5c6f4c5518ac52","ref":"refs/heads/main","pushedAt":"2023-11-18T00:42:23.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Update build.sh to make web tokenizer work with chrome extension (#20)\n\nweb-llm's chrome extension not working with manifest version 3 and Google will be stopping manifest version 2 in January, 2024\r\n\r\nThe chrome extension isn't working because of use of evals in code. using -s NO_DYNAMIC_EXECUTION=1 removes eval() and new Function() from generated code\r\n\r\nrefer: discussions in https://github.com/mlc-ai/web-llm/issues/214","shortMessageHtmlLink":"Update build.sh to make web tokenizer work with chrome extension (#20)"}},{"before":"22b89324a0eebf2e26afa6de3ba7e34e4b382e86","after":"4456fa8c8a9747074403f1f0804d99bc3061a8fa","ref":"refs/heads/main","pushedAt":"2023-11-16T08:42:49.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"Hzfengsy","name":"Siyuan Feng","path":"/Hzfengsy","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/25500082?s=80&v=4"},"commit":{"message":"fix rwkv world tokenzier (#19)\n\n* support rwkv world tokenizer\r\n\r\n* refine\r\n\r\n* rename\r\n\r\n* refine\r\n\r\n* switch msgpack version\r\n\r\n* refine\r\n\r\n* refine\r\n\r\n* fix rwkv world tokenizer bug\r\n\r\n* fix comment","shortMessageHtmlLink":"fix rwkv world tokenzier (#19)"}},{"before":"e47442f1aa766c27f17444218783ba1903890ba9","after":"22b89324a0eebf2e26afa6de3ba7e34e4b382e86","ref":"refs/heads/main","pushedAt":"2023-10-28T15:25:54.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Update to reduce memory realloc (#18)","shortMessageHtmlLink":"Update to reduce memory realloc (#18)"}},{"before":"0e23aecd49f405a9e5569adf96b8a86d12de0cca","after":"5f2bbf24101b6868fa2e85e968ae93b673684a1c","ref":"refs/heads/rust","pushedAt":"2023-10-28T15:25:21.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Update to reduce memory realloc","shortMessageHtmlLink":"Update to reduce memory realloc"}},{"before":"1eae242d3da617a2cf687753ede61345e999852c","after":"e47442f1aa766c27f17444218783ba1903890ba9","ref":"refs/heads/main","pushedAt":"2023-09-06T09:41:50.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"Hzfengsy","name":"Siyuan Feng","path":"/Hzfengsy","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/25500082?s=80&v=4"},"commit":{"message":"[FIX] Remove boost dependency from submodule msgpack (#16)","shortMessageHtmlLink":"[FIX] Remove boost dependency from submodule msgpack (#16)"}},{"before":"eec72a6b234b92609cd95050198c0cac68d25d69","after":"1eae242d3da617a2cf687753ede61345e999852c","ref":"refs/heads/main","pushedAt":"2023-09-06T05:59:14.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"Hzfengsy","name":"Siyuan Feng","path":"/Hzfengsy","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/25500082?s=80&v=4"},"commit":{"message":"msgpack as 3rdparty library instead of fetch_context (#15)\n\n* support rwkv world tokenizer\r\n\r\n* refine\r\n\r\n* rename\r\n\r\n* refine\r\n\r\n* switch msgpack version\r\n\r\n* refine\r\n\r\n* refine","shortMessageHtmlLink":"msgpack as 3rdparty library instead of fetch_context (#15)"}},{"before":"4f42c9fa74946d70af86671a3804b6f2433e5dac","after":"eec72a6b234b92609cd95050198c0cac68d25d69","ref":"refs/heads/main","pushedAt":"2023-09-01T11:34:49.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"Hzfengsy","name":"Siyuan Feng","path":"/Hzfengsy","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/25500082?s=80&v=4"},"commit":{"message":"add rwkv world tokenizer (#14)\n\n* support rwkv world tokenizer\r\n\r\n* refine\r\n\r\n* rename","shortMessageHtmlLink":"add rwkv world tokenizer (#14)"}},{"before":"6e3a37e2ce4165fb70635a684ad300034fcb63dc","after":"4f42c9fa74946d70af86671a3804b6f2433e5dac","ref":"refs/heads/main","pushedAt":"2023-08-15T15:23:22.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Update rust to support latest tokenizers (#11)","shortMessageHtmlLink":"Update rust to support latest tokenizers (#11)"}},{"before":null,"after":"0e23aecd49f405a9e5569adf96b8a86d12de0cca","ref":"refs/heads/rust","pushedAt":"2023-08-15T15:22:12.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Update rust to support latest tokenizers","shortMessageHtmlLink":"Update rust to support latest tokenizers"}},{"before":"c53bc0444dbe2ea1f66e364cd576a6c1e23539b4","after":"6e3a37e2ce4165fb70635a684ad300034fcb63dc","ref":"refs/heads/main","pushedAt":"2023-07-16T15:35:34.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Update CMakeLists.txt (#6)","shortMessageHtmlLink":"Update CMakeLists.txt (#6)"}},{"before":"8465f9bb789c9610abc1bc1278593b320ee9699a","after":"c53bc0444dbe2ea1f66e364cd576a6c1e23539b4","ref":"refs/heads/main","pushedAt":"2023-06-03T17:57:00.772Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Fix windows build by linking against ntdll (#5)","shortMessageHtmlLink":"Fix windows build by linking against ntdll (#5)"}},{"before":"a9f93597e6ee1738509c4a60f5d156d53f5954cd","after":"8465f9bb789c9610abc1bc1278593b320ee9699a","ref":"refs/heads/main","pushedAt":"2023-05-26T21:03:46.569Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Fix readme typo (#4)","shortMessageHtmlLink":"Fix readme typo (#4)"}},{"before":"bd24b8a5dd971fd1a9308e10bc20741cedb84fe7","after":"a9f93597e6ee1738509c4a60f5d156d53f5954cd","ref":"refs/heads/main","pushedAt":"2023-05-24T20:17:12.359Z","pushType":"push","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Update to parcel2","shortMessageHtmlLink":"Update to parcel2"}},{"before":"19d3a00d0f97dc8a26f4b7fc9904c3e6a5a8c92e","after":"bd24b8a5dd971fd1a9308e10bc20741cedb84fe7","ref":"refs/heads/main","pushedAt":"2023-05-22T16:34:56.472Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Update CMakeLists.txt (#2)\n\nAdd ${TOKENIZERS_C_LINK_LIBS} and append ${CMAKE_DL_LIBS} for Linux platform (https://github.com/mlc-ai/tokenizers-cpp/issues/1#issuecomment-1556206254)\r\n\r\nmove some of the TOKENIZERS_CPP_LINK_LIBS deps to TOKENIZERS_C_LINK_LIBS.","shortMessageHtmlLink":"Update CMakeLists.txt (#2)"}},{"before":"5703f8da64201d03e4d8d950ebbc655b46f000aa","after":"19d3a00d0f97dc8a26f4b7fc9904c3e6a5a8c92e","ref":"refs/heads/main","pushedAt":"2023-05-22T14:48:56.261Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Merge pull request #3 from mlc-ai/npm\n\nAdd npm package","shortMessageHtmlLink":"Merge pull request #3 from mlc-ai/npm"}},{"before":"49e29dac12b66d714295e180e2cac887adb00974","after":"75353f534be2beb92075e644d146f2dcf336eb19","ref":"refs/heads/npm","pushedAt":"2023-05-22T14:48:47.318Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Add npm package","shortMessageHtmlLink":"Add npm package"}},{"before":null,"after":"49e29dac12b66d714295e180e2cac887adb00974","ref":"refs/heads/npm","pushedAt":"2023-05-22T14:45:06.544Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"tqchen","name":"Tianqi Chen","path":"/tqchen","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/2577440?s=80&v=4"},"commit":{"message":"Add npm package","shortMessageHtmlLink":"Add npm package"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAEcRsbxwA","startCursor":null,"endCursor":null}},"title":"Activity · mlc-ai/tokenizers-cpp"}