{"id":4577,"date":"2025-02-12T18:15:42","date_gmt":"2025-02-12T10:15:42","guid":{"rendered":"https:\/\/infernews.com\/?p=4577"},"modified":"2025-02-12T18:15:43","modified_gmt":"2025-02-12T10:15:43","slug":"tokenizers-%e5%88%86%e8%a9%9e%e5%99%a8%e7%9a%84%e6%91%98%e8%a6%81","status":"publish","type":"post","link":"https:\/\/infernews.com\/blog\/tokenizers-%e5%88%86%e8%a9%9e%e5%99%a8%e7%9a%84%e6%91%98%e8%a6%81\/","title":{"rendered":"Tokenizers \u5206\u8a5e\u5668\u7684\u6458\u8981"},"content":{"rendered":"\n<p>\u5206\u8a5e\u5c0d\u8a13\u7df4\u6a21\u578b\u76f8\u7576\u91cd\u8981\u3002\u7c21\u55ae\u4f86\u8aaa\uff0c\u5c31\u662f\u5c07\u4e00\u6bb5\u6587\u672c\u5206\u5272\u6210\u5f88\u591a\u55ae\u8a5e\u6216\u8005\u5b50\u55ae\u8a5e\uff0c \u9019\u4e9b\u55ae\u8a5e\u6216\u8005\u5b50\u55ae\u8a5e\u5c07\u6703\u901a\u904e\u4e00\u500b\u8868\u683c\u88ab\u8a8d\u77e5\u70ba\u540d\u8a5e\uff0c\u5c08\u6709\u540d\u8a5e\uff0c\u6216\u662f\u52a9\u8a9e\u8a5e\u4e4b\u985e\uff0c\u4e5f\u5c31\u662f\u5c07\u6587\u7ae0\u7684\u8a5e\u5f59\u8f49\u70ba\u6709\u610f\u7fa9\u7684\u95dc\u5efa\u6578\u503c\u3002\u9019\u500b <a href=\"https:\/\/huggingface.co\/docs\/transformers\/v4.48.2\/zh\/tokenizer_summary\" data-type=\"link\" data-id=\"https:\/\/huggingface.co\/docs\/transformers\/v4.48.2\/zh\/tokenizer_summary\" target=\"_blank\" rel=\"noreferrer noopener\">HuggingFace<\/a> \u7684\u4ecb\u7d39\u5305\u542b\u4e86 CoLab \u5605\u7b46\u8a18\uff0c\u4f60\u53ef\u4ee5\u76f4\u63a5\u8a66\u7528\u4e0d\u540c\u7684\u5206\u8a5e\u6280\u8853\u3002<\/p>\n\n\n\n<figure class=\"wp-block-audio\"><audio controls src=\"\/blog\/wp-content\/uploads\/2025\/02\/1739354886996458615-236756806050039.mp3\"><\/audio><\/figure>\n\n\n\n<figure class=\"wp-block-image size-full\"><a href=\"https:\/\/huggingface.co\/docs\/transformers\/v4.48.2\/zh\/tokenizer_summary\" target=\"_blank\" rel=\" noreferrer noopener\"><img data-dominant-color=\"eaf2f9\" data-has-transparency=\"false\" style=\"--dominant-color: #eaf2f9;\" loading=\"lazy\" decoding=\"async\" width=\"800\" height=\"251\" src=\"\/blog\/wp-content\/uploads\/2025\/02\/668850-20161115201058029-666718440.png\" alt=\"\" class=\"wp-image-4578 not-transparent\" srcset=\"https:\/\/infernews.com\/blog\/wp-content\/uploads\/2025\/02\/668850-20161115201058029-666718440.png 800w, https:\/\/infernews.com\/blog\/wp-content\/uploads\/2025\/02\/668850-20161115201058029-666718440-300x94.png 300w, https:\/\/infernews.com\/blog\/wp-content\/uploads\/2025\/02\/668850-20161115201058029-666718440-768x241.png 768w, https:\/\/infernews.com\/blog\/wp-content\/uploads\/2025\/02\/668850-20161115201058029-666718440-640x201.png 640w\" sizes=\"auto, (max-width: 800px) 100vw, 800px\" \/><\/a><\/figure>\n\n\n\n<p><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u5206\u8a5e\u5c0d\u8a13\u7df4\u6a21\u578b\u76f8\u7576\u91cd\u8981\u3002\u7c21\u55ae\u4f86\u8aaa\uff0c\u5c31\u662f\u5c07\u4e00\u6bb5\u6587\u672c\u5206\u5272\u6210\u5f88\u591a\u55ae\u8a5e\u6216\u8005\u5b50\u55ae\u8a5e\uff0c \u9019\u4e9b\u55ae\u8a5e\u6216\u8005\u5b50\u55ae\u8a5e\u5c07\u6703\u901a\u904e\u4e00\u500b\u8868\u683c\u88ab\u8a8d\u77e5\u70ba\u540d\u8a5e\uff0c\u5c08\u6709\u540d\u8a5e\uff0c\u6216\u662f\u52a9\u8a9e\u8a5e\u4e4b\u985e\uff0c\u4e5f\u5c31\u662f\u5c07\u6587\u7ae0\u7684\u8a5e\u5f59\u8f49\u70ba\u6709\u610f\u7fa9\u7684\u95dc\u5efa\u6578\u503c\u3002\u9019\u500b HuggingFace \u7684\u4ecb\u7d39\u5305\u542b\u4e86 CoLab \u5605\u7b46\u8a18\uff0c\u4f60\u53ef\u4ee5\u76f4\u63a5\u8a66\u7528\u4e0d\u540c\u7684\u5206\u8a5e\u6280\u8853\u3002<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"googlesitekit_rrm_CAowvqSiDA:productID":"","footnotes":""},"categories":[27],"tags":[],"class_list":["post-4577","post","type-post","status-publish","format-standard","hentry","category-paper"],"_links":{"self":[{"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/posts\/4577","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/comments?post=4577"}],"version-history":[{"count":0,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/posts\/4577\/revisions"}],"wp:attachment":[{"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/media?parent=4577"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/categories?post=4577"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/tags?post=4577"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}