{"id":12866,"date":"2026-10-03T00:22:32","date_gmt":"2026-10-02T16:22:32","guid":{"rendered":"https:\/\/infernews.com\/blog\/?page_id=12866"},"modified":"2026-10-03T00:32:24","modified_gmt":"2026-10-02T16:32:24","slug":"%e8%a7%a3%e6%a7%8b-llm-%e6%8e%a8%e8%ab%96%e5%ba%95%e5%b1%a4","status":"publish","type":"page","link":"https:\/\/infernews.com\/blog\/%e8%a7%a3%e6%a7%8b-llm-%e6%8e%a8%e8%ab%96%e5%ba%95%e5%b1%a4\/","title":{"rendered":"\u89e3\u69cb LLM \u63a8\u8ad6\u5e95\u5c64"},"content":{"rendered":"\n<figure class=\"wp-block-image size-full\"><img data-dominant-color=\"636c75\" data-has-transparency=\"false\" style=\"--dominant-color: #636c75;\" loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"397\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" src=\"https:\/\/infernews.com\/blog\/wp-content\/uploads\/2026\/10\/image-2.png\" alt=\"\" class=\"wp-image-12864 not-transparent\" srcset=\"https:\/\/infernews.com\/blog\/wp-content\/uploads\/2026\/10\/image-2.png 1024w, https:\/\/infernews.com\/blog\/wp-content\/uploads\/2026\/10\/image-2-300x116.png 300w, https:\/\/infernews.com\/blog\/wp-content\/uploads\/2026\/10\/image-2-766x297.png 766w\" \/><\/figure>\n\n\n\n<style>\nhtml {\n  scroll-behavior: smooth;\n}\n#wp-scoped-article {\n  font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Noto Sans TC\", sans-serif;\n  color: #1e293b;\n  line-height: 1.8;\n  font-size: 22px; \n  margin: 1.5rem 0;\n  box-sizing: border-box;\n  -webkit-font-smoothing: antialiased;\n}\n#wp-scoped-article * {\n  box-sizing: border-box;\n}\n#wp-scoped-article h1 {\n  font-size: 1.8rem;\n  font-weight: 800;\n  color: #0f172a;\n  line-height: 1.35;\n  margin-bottom: 1.25rem;\n}\n#wp-scoped-article h2 {\n  font-size: 1.5rem;\n  font-weight: 700;\n  color: #0f172a;\n  margin-top: 2.25rem;\n  margin-bottom: 1rem;\n  padding-bottom: 0.5rem;\n  border-bottom: 2px solid #e2e8f0;\n  display: flex;\n  align-items: center;\n  gap: 0.5rem;\n  scroll-margin-top: 80px; \n}\n#wp-scoped-article h3 {\n  font-size: 1.25rem;\n  font-weight: 600;\n  color: #1e293b;\n  margin-top: 1.6rem;\n  margin-bottom: 0.75rem;\n  scroll-margin-top: 80px; \n}\n#wp-scoped-article p {\n  margin-bottom: 1.2rem;\n  color: #334155;\n}\n#wp-scoped-article a {\n  color: #2563eb;\n  text-decoration: none;\n}\n#wp-scoped-article a:hover {\n  text-decoration: underline;\n}\n#wp-scoped-article .in-toc a {\n  text-decoration: none;\n  color: #2563eb;\n}\n#wp-scoped-article .in-toc a:hover {\n  text-decoration: none;\n  opacity: 0.8;\n}\n#wp-scoped-article .in-toc {\n  background: #f8fafc;\n  border: 1px solid #e2e8f0;\n  border-radius: 8px;\n  padding: 1.5rem;\n  margin: 1.5rem 0;\n}\n#wp-scoped-article .in-toc ul {\n  list-style: none;\n  padding-left: 0;\n  margin: 0;\n}\n#wp-scoped-article .in-toc li {\n  margin-bottom: 0.5rem;\n}\n#wp-scoped-article .in-tag-group {\n  display: flex;\n  flex-wrap: wrap;\n  gap: 6px;\n  margin-bottom: 1.25rem;\n}\n#wp-scoped-article .in-tag {\n  display: inline-block;\n  padding: 2px 10px;\n  border-radius: 9999px;\n  font-size: 17px;\n  font-weight: 600;\n  background: #eff6ff;\n  color: #1d4ed8;\n  border: 1px solid #bfdbfe;\n}\n#wp-scoped-article .in-callout-danger {\n  background-color: #fef2f2;\n  border-left: 5px solid #ef4444;\n  border-radius: 0 8px 8px 0;\n  padding: 1.15rem 1.25rem;\n  margin: 1.5rem 0;\n  font-size: 1rem;\n}\n#wp-scoped-article .in-callout-info {\n  background-color: #eff6ff;\n  border-left: 5px solid #3b82f6;\n  border-radius: 0 8px 8px 0;\n  padding: 1.15rem 1.25rem;\n  margin: 1.5rem 0;\n  font-size: 1rem;\n}\n#wp-scoped-article .in-badge {\n  display: inline-flex;\n  align-items: center;\n  justify-content: center;\n  width: 28px;\n  height: 28px;\n  border-radius: 6px;\n  background: #2563eb;\n  color: #ffffff;\n  font-size: 18px;\n  font-weight: 700;\n  margin-right: 6px;\n  flex-shrink: 0;\n}\n#wp-scoped-article .in-terminal-box {\n  background: #0d1117;\n  border: 1px solid #30363d;\n  border-radius: 8px;\n  margin: 1.5rem 0;\n  overflow: hidden;\n  box-shadow: 0 4px 12px rgba(0, 0, 0, 0.12);\n}\n#wp-scoped-article .in-terminal-header {\n  background: #161b22;\n  padding: 8px 14px;\n  display: flex;\n  align-items: center;\n  justify-content: space-between;\n  border-bottom: 1px solid #30363d;\n  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;\n  font-size: 14px;\n  color: #8b949e;\n}\n#wp-scoped-article .in-terminal-dots {\n  display: flex;\n  gap: 6px;\n}\n#wp-scoped-article .in-dot {\n  width: 10px;\n  height: 10px;\n  border-radius: 50%;\n  display: inline-block;\n}\n#wp-scoped-article .in-dot-red { background: #ff5f56; }\n#wp-scoped-article .in-dot-yellow { background: #ffbd2e; }\n#wp-scoped-article .in-dot-green { background: #27c93f; }\n#wp-scoped-article pre.in-code {\n  margin: 0;\n  padding: 16px;\n  background: #0d1117;\n  color: #c9d1d9;\n  font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;\n  font-size: 15.5px;\n  line-height: 1.6;\n  overflow-x: auto;\n  white-space: pre;\n  border: none;\n}\n#wp-scoped-article .in-table-wrapper {\n  overflow-x: auto;\n  margin: 1.5rem 0;\n  border: 1px solid #e2e8f0;\n  border-radius: 8px;\n}\n#wp-scoped-article table.in-table {\n  width: 100%;\n  border-collapse: collapse;\n  font-size: 20px; \n  text-align: left;\n  margin: 0;\n}\n#wp-scoped-article table.in-table th {\n  background-color: #f8fafc;\n  color: #334155;\n  font-weight: 600;\n  padding: 12px 16px;\n  border-bottom: 1px solid #e2e8f0;\n  font-size: 20px; \n}\n#wp-scoped-article table.in-table td {\n  padding: 12px 16px;\n  border-bottom: 1px solid #f1f5f9;\n  color: #475569;\n  font-size: 20px; \n}\n#wp-scoped-article .in-checklist {\n  display: grid;\n  grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));\n  gap: 10px;\n  margin: 1.5rem 0;\n}\n#wp-scoped-article .in-checklist-item {\n  display: flex;\n  align-items: center;\n  gap: 10px;\n  padding: 10px 14px;\n  background: #f8fafc;\n  border: 1px solid #e2e8f0;\n  border-radius: 8px;\n  font-size: 20px;\n}\n#wp-scoped-article .in-check-icon {\n  color: #10b981;\n  font-weight: bold;\n}\n#wp-scoped-article .in-math {\n  font-family: \"Cambria Math\", \"Times New Roman\", Times, serif;\n  font-style: italic;\n  font-weight: 500;\n}\n#wp-scoped-article .in-math-bold {\n  font-family: \"Cambria Math\", \"Times New Roman\", Times, serif;\n  font-weight: 700;\n}\n#wp-scoped-article .in-fraction {\n  display: inline-flex;\n  flex-direction: column;\n  vertical-align: middle;\n  text-align: center;\n  padding: 0 4px;\n  font-size: 0.9em;\n  line-height: 1.1;\n}\n#wp-scoped-article .in-fraction-top {\n  border-bottom: 1px solid currentColor;\n  padding-bottom: 2px;\n}\n#wp-scoped-article .in-fraction-bottom {\n  padding-top: 2px;\n}\n<\/style>\n\n<div id=\"wp-scoped-article\">\n  <div class=\"in-tag-group\">\n    <span class=\"in-tag\">\u4eba\u5de5\u667a\u6167<\/span>\n    <span class=\"in-tag\">\u5927\u8a9e\u8a00\u6a21\u578b<\/span>\n    <span class=\"in-tag\">\u7dda\u6027\u4ee3\u6578<\/span>\n    <span class=\"in-tag\">\u63a8\u8ad6\u67b6\u69cb<\/span>\n    <span class=\"in-tag\">\u6578\u5b78\u5e95\u5c64<\/span>\n  <\/div>\n\n  <h1>\u5de5\u7a0b\u5e2b\u5fc5\u5099\u7684\u9ad8\u4e2d\u6578\u5b78\u601d\u7dad\u5be6\u52d9<\/h1>\n\n  <p>\u8a31\u591a\u8edf\u9ad4\u5de5\u7a0b\u5e2b\u5728\u8de8\u5165\u5927\u578b\u8a9e\u8a00\u6a21\u578b\uff08LLM\uff09\u6280\u8853\u9818\u57df\u6642\uff0c\u5f80\u5f80\u88ab\u8ad6\u6587\u4e2d\u5bc6\u5bc6\u9ebb\u9ebb\u7684\u504f\u5fae\u5206\u3001\u53cd\u5411\u50b3\u64ad\u8207\u7d71\u8a08\u5b78\u7fd2\u7406\u8ad6\u52f8\u9000\u3002\u7136\u800c\uff0c\u8cc7\u6df1\u6280\u8853\u4f5c\u8005 Giles Thomas \u6307\u51fa\uff1a\u82e5\u4f60\u7684\u76ee\u6a19\u662f<b>\u900f\u5fb9\u7406\u89e3 LLM \u5728\u63a8\u8ad6\uff08Inference\uff09\u968e\u6bb5\u7684\u904b\u4f5c\u6a5f\u5236<\/b>\uff0c\u800c\u975e\u5f9e\u982d\u7814\u767c\u57fa\u790e\u67b6\u69cb\uff0c\u4f60\u6240\u9700\u8981\u7684\u6578\u5b78\u5de5\u5177\u5e7e\u4e4e\u4e0d\u8d85\u51fa\u9ad8\u4e2d\u7bc4\u7587\u2014\u2014\u6838\u5fc3\u5b8c\u5168\u570d\u7e5e\u5728<b>\u5411\u91cf\uff08Vectors\uff09<\/b>\u3001<b>\u9ad8\u7dad\u7a7a\u9593\u5e7e\u4f55<\/b>\u3001<b>\u5167\u7a4d\uff08Dot Product\uff09<\/b>\u4ee5\u53ca<b>\u77e9\u9663\u4e58\u6cd5\u6295\u5f71\uff08Matrix Projections\uff09<\/b>\u3002<\/p>\n\n  <div class=\"in-callout-info\">\n    <strong>\u5c0e\u8b80\u6838\u5fc3\u63d0\u8981\uff1a<\/strong> \u672c\u6587\u91dd\u5c0d Giles Thomas \u6df1\u5165\u5256\u6790 Sebastian Raschka \u8457\u4f5c\u300a\u5f9e\u96f6\u69cb\u5efa\u5927\u8a9e\u8a00\u6a21\u578b\uff08Build a Large Language Model from Scratch\uff09\u300b\u7b2c 19 \u90e8\u5206\u7684\u5fc3\u5f97\u9032\u884c\u5b8c\u6574\u67b6\u69cb\u89e3\u69cb\uff0c\u805a\u7126\u65bc\u6a21\u578b\u5982\u4f55\u5c07\u96e2\u6563\u6587\u5b57\u6620\u5c04\u70ba\u9023\u7e8c\u5e7e\u4f55\u7a7a\u9593\uff0c\u4e26\u5b8c\u6210\u9810\u6e2c\u8207\u8f49\u63db\u3002\n  <\/div>\n\n  <div class=\"in-toc\">\n    <strong>\u6587\u7ae0\u76ee\u9304<\/strong>\n    <ul>\n      <li><a href=\"#section-1\">1. \u5411\u91cf\u672c\u8cea\uff1a\u5f9e\u6578\u503c\u9663\u5217\u5230\u9ad8\u7dad\u7a7a\u9593\u5ea7\u6a19<\/a><\/li>\n      <li><a href=\"#section-2\">2. \u8a5e\u5f59\u7a7a\u9593\uff08Vocab Space\uff09\u8207 Softmax \u6b78\u4e00\u5316<\/a><\/li>\n      <li><a href=\"#section-3\">3. \u5d4c\u5165\u7a7a\u9593\uff08Embedding Space\uff09\u8207\u8a9e\u610f\u5e7e\u4f55\u805a\u96c6<\/a><\/li>\n      <li><a href=\"#section-4\">4. \u5167\u7a4d\uff08Dot Product\uff09\u8207\u9918\u5f26\u76f8\u4f3c\u5ea6\u5ea6\u91cf<\/a><\/li>\n      <li><a href=\"#section-5\">5. \u77e9\u9663\u4e58\u6cd5\u672c\u8cea\uff1a\u9ad8\u7dad\u7a7a\u9593\u4e2d\u7684\u5e7e\u4f55\u8b8a\u63db\u8207\u6295\u5f71<\/a><\/li>\n      <li><a href=\"#section-6\">6. LLM \u63a8\u8ad6\u6838\u5fc3\u6578\u5b78\u6982\u5ff5\u901f\u67e5\u5c0d\u7167\u8868<\/a><\/li>\n    <\/ul>\n  <\/div>\n\n  <h2 id=\"section-1\"><span class=\"in-badge\">1<\/span> \u5411\u91cf\u672c\u8cea\uff1a\u5f9e\u6578\u503c\u9663\u5217\u5230\u9ad8\u7dad\u7a7a\u9593\u5ea7\u6a19<\/h2>\n  <p>\u5728\u50b3\u7d71\u8edf\u9ad4\u958b\u767c\u4e2d\uff0c\u5de5\u7a0b\u5e2b\u901a\u5e38\u5c07\u300c\u5411\u91cf\uff08Vector\uff09\u300d\u55ae\u7d14\u8996\u70ba\u6d6e\u9ede\u6578\u6216\u6574\u6578\u7684\u9663\u5217\uff08Array of Numbers\uff09\u3002\u4f46\u5728\u5e7e\u4f55\u8207\u7dda\u6027\u4ee3\u6578\u4e2d\uff0c\u4e00\u500b\u9577\u5ea6\u70ba <span class=\"in-math\">n<\/span> \u7684\u5411\u91cf\u5177\u6709\u66f4\u8c50\u5bcc\u7684\u7269\u7406\u610f\u7fa9\uff1a\u5b83\u662f <span class=\"in-math\">n<\/span> \u7dad\u7a7a\u9593\u4e2d\u81ea\u539f\u9ede\u51fa\u767c\u7684<b>\u8ddd\u96e2\u8207\u65b9\u5411<\/b>\uff0c\u540c\u6642\u4e5f\u4ee3\u8868\u8a72\u7a7a\u9593\u4e2d\u7684\u552f\u4e00<b>\u9ede\u5ea7\u6a19<\/b>\u3002<\/p>\n  <p>\u5728\u4e8c\u7dad\u5e73\u9762\u4e2d\uff0c\u5411\u91cf (2, \u22123) \u4ee3\u8868\u5411\u53f3\u79fb\u52d5 2 \u55ae\u4f4d\u3001\u5411\u4e0b\u79fb\u52d5 3 \u55ae\u4f4d\uff1b\u5728\u4e09\u7dad\u7a7a\u9593\u4e2d\uff0c(5, 1, \u22127) \u5247\u5b9a\u7fa9\u4e86\u7acb\u9ad4\u7a7a\u9593\u4e2d\u7684\u7279\u5b9a\u4f4d\u7f6e\u3002\u7576\u7dad\u5ea6\u64f4\u5c55\u5230\u6578\u767e\u751a\u81f3\u6578\u842c\u7dad\u6642\uff0c\u4eba\u985e\u5927\u8166\u5df2\u7121\u6cd5\u76f4\u63a5\u8996\u89ba\u5316\uff0c\u4f46\u5176\u4ee3\u6578\u672c\u8cea\u8207\u5e7e\u4f55\u6cd5\u5247\u5b8c\u5168\u4e0d\u8b8a\u3002\u5728 LLM \u5167\u90e8\uff0c\u6240\u6709\u7279\u5fb5\u3001\u6a5f\u7387\u8207\u8a9e\u610f\u8cc7\u8a0a\uff0c\u5168\u90e8\u900f\u904e\u9019\u7a2e\u9ad8\u7dad\u5ea7\u6a19\u4f86\u7cbe\u6e96\u8868\u9054\u3002<\/p>\n\n  <div class=\"in-checklist\">\n    <div class=\"in-checklist-item\"><span class=\"in-check-icon\">\u2713<\/span> <span>\u4e00\u7dad\u9663\u5217\u5373\u9ad8\u7dad\u7a7a\u9593\u4e2d\u7684\u5b9a\u9ede\u5ea7\u6a19<\/span><\/div>\n    <div class=\"in-checklist-item\"><span class=\"in-check-icon\">\u2713<\/span> <span>\u5411\u91cf\u593e\u89d2\u8207\u9577\u5ea6\u6c7a\u5b9a\u5e7e\u4f55\u95dc\u4fc2<\/span><\/div>\n    <div class=\"in-checklist-item\"><span class=\"in-check-icon\">\u2713<\/span> <span>\u9ad8\u7dad\u4ee3\u6578\u6cd5\u5247\u8207\u4e8c\u7dad\u5e73\u9762\u5e7e\u4f55\u5b8c\u5168\u4e00\u81f4<\/span><\/div>\n  <\/div>\n\n  <h2 id=\"section-2\"><span class=\"in-badge\">2<\/span> \u8a5e\u5f59\u7a7a\u9593\uff08Vocab Space\uff09\u8207 Softmax \u6b78\u4e00\u5316<\/h2>\n  <p>\u7576 LLM \u8655\u7406\u8f38\u5165\u5e8f\u5217\u4e26\u6e96\u5099\u9810\u6e2c\u4e0b\u4e00\u500b Token \u6642\uff0c\u6a21\u578b\u6700\u5f8c\u4e00\u5c64\u8f38\u51fa\u7684\u662f\u672a\u6b78\u4e00\u5316\u7684\u5c0d\u6578\u6a5f\u7387\uff0c\u5373 <b>Logits \u5411\u91cf<\/b>\u3002\u4ee5\u7d93\u5178\u7684 GPT-2 \u6a21\u578b\u70ba\u4f8b\uff0c\u5176\u5206\u8a5e\u5668\uff08Tokenizer\uff09\u8a5e\u5f59\u8868\u5927\u5c0f\u70ba 50,257\uff0c\u9019\u4ee3\u8868\u6bcf\u500b Logits \u5411\u91cf\u7684\u9577\u5ea6\u6070\u597d\u662f 50,257\uff0c\u5c0d\u61c9\u65bc 50,257 \u7dad\u7684\u9f90\u5927\u7a7a\u9593\u3002<\/p>\n  <p>\u6bcf\u500b\u7dad\u5ea6\uff08Slot\uff09\u7684\u5206\u6578\u4ee3\u8868\u5c0d\u61c9 Token \u4f5c\u70ba\u4e0b\u4e00\u500b\u5b57\u7684\u76f8\u5c0d\u50be\u5411\u3002\u4f8b\u5982\u7de8\u865f 464 \u7684\u4f4d\u7f6e\u4ee3\u8868\u55ae\u8a5e\u300cThe\u300d\uff0c\u5176\u6578\u503c\u9ad8\u4f4e\u4ee3\u8868\u63a5\u7e8c\u8a72\u8a5e\u7684\u5f37\u70c8\u7a0b\u5ea6\u3002\u7136\u800c\uff0c\u9019\u7a2e\u672a\u7d93\u8655\u7406\u7684\u7a7a\u9593\u975e\u5e38\u300c\u6df7\u4e82\u300d\uff0c\u4f8b\u5982\u5411\u91cf (1, 2, 3) \u8207 (\u22129, \u22128, \u22127) \u5177\u6709\u5b8c\u5168\u76f8\u540c\u7684\u76f8\u5c0d\u6392\u5e8f\uff0c\u4f46\u6578\u503c\u7d1a\u8ddd\u4e0d\u540c\u3002<\/p>\n\n  <div class=\"in-terminal-box\">\n    <div class=\"in-terminal-header\">\n      <div class=\"in-terminal-dots\">\n        <span class=\"in-dot in-dot-red\"><\/span>\n        <span class=\"in-dot in-dot-yellow\"><\/span>\n        <span class=\"in-dot in-dot-green\"><\/span>\n      <\/div>\n      <span>softmax_normalization.py<\/span>\n    <\/div>\n    <pre class=\"in-code\">import torch\nimport torch.nn.functional as F\n\n# \u5169\u500b\u770b\u4f3c\u6578\u503c\u76f8\u5dee\u6975\u5927\u7684\u672a\u6b78\u4e00\u5316 Logits\nlogits_a = torch.tensor([1.0, 2.0, 3.0])\nlogits_b = torch.tensor([-9.0, -8.0, -7.0])\n\n# \u7d93\u904e Softmax \u6620\u5c04\u5230\u6a19\u6e96\u6a5f\u7387\u5206\u4f48\nprob_a = F.softmax(logits_a, dim=-1)\nprob_b = F.softmax(logits_b, dim=-1)\n\nprint(\"\u6a5f\u7387\u5206\u4f48 A:\", prob_a.numpy().round(3))  # [0.09, 0.244, 0.665]\nprint(\"\u6a5f\u7387\u5206\u4f48 B:\", prob_b.numpy().round(3))  # [0.09, 0.244, 0.665]<\/pre>\n  <\/div>\n\n  <p>\u900f\u904e <b>Softmax \u51fd\u6578<\/b>\uff0c\u6211\u5011\u80fd\u5c07\u6df7\u4e82\u7684 Logits \u5411\u91cf\u58d3\u5236\u5728 [0, 1] \u5340\u9593\u5167\uff0c\u4e26\u4e14\u4fdd\u8b49\u6240\u6709\u7dad\u5ea6\u6578\u503c\u52a0\u7e3d\u56b4\u683c\u7b49\u65bc 1\uff0c\u8f49\u63db\u70ba\u6a19\u6e96\u7684\u6a5f\u7387\u5206\u4f48\u3002\u9019\u4f7f\u5f97\u6a21\u578b\u80fd\u5920\u4ee5\u6e05\u6670\u7684\u6a5f\u7387\u6b0a\u91cd\u9032\u884c\u63a1\u6a23\uff08Sampling\uff09\u6216\u8caa\u5a6a\u641c\u5c0b\uff08Greedy Search\uff09\u3002<\/p>\n\n  <h2 id=\"section-3\"><span class=\"in-badge\">3<\/span> \u5d4c\u5165\u7a7a\u9593\uff08Embedding Space\uff09\u8207\u8a9e\u610f\u5e7e\u4f55\u805a\u96c6<\/h2>\n  <p>\u9664\u4e86\u8a5e\u5f59\u9810\u6e2c\u7a7a\u9593\u5916\uff0cLLM \u6700\u6838\u5fc3\u7684\u6982\u5ff5\u5728\u65bc<b>\u5d4c\u5165\u7a7a\u9593\uff08Embedding Space\uff09<\/b>\u3002\u5728\u9019\u500b\u9ad8\u7dad\u7a7a\u9593\u4e2d\uff0c\u5411\u91cf\u4e0d\u518d\u4ee3\u8868\u8a5e\u5f59\u6a5f\u7387\uff0c\u800c\u662f\u4ee3\u8868\u300c\u610f\u7fa9\uff08Semantics\uff09\u300d\u3002<\/p>\n  <p>\u5728\u8a13\u7df4\u826f\u597d\u7684\u5d4c\u5165\u7a7a\u9593\u4e2d\uff0c\u5177\u6709\u76f8\u4f3c\u6982\u5ff5\u6216\u95dc\u806f\u6027\u7684\u8a5e\u5f59\u6703\u805a\u96c6\u5728\u76f8\u8fd1\u7684\u7a7a\u9593\u5340\u57df\u3002\u4f8b\u5982\uff0c\u300c\u5bb6\u8c93\u300d\u3001\u300c\u7345\u5b50\u300d\u8207\u300c\u8001\u864e\u300d\u6703\u805a\u96c6\u5728\u8c93\u79d1\u52d5\u7269\u53e2\u96c6\u4e2d\uff1b\u800c\u300c\u67f4\u72ac\u300d\u3001\u300c\u72fc\u300d\u8207\u300c\u571f\u72fc\u300d\u5247\u5206\u4f48\u5728\u72ac\u79d1\u53e2\u96c6\u4e2d\u3002\u9019\u5169\u8005\u53c8\u5171\u540c\u5ea7\u843d\u65bc\u300c\u54fa\u4e73\u52d5\u7269\u300d\u7684\u5ee3\u95ca\u5e7e\u4f55\u8d85\u66f2\u9762\u4e4b\u4e2d\u3002<\/p>\n\n  <div class=\"in-callout-danger\">\n    <strong>\u91cd\u8981\u89c0\u5ff5\u8fa8\u6790\uff1a<\/strong> \u5728\u8a9e\u610f\u5d4c\u5165\u7a7a\u9593\u4e2d\uff0c\u6211\u5011\u901a\u5e38<strong>\u4e0d\u5728\u610f\u5411\u91cf\u7684\u7d55\u5c0d\u9577\u5ea6\uff08\u6a21\u9577\uff09<\/strong>\uff0c\u800c\u662f\u6975\u5ea6\u95dc\u6ce8\u5411\u91cf\u5728\u7a7a\u9593\u4e2d\u6240\u6307\u5411\u7684<strong>\u89d2\u5ea6\u8207\u65b9\u5411<\/strong>\u3002\u5728\u4e8c\u7dad\u793a\u7bc4\u4e2d\uff0c(1, 2) \u8207 (8, 16) \u6307\u5411\u540c\u4e00\u500b\u65b9\u5411\uff0c\u56e0\u6b64\u5728\u8a9e\u610f\u6bd4\u8f03\u6642\u5f80\u5f80\u88ab\u8996\u70ba\u7b49\u50f9\u7279\u5fb5\u3002\n  <\/div>\n\n  <h2 id=\"section-4\"><span class=\"in-badge\">4<\/span> \u5167\u7a4d\uff08Dot Product\uff09\u8207\u9918\u5f26\u76f8\u4f3c\u5ea6\u5ea6\u91cf<\/h2>\n  <p>\u82e5\u8981\u8861\u91cf\u9ad8\u7dad\u7a7a\u9593\u4e2d\u5169\u500b\u8a9e\u610f\u5411\u91cf\u6709\u591a\u76f8\u4f3c\uff0c\u6700\u57fa\u790e\u4e14\u8a08\u7b97\u6548\u7387\u6700\u9ad8\u7684\u6578\u5b78\u904b\u7b97\u5c31\u662f<b>\u5167\u7a4d\uff08Dot Product\uff09<\/b>\u3002\u7d66\u5b9a\u5169\u500b\u76f8\u540c\u9577\u5ea6\u7684\u5411\u91cf\uff0c\u5176\u8a08\u7b97\u65b9\u5f0f\u5373\u70ba\u5404\u5143\u7d20\u5c0d\u61c9\u76f8\u4e58\u5f8c\u5168\u90e8\u52a0\u7e3d\uff1a<\/p>\n\n  <div class=\"in-terminal-box\">\n    <div class=\"in-terminal-header\">\n      <div class=\"in-terminal-dots\">\n        <span class=\"in-dot in-dot-red\"><\/span>\n        <span class=\"in-dot in-dot-yellow\"><\/span>\n        <span class=\"in-dot in-dot-green\"><\/span>\n      <\/div>\n      <span>dot_product_similarity.py<\/span>\n    <\/div>\n    <pre class=\"in-code\"># \u5411\u91cf\u5167\u7a4d\u7bc4\u4f8b\uff1a(a \u00b7 b) = sum(a_i * b_i)\n# \u5411\u91cf A: [2, 3, 4]\n# \u5411\u91cf B: [1, 0, 2]\ndot_product = (2 * 1) + (3 * 0) + (4 * 2)  # 2 + 0 + 8 = 10\n\n# \u82e5\u4e8b\u5148\u9032\u884c L2 \u6b78\u4e00\u5316\uff08\u9577\u5ea6\u8b8a\u70ba 1\uff09\uff0c\u5167\u7a4d\u5373\u7cbe\u78ba\u7b49\u65bc\u9918\u5f26\u76f8\u4f3c\u5ea6\uff08Cosine Similarity\uff09\n# Cosine Similarity = cos(\u03b8) = (A \u00b7 B) \/ (||A|| * ||B||)<\/pre>\n  <\/div>\n\n  <p>\u7576\u5169\u500b\u5411\u91cf\u9577\u5ea6\u76f8\u8fd1\u6642\uff0c\u5167\u7a4d\u6578\u503c\u8d8a\u5927\uff0c\u4ee3\u8868\u5169\u8005\u65b9\u5411\u8d8a\u4e00\u81f4\u3001\u8a9e\u610f\u8d8a\u8cbc\u8fd1\u3002\u5982\u679c\u5728\u904b\u7b97\u524d\u5148\u5c07\u5411\u91cf\u9032\u884c L2 \u6b78\u4e00\u5316\uff0c\u5167\u7a4d\u4fbf\u76f4\u63a5\u7b49\u540c\u65bc<b>\u9918\u5f26\u76f8\u4f3c\u5ea6\uff08Cosine Similarity\uff09<\/b>\u3002\u9019\u9805\u904b\u7b97\u6b63\u662f Transformer \u67b6\u69cb\u4e2d<b>\u81ea\u6ce8\u610f\u529b\u6a5f\u5236\uff08Self-Attention\uff09<\/b>\u8a08\u7b97 Query \u8207 Key \u6b0a\u91cd\u914d\u5c0d\u7684\u6838\u5fc3\u57fa\u790e\u3002<\/p>\n\n  <h2 id=\"section-5\"><span class=\"in-badge\">5<\/span> \u77e9\u9663\u4e58\u6cd5\u672c\u8cea\uff1a\u9ad8\u7dad\u7a7a\u9593\u4e2d\u7684\u5e7e\u4f55\u8b8a\u63db\u8207\u6295\u5f71<\/h2>\n  <p>\u77e9\u9663\u5728\u672c\u8cea\u4e0a\u5c31\u662f\u591a\u500b\u5411\u91cf\u7684\u5806\u758a\u3002\u5728\u795e\u7d93\u7db2\u8def\u4e2d\uff0c\u77e9\u9663\u4e58\u6cd5\u7d55\u975e\u7d14\u7cb9\u7684\u4ee3\u6578\u6392\u5217\u7d44\u5408\uff0c\u5176\u771f\u6b63\u7684\u7269\u7406\u610f\u7fa9\u5728\u65bc<b>\u7a7a\u9593\u5e7e\u4f55\u8b8a\u63db\u8207\u6295\u5f71\uff08Geometric Transformation &amp; Projection\uff09<\/b>\u3002<\/p>\n  <p>\u4ee5\u6700\u7d93\u5178\u7684\u65cb\u8f49\u77e9\u9663\u70ba\u4f8b\uff0c\u7576\u6211\u5011\u5c07\u5ea7\u6a19\u5411\u91cf\u4e58\u4e0a\u65cb\u8f49\u77e9\u9663\u6642\uff0c\u7b49\u65bc\u5c07\u6574\u500b\u5ea7\u6a19\u7cfb\u9032\u884c\u65cb\u8f49\uff1b\u540c\u7406\uff0c\u795e\u7d93\u7db2\u8def\u4e2d\u6bcf\u4e00\u5c64\u5168\u9023\u63a5\u5c64\u6216\u6ce8\u610f\u529b\u6295\u5f71\u5c64\uff0c\u672c\u8cea\u4e0a\u90fd\u662f\u5c07\u7279\u5fb5\u5411\u91cf\u5f9e\u4e00\u500b\u9ad8\u7dad\u5ea6\u7a7a\u9593\u6295\u5f71\u5230\u53e6\u4e00\u500b\u5177\u6709\u7279\u5b9a\u5e7e\u4f55\u7d50\u69cb\u7684\u7279\u5fb5\u7a7a\u9593\uff1a<\/p>\n\n  <div class=\"in-terminal-box\">\n    <div class=\"in-terminal-header\">\n      <div class=\"in-terminal-dots\">\n        <span class=\"in-dot in-dot-red\"><\/span>\n        <span class=\"in-dot in-dot-yellow\"><\/span>\n        <span class=\"in-dot in-dot-green\"><\/span>\n      <\/div>\n      <span>projection_math.txt<\/span>\n    <\/div>\n    <pre class=\"in-code\">\u7dda\u6027\u8b8a\u63db\u57fa\u672c\u5f62\u5f0f\uff1a\nY = X \u00b7 W  (\u5176\u4e2d X \u70ba\u8f38\u5165\u7279\u5fb5\u77e9\u9663\uff0cW \u70ba\u6b0a\u91cd\u8b8a\u63db\u77e9\u9663)\n\n\u5e7e\u4f55\u4f5c\u7528\uff1a\n1. \u65cb\u8f49\uff08Rotation\uff09\u8207\u7e2e\u653e\uff08Scaling\uff09\u7279\u5fb5\u5206\u4f48\n2. \u7dad\u5ea6\u5347\u964d\uff08Projection between Spaces\uff0c\u5982\u5f9e Hidden State \u6620\u5c04\u81f3 Vocab Space\uff09\n3. \u8a9e\u610f\u63d0\u53d6\uff08\u900f\u904e\u6295\u5f71\u8403\u53d6\u7279\u5b9a\u8a9e\u610f\u7dad\u5ea6\uff0c\u4f8b\u5982\u6ce8\u610f\u529b\u6b0a\u91cd\u77e9\u9663 W_q, W_k, W_v\uff09<\/pre>\n  <\/div>\n\n  <h2 id=\"section-6\"><span class=\"in-badge\">6<\/span> LLM \u63a8\u8ad6\u6838\u5fc3\u6578\u5b78\u6982\u5ff5\u901f\u67e5\u5c0d\u7167\u8868<\/h2>\n  <div class=\"in-table-wrapper\">\n    <table class=\"in-table\">\n      <thead>\n        <tr>\n          <th>\u6578\u5b78\u6982\u5ff5<\/th>\n          <th>\u4ee3\u6578\u5b9a\u7fa9<\/th>\n          <th>LLM \u63a8\u8ad6\u5be6\u52d9\u5c0d\u61c9<\/th>\n          <th>\u5e7e\u4f55\u7269\u7406\u610f\u7fa9<\/th>\n        <\/tr>\n      <\/thead>\n      <tbody>\n        <tr>\n          <td><strong>\u5411\u91cf (Vector)<\/strong><\/td>\n          <td>\u6709\u5e8f\u6578\u503c\u9663\u5217 [<span class=\"in-math\">x<\/span><sub>1<\/sub>, <span class=\"in-math\">x<\/span><sub>2<\/sub>, \u2026, <span class=\"in-math\">x<\/span><sub><span class=\"in-math\">n<\/span><\/sub>]<\/td>\n          <td>Token \u5d4c\u5165\u7279\u5fb5\u3001\u96b1\u85cf\u5c64\u72c0\u614b\u3001Logits \u8a55\u5206<\/td>\n          <td><span class=\"in-math\">n<\/span> \u7dad\u7279\u5fb5\u7a7a\u9593\u4e2d\u7684\u65b9\u5411\u8207\u4f4d\u7f6e\u5ea7\u6a19<\/td>\n        <\/tr>\n        <tr>\n          <td><strong>Softmax \u51fd\u6578<\/strong><\/td>\n          <td>\n            <span class=\"in-math\">\u03c3<\/span>(<span class=\"in-math\">z<\/span><sub><span class=\"in-math\">i<\/span><\/sub>) = \n            <span class=\"in-fraction\">\n              <span class=\"in-fraction-top\"><span class=\"in-math\">e<\/span><sup><span class=\"in-math\">z<\/span><sub><span class=\"in-math\">i<\/span><\/sub><\/sup><\/span>\n              <span class=\"in-fraction-bottom\">\u2211<sub><span class=\"in-math\">j<\/span><\/sub> <span class=\"in-math\">e<\/span><sup><span class=\"in-math\">z<\/span><sub><span class=\"in-math\">j<\/span><\/sub><\/sup><\/span>\n            <\/span>\n          <\/td>\n          <td>\u5c07\u8f38\u51fa Logits \u8f49\u63db\u70ba\u5404 Token \u7684\u63a1\u6a23\u6a5f\u7387\u5206\u4f48<\/td>\n          <td>\u5c07\u5168\u57df\u7121\u754c\u6578\u503c\u58d3\u7e2e\u4e26\u6b78\u4e00\u5316\u81f3\u6a19\u6e96\u6a5f\u7387\u8d85\u5e73\u9762<\/td>\n        <\/tr>\n        <tr>\n          <td><strong>\u5167\u7a4d (Dot Product)<\/strong><\/td>\n          <td><span class=\"in-math-bold\">A<\/span> \u00b7 <span class=\"in-math-bold\">B<\/span> = \u2211 <span class=\"in-math\">A<\/span><sub><span class=\"in-math\">i<\/span><\/sub><span class=\"in-math\">B<\/span><sub><span class=\"in-math\">i<\/span><\/sub><\/td>\n          <td>\u8a08\u7b97\u6ce8\u610f\u529b\u6a5f\u5236\u4e2d\u7684\u6ce8\u610f\u529b\u5206\u6578 (<span class=\"in-math-bold\">Q<\/span> \u00b7 <span class=\"in-math-bold\">K<\/span><sup>T<\/sup>)<\/td>\n          <td>\u8861\u91cf\u5169\u5411\u91cf\u65b9\u5411\u7684\u4e00\u81f4\u6027\u8207\u8a9e\u610f\u76f8\u95dc\u7a0b\u5ea6<\/td>\n        <\/tr>\n        <tr>\n          <td><strong>\u77e9\u9663\u6295\u5f71 (Projection)<\/strong><\/td>\n          <td><span class=\"in-math-bold\">Y<\/span> = <span class=\"in-math-bold\">X<\/span><span class=\"in-math-bold\">W<\/span> + <span class=\"in-math-bold\">b<\/span><\/td>\n          <td>\u7279\u5fb5\u6620\u5c04\u3001\u524d\u994b\u795e\u7d93\u7db2\u8def\u5c64\u3001\u6ce8\u610f\u529b\u6b0a\u91cd\u7dda\u6027\u8b8a\u63db<\/td>\n          <td>\u7a7a\u9593\u65cb\u8f49\u3001\u62c9\u4f38\u3001\u7dad\u5ea6\u58d3\u7e2e\u6216\u9ad8\u7dad\u7a7a\u9593\u91cd\u69cb<\/td>\n        <\/tr>\n      <\/tbody>\n    <\/table>\n  <\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>\u4eba\u5de5\u667a\u6167 \u5927\u8a9e\u8a00\u6a21\u578b \u7dda\u6027\u4ee3\u6578 \u63a8\u8ad6\u67b6\u69cb \u6578\u5b78\u5e95\u5c64 \u5de5\u7a0b\u5e2b\u5fc5\u5099\u7684\u9ad8\u4e2d\u6578\u5b78\u601d\u7dad\u5be6\u52d9 \u8a31\u591a\u8edf\u9ad4\u5de5\u7a0b\u5e2b\u5728\u8de8\u5165\u5927\u578b\u8a9e\u8a00\u6a21\u578b\uff08LLM\uff09\u6280\u8853\u9818\u57df\u6642\uff0c\u5f80\u5f80\u88ab\u8ad6\u6587\u4e2d\u5bc6\u5bc6\u9ebb\u9ebb\u7684\u504f\u5fae\u5206\u3001\u53cd\u5411\u50b3\u64ad\u8207\u7d71\u8a08\u5b78\u7fd2\u7406\u8ad6\u52f8\u9000\u3002\u7136\u800c\uff0c\u8cc7\u6df1\u6280\u8853\u4f5c\u8005 Giles Thomas \u6307\u51fa\uff1a\u82e5\u4f60\u7684\u76ee\u6a19\u662f\u900f\u5fb9\u7406\u89e3 LLM \u5728\u63a8\u8ad6\uff08Inference\uff09\u968e\u6bb5\u7684\u904b\u4f5c\u6a5f\u5236\uff0c\u800c\u975e\u5f9e\u982d\u7814\u767c\u57fa\u790e\u67b6\u69cb\uff0c\u4f60\u6240\u9700\u8981\u7684\u6578\u5b78\u5de5\u5177\u5e7e\u4e4e\u4e0d\u8d85\u51fa\u9ad8\u4e2d\u7bc4\u7587\u2014\u2014\u6838\u5fc3\u5b8c\u5168\u570d\u7e5e\u5728\u5411\u91cf\uff08Vectors\uff09\u3001\u9ad8\u7dad\u7a7a\u9593\u5e7e\u4f55\u3001\u5167\u7a4d\uff08Dot Product\uff09\u4ee5\u53ca\u77e9\u9663\u4e58\u6cd5\u6295\u5f71\uff08Matrix Projections\uff09\u3002 \u5c0e\u8b80\u6838\u5fc3\u63d0\u8981\uff1a \u672c\u6587\u91dd\u5c0d Giles Thomas \u6df1\u5165\u5256\u6790 Sebastian Raschka \u8457\u4f5c\u300a\u5f9e\u96f6\u69cb\u5efa\u5927\u8a9e\u8a00\u6a21\u578b\uff08Build a Large Language Model from Scratch\uff09\u300b\u7b2c 19 \u90e8\u5206\u7684\u5fc3\u5f97\u9032\u884c\u5b8c\u6574\u67b6\u69cb\u89e3\u69cb\uff0c\u805a\u7126\u65bc\u6a21\u578b\u5982\u4f55\u5c07\u96e2\u6563\u6587\u5b57\u6620\u5c04\u70ba\u9023\u7e8c\u5e7e\u4f55\u7a7a\u9593\uff0c\u4e26\u5b8c\u6210\u9810\u6e2c\u8207\u8f49\u63db\u3002 \u6587\u7ae0\u76ee\u9304 1. \u5411\u91cf\u672c\u8cea\uff1a\u5f9e\u6578\u503c\u9663\u5217\u5230\u9ad8\u7dad\u7a7a\u9593\u5ea7\u6a19 2. \u8a5e\u5f59\u7a7a\u9593\uff08Vocab Space\uff09\u8207 Softmax \u6b78\u4e00\u5316 3. \u5d4c\u5165\u7a7a\u9593\uff08Embedding Space\uff09\u8207\u8a9e\u610f\u5e7e\u4f55\u805a\u96c6 4. \u5167\u7a4d\uff08Dot Product\uff09\u8207\u9918\u5f26\u76f8\u4f3c\u5ea6\u5ea6\u91cf 5. \u77e9\u9663\u4e58\u6cd5\u672c\u8cea\uff1a\u9ad8\u7dad\u7a7a\u9593\u4e2d\u7684\u5e7e\u4f55\u8b8a\u63db\u8207\u6295\u5f71 6. LLM \u63a8\u8ad6\u6838\u5fc3\u6578\u5b78\u6982\u5ff5\u901f\u67e5\u5c0d\u7167\u8868 1 \u5411\u91cf\u672c\u8cea\uff1a\u5f9e\u6578\u503c\u9663\u5217\u5230\u9ad8\u7dad\u7a7a\u9593\u5ea7\u6a19 \u5728\u50b3\u7d71\u8edf\u9ad4\u958b\u767c\u4e2d\uff0c\u5de5\u7a0b\u5e2b\u901a\u5e38\u5c07\u300c\u5411\u91cf\uff08Vector\uff09\u300d\u55ae\u7d14\u8996\u70ba\u6d6e\u9ede\u6578\u6216\u6574\u6578\u7684\u9663\u5217\uff08Array of Numbers\uff09\u3002\u4f46\u5728\u5e7e\u4f55\u8207\u7dda\u6027\u4ee3\u6578\u4e2d\uff0c\u4e00\u500b\u9577\u5ea6\u70ba n [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"parent":0,"menu_order":0,"comment_status":"closed","ping_status":"closed","template":"","meta":{"wpai_generated_summary":"","wpai_meta_description":"","footnotes":""},"class_list":["post-12866","page","type-page","status-publish","hentry"],"_links":{"self":[{"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/pages\/12866","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/pages"}],"about":[{"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/types\/page"}],"author":[{"embeddable":true,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/comments?post=12866"}],"version-history":[{"count":4,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/pages\/12866\/revisions"}],"predecessor-version":[{"id":12871,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/pages\/12866\/revisions\/12871"}],"wp:attachment":[{"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/media?parent=12866"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}