{"id":3177,"date":"2024-07-02T02:32:08","date_gmt":"2024-07-01T18:32:08","guid":{"rendered":"https:\/\/infernews.com\/?p=3177"},"modified":"2026-10-24T23:08:07","modified_gmt":"2026-10-24T15:08:07","slug":"%e4%ba%86%e8%a7%a3-transformer-attention-is-all-you-need","status":"publish","type":"post","link":"https:\/\/infernews.com\/blog\/%e4%ba%86%e8%a7%a3-transformer-attention-is-all-you-need\/","title":{"rendered":"\u4e86\u89e3 Transformer &#8220;Attention is All You Need&#8221;"},"content":{"rendered":"\n<figure class=\"wp-block-image size-full\"><img data-dominant-color=\"c4cbd9\" data-has-transparency=\"false\" style=\"--dominant-color: #c4cbd9;\" loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"397\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" src=\"https:\/\/infernews.com\/blog\/wp-content\/uploads\/2024\/07\/transformer.jpg\" alt=\"\" class=\"wp-image-13886 not-transparent\" srcset=\"https:\/\/infernews.com\/blog\/wp-content\/uploads\/2024\/07\/transformer.jpg 1024w, https:\/\/infernews.com\/blog\/wp-content\/uploads\/2024\/07\/transformer-300x116.jpg 300w, https:\/\/infernews.com\/blog\/wp-content\/uploads\/2024\/07\/transformer-766x297.jpg 766w\" \/><\/figure>\n\n\n\n<div id=\"wp-transformer-article\">\n  <style>\n    \/* \u56b4\u683c\u9694\u96e2\u7684 WordPress \u5bb9\u5668\u6a23\u5f0f *\/\n    #wp-transformer-article {\n      max-width: 900px;\n      margin: 1.5rem auto;\n      font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Helvetica Neue\", Arial, sans-serif;\n      color: #1f2937;\n      line-height: 1.7;\n    }\n    #wp-transformer-article h2 {\n      font-size: 1.65rem;\n      font-weight: 800;\n      color: #0f172a;\n      margin-top: 2.5rem;\n      margin-bottom: 1rem;\n      border-bottom: 2px solid #f1f5f9;\n      padding-bottom: 0.5rem;\n    }\n    #wp-transformer-article h3 {\n      font-size: 1.25rem;\n      font-weight: 700;\n      color: #1e293b;\n      margin-top: 1.75rem;\n      margin-bottom: 0.75rem;\n    }\n    #wp-transformer-article p {\n      margin-bottom: 1.2rem;\n      font-size: 1.05rem;\n    }\n    #wp-transformer-article ul, #wp-transformer-article ol {\n      margin-bottom: 1.25rem;\n      padding-left: 1.5rem;\n    }\n    #wp-transformer-article li {\n      margin-bottom: 0.4rem;\n    }\n    #wp-transformer-article code {\n      font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;\n      background: #f1f5f9;\n      color: #b45309;\n      padding: 0.2rem 0.4rem;\n      border-radius: 4px;\n      font-size: 0.9em;\n    }\n    #wp-transformer-article blockquote {\n      border-left: 4px solid #f59e0b;\n      background: #fffbeb;\n      margin: 1.5rem 0;\n      padding: 1rem 1.25rem;\n      border-radius: 0 8px 8px 0;\n      color: #78350f;\n    }\n    \/* \u4e92\u52d5\u5143\u4ef6\u5bb9\u5668\u6a23\u5f0f *\/\n    .wp-tr-interactive-box {\n      border: 1px solid #e2e8f0;\n      border-radius: 14px;\n      background: #ffffff;\n      box-shadow: 0 4px 16px rgba(0, 0, 0, 0.06);\n      margin: 2rem 0;\n      padding: 1.25rem;\n    }\n    .wp-tr-header-badge {\n      display: inline-block;\n      background: linear-gradient(135deg, #f59e0b, #d97706);\n      color: #ffffff;\n      font-size: 0.75rem;\n      font-weight: 700;\n      padding: 0.25rem 0.65rem;\n      border-radius: 999px;\n      text-transform: uppercase;\n      letter-spacing: 0.05em;\n      margin-bottom: 0.5rem;\n    }\n    .wp-tr-btn {\n      padding: 0.4rem 0.8rem;\n      border-radius: 6px;\n      border: 1px solid #cbd5e1;\n      background: #f8fafc;\n      color: #334155;\n      font-size: 0.8rem;\n      font-weight: 600;\n      cursor: pointer;\n      transition: all 0.15s ease;\n    }\n    .wp-tr-btn:hover {\n      background: #e2e8f0;\n      color: #0f172a;\n    }\n    .wp-tr-btn.active {\n      background: #f59e0b;\n      color: #ffffff;\n      border-color: #d97706;\n    }\n    \/* \u81ea\u6ce8\u610f\u529b\u71b1\u529b\u8868\u683c *\/\n    .wp-tr-matrix-table {\n      width: 100%;\n      border-collapse: collapse;\n      text-align: center;\n      margin-top: 1rem;\n      font-size: 0.78rem;\n    }\n    .wp-tr-matrix-table th {\n      padding: 6px;\n      font-weight: 600;\n      background: #f8fafc;\n      border: 1px solid #e2e8f0;\n    }\n    .wp-tr-matrix-table td {\n      padding: 4px;\n      border: 1px solid #e2e8f0;\n      cursor: pointer;\n      transition: all 0.1s ease;\n      position: relative;\n    }\n    .wp-tr-matrix-table td:hover {\n      outline: 2px solid #f59e0b;\n      z-index: 2;\n    }\n    .wp-tr-cell-inner {\n      border-radius: 4px;\n      padding: 4px 2px;\n      font-weight: 700;\n      font-family: monospace;\n    }\n    \/* \u67b6\u69cb\u5716\u6a23\u5f0f *\/\n    .wp-tr-arch-grid {\n      display: grid;\n      grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));\n      gap: 1.25rem;\n      align-items: start;\n    }\n    .wp-tr-arch-block {\n      border: 1px solid #cbd5e1;\n      border-radius: 8px;\n      padding: 0.65rem;\n      margin-bottom: 0.5rem;\n      background: #f8fafc;\n      cursor: pointer;\n      font-size: 0.85rem;\n      font-weight: 600;\n      transition: all 0.15s ease;\n    }\n    .wp-tr-arch-block:hover {\n      border-color: #f59e0b;\n      background: #fffbeb;\n      transform: translateY(-1px);\n    }\n    .wp-tr-arch-block.active {\n      background: #f59e0b;\n      color: #ffffff;\n      border-color: #d97706;\n      box-shadow: 0 4px 12px rgba(245, 158, 11, 0.3);\n    }\n    .wp-tr-inspector {\n      background: #f8fafc;\n      border: 1px solid #e2e8f0;\n      border-radius: 10px;\n      padding: 1rem;\n      font-size: 0.85rem;\n    }\n    \/* \u7a0b\u5f0f\u78bc\u5340\u584a\u6a23\u5f0f *\/\n    .wp-tr-code-block {\n      background: #0f172a;\n      color: #f8fafc;\n      border-radius: 10px;\n      padding: 1rem;\n      font-family: monospace;\n      font-size: 0.85rem;\n      overflow-x: auto;\n      position: relative;\n    }\n    .wp-tr-copy-btn {\n      position: absolute;\n      top: 10px;\n      right: 10px;\n      background: #334155;\n      color: #ffffff;\n      border: none;\n      border-radius: 6px;\n      padding: 4px 10px;\n      font-size: 0.75rem;\n      cursor: pointer;\n    }\n    .wp-tr-copy-btn:hover {\n      background: #f59e0b;\n      color: #0f172a;\n    }\n  <\/style>\n\n  <!-- \u6587\u7ae0\u958b\u982d -->\n  <h1>Transformer \u67b6\u69cb\u6838\u5fc3\u6982\u5ff5\u8207\u6578\u5b78\u6df1\u5ea6\u89e3\u6790<\/h1>\n  <p style=\"color: #64748b; font-size: 0.95rem;\">\n    \u7d93\u5178\u8ad6\u6587\u7cbe\u8b80 \u00b7 Vaswani et al. (NeurIPS 2017) \u00b7 \u7e41\u9ad4\u4e2d\u6587\u6df1\u5ea6\u7cbe\u6821 \u00b7 \u95b1\u8b80\u6642\u9593\u7d04 18 \u5206\u9418\n  <\/p>\n\n\n  <h2>01. \u5faa\u74b0\u795e\u7d93\u7db2\u7d61\u7684\u74f6\u9838\u8207\u6ce8\u610f\u529b\u7684\u8a95\u751f<\/h2>\n  <p>\u5728 2017 \u5e74\u4e4b\u524d\uff0c\u5e7e\u4e4e\u6240\u6709\u9802\u5c16\u7684\u81ea\u7136\u8a9e\u8a00\u8655\u7406\uff08NLP\uff09\u6a21\u578b\u90fd\u662f\u570d\u7e5e\u8457\u5faa\u74b0\u795e\u7d93\u7db2\u7d61\uff08RNN\uff09\u69cb\u5efa\u3002RNN \u7684\u6838\u5fc3\u8a2d\u8a08\u662f\u300c\u6309\u6642\u9593\u6b65\u9577\u5faa\u5e8f\u905e\u6b78\u300d\uff1a\u8655\u7406\u7576\u524d\u7b2c <code>t<\/code> \u500b\u8a5e\u6642\uff0c\u5fc5\u9808\u4ef0\u8cf4\u524d\u4e00\u500b\u6642\u9593\u6b65\u7684\u96b1\u85cf\u72c0\u614b <code>h_{t-1}<\/code>\u3002\u9019\u7a2e\u905e\u8ff4\u7d50\u69cb\u5b58\u5728\u4e09\u5927\u81f4\u547d\u7f3a\u9677\uff1a<\/p>\n  <ul>\n    <li><strong>\u7121\u6cd5\u9ad8\u5ea6\u4e26\u884c\u904b\u7b97\uff1a<\/strong>\u8a08\u7b97\u5fc5\u9808\u4f9d\u5e8f\u7b49\u5f85\uff0c\u5e8f\u5217\u9577\u5ea6\u6642\u9593\u8907\u96dc\u5ea6\u5f37\u5236\u70ba <code>O(n)<\/code>\uff0c\u7121\u6cd5\u767c\u63ee\u73fe\u4ee3 GPU \u77e9\u9663\u7b97\u529b\u3002<\/li>\n    <li><strong>\u9577\u8ddd\u96e2\u4f9d\u8cf4\u8cc7\u8a0a\u907a\u5fd8\uff1a<\/strong>\u6578\u5341\u6b65\u4e4b\u5f8c\u5bb9\u6613\u767c\u751f\u68af\u5ea6\u6d88\u5931\u6216\u7206\u70b8\uff0c\u96e3\u4ee5\u6355\u6349\u8de8\u6bb5\u843d\u8a9e\u610f\u3002<\/li>\n    <li><strong>\u56fa\u5b9a\u5411\u91cf\u8cc7\u8a0a\u74f6\u9838\uff1a<\/strong>\u5f37\u884c\u5c07\u6574\u53e5\u8a71\u58d3\u7e2e\u81f3\u55ae\u4e00\u56fa\u5b9a\u7dad\u5ea6\u5411\u91cf\u4e2d\u3002<\/li>\n  <\/ul>\n  <p>\u6b63\u5982\u8ad6\u6587\u6a19\u984c\u6240\u5ba3\u793a\u7684\uff1a<strong>\u300cAttention Is All You Need\u300d<\/strong>\u2014\u2014\u6368\u68c4\u5faa\u74b0\u8207\u5377\u7a4d\uff0c\u50c5\u9760\u6ce8\u610f\u529b\u6a5f\u5236\u5be6\u73fe\u4e00\u6b65\u5230\u4f4d\u5168\u5c40\u76f4\u9023\uff01<\/p>\n\n  <h2>02. \u5b8f\u89c0\u67b6\u69cb\u5168\u8c8c\uff1aEncoder-Decoder \u7684\u5806\u758a\u667a\u6167<\/h2>\n  <p>\u539f\u59cb Transformer \u7531\u5de6\u5074 6 \u5c64\u7de8\u78bc\u5668\uff08Encoder Stack\uff09\u8207\u53f3\u5074 6 \u5c64\u89e3\u78bc\u5668\uff08Decoder Stack\uff09\u5354\u540c\u7d44\u6210\u3002\u9ede\u64ca\u4e0b\u65b9\u4e92\u52d5\u65b9\u584a\uff0c\u5373\u6642\u67e5\u770b\u5404\u5c64\u5f35\u91cf\u5f62\u72c0\u8207\u6578\u5b78\u904b\u7b97\uff1a<\/p>\n\n  <!-- \u4e92\u52d5\u5143\u4ef6 1\uff1a\u67b6\u69cb\u6aa2\u8996\u5668 -->\n  <div class=\"wp-tr-interactive-box\" id=\"wp-tr-arch-app\">\n    <span class=\"wp-tr-header-badge\">\u4e92\u52d5\u67b6\u69cb\u6aa2\u8996\u5668 (\u9ede\u64ca\u7d44\u4ef6\u67e5\u770b\u7d30\u7bc0)<\/span>\n    <div class=\"wp-tr-arch-grid\">\n      <div>\n        <div style=\"font-weight: 700; color: #4338ca; font-size: 0.9rem; margin-bottom: 0.5rem;\">\u7de8\u78bc\u5668\u5806\u758a (Encoder Nx=6)<\/div>\n        <div class=\"wp-tr-arch-block\" onclick=\"wpTrSelectArch('add-norm')\">1. Add &amp; Norm (\u6b98\u5dee + \u5c64\u6b78\u4e00\u5316)<\/div>\n        <div class=\"wp-tr-arch-block\" onclick=\"wpTrSelectArch('ffn')\">2. Feed Forward (\u524d\u994b\u7db2\u7d61 512\u21922048\u2192512)<\/div>\n        <div class=\"wp-tr-arch-block active\" id=\"wp-arch-mha\" onclick=\"wpTrSelectArch('enc-mha')\">3. Multi-Head Attention (\u591a\u982d\u81ea\u6ce8\u610f\u529b)<\/div>\n        <div class=\"wp-tr-arch-block\" onclick=\"wpTrSelectArch('pos-enc')\">4. Positional Encoding (\u4f4d\u7f6e\u7de8\u78bc)<\/div>\n        <div class=\"wp-tr-arch-block\" onclick=\"wpTrSelectArch('input-emb')\">5. Input Embedding (\u8f38\u5165\u8a5e\u5d4c\u5165 d=512)<\/div>\n      <\/div>\n      <div>\n        <div style=\"font-weight: 700; color: #047857; font-size: 0.9rem; margin-bottom: 0.5rem;\">\u89e3\u78bc\u5668\u5806\u758a (Decoder Nx=6)<\/div>\n        <div class=\"wp-tr-arch-block\" onclick=\"wpTrSelectArch('linear-softmax')\">1. Linear &amp; Softmax (\u9810\u6e2c\u8a5e\u8868\u6a5f\u7387)<\/div>\n        <div class=\"wp-tr-arch-block\" onclick=\"wpTrSelectArch('ffn')\">2. Feed Forward + Add &amp; Norm<\/div>\n        <div class=\"wp-tr-arch-block\" onclick=\"wpTrSelectArch('dec-cross')\">3. Cross Multi-Head Attention (\u4ea4\u53c9\u6ce8\u610f\u529b)<\/div>\n        <div class=\"wp-tr-arch-block\" onclick=\"wpTrSelectArch('dec-masked')\">4. Masked Multi-Head Attention (\u56e0\u679c\u906e\u7f69)<\/div>\n        <div class=\"wp-tr-arch-block\" onclick=\"wpTrSelectArch('input-emb')\">5. Output Embedding (\u76ee\u6a19\u8a5e\u5d4c\u5165)<\/div>\n      <\/div>\n    <\/div>\n\n    <!-- Inspector Output Panel -->\n    <div class=\"wp-tr-inspector\" id=\"wp-arch-detail\" style=\"margin-top: 1rem;\">\n      <div style=\"font-weight: 700; font-size: 1rem; color: #0f172a;\" id=\"wp-arch-title\">\u7de8\u78bc\u5668\u591a\u982d\u81ea\u6ce8\u610f\u529b (Multi-Head Self-Attention)<\/div>\n      <div style=\"color: #6366f1; font-family: monospace; font-size: 0.8rem; margin: 0.25rem 0;\" id=\"wp-arch-dim\">\u5f35\u91cf\u7dad\u5ea6\uff1aQ, K, V \u7686\u4f86\u81ea\u7de8\u78bc\u5668\u524d\u5c64: [Batch, Seq_Len, 512]<\/div>\n      <div style=\"margin-top: 0.5rem;\" id=\"wp-arch-desc\">8 \u500b\u7368\u7acb\u7684\u6ce8\u610f\u529b\u982d\u540c\u6642\u6295\u5f71\u81f3\u5b50\u7a7a\u9593 (d_k=64)\uff0c\u5206\u5225\u6355\u7372\u8a9e\u6cd5\u6a39\u95dc\u4fc2\u3001\u6838\u5fc3\u4e3b\u8a5e\u4fee\u98fe\u8a9e\u3001\u9060\u8ddd\u96e2\u5171\u6307\u7b49\u4e0d\u540c\u5c64\u6b21\u7684\u8a9e\u8a00\u7279\u5fb5\u3002<\/div>\n    <\/div>\n  <\/div>\n\n  <h2>03. \u6838\u5fc3\u5f15\u64ce\uff1a\u81ea\u6ce8\u610f\u529b (Self-Attention) \u8207\u9664\u4ee5 \u221ad_k \u7684\u6578\u5b78\u5fc5\u7136<\/h2>\n  <p>\u81ea\u6ce8\u610f\u529b\u6a5f\u5236\u516c\u5f0f\uff1a<code>Attention(Q, K, V) = softmax( (Q \u00b7 K^T) \/ \u221ad_k ) \u00b7 V<\/code><\/p>\n  <ul>\n    <li><strong>Query (Q)\uff1a\u300c\u6211\u6b63\u5728\u5c0b\u627e\u4ec0\u9ebc\uff1f\u300d<\/strong>\u4ee3\u8868\u7576\u524d\u4f4d\u7f6e\u8a5e\u5f59\u7684\u6aa2\u7d22\u8a0a\u865f\u3002<\/li>\n    <li><strong>Key (K)\uff1a\u300c\u6211\u662f\u4ec0\u9ebc\u7279\u5fb5\u6a19\u7c64\uff1f\u300d<\/strong>\u4ee3\u8868\u6240\u6709\u8a5e\u5f59\u7684\u7d22\u5f15\u6a19\u7c64\u3002<\/li>\n    <li><strong>Value (V)\uff1a\u300c\u6211\u5305\u542b\u7684\u771f\u5be6\u5167\u5bb9\u300d<\/strong>\u4ee3\u8868\u52a0\u6b0a\u5e73\u5747\u5f8c\u805a\u5408\u7684\u8a5e\u5f59\u8a9e\u610f\u3002<\/li>\n  <\/ul>\n\n  <!-- \u4e92\u52d5\u5143\u4ef6 2\uff1a\u81ea\u6ce8\u610f\u529b\u6c99\u76d2\u8207 Hover \u6aa2\u67e5 -->\n  <div class=\"wp-tr-interactive-box\" id=\"wp-tr-attn-app\">\n    <span class=\"wp-tr-header-badge\">\u4e92\u52d5\u81ea\u6ce8\u610f\u529b\u6c99\u76d2 (\u652f\u63f4 Hover \u61f8\u505c\u67e5\u770b\u6578\u503c)<\/span>\n    <div style=\"display: flex; gap: 0.5rem; flex-wrap: wrap; margin-bottom: 0.75rem;\">\n      <button class=\"wp-tr-btn active\" id=\"wp-btn-h1\" onclick=\"wpTrSetHead(1)\">Head 1 (\u8a9e\u6cd5\/\u4e3b\u8a5e)<\/button>\n      <button class=\"wp-tr-btn\" id=\"wp-btn-h2\" onclick=\"wpTrSetHead(2)\">Head 2 (\u4ee3\u540d\u8a5e &#8216;it&#8217; \u6307\u4ee3)<\/button>\n      <button class=\"wp-tr-btn\" id=\"wp-btn-h3\" onclick=\"wpTrSetHead(3)\">Head 3 (\u76f8\u9130\u5c40\u90e8\u8a5e\u7a97)<\/button>\n    <\/div>\n\n    <!-- Controls -->\n    <div style=\"background: #f8fafc; padding: 0.75rem; border-radius: 8px; border: 1px solid #e2e8f0; font-size: 0.82rem; display: flex; gap: 1rem; align-items: center; flex-wrap: wrap;\">\n      <div>\n        <label>\u6eab\u5ea6\u4fc2\u6578 (\u03c4): <strong id=\"wp-temp-val\">1.0<\/strong><\/label>\n        <input type=\"range\" min=\"0.2\" max=\"2.5\" step=\"0.1\" value=\"1.0\" id=\"wp-temp-slider\" oninput=\"wpTrUpdateAttention()\" style=\"vertical-align: middle; margin-left: 6px;\">\n      <\/div>\n      <div>\n        <label><input type=\"checkbox\" id=\"wp-mask-chk\" onchange=\"wpTrUpdateAttention()\"> \u555f\u7528\u56e0\u679c\u4e0b\u4e09\u89d2\u906e\u7f69 (Causal Mask)<\/label>\n      <\/div>\n    <\/div>\n\n    <!-- Table Container -->\n    <div style=\"overflow-x: auto;\">\n      <table class=\"wp-tr-matrix-table\" id=\"wp-attn-matrix\">\n        <!-- \u7531 JavaScript \u52d5\u614b\u7e6a\u88fd\u8207\u76e3\u807d Hover -->\n      <\/table>\n    <\/div>\n\n    <!-- Hover Detail Display Card -->\n    <div style=\"margin-top: 1rem; background: #fffbeb; border: 1px solid #fde68a; border-radius: 8px; padding: 0.75rem; font-size: 0.82rem;\" id=\"wp-hover-info\">\n      <strong>\u6578\u503c\u6aa2\u67e5\uff1a<\/strong>\u8acb\u5c07\u6ed1\u9f20\u79fb\u81f3\u4e0a\u65b9\u4efb\u610f\u683c\u5b50\uff0c\u6216\u9ede\u64ca Token \u67e5\u770b\u5373\u6642\u8a08\u7b97\u4e4b Q\u00b7K^T \u9ede\u7a4d\u5206\u6578\u8207 Softmax \u6b0a\u91cd\uff01\n    <\/div>\n  <\/div>\n\n  <h2>04. \u591a\u982d\u6a5f\u5236\uff1a\u591a\u7dad\u8a9e\u610f\u5b50\u7a7a\u9593\u7684\u806f\u5408\u6295\u5f71<\/h2>\n  <p>\u516c\u5f0f\uff1a<code>MultiHead(Q, K, V) = Concat(head_1, ..., head_h) \u00b7 W^O<\/code><\/p>\n  <p>\u5728\u6a19\u6e96 Transformer \u4e2d\uff0c\u5c07\u7e3d\u7dad\u5ea6 <code>d_model = 512<\/code> \u5747\u5206\u7d66 <code>h = 8<\/code> \u500b\u6ce8\u610f\u529b\u982d\uff0c\u6bcf\u500b\u982d\u7684\u7dad\u5ea6\u70ba <code>d_k = 64<\/code>\u30028 \u500b\u982d\u5404\u81ea\u5728 64 \u7dad\u5b50\u7a7a\u9593\u7368\u7acb\u904b\u7b97\uff0c\u8a08\u7b97\u91cf\u8207\u55ae\u982d\u5b8c\u5168\u4e00\u81f4\uff0c\u537b\u7372\u5f97\u4e86 8 \u500d\u7684\u7279\u5fb5\u8868\u5fb5\u591a\u6a23\u6027\uff01<\/p>\n\n  <h2>05. \u7a7a\u9593\u611f\u77e5\uff1a\u6b63\u5f26\u8207\u9918\u5f26\u4f4d\u7f6e\u7de8\u78bc<\/h2>\n  <p>\u516c\u5f0f\uff1a<code>PE(pos, 2i) = sin(pos \/ 10000^{2i\/d})<\/code>\uff0c<code>PE(pos, 2i+1) = cos(pos \/ 10000^{2i\/d})<\/code><\/p>\n\n  <!-- \u4e92\u52d5\u5143\u4ef6 3\uff1a\u4f4d\u7f6e\u7de8\u78bc\u5e7e\u4f55\u6ce2\u5f62 -->\n  <div class=\"wp-tr-interactive-box\" id=\"wp-tr-pe-app\">\n    <span class=\"wp-tr-header-badge\">\u4f4d\u7f6e\u7de8\u78bc\u6ce2\u5f62\u5373\u6642\u9810\u89bd<\/span>\n    <div style=\"display: flex; gap: 1rem; flex-wrap: wrap; font-size: 0.85rem; margin-bottom: 0.75rem;\">\n      <div>\n        <label>\u89c0\u6e2c\u7dad\u5ea6\uff1a<\/label>\n        <select id=\"wp-pe-dim\" onchange=\"wpTrRenderPE()\" style=\"padding: 2px 8px; border-radius: 4px; border: 1px solid #cbd5e1;\">\n          <option value=\"0\">\u7b2c 0 \u7dad\u5ea6 (Sin &#8211; \u9ad8\u983b\u632f\u76ea)<\/option>\n          <option value=\"4\">\u7b2c 4 \u7dad\u5ea6 (Sin &#8211; \u4e2d\u983b)<\/option>\n          <option value=\"16\">\u7b2c 16 \u7dad\u5ea6 (Sin &#8211; \u4f4e\u983b\u5e73\u7de9)<\/option>\n        <\/select>\n      <\/div>\n      <div>\n        <label>\u5e8f\u5217\u9577\u5ea6\uff1a<span id=\"wp-pe-len-val\">16<\/span> \u8a5e<\/label>\n        <input type=\"range\" min=\"8\" max=\"24\" step=\"2\" value=\"16\" id=\"wp-pe-len\" oninput=\"wpTrRenderPE()\" style=\"vertical-align: middle; margin-left: 6px;\">\n      <\/div>\n    <\/div>\n    <div style=\"background: #ffffff; border: 1px solid #e2e8f0; border-radius: 8px; padding: 10px;\">\n      <svg id=\"wp-pe-svg\" viewBox=\"0 0 320 100\" style=\"width: 100%; height: 120px; overflow: visible;\"><\/svg>\n    <\/div>\n    <p style=\"font-size: 0.8rem; color: #64748b; margin-top: 0.5rem;\">\n      \u6839\u64da\u4e09\u89d2\u548c\u89d2\u516c\u5f0f <code>sin(\u03c9(pos+k)) = sin(\u03c9\u00b7pos)cos(\u03c9\u00b7k) + cos(\u03c9\u00b7pos)sin(\u03c9\u00b7k)<\/code>\uff0c\u76f8\u5c0d\u4f4d\u79fb <code>k<\/code> \u53ef\u7531 2\u00d72 \u7dda\u6027\u65cb\u8f49\u77e9\u9663\u63a8\u5c0e\uff0c\u4f7f\u7db2\u7d61\u8f15\u9b06\u638c\u63e1\u8a5e\u8a9e\u76f8\u5c0d\u8ddd\u96e2\uff01\n    <\/p>\n  <\/div>\n\n  <h2>06. \u9010\u4f4d\u7f6e\u524d\u994b\u795e\u7d93\u7db2\u7d61 (FFN)<\/h2>\n  <p>\u516c\u5f0f\uff1a<code>FFN(x) = max(0, xW_1 + b_1)W_2 + b_2<\/code>\u3002\u96b1\u85cf\u5c64\u5148\u5c07\u7dad\u5ea6\u653e\u5927 4 \u500d\u81f3 2048\uff0c\u7d93\u6fc0\u6d3b\u51fd\u6578\u5f8c\u964d\u56de 512\u3002FFN \u626e\u6f14\u5927\u6a21\u578b\u7684\u4e8b\u5be6\u8a18\u61b6\u5eab\uff0c\u7b2c\u4e00\u5c64\u6fc0\u767c\u6982\u5ff5\u6a21\u5f0f\uff0c\u7b2c\u4e8c\u5c64\u8abf\u7528\u76f8\u95dc\u4e8b\u5be6\u77e5\u8b58\u3002<\/p>\n\n  <h2>07. \u89e3\u78bc\u5967\u79d8\uff1a\u56e0\u679c\u906e\u7f69\u8207 Cross-Attention<\/h2>\n  <ul>\n    <li><strong>\u56e0\u679c\u906e\u7f69 (Causal Mask)\uff1a<\/strong>\u5728\u672a\u4f86\u6642\u9593\u6b65\u586b\u5165 <code>-1e9<\/code>\uff0c\u5f37\u5236\u672a\u4f86\u6b0a\u91cd\u70ba 0\uff0c\u9632\u6b62\u81ea\u56de\u6b78\u751f\u6210\u6642\u5077\u770b\u7b54\u6848\u3002<\/li>\n    <li><strong>\u4ea4\u53c9\u6ce8\u610f\u529b (Cross-Attention)\uff1a<\/strong>Query \u4f86\u81ea\u89e3\u78bc\u5668\uff0cKey \u8207 Value \u4f86\u81ea\u7de8\u78bc\u5668\u9802\u5c64\uff0c\u5efa\u7acb\u8de8\u8a9e\u8a00\/\u8de8\u6a21\u614b\u5c0d\u9f4a\u901a\u9053\u3002<\/li>\n  <\/ul>\n\n  <h2>08. \u73fe\u4ee3 PyTorch \u6838\u5fc3\u7cbe\u7c21\u5be6\u4f5c<\/h2>\n  <div class=\"wp-tr-code-block\">\n    <button class=\"wp-tr-copy-btn\" onclick=\"wpTrCopyPyTorch(this)\">\u8907\u88fd\u4ee3\u78bc<\/button>\n    <div class=\"codecopy-container\">\n                <div class=\"codecopy-header\">\n                    \n                    <button class=\"codecopy-btn\" data-target=\"codecopy-6aeebe092927b\" title=\"Copy code to clipboard\">\n                <svg width=\"16\" height=\"16\" viewBox=\"0 0 24 24\" fill=\"none\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\">\n                    <path d=\"M16 1H4C2.9 1 2 1.9 2 3V17H4V3H16V1ZM19 5H8C6.9 5 6 5.9 6 7V21C6 22.1 6.9 23 8 23H19C20.1 23 21 22.1 21 21V7C21 5.9 20.1 5 19 5ZM19 21H8V7H19V21Z\" fill=\"currentColor\"\/>\n                <\/svg>\n                <span class=\"codecopy-text\">Copy<\/span>\n            <\/button>\n                <\/div>\n                <pre class=\"line-numbers\" id=\"codecopy-6aeebe092927b\"><code class=\"language-text\">import math\nimport torch\nimport torch.nn as nn\n\nclass ScaledDotProductAttention(nn.Module):\n    def __init__(self, temperature: float = 1.0):\n        super().__init__()\n        self.temperature = temperature\n        self.softmax = nn.Softmax(dim=-1)\n\n    def forward(self, q, k, v, mask=None):\n        d_k = q.size(-1)\n        scores = torch.matmul(q, k.transpose(-2, -1)) \/ (math.sqrt(d_k) * self.temperature)\n        if mask is not None:\n            scores = scores.masked_fill(mask == 0, -1e9)\n        attn_weights = self.softmax(scores)\n        return torch.matmul(attn_weights, v), attn_weights\n\nclass MultiHeadAttention(nn.Module):\n    def __init__(self, d_model: int = 512, n_heads: int = 8):\n        super().__init__()\n        self.d_model = d_model\n        self.n_heads = n_heads\n        self.d_k = d_model \/\/ n_heads\n        self.w_q = nn.Linear(d_model, d_model, bias=False)\n        self.w_k = nn.Linear(d_model, d_model, bias=False)\n        self.w_v = nn.Linear(d_model, d_model, bias=False)\n        self.attention = ScaledDotProductAttention()\n        self.w_out = nn.Linear(d_model, d_model, bias=False)\n\n    def forward(self, q, k, v, mask=None):\n        B = q.size(0)\n        q_h = self.w_q(q).view(B, -1, self.n_heads, self.d_k).transpose(1, 2)\n        k_h = self.w_k(k).view(B, -1, self.n_heads, self.d_k).transpose(1, 2)\n        v_h = self.w_v(v).view(B, -1, self.n_heads, self.d_k).transpose(1, 2)\n        out, weights = self.attention(q_h, k_h, v_h, mask=mask)\n        concat = out.transpose(1, 2).contiguous().view(B, -1, self.d_model)\n        return self.w_out(concat), weights<\/code><\/pre>\n            <\/div>\n  <\/div>\n\n  <h2>09. \u6a21\u578b\u6f14\u9032\uff1aBERT\u3001GPT \u5230\u73fe\u4ee3 LLM<\/h2>\n  <ul>\n    <li><strong>Encoder-Only (BERT)\uff1a<\/strong>\u96d9\u5411\u5168\u6ce8\u610f\u529b\uff0c\u4e3b\u6253\u8a9e\u610f\u7406\u89e3\u8207\u6aa2\u7d22\u5411\u91cf\u5316\u3002<\/li>\n    <li><strong>Decoder-Only (GPT-4, Claude, DeepSeek)\uff1a<\/strong>\u56e0\u679c\u55ae\u5411\u81ea\u56de\u6b78\uff0c\u7d71\u6cbb\u901a\u7528\u8a9e\u8a00\u751f\u6210\u8207\u63a8\u7406\u3002<\/li>\n    <li><strong>Encoder-Decoder (T5, Whisper)\uff1a<\/strong>\u8de8\u5e8f\u5217\u5c0d\u9f4a\uff0c\u4e3b\u5c0e\u6a5f\u5668\u7ffb\u8b6f\u8207\u8a9e\u97f3\u8f49\u6587\u5b57\u3002<\/li>\n  <\/ul>\n\n  <h2>10. \u6df1\u5ea6 FAQ<\/h2>\n  <p><strong>Q: \u70ba\u4f55\u63a8\u8ad6\u6642\u9700\u8981 KV Cache\uff1f<\/strong><br>\u907f\u514d\u91cd\u8907\u8a08\u7b97\u6b77\u53f2 Key\/Value\uff0c\u5c07\u6bcf\u6b65\u751f\u6210\u8907\u96dc\u5ea6\u5f9e <code>O(n^2)<\/code> \u964d\u70ba <code>O(n)<\/code>\u3002<\/p>\n  <p><strong>Q: \u4ec0\u9ebc\u662f FlashAttention\uff1f<\/strong><br>\u5229\u7528 GPU SRAM\/HBM \u5206\u584a Tiling \u8a08\u7b97\uff0c\u4e0d\u628a\u5de8\u5927\u77e9\u9663\u5beb\u56de\u986f\u5b58\uff0c\u5927\u5e45\u964d\u4f4e\u9577\u6587\u672c\u8a18\u61b6\u9ad4\u958b\u92b7\u3002<\/p>\n<\/div>\n\n<!-- ================= \u7368\u7acb\u4e92\u52d5 JS \u908f\u8f2f (\u5b8c\u5168\u539f\u751f\uff0c\u7121\u5916\u90e8\u4f9d\u8cf4) ================= -->\n<script>\n(function() {\n  \/\/ 1. \u67b6\u69cb\u6aa2\u8996\u5668\u908f\u8f2f\n  const archData = {\n    'add-norm': { title: '\u6b98\u5dee\u9023\u63a5\u8207\u5c64\u6b78\u4e00\u5316 (Add & Norm)', dim: '[Batch, Seq_Len, 512]', desc: '\u4ee5 LayerNorm(x + Sublayer(x)) \u4fdd\u969c\u68af\u5ea6\u7121\u963b\u7919\u7a7f\u900f 6 \u5c64\u7d50\u69cb\uff0c\u675c\u7d55\u68af\u5ea6\u6d88\u5931\u3002' },\n    'ffn': { title: '\u9010\u4f4d\u7f6e\u524d\u994b\u7db2\u7d61 (FFN)', dim: '512 \u2192 2048 \u2192 512', desc: '\u5169\u5c64\u7dda\u6027\u6295\u5f71\u642d\u914d\u6fc0\u6d3b\u51fd\u6578\uff0c\u626e\u6f14\u5927\u6a21\u578b\u7684\u4e8b\u5be6\u8a18\u61b6\u5132\u5b58\u5eab\u3002' },\n    'enc-mha': { title: '\u7de8\u78bc\u5668\u591a\u982d\u81ea\u6ce8\u610f\u529b (Multi-Head Attention)', dim: '[Batch, Seq_Len, 512]', desc: '8 \u500b 64 \u7dad\u982d\u90e8\u540c\u6642\u6295\u5f71\uff0c\u6355\u6349\u4e3b\u8b02\u8a9e\u6cd5\u3001\u5171\u6307\u95dc\u806f\u7b49\u591a\u91cd\u8a9e\u7fa9\u3002' },\n    'pos-enc': { title: '\u4f4d\u7f6e\u7de8\u78bc (Positional Encoding)', dim: '[Seq_Len, 512]', desc: '\u5229\u7528 sin\/cos \u5e7e\u4f55\u983b\u7387\u6ce8\u5165\u7cbe\u78ba\u7d55\u5c0d\u8207\u76f8\u5c0d\u8a5e\u5e8f\u5ea7\u6a19\u3002' },\n    'input-emb': { title: '\u8f38\u5165\u8a5e\u5d4c\u5165\u5c64 (Input Embedding)', dim: '[Batch, Seq] \u2192 [Batch, Seq, 512]', desc: '\u96e2\u6563\u8a5e\u7d22\u5f15\u8f49\u7a20\u5bc6\u5411\u91cf\uff0c\u4e58\u4ee5 \u221ad_model \u5e73\u8861\u6578\u503c\u5c3a\u5ea6\u3002' },\n    'linear-softmax': { title: '\u7dda\u6027\u8f38\u51fa\u8207 Softmax (Output Probabilities)', dim: '512 \u2192 Vocab_Size (37,000)', desc: '\u5c07\u89e3\u78bc\u5411\u91cf\u6295\u5f71\u56de\u8a5e\u8868\uff0c\u9810\u6e2c\u4e0b\u4e00\u500b Token \u7684\u6a5f\u7387\u5206\u4f48\u3002' },\n    'dec-cross': { title: '\u4ea4\u53c9\u6ce8\u610f\u529b (Cross-Attention)', dim: 'Query\u4f86\u81ea\u89e3\u78bc\u5668\uff0cKey\/Value\u4f86\u81ea\u7de8\u78bc\u5668', desc: '\u9023\u63a5\u4f86\u6e90\u8a9e\u53e5\u8207\u76ee\u6a19\u8a9e\u53e5\uff0c\u5be6\u73fe\u8de8\u8a9e\u8a00\u8207\u8de8\u6a21\u614b\u7cbe\u78ba\u5c0d\u9f4a\u3002' },\n    'dec-masked': { title: '\u89e3\u78bc\u5668\u56e0\u679c\u906e\u7f69\u81ea\u6ce8\u610f\u529b (Masked MHA)', dim: '[Batch, Target_Len, 512]', desc: '\u5728\u672a\u4f86\u4f4d\u7f6e\u586b\u5165 -\u221e\uff0c\u9632\u6b62\u89e3\u78bc\u5668\u5728\u81ea\u56de\u6b78\u6642\u5077\u770b\u672a\u4f86\u7b54\u6848\u3002' }\n  };\n\n  window.wpTrSelectArch = function(key) {\n    const data = archData[key];\n    if (!data) return;\n    document.getElementById('wp-arch-title').textContent = data.title;\n    document.getElementById('wp-arch-dim').textContent = '\u5f35\u91cf\u7dad\u5ea6\uff1a' + data.dim;\n    document.getElementById('wp-arch-desc').textContent = data.desc;\n  };\n\n  \/\/ 2. \u81ea\u6ce8\u610f\u529b\u6c99\u76d2\u908f\u8f2f\n  const tokens = ['The', 'animal', \"didn't\", 'cross', 'street', 'because', 'it', 'was', 'too', 'tired'];\n  let currentHead = 1;\n\n  window.wpTrSetHead = function(h) {\n    currentHead = h;\n    document.querySelectorAll('#wp-tr-attn-app .wp-tr-btn').forEach((b, idx) => {\n      b.classList.toggle('active', idx + 1 === h);\n    });\n    wpTrUpdateAttention();\n  };\n\n  window.wpTrUpdateAttention = function() {\n    const temp = parseFloat(document.getElementById('wp-temp-slider').value);\n    document.getElementById('wp-temp-val').textContent = temp.toFixed(1);\n    const isMasked = document.getElementById('wp-mask-chk').checked;\n\n    const table = document.getElementById('wp-attn-matrix');\n    let html = '<thead><tr><th>Q \\ K<\/th>';\n    tokens.forEach(t => { html += '<th>' + t + '<\/th>'; });\n    html += '<\/tr><\/thead><tbody>';\n\n    for (let i = 0; i < tokens.length; i++) {\n      html += '<tr><th>' + tokens[i] + '<\/th>';\n      \/\/ \u6a21\u64ec\u9ede\u7a4d\u5f97\u5206\n      let scores = [];\n      for (let j = 0; j < tokens.length; j++) {\n        let score = Math.sin((i + 1) * (j + 1) * 0.3) * 1.5;\n        if (currentHead === 2 &#038;&#038; tokens[i] === 'it') {\n          if (tokens[j] === 'animal') score += 4.5;\n          if (tokens[j] === 'tired') score += 2.5;\n        }\n        if (isMasked &#038;&#038; j > i) score = -999;\n        scores.push(score \/ temp);\n      }\n      \/\/ Softmax\n      const maxS = Math.max(...scores);\n      const exps = scores.map(s => Math.exp(s - maxS));\n      const sum = exps.reduce((a, b) => a + b, 0);\n      const weights = exps.map(e => e \/ sum);\n\n      for (let j = 0; j < tokens.length; j++) {\n        const w = weights[j];\n        const pct = (w * 100).toFixed(0) + '%';\n        const masked = isMasked &#038;&#038; j > i;\n        const bg = masked ? '#f1f5f9' : 'rgba(245, 158, 11, ' + Math.max(w, 0.05) + ')';\n        const fg = w > 0.4 ? '#ffffff' : '#1f2937';\n        html += '<td onmouseenter=\"wpTrHoverCell(' + i + ',' + j + ',' + w + ')\" style=\"background:' + bg + '; color:' + fg + ';\">' + (masked ? '0' : pct) + '<\/td>';\n      }\n      html += '<\/tr>';\n    }\n    html += '<\/tbody>';\n    table.innerHTML = html;\n  };\n\n  window.wpTrHoverCell = function(q, k, weight) {\n    const info = document.getElementById('wp-hover-info');\n    info.innerHTML = '<strong>\u6578\u503c\u6aa2\u67e5\uff1a<\/strong>Query: <code>[' + tokens[q] + ']<\/code> \u00b7 Key: <code>[' + tokens[k] + ']<\/code> \u00b7 \u6ce8\u610f\u529b\u5206\u914d\u6a5f\u7387\uff1a<strong style=\"color:#d97706; font-size:1.05em;\">' + (weight * 100).toFixed(2) + '%<\/strong>';\n  };\n\n  \/\/ 3. \u4f4d\u7f6e\u7de8\u78bc\u6ce2\u5f62\u5716\n  window.wpTrRenderPE = function() {\n    const dim = parseInt(document.getElementById('wp-pe-dim').value);\n    const len = parseInt(document.getElementById('wp-pe-len').value);\n    document.getElementById('wp-pe-len-val').textContent = len;\n\n    const svg = document.getElementById('wp-pe-svg');\n    const divTerm = Math.pow(10000, dim \/ 32);\n    let points = [];\n\n    for (let pos = 0; pos < len; pos++) {\n      const val = Math.sin(pos \/ divTerm);\n      const x = (pos \/ (len - 1)) * 300 + 10;\n      const y = 50 - val * 40;\n      points.push({ x, y, pos, val });\n    }\n\n    let svgHtml = '<line x1=\"10\" y1=\"50\" x2=\"310\" y2=\"50\" stroke=\"#cbd5e1\" stroke-dasharray=\"4\"\/>';\n    svgHtml += '<polyline fill=\"none\" stroke=\"#f59e0b\" stroke-width=\"2.5\" stroke-linecap=\"round\" points=\"' + points.map(p => p.x + ',' + p.y).join(' ') + '\"\/>';\n    points.forEach(p => {\n      svgHtml += '<circle cx=\"' + p.x + '\" cy=\"' + p.y + '\" r=\"3\" fill=\"#4f46e5\"><title>\u4f4d\u7f6e ' + p.pos + ': ' + p.val.toFixed(3) + '<\/title><\/circle>';\n    });\n    svg.innerHTML = svgHtml;\n  };\n\n  \/\/ 4. \u4ee3\u78bc\u8907\u88fd\n  window.wpTrCopyPyTorch = function(btn) {\n    const code = btn.nextElementSibling.textContent;\n    navigator.clipboard.writeText(code).then(() => {\n      btn.textContent = '\u5df2\u8907\u88fd\uff01';\n      setTimeout(() => { btn.textContent = '\u8907\u88fd\u4ee3\u78bc'; }, 2000);\n    });\n  };\n\n  \/\/ \u521d\u59cb\u5316\n  wpTrUpdateAttention();\n  wpTrRenderPE();\n})();\n<\/script>\n","protected":false},"excerpt":{"rendered":"<br \/>\n<blockquote><p>\n    \u5728 2017 \u5e74\u4e4b\u524d\uff0c\u5e8f\u5217\u5efa\u6a21\u9577\u671f\u88ab RNN\u3001LSTM \u8207 GRU \u7684\u905e\u8ff4\u6642\u5e8f\u93c8\u7262\u7262\u7d71\u6cbb\uff0c\u4f46\u5176\u5faa\u5e8f\u8a08\u7b97\u672c\u8cea\u5e36\u4f86\u4e86\u7121\u6cd5\u4e26\u884c\u7684\u8a13\u7df4\u5929\u82b1\u677f\u8207\u9577\u8ddd\u96e2\u8a18\u61b6\u8870\u6e1b\u96e3\u984c\u3002Google \u5718\u968a\u5728\u8ad6\u6587\u300aAttention Is All You Need\u300b\u4e2d\u5927\u81bd\u63d0\u51fa\u4e86\u300c\u5b8c\u5168\u6368\u68c4\u5faa\u74b0\u8207\u5377\u7a4d\u7d50\u69cb\uff0c\u50c5\u4f9d\u9760\u6ce8\u610f\u529b\u6a5f\u5236\u9023\u7d50\u5168\u5c40\u300d\u7684\u5168\u65b0\u67b6\u69cb\u2014\u2014<strong>Transformer<\/strong>\u3002\u672c\u7bc7\u6559\u5b78\u70ba\u60a8\u62c6\u89e3\u81ea\u6ce8\u610f\u529b\u3001\u591a\u982d\u6295\u5f71\u3001\u4f4d\u7f6e\u7de8\u78bc\u8207 KV Cache \u7b49\u6838\u5fc3\u5967\u79d8\u3002\n  <\/p><\/blockquote>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"wpai_generated_summary":"","wpai_meta_description":"","footnotes":""},"categories":[27],"tags":[70],"class_list":["post-3177","post","type-post","status-publish","format-standard","hentry","category-paper","tag-transformer"],"_links":{"self":[{"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/posts\/3177","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/comments?post=3177"}],"version-history":[{"count":7,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/posts\/3177\/revisions"}],"predecessor-version":[{"id":13887,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/posts\/3177\/revisions\/13887"}],"wp:attachment":[{"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/media?parent=3177"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/categories?post=3177"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/infernews.com\/blog\/wp-json\/wp\/v2\/tags?post=3177"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}